From 77257a18dabb758b794011f6222580a3f71512a1 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Tue, 5 Feb 2019 18:46:46 -0500 Subject: [PATCH] Add build scripts and documentation for building releases Adds builds scripts that are used to build releases deterministically. Also adds documentation that explains the release process and what the build scripts do. --- .python-version | 2 +- contrib/README.md | 21 +++++++++++++ contrib/build.Dockerfile | 48 ++++++++++++++++++++++++++++ contrib/build_bin.sh | 26 +++++++++++++++ contrib/build_dist.sh | 15 +++++++++ contrib/build_wine.sh | 54 ++++++++++++++++++++++++++++++++ contrib/generate_setup.sh | 32 +++++++++++++++++++ contrib/reproducible-python.diff | 13 ++++++++ docs/release-process.md | 51 ++++++++++++++++++++++++++++++ 9 files changed, 261 insertions(+), 1 deletion(-) create mode 100644 contrib/README.md create mode 100644 contrib/build.Dockerfile create mode 100755 contrib/build_bin.sh create mode 100755 contrib/build_dist.sh create mode 100755 contrib/build_wine.sh create mode 100755 contrib/generate_setup.sh create mode 100644 contrib/reproducible-python.diff create mode 100644 docs/release-process.md diff --git a/.python-version b/.python-version index c492825..424e179 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.5.6 +3.6.8 diff --git a/contrib/README.md b/contrib/README.md new file mode 100644 index 0000000..f484e2d --- /dev/null +++ b/contrib/README.md @@ -0,0 +1,21 @@ +# Assorted tools + +## `build_bin.sh` + +Creates a virtualenv with the locked dependencies using Poetry. Then uses pyinstaller to create a standalone binary for the OS type currently running. + +## `build_dist.sh` + +Creates a virtualenv with the locked dependencies using Poetry. Then uses Poetry to produce deterministic builds of the wheel and sdist for upload to PyPi + +`faketime` needs to be installed + +## `build_wine.sh` + +Sets up Wine with Python and everything needed to build Windows binaries. Creates a virtualenv with the locked dependencies using Poetry. Then uses pyinstaller to create a standalone Windows binary. + +`wine` needs to be installed + +## `generate_setup.sh` + +Builds the source distribution and extracts the setup.py from it. diff --git a/contrib/build.Dockerfile b/contrib/build.Dockerfile new file mode 100644 index 0000000..d588ed1 --- /dev/null +++ b/contrib/build.Dockerfile @@ -0,0 +1,48 @@ +FROM debian:stretch-slim + +SHELL ["/bin/bash", "-c"] + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update +RUN apt-get install -y \ + apt-transport-https \ + git \ + make \ + build-essential \ + libssl-dev \ + zlib1g-dev \ + libbz2-dev \ + libreadline-dev \ + libsqlite3-dev \ + wget \ + curl \ + llvm \ + libncurses5-dev \ + xz-utils \ + libxml2-dev \ + libxmlsec1-dev \ + libffi-dev \ + liblzma-dev \ + libusb-1.0-0-dev \ + libudev-dev \ + faketime + +RUN curl https://pyenv.run | bash +ENV PATH="/root/.pyenv/bin:$PATH" +COPY contrib/reproducible-python.diff /opt/reproducible-python.diff +ENV PYTHON_CONFIGURE_OPTS="--enable-shared" +ENV BUILD_DATE="Jan 1 2019" +ENV BUILD_TIME="00:00:00" +RUN eval "$(pyenv init -)" && eval "$(pyenv virtualenv-init -)" && cat /opt/reproducible-python.diff | pyenv install -kp 3.6.8 + +RUN dpkg --add-architecture i386 +RUN wget -nc https://dl.winehq.org/wine-builds/winehq.key +RUN apt-key add winehq.key +RUN echo "deb https://dl.winehq.org/wine-builds/debian/ stretch main" >> /etc/apt/sources.list +RUN apt-get update +RUN apt-get install --install-recommends -y \ + wine-stable-amd64 \ + wine-stable-i386 \ + wine-stable \ + winehq-stable \ + p7zip-full diff --git a/contrib/build_bin.sh b/contrib/build_bin.sh new file mode 100755 index 0000000..151f5b9 --- /dev/null +++ b/contrib/build_bin.sh @@ -0,0 +1,26 @@ +#! /bin/bash +# Script for building standalone binary releases deterministically + +eval "$(pyenv init -)" +eval "$(pyenv virtualenv-init -)" +pip install -U pip +pip install poetry + +# Setup poetry and install the dependencies +poetry install + +# We now need to remove debugging symbols and build id from the hidapi SO file +so_dir=`dirname $(dirname $(poetry run which python))`/lib/python3.6/site-packages +find ${so_dir} -name '*.so' -type f -execdir strip '{}' \; +if [[ $OSTYPE != *"darwin"* ]]; then + find ${so_dir} -name '*.so' -type f -execdir strip -R .note.gnu.build-id '{}' \; +fi + +# We also need to change the timestamps of all of the base library files +lib_dir=`pyenv root`/versions/3.6.8/lib/python3.6 +TZ=UTC find ${lib_dir} -name '*.py' -type f -execdir touch -t "201901010000.00" '{}' \; + +# Make the standalone binary +export PYTHONHASHSEED=42 +poetry run pyinstaller hwi.spec +unset PYTHONHASHSEED diff --git a/contrib/build_dist.sh b/contrib/build_dist.sh new file mode 100755 index 0000000..b6b7d6b --- /dev/null +++ b/contrib/build_dist.sh @@ -0,0 +1,15 @@ +#! /bin/bash +# Script for building pypi distribution archives deterministically + +eval "$(pyenv init -)" +eval "$(pyenv virtualenv-init -)" +pip install -U pip +pip install poetry + +# Setup poetry and install the dependencies +poetry install + +# Make the distribution archives for pypi +poetry build -f wheel +# faketime is needed to make sdist detereministic +TZ=UTC faketime -f "2019-01-01 00:00:00" poetry build -f sdist diff --git a/contrib/build_wine.sh b/contrib/build_wine.sh new file mode 100755 index 0000000..3a92d87 --- /dev/null +++ b/contrib/build_wine.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# Script which sets up Wine and builds the Windows standalone binary + +set -e + +PYTHON_VERSION=3.6.8 + +PYTHON_FOLDER="python3" +PYHOME="c:/$PYTHON_FOLDER" +PYTHON="wine $PYHOME/python.exe -OO -B" + +LIBUSB_URL=https://github.com/libusb/libusb/releases/download/v1.0.22/libusb-1.0.22.7z +LIBUSB_HASH="671f1a420757b4480e7fadc8313d6fb3cbb75ca00934c417c1efa6e77fb8779b" + +wine 'wineboot' + +# Install Python +# Get the PGP keys +wget -N -c "https://www.python.org/static/files/pubkeys.txt" +gpg --import pubkeys.txt +rm pubkeys.txt + +# Install python components +for msifile in core dev exe lib pip tools; do + wget -N -c "https://www.python.org/ftp/python/$PYTHON_VERSION/amd64/${msifile}.msi" + wget -N -c "https://www.python.org/ftp/python/$PYTHON_VERSION/amd64/${msifile}.msi.asc" + gpg --verify "${msifile}.msi.asc" "${msifile}.msi" + wine msiexec /i "${msifile}.msi" /qb TARGETDIR=$PYHOME + rm $msifile.msi* +done + +# Get libusb +wget -N -c -O libusb.7z "$LIBUSB_URL" +echo "$LIBUSB_HASH libusb.7z" | sha256sum -c +7za x -olibusb libusb.7z -aoa +cp libusb/MS64/dll/libusb-1.0.dll ~/.wine/drive_c/python3/ +rm -r libusb* + +# Update pip +$PYTHON -m pip install -U pip + +# Install Poetry and things needed for pyinstaller +$PYTHON -m pip install poetry + +# We also need to change the timestamps of all of the base library files +lib_dir=~/.wine/drive_c/python3/Lib +TZ=UTC find ${lib_dir} -name '*.py' -type f -execdir touch -t "201901010000.00" '{}' \; + +# Do the build +POETRY="wine $PYHOME/Scripts/poetry.exe" +$POETRY install -E windist +export PYTHONHASHSEED=42 +$POETRY run pyinstaller hwi.spec +unset PYTHONHASHSEED diff --git a/contrib/generate_setup.sh b/contrib/generate_setup.sh new file mode 100755 index 0000000..8786487 --- /dev/null +++ b/contrib/generate_setup.sh @@ -0,0 +1,32 @@ +#! /bin/bash +# Generates the setup.py file + +set -e + +# Setup poetry and install the dependencies +poetry install + +# Build the source distribution +poetry build -f sdist + +# Extract setup.py from the distribution +unset -v tarball +for file in dist/* +do + if [[ $file -nt $tarball && $file == *".tar.gz" ]] + then + tarball=$file + fi +done +unset -v toextract +for file in `tar -tf $tarball` +do + if [[ $file == *"setup.py" ]] + then + toextract=$file + fi +done +tar -xf $tarball $toextract +mv $toextract . +dir=`echo $toextract | cut -f1 -d"/"` +rm -r $dir diff --git a/contrib/reproducible-python.diff b/contrib/reproducible-python.diff new file mode 100644 index 0000000..13721e0 --- /dev/null +++ b/contrib/reproducible-python.diff @@ -0,0 +1,13 @@ +# DP: Build getbuildinfo.o with DATE/TIME values when defined + +--- Makefile.pre.in ++++ Makefile.pre.in +@@ -741,6 +741,8 @@ Modules/getbuildinfo.o: $(PARSER_OBJS) \ + -DGITVERSION="\"`LC_ALL=C $(GITVERSION)`\"" \ + -DGITTAG="\"`LC_ALL=C $(GITTAG)`\"" \ + -DGITBRANCH="\"`LC_ALL=C $(GITBRANCH)`\"" \ ++ $(if $(BUILD_DATE),-DDATE='"$(BUILD_DATE)"') \ ++ $(if $(BUILD_TIME),-DTIME='"$(BUILD_TIME)"') \ + -o $@ $(srcdir)/Modules/getbuildinfo.c + + Modules/getpath.o: $(srcdir)/Modules/getpath.c Makefile diff --git a/docs/release-process.md b/docs/release-process.md new file mode 100644 index 0000000..62af0e6 --- /dev/null +++ b/docs/release-process.md @@ -0,0 +1,51 @@ +# Release Process + +1. Bump version number in `pyproject.toml`, generate the setup.py file, and git tag release +2. Build distribution archives for PyPi with `contrib/build_dist.sh` +3. For MacOS and Linux, use `contrib/build_bin.sh`. This needs to be run on a MacOS machine for the MacOS binary and on a Linux machine for the linux one. +4. For Windows, use `contrib/build_wine.sh` to build the Windows binary using wine +5. Upload distribution archives to PyPi +6. Upload distribution archives and standalone binaries to Github + +## Deterministic builds with Docker + +Create the docker image: + +``` +docker build --no-cache -t hwi-builder -f contrib/build.Dockerfile . +``` + +Build everything + +``` +docker run -it --name hwi-builder -v $PWD:/opt/hwi --rm --workdir /opt/hwi hwi-builder /bin/bash -c "contrib/build_bin.sh && contrib/build_dist.sh && contrib/build_wine.sh" +``` + +## Building macOS binary + +Note that the macOS build is non-deterministic. + +First install [pyenv](https://github.com/pyenv/pyenv) using whichever method you prefer. + +Then a deterministic build of Python 3.6.8 needs to be installed. This can be done with the patch in `contrib/reproducible-python.diff`. First `cd` into HWI's source tree. Then use: + +``` +cat contrib/reproducible-python.diff | PYTHON_CONFIGURE_OPTS="--enable-framework" BUILD_DATE="Jan 1 2019" BUILD_TIME="00:00:00" pyenv install -kp 3.6.8 +``` + +Make sure that python 3.6.8 is active + +``` +$ python --version +Python 3.6.8 +``` + +Now install [Poetry](https://github.com/sdispater/poetry) with `pip install poetry` + +Additional dependencies can be installed with: + +``` +brew install libusb +``` + +Build the binaries by using `contrib/build_bin.sh`.