From 1f3a01976b003630cf488e611a27b9d7272e1d03 Mon Sep 17 00:00:00 2001 From: Tim Yuen Date: Fri, 9 Sep 2022 21:32:13 -0400 Subject: [PATCH 1/6] add build workflow for windows, mac, and linux --- .github/workflows/create-binaries.yaml | 39 ++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/create-binaries.yaml diff --git a/.github/workflows/create-binaries.yaml b/.github/workflows/create-binaries.yaml new file mode 100644 index 0000000..352f345 --- /dev/null +++ b/.github/workflows/create-binaries.yaml @@ -0,0 +1,39 @@ +name: Package using PyInstaller +on: + push: + tags: + - v** + workflow_dispatch: + +env: + dist-folder-path: dist/OpenSeeFace + +jobs: + build-all: + strategy: + fail-fast: false + matrix: + os: [windows-latest, ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - name: Pull OpenSeeFace + uses: actions/checkout@v3 + + - name: Create binary + shell: bash + run: | + chmod +x make_exe.sh + ./make_exe.sh + + - name: Move and zip files + shell: bash # Using bash for all OSs should be fine, even on Windows + run: | + mv dist/facetracker ${{ env.dist-folder-path }} + cp -r models ${{ env.dist-folder-path }} + + - name: Upload artifact + uses: actions/upload-artifact@v3 + with: + name: ${{ matrix.os }} + path: ${{ env.dist-folder-path }} + if-no-files-found: error From d6062c9664554789fe699f697384a868ede9354e Mon Sep 17 00:00:00 2001 From: Tim Yuen Date: Sat, 10 Sep 2022 16:55:47 -0400 Subject: [PATCH 2/6] fix actions/upload-artifact generating tar bombs, change how make_exe.bat works --- .github/workflows/create-binaries.yaml | 24 +++++++++++++++++++++--- make_exe.bat | 8 ++++---- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/.github/workflows/create-binaries.yaml b/.github/workflows/create-binaries.yaml index 352f345..a705d7b 100644 --- a/.github/workflows/create-binaries.yaml +++ b/.github/workflows/create-binaries.yaml @@ -6,7 +6,8 @@ on: workflow_dispatch: env: - dist-folder-path: dist/OpenSeeFace + dist-folder-path: dist/OpenSeeFace # The directory generated by PyInstaller + artifact-path: OpenSeeFace # Needed since upload-artifacts pulls files out of parent directories jobs: build-all: @@ -19,21 +20,38 @@ jobs: - name: Pull OpenSeeFace uses: actions/checkout@v3 - - name: Create binary + - name: Create binary Unix + if: matrix.os != 'windows-latest' shell: bash run: | chmod +x make_exe.sh ./make_exe.sh + - name: Create binary Windows + if: matrix.os == 'windows-latest' + shell: cmd + run: | + choco install vcredist140 + + xcopy c:\Windows\System32\msvcp140.dll . + xcopy c:\Windows\System32\vcomp140.dll . + xcopy c:\Windows\System32\concrt140.dll . + xcopy c:\Windows\System32\vccorlib140.dll . + + make_exe.bat + - name: Move and zip files shell: bash # Using bash for all OSs should be fine, even on Windows run: | mv dist/facetracker ${{ env.dist-folder-path }} cp -r models ${{ env.dist-folder-path }} + mkdir ${{ env.artifact-path }} + mv ${{ env.dist-folder-path }} ${{ env.artifact-path }} + - name: Upload artifact uses: actions/upload-artifact@v3 with: name: ${{ matrix.os }} - path: ${{ env.dist-folder-path }} + path: ${{ env.artifact-path }} if-no-files-found: error diff --git a/make_exe.bat b/make_exe.bat index a5ad490..36c6f62 100644 --- a/make_exe.bat +++ b/make_exe.bat @@ -1,4 +1,6 @@ -pyinstaller facetracker.spec --onedir ^ +pip install onnxruntime opencv-python==4.5.4.60 pillow numpy pyinstaller + +pyinstaller facetracker.py --onedir ^ --add-binary dshowcapture/dshowcapture_x86.dll;. ^ --add-binary dshowcapture/dshowcapture_x64.dll;. ^ --add-binary dshowcapture/libminibmcapture32.dll;. ^ @@ -11,6 +13,4 @@ pyinstaller facetracker.spec --onedir ^ --add-binary vccorlib140.dll;. ^ --add-binary run.bat;. -del dist\facetracker\cv2\opencv_videoio_ffmpeg420_64.dll - -pause \ No newline at end of file +del dist\facetracker\cv2\opencv_videoio_* \ No newline at end of file From 022a0520fa7624c5225c27b5343c4d656080141b Mon Sep 17 00:00:00 2001 From: Tim Yuen Date: Sat, 10 Sep 2022 23:24:22 -0400 Subject: [PATCH 3/6] Update make_exe.bat to use a venv, explicitly add onnxruntime dll to pyinstaller binary, add debug prints during build --- .github/workflows/create-binaries.yaml | 2 +- make_exe.bat | 28 +++++++++++++++++++------- make_exe.sh | 8 +++++--- 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/.github/workflows/create-binaries.yaml b/.github/workflows/create-binaries.yaml index a705d7b..d799714 100644 --- a/.github/workflows/create-binaries.yaml +++ b/.github/workflows/create-binaries.yaml @@ -40,7 +40,7 @@ jobs: make_exe.bat - - name: Move and zip files + - name: Move files for uploading shell: bash # Using bash for all OSs should be fine, even on Windows run: | mv dist/facetracker ${{ env.dist-folder-path }} diff --git a/make_exe.bat b/make_exe.bat index 36c6f62..8c55e0b 100644 --- a/make_exe.bat +++ b/make_exe.bat @@ -1,16 +1,30 @@ +echo "Started" + +:: A venv is used so it's easy to add the onnxruntime dll to the binary + +echo "Creating venv" +python -m venv venv + +echo "Activating venv" +call venv\Scripts\activate.bat + +echo "Installing dependencies" +pip install wheel + pip install onnxruntime opencv-python==4.5.4.60 pillow numpy pyinstaller +echo "Running pyinstaller" pyinstaller facetracker.py --onedir ^ - --add-binary dshowcapture/dshowcapture_x86.dll;. ^ - --add-binary dshowcapture/dshowcapture_x64.dll;. ^ - --add-binary dshowcapture/libminibmcapture32.dll;. ^ - --add-binary dshowcapture/libminibmcapture64.dll;. ^ - --add-binary escapi/escapi_x86.dll;. ^ - --add-binary escapi/escapi_x64.dll;. ^ + --add-binary dshowcapture/*.dll;. ^ + --add-binary escapi/*.dll;. ^ + --add-binary venv/lib/site-packages/onnxruntime/capi/*.dll;onnxruntime\capi ^ --add-binary msvcp140.dll;. ^ --add-binary vcomp140.dll;. ^ --add-binary concrt140.dll;. ^ --add-binary vccorlib140.dll;. ^ --add-binary run.bat;. -del dist\facetracker\cv2\opencv_videoio_* \ No newline at end of file +echo "Deleting opencv dll" +del dist\facetracker\cv2\opencv_videoio_* + +echo "Finished" diff --git a/make_exe.sh b/make_exe.sh index dd2fd70..834d486 100755 --- a/make_exe.sh +++ b/make_exe.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -e + PYTHON_BINARY=$(which python3) if [[ -z "$PYTHON_BINARY" ]]; then @@ -12,18 +14,18 @@ if [[ -z "$PYTHON_BINARY" ]]; then fi echo "Creating venv" - "$PYTHON_BINARY" -m venv venv source venv/bin/activate echo "Installing packages" +pip install wheel # Make sure this is installed beforehand pip install onnxruntime opencv-python==4.5.4.60 pillow numpy pyinstaller echo "Creating binary" - -pyinstaller --clean facetracker.py +pyinstaller --onedir --clean facetracker.py \ + --add-binary venv/lib/python3.*/site-packages/onnxruntime/capi/*.so:onnxruntime/capi echo "Files should be available in the dist/ folder" echo "Done!" From 0443549277386f720c8cd14936c7bea00f895122 Mon Sep 17 00:00:00 2001 From: Tim Yuen Date: Sun, 11 Sep 2022 13:11:03 -0400 Subject: [PATCH 4/6] Fix unix builds not having their executable bit set in ci, make windows build script perform a clean build every time --- .github/workflows/create-binaries.yaml | 6 ++++++ make_exe.bat | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/create-binaries.yaml b/.github/workflows/create-binaries.yaml index d799714..d2214fa 100644 --- a/.github/workflows/create-binaries.yaml +++ b/.github/workflows/create-binaries.yaml @@ -49,6 +49,12 @@ jobs: mkdir ${{ env.artifact-path }} mv ${{ env.dist-folder-path }} ${{ env.artifact-path }} + # Sometimes the executable bit doesn't get set for Unix builds for some reason + - name: Set executable bit + if: matrix.os != 'windows-latest' + shell: bash + run: chmod +x ${{ env.artifact-path }}/* + - name: Upload artifact uses: actions/upload-artifact@v3 with: diff --git a/make_exe.bat b/make_exe.bat index 8c55e0b..a7b6ef8 100644 --- a/make_exe.bat +++ b/make_exe.bat @@ -14,7 +14,8 @@ pip install wheel pip install onnxruntime opencv-python==4.5.4.60 pillow numpy pyinstaller echo "Running pyinstaller" -pyinstaller facetracker.py --onedir ^ +pyinstaller facetracker.py --clean ^ + --onedir ^ --add-binary dshowcapture/*.dll;. ^ --add-binary escapi/*.dll;. ^ --add-binary venv/lib/site-packages/onnxruntime/capi/*.dll;onnxruntime\capi ^ From be48eadbc60ab3534dc4b942f66a43d76aeea882 Mon Sep 17 00:00:00 2001 From: Tim Yuen Date: Sun, 11 Sep 2022 18:42:23 -0400 Subject: [PATCH 5/6] tar files before uploading, properly set execution bit, remove redundant file moving logic, split into windows and unix jobs --- .github/workflows/create-binaries.yaml | 86 ++++++++++++++++---------- 1 file changed, 54 insertions(+), 32 deletions(-) diff --git a/.github/workflows/create-binaries.yaml b/.github/workflows/create-binaries.yaml index d2214fa..02bb4b9 100644 --- a/.github/workflows/create-binaries.yaml +++ b/.github/workflows/create-binaries.yaml @@ -5,30 +5,16 @@ on: - v** workflow_dispatch: -env: - dist-folder-path: dist/OpenSeeFace # The directory generated by PyInstaller - artifact-path: OpenSeeFace # Needed since upload-artifacts pulls files out of parent directories - jobs: - build-all: - strategy: - fail-fast: false - matrix: - os: [windows-latest, ubuntu-latest, macos-latest] - runs-on: ${{ matrix.os }} + build-windows: + env: + dist-path: dist/OpenSeeFace + runs-on: windows-latest steps: - name: Pull OpenSeeFace uses: actions/checkout@v3 - - name: Create binary Unix - if: matrix.os != 'windows-latest' - shell: bash - run: | - chmod +x make_exe.sh - ./make_exe.sh - - - name: Create binary Windows - if: matrix.os == 'windows-latest' + - name: Create binary shell: cmd run: | choco install vcredist140 @@ -41,23 +27,59 @@ jobs: make_exe.bat - name: Move files for uploading - shell: bash # Using bash for all OSs should be fine, even on Windows - run: | - mv dist/facetracker ${{ env.dist-folder-path }} - cp -r models ${{ env.dist-folder-path }} - - mkdir ${{ env.artifact-path }} - mv ${{ env.dist-folder-path }} ${{ env.artifact-path }} - - # Sometimes the executable bit doesn't get set for Unix builds for some reason - - name: Set executable bit - if: matrix.os != 'windows-latest' shell: bash - run: chmod +x ${{ env.artifact-path }}/* + run: | + # Prevents the parent dist/ directory from being included + mv dist/facetracker ${{ env.dist-path }} + cp -r models ${{ env.dist-path }} + + - name: Upload artifact + uses: actions/upload-artifact@v3 + with: + name: windows-latest + path: dist # Add the parent directory to prevent a tarbomb + if-no-files-found: error + + # Files must be packed into a tar in order for file permissions to be preserved + build-unix: + env: + artifact-path: OpenSeeFace + tar-path: OpenSeeFace.tar + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - name: Pull OpenSeeFace + uses: actions/checkout@v3 + + - name: Create binary + shell: bash + run: | + chmod +x make_exe.sh + ./make_exe.sh + + - name: Move files for uploading + shell: bash + run: | + # Prevents the parent dist/ directory from being included + mv dist/facetracker ${{ env.artifact-path }} + cp -r models ${{ env.artifact-path }} + + - name: Fix permissions from PyInstaller + shell: bash + run: | + find ${{ env.artifact-path }} -type f -exec chmod -x {} + + chmod +x ${{ env.artifact-path }}/facetracker + + - name: Pack files into tar + shell: bash + run: tar -cvf ${{ env.tar-path }} ${{ env.artifact-path }} - name: Upload artifact uses: actions/upload-artifact@v3 with: name: ${{ matrix.os }} - path: ${{ env.artifact-path }} + path: ${{ env.tar-path }} if-no-files-found: error From 822e9e77535f3ffebc875368ebaccc27195cb5a9 Mon Sep 17 00:00:00 2001 From: Tim Yuen Date: Mon, 12 Sep 2022 14:57:51 -0400 Subject: [PATCH 6/6] add action to append date to artifact name during upload --- .../actions/upload-dated-artifact/action.yaml | 26 +++++++++++++++++++ .github/workflows/create-binaries.yaml | 16 +++++++----- 2 files changed, 35 insertions(+), 7 deletions(-) create mode 100644 .github/actions/upload-dated-artifact/action.yaml diff --git a/.github/actions/upload-dated-artifact/action.yaml b/.github/actions/upload-dated-artifact/action.yaml new file mode 100644 index 0000000..7fc3fbe --- /dev/null +++ b/.github/actions/upload-dated-artifact/action.yaml @@ -0,0 +1,26 @@ +name: Upload dated artifact +description: Uploads an artifact with a datetime + +inputs: + name: + description: The name of the artifact + required: true + path: + description: The path to the file/folder to upload + required: true + +runs: + using: composite + steps: + # https://stackoverflow.com/questions/60942067/get-current-date-and-time-in-github-workflows + - name: Get current datetime + id: date + shell: bash + run: echo "::set-output name=date::$(date '+%Y-%m-%d_%H-%M-%S')" + + - name: Upload artifact + uses: actions/upload-artifact@v3 + with: + name: ${{ inputs.name }}_${{ steps.date.outputs.date }} + path: ${{ inputs.path }} + if-no-files-found: error diff --git a/.github/workflows/create-binaries.yaml b/.github/workflows/create-binaries.yaml index 02bb4b9..26572bc 100644 --- a/.github/workflows/create-binaries.yaml +++ b/.github/workflows/create-binaries.yaml @@ -34,11 +34,10 @@ jobs: cp -r models ${{ env.dist-path }} - name: Upload artifact - uses: actions/upload-artifact@v3 + uses: ./.github/actions/upload-dated-artifact with: - name: windows-latest + name: windows path: dist # Add the parent directory to prevent a tarbomb - if-no-files-found: error # Files must be packed into a tar in order for file permissions to be preserved build-unix: @@ -48,7 +47,11 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] + include: + - name: linux + os: ubuntu-latest + - name: mac + os: macos-latest runs-on: ${{ matrix.os }} steps: - name: Pull OpenSeeFace @@ -78,8 +81,7 @@ jobs: run: tar -cvf ${{ env.tar-path }} ${{ env.artifact-path }} - name: Upload artifact - uses: actions/upload-artifact@v3 + uses: ./.github/actions/upload-dated-artifact with: - name: ${{ matrix.os }} + name: ${{ matrix.name }} path: ${{ env.tar-path }} - if-no-files-found: error