X-Git-Url: https://jfr.im/git/yt-dlp.git/blobdiff_plain/632b8ee54eb2df8ac6e20746a0bd95b7ebb053aa..5fdd13006a1c5d78642c8d3c4c7df0448273c2ae:/.github/workflows/build.yml

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 036ce4348..9a1a22e8f 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -12,6 +12,9 @@ on:
       unix:
         default: true
         type: boolean
+      linux_static:
+        default: true
+        type: boolean
       linux_arm:
         default: true
         type: boolean
@@ -27,9 +30,6 @@ on:
       windows32:
         default: true
         type: boolean
-      meta_files:
-        default: true
-        type: boolean
       origin:
         required: false
         default: ''
@@ -52,7 +52,11 @@ on:
         default: stable
         type: string
       unix:
-        description: yt-dlp, yt-dlp.tar.gz, yt-dlp_linux, yt-dlp_linux.zip
+        description: yt-dlp, yt-dlp.tar.gz
+        default: true
+        type: boolean
+      linux_static:
+        description: yt-dlp_linux
         default: true
         type: boolean
       linux_arm:
@@ -75,10 +79,6 @@ on:
         description: yt-dlp_x86.exe
         default: true
         type: boolean
-      meta_files:
-        description: SHA2-256SUMS, SHA2-512SUMS, _update_spec
-        default: true
-        type: boolean
       origin:
         description: Origin
         required: false
@@ -107,70 +107,72 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
-      - uses: conda-incubator/setup-miniconda@v2
+          fetch-depth: 0  # Needed for changelog
+      - uses: actions/setup-python@v5
         with:
-          miniforge-variant: Mambaforge
-          use-mamba: true
-          channels: conda-forge
-          auto-update-conda: true
-          activate-environment: ""
-          auto-activate-base: false
+          python-version: "3.10"
       - name: Install Requirements
         run: |
           sudo apt -y install zip pandoc man sed
-          reqs=$(mktemp)
-          cat > "$reqs" << EOF
-          python=3.10.*
-          pyinstaller
-          cffi
-          brotli-python
-          secretstorage
-          EOF
-          sed -E '/^(brotli|secretstorage).*/d' requirements.txt >> "$reqs"
-          mamba create -n build --file "$reqs"
-
       - name: Prepare
         run: |
           python devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
+          python devscripts/update_changelog.py -vv
           python devscripts/make_lazy_extractors.py
       - name: Build Unix platform-independent binary
         run: |
           make all tar
-      - name: Build Unix standalone binary
-        shell: bash -l {0}
-        run: |
-          unset LD_LIBRARY_PATH  # Harmful; set by setup-python
-          conda activate build
-          python pyinst.py --onedir
-          (cd ./dist/yt-dlp_linux && zip -r ../yt-dlp_linux.zip .)
-          python pyinst.py
-          mv ./dist/yt-dlp_linux ./yt-dlp_linux
-          mv ./dist/yt-dlp_linux.zip ./yt-dlp_linux.zip
-
       - name: Verify --update-to
         if: vars.UPDATE_TO_VERIFICATION
         run: |
-          binaries=("yt-dlp" "yt-dlp_linux")
-          for binary in "${binaries[@]}"; do
-            chmod +x ./${binary}
-            cp ./${binary} ./${binary}_downgraded
-            version="$(./${binary} --version)"
-            ./${binary}_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04
-            downgraded_version="$(./${binary}_downgraded --version)"
-            [[ "$version" != "$downgraded_version" ]]
-          done
-
+          chmod +x ./yt-dlp
+          cp ./yt-dlp ./yt-dlp_downgraded
+          version="$(./yt-dlp --version)"
+          ./yt-dlp_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04
+          downgraded_version="$(./yt-dlp_downgraded --version)"
+          [[ "$version" != "$downgraded_version" ]]
       - name: Upload artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
+          name: build-bin-${{ github.job }}
           path: |
             yt-dlp
             yt-dlp.tar.gz
-            yt-dlp_linux
-            yt-dlp_linux.zip
+          compression-level: 0
+
+  linux_static:
+    needs: process
+    if: inputs.linux_static
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build static executable
+        env:
+          channel: ${{ inputs.channel }}
+          origin: ${{ needs.process.outputs.origin }}
+          version: ${{ inputs.version }}
+        run: |
+          mkdir ~/build
+          cd bundle/docker
+          docker compose up --build static
+          sudo chown "${USER}:docker" ~/build/yt-dlp_linux
+      - name: Verify --update-to
+        if: vars.UPDATE_TO_VERIFICATION
+        run: |
+          chmod +x ~/build/yt-dlp_linux
+          cp ~/build/yt-dlp_linux ~/build/yt-dlp_linux_downgraded
+          version="$(~/build/yt-dlp_linux --version)"
+          ~/build/yt-dlp_linux_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04
+          downgraded_version="$(~/build/yt-dlp_linux_downgraded --version)"
+          [[ "$version" != "$downgraded_version" ]]
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: build-bin-${{ github.job }}
+          path: |
+            ~/build/yt-dlp_linux
+          compression-level: 0
 
   linux_arm:
     needs: process
@@ -201,17 +203,18 @@ jobs:
           dockerRunArgs: --volume "${PWD}/repo:/repo"
           install: | # Installing Python 3.10 from the Deadsnakes repo raises errors
             apt update
-            apt -y install zlib1g-dev python3.8 python3.8-dev python3.8-distutils python3-pip
+            apt -y install zlib1g-dev libffi-dev python3.8 python3.8-dev python3.8-distutils python3-pip
             python3.8 -m pip install -U pip setuptools wheel
-            # Cannot access requirements.txt from the repo directory at this stage
-            python3.8 -m pip install -U Pyinstaller mutagen pycryptodomex websockets brotli certifi secretstorage
+            # Cannot access any files from the repo directory at this stage
+            python3.8 -m pip install -U Pyinstaller mutagen pycryptodomex websockets brotli certifi secretstorage cffi
 
           run: |
             cd repo
-            python3.8 -m pip install -U Pyinstaller secretstorage -r requirements.txt  # Cached version may be out of date
+            python3.8 devscripts/install_deps.py -o --include build
+            python3.8 devscripts/install_deps.py --include pyinstaller --include secretstorage  # Cached version may be out of date
             python3.8 devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
             python3.8 devscripts/make_lazy_extractors.py
-            python3.8 pyinst.py
+            python3.8 -m bundle.pyinstaller
 
             if ${{ vars.UPDATE_TO_VERIFICATION && 'true' || 'false' }}; then
               arch="${{ (matrix.architecture == 'armv7' && 'armv7l') || matrix.architecture }}"
@@ -224,15 +227,17 @@ jobs:
             fi
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
+          name: build-bin-linux_${{ matrix.architecture }}
           path: | # run-on-arch-action designates armv7l as armv7
             repo/dist/yt-dlp_linux_${{ (matrix.architecture == 'armv7' && 'armv7l') || matrix.architecture }}
+          compression-level: 0
 
   macos:
     needs: process
     if: inputs.macos
-    runs-on: macos-11
+    runs-on: macos-12
 
     steps:
       - uses: actions/checkout@v4
@@ -240,9 +245,38 @@ jobs:
       - name: Install Requirements
         run: |
           brew install coreutils
-          python3 -m pip install -U --user pip setuptools wheel
+          python3 devscripts/install_deps.py --user -o --include build
+          python3 devscripts/install_deps.py --print --include pyinstaller > requirements.txt
           # We need to ignore wheels otherwise we break universal2 builds
-          python3 -m pip install -U --user --no-binary :all: Pyinstaller -r requirements.txt
+          python3 -m pip install -U --user --no-binary :all: -r requirements.txt
+          # We need to fuse our own universal2 wheels for curl_cffi
+          python3 -m pip install -U --user delocate
+          mkdir curl_cffi_whls curl_cffi_universal2
+          python3 devscripts/install_deps.py --print -o --include curl-cffi > requirements.txt
+          for platform in "macosx_11_0_arm64" "macosx_11_0_x86_64"; do
+            python3 -m pip download \
+              --only-binary=:all: \
+              --platform "${platform}" \
+              --pre -d curl_cffi_whls \
+              -r requirements.txt
+          done
+          ( # Overwrite x86_64-only libs with fat/universal2 libs or else Pyinstaller will do the opposite
+            # See https://github.com/yt-dlp/yt-dlp/pull/10069
+            cd curl_cffi_whls
+            mkdir -p curl_cffi/.dylibs
+            python_libdir=$(python3 -c 'import sys; from pathlib import Path; print(Path(sys.path[1]).parent)')
+            for dylib in lib{ssl,crypto}.3.dylib; do
+              cp "${python_libdir}/${dylib}" "curl_cffi/.dylibs/${dylib}"
+              for wheel in curl_cffi*macos*x86_64.whl; do
+                zip "${wheel}" "curl_cffi/.dylibs/${dylib}"
+              done
+            done
+          )
+          python3 -m delocate.cmd.delocate_fuse curl_cffi_whls/curl_cffi*.whl -w curl_cffi_universal2
+          python3 -m delocate.cmd.delocate_fuse curl_cffi_whls/cffi*.whl -w curl_cffi_universal2
+          cd curl_cffi_universal2
+          for wheel in ./*cffi*.whl; do mv -n -- "${wheel}" "${wheel/x86_64/universal2}"; done
+          python3 -m pip install -U --user ./*cffi*.whl
 
       - name: Prepare
         run: |
@@ -250,9 +284,9 @@ jobs:
           python3 devscripts/make_lazy_extractors.py
       - name: Build
         run: |
-          python3 pyinst.py --target-architecture universal2 --onedir
+          python3 -m bundle.pyinstaller --target-architecture universal2 --onedir
           (cd ./dist/yt-dlp_macos && zip -r ../yt-dlp_macos.zip .)
-          python3 pyinst.py --target-architecture universal2
+          python3 -m bundle.pyinstaller --target-architecture universal2
 
       - name: Verify --update-to
         if: vars.UPDATE_TO_VERIFICATION
@@ -265,16 +299,18 @@ jobs:
           [[ "$version" != "$downgraded_version" ]]
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
+          name: build-bin-${{ github.job }}
           path: |
             dist/yt-dlp_macos
             dist/yt-dlp_macos.zip
+          compression-level: 0
 
   macos_legacy:
     needs: process
     if: inputs.macos_legacy
-    runs-on: macos-latest
+    runs-on: macos-12
 
     steps:
       - uses: actions/checkout@v4
@@ -287,14 +323,14 @@ jobs:
           # Hack to get the latest patch version. Uncomment if needed
           #brew install python@3.10
           #export PYTHON_VERSION=$( $(brew --prefix)/opt/python@3.10/bin/python3 --version | cut -d ' ' -f 2 )
-          curl https://www.python.org/ftp/python/${PYTHON_VERSION}/python-${PYTHON_VERSION}-macos11.pkg -o "python.pkg"
+          curl "https://www.python.org/ftp/python/${PYTHON_VERSION}/python-${PYTHON_VERSION}-macos11.pkg" -o "python.pkg"
           sudo installer -pkg python.pkg -target /
           python3 --version
       - name: Install Requirements
         run: |
           brew install coreutils
-          python3 -m pip install -U --user pip setuptools wheel
-          python3 -m pip install -U --user Pyinstaller -r requirements.txt
+          python3 devscripts/install_deps.py --user -o --include build
+          python3 devscripts/install_deps.py --user --include pyinstaller
 
       - name: Prepare
         run: |
@@ -302,7 +338,7 @@ jobs:
           python3 devscripts/make_lazy_extractors.py
       - name: Build
         run: |
-          python3 pyinst.py
+          python3 -m bundle.pyinstaller
           mv dist/yt-dlp_macos dist/yt-dlp_macos_legacy
 
       - name: Verify --update-to
@@ -316,10 +352,12 @@ jobs:
           [[ "$version" != "$downgraded_version" ]]
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
+          name: build-bin-${{ github.job }}
           path: |
             dist/yt-dlp_macos_legacy
+          compression-level: 0
 
   windows:
     needs: process
@@ -328,13 +366,14 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with: # 3.8 is used for Win7 support
           python-version: "3.8"
       - name: Install Requirements
         run: | # Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds
-          python -m pip install -U pip setuptools wheel py2exe
-          pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-5.8.0-py3-none-any.whl" -r requirements.txt
+          python devscripts/install_deps.py -o --include build
+          python devscripts/install_deps.py --include curl-cffi
+          python -m pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-6.7.0-py3-none-any.whl"
 
       - name: Prepare
         run: |
@@ -342,12 +381,20 @@ jobs:
           python devscripts/make_lazy_extractors.py
       - name: Build
         run: |
-          python setup.py py2exe
-          Move-Item ./dist/yt-dlp.exe ./dist/yt-dlp_min.exe
-          python pyinst.py
-          python pyinst.py --onedir
+          python -m bundle.pyinstaller
+          python -m bundle.pyinstaller --onedir
+          Move-Item ./dist/yt-dlp.exe ./dist/yt-dlp_real.exe
           Compress-Archive -Path ./dist/yt-dlp/* -DestinationPath ./dist/yt-dlp_win.zip
 
+      - name: Install Requirements (py2exe)
+        run: |
+          python devscripts/install_deps.py --include py2exe
+      - name: Build (py2exe)
+        run: |
+          python -m bundle.py2exe
+          Move-Item ./dist/yt-dlp.exe ./dist/yt-dlp_min.exe
+          Move-Item ./dist/yt-dlp_real.exe ./dist/yt-dlp.exe
+
       - name: Verify --update-to
         if: vars.UPDATE_TO_VERIFICATION
         run: |
@@ -362,12 +409,14 @@ jobs:
           }
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
+          name: build-bin-${{ github.job }}
           path: |
             dist/yt-dlp.exe
             dist/yt-dlp_min.exe
             dist/yt-dlp_win.zip
+          compression-level: 0
 
   windows32:
     needs: process
@@ -376,14 +425,15 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.8"
           architecture: "x86"
       - name: Install Requirements
         run: |
-          python -m pip install -U pip setuptools wheel
-          pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-5.8.0-py3-none-any.whl" -r requirements.txt
+          python devscripts/install_deps.py -o --include build
+          python devscripts/install_deps.py
+          python -m pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-6.7.0-py3-none-any.whl"
 
       - name: Prepare
         run: |
@@ -391,7 +441,7 @@ jobs:
           python devscripts/make_lazy_extractors.py
       - name: Build
         run: |
-          python pyinst.py
+          python -m bundle.pyinstaller
 
       - name: Verify --update-to
         if: vars.UPDATE_TO_VERIFICATION
@@ -407,16 +457,19 @@ jobs:
           }
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
+          name: build-bin-${{ github.job }}
           path: |
             dist/yt-dlp_x86.exe
+          compression-level: 0
 
   meta_files:
-    if: inputs.meta_files && always() && !cancelled()
+    if: always() && !cancelled()
     needs:
       - process
       - unix
+      - linux_static
       - linux_arm
       - macos
       - macos_legacy
@@ -424,13 +477,18 @@ jobs:
       - windows32
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
+        with:
+          path: artifact
+          pattern: build-bin-*
+          merge-multiple: true
 
       - name: Make SHA2-SUMS files
         run: |
           cd ./artifact/
-          sha256sum * > ../SHA2-256SUMS
-          sha512sum * > ../SHA2-512SUMS
+          # make sure SHA sums are also printed to stdout
+          sha256sum -- * | tee ../SHA2-256SUMS
+          sha512sum -- * | tee ../SHA2-512SUMS
 
       - name: Make Update spec
         run: |
@@ -459,8 +517,11 @@ jobs:
           done
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
+          name: build-${{ github.job }}
           path: |
-            SHA*SUMS*
             _update_spec
+            SHA*SUMS*
+          compression-level: 0
+          overwrite: true