.github/workflows/build.yml

   1 name: Build Artifacts
   2 on:
   3   workflow_call:
   4     inputs:
   5       version:
   6         required: true
   7         type: string
   8       channel:
   9         required: false
  10         default: stable
  11         type: string
  12       unix:
  13         default: true
  14         type: boolean
  15       linux_arm:
  16         default: true
  17         type: boolean
  18       macos:
  19         default: true
  20         type: boolean
  21       macos_legacy:
  22         default: true
  23         type: boolean
  24       windows:
  25         default: true
  26         type: boolean
  27       windows32:
  28         default: true
  29         type: boolean
  30       meta_files:
  31         default: true
  32         type: boolean
  33       origin:
  34         required: false
  35         default: ''
  36         type: string
  37     secrets:
  38       GPG_SIGNING_KEY:
  39         required: false
  40
  41   workflow_dispatch:
  42     inputs:
  43       version:
  44         description: |
  45           VERSION: yyyy.mm.dd[.rev] or rev
  46         required: true
  47         type: string
  48       channel:
  49         description: |
  50           SOURCE of this build's updates: stable/nightly/master/<repo>
  51         required: true
  52         default: stable
  53         type: string
  54       unix:
  55         description: yt-dlp, yt-dlp.tar.gz, yt-dlp_linux, yt-dlp_linux.zip
  56         default: true
  57         type: boolean
  58       linux_arm:
  59         description: yt-dlp_linux_aarch64, yt-dlp_linux_armv7l
  60         default: true
  61         type: boolean
  62       macos:
  63         description: yt-dlp_macos, yt-dlp_macos.zip
  64         default: true
  65         type: boolean
  66       macos_legacy:
  67         description: yt-dlp_macos_legacy
  68         default: true
  69         type: boolean
  70       windows:
  71         description: yt-dlp.exe, yt-dlp_min.exe, yt-dlp_win.zip
  72         default: true
  73         type: boolean
  74       windows32:
  75         description: yt-dlp_x86.exe
  76         default: true
  77         type: boolean
  78       meta_files:
  79         description: SHA2-256SUMS, SHA2-512SUMS, _update_spec
  80         default: true
  81         type: boolean
  82       origin:
  83         description: Origin
  84         required: false
  85         default: 'current repo'
  86         type: choice
  87         options:
  88         - 'current repo'
  89
  90 permissions:
  91   contents: read
  92
  93 jobs:
  94   process:
  95     runs-on: ubuntu-latest
  96     outputs:
  97       origin: ${{ steps.process_origin.outputs.origin }}
  98     steps:
  99       - name: Process origin
 100         id: process_origin
 101         run: |
 102           echo "origin=${{ inputs.origin == 'current repo' && github.repository || inputs.origin }}" | tee "$GITHUB_OUTPUT"
 103
 104   unix:
 105     needs: process
 106     if: inputs.unix
 107     runs-on: ubuntu-latest
 108     steps:
 109       - uses: actions/checkout@v4
 110         with:
 111           fetch-depth: 0  # Needed for changelog
 112       - uses: actions/setup-python@v5
 113         with:
 114           python-version: "3.10"
 115       - uses: conda-incubator/setup-miniconda@v3
 116         with:
 117           miniforge-variant: Mambaforge
 118           use-mamba: true
 119           channels: conda-forge
 120           auto-update-conda: true
 121           activate-environment: ""
 122           auto-activate-base: false
 123       - name: Install Requirements
 124         run: |
 125           sudo apt -y install zip pandoc man sed
 126           cat > ./requirements.txt << EOF
 127           python=3.10.*
 128           pyinstaller
 129           brotli-python
 130           EOF
 131           python devscripts/install_deps.py --print \
 132             --exclude brotli --exclude brotlicffi \
 133             --include secretstorage >> ./requirements.txt
 134           mamba create -n build --file ./requirements.txt
 135
 136       - name: Prepare
 137         run: |
 138           python devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
 139           python devscripts/update_changelog.py -vv
 140           python devscripts/make_lazy_extractors.py
 141       - name: Build Unix platform-independent binary
 142         run: |
 143           make all tar
 144       - name: Build Unix standalone binary
 145         shell: bash -l {0}
 146         run: |
 147           unset LD_LIBRARY_PATH  # Harmful; set by setup-python
 148           conda activate build
 149           python -m bundle.pyinstaller --onedir
 150           (cd ./dist/yt-dlp_linux && zip -r ../yt-dlp_linux.zip .)
 151           python -m bundle.pyinstaller
 152           mv ./dist/yt-dlp_linux ./yt-dlp_linux
 153           mv ./dist/yt-dlp_linux.zip ./yt-dlp_linux.zip
 154
 155       - name: Verify --update-to
 156         if: vars.UPDATE_TO_VERIFICATION
 157         run: |
 158           binaries=("yt-dlp" "yt-dlp_linux")
 159           for binary in "${binaries[@]}"; do
 160             chmod +x ./${binary}
 161             cp ./${binary} ./${binary}_downgraded
 162             version="$(./${binary} --version)"
 163             ./${binary}_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04
 164             downgraded_version="$(./${binary}_downgraded --version)"
 165             [[ "$version" != "$downgraded_version" ]]
 166           done
 167
 168       - name: Upload artifacts
 169         uses: actions/upload-artifact@v4
 170         with:
 171           name: build-bin-${{ github.job }}
 172           path: |
 173             yt-dlp
 174             yt-dlp.tar.gz
 175             yt-dlp_linux
 176             yt-dlp_linux.zip
 177           compression-level: 0
 178
 179   linux_arm:
 180     needs: process
 181     if: inputs.linux_arm
 182     permissions:
 183       contents: read
 184       packages: write # for creating cache
 185     runs-on: ubuntu-latest
 186     strategy:
 187       matrix:
 188         architecture:
 189           - armv7
 190           - aarch64
 191
 192     steps:
 193       - uses: actions/checkout@v4
 194         with:
 195           path: ./repo
 196       - name: Virtualized Install, Prepare & Build
 197         uses: yt-dlp/run-on-arch-action@v2
 198         with:
 199           # Ref: https://github.com/uraimo/run-on-arch-action/issues/55
 200           env: |
 201             GITHUB_WORKFLOW: build
 202           githubToken: ${{ github.token }} # To cache image
 203           arch: ${{ matrix.architecture }}
 204           distro: ubuntu18.04 # Standalone executable should be built on minimum supported OS
 205           dockerRunArgs: --volume "${PWD}/repo:/repo"
 206           install: | # Installing Python 3.10 from the Deadsnakes repo raises errors
 207             apt update
 208             apt -y install zlib1g-dev libffi-dev python3.8 python3.8-dev python3.8-distutils python3-pip
 209             python3.8 -m pip install -U pip setuptools wheel
 210             # Cannot access any files from the repo directory at this stage
 211             python3.8 -m pip install -U Pyinstaller mutagen pycryptodomex websockets brotli certifi secretstorage cffi
 212
 213           run: |
 214             cd repo
 215             python3.8 devscripts/install_deps.py -o --include build
 216             python3.8 devscripts/install_deps.py --include pyinstaller --include secretstorage  # Cached version may be out of date
 217             python3.8 devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
 218             python3.8 devscripts/make_lazy_extractors.py
 219             python3.8 -m bundle.pyinstaller
 220
 221             if ${{ vars.UPDATE_TO_VERIFICATION && 'true' || 'false' }}; then
 222               arch="${{ (matrix.architecture == 'armv7' && 'armv7l') || matrix.architecture }}"
 223               chmod +x ./dist/yt-dlp_linux_${arch}
 224               cp ./dist/yt-dlp_linux_${arch} ./dist/yt-dlp_linux_${arch}_downgraded
 225               version="$(./dist/yt-dlp_linux_${arch} --version)"
 226               ./dist/yt-dlp_linux_${arch}_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04
 227               downgraded_version="$(./dist/yt-dlp_linux_${arch}_downgraded --version)"
 228               [[ "$version" != "$downgraded_version" ]]
 229             fi
 230
 231       - name: Upload artifacts
 232         uses: actions/upload-artifact@v4
 233         with:
 234           name: build-bin-linux_${{ matrix.architecture }}
 235           path: | # run-on-arch-action designates armv7l as armv7
 236             repo/dist/yt-dlp_linux_${{ (matrix.architecture == 'armv7' && 'armv7l') || matrix.architecture }}
 237           compression-level: 0
 238
 239   macos:
 240     needs: process
 241     if: inputs.macos
 242     runs-on: macos-11
 243
 244     steps:
 245       - uses: actions/checkout@v4
 246       # NB: Building universal2 does not work with python from actions/setup-python
 247       - name: Install Requirements
 248         run: |
 249           brew install coreutils
 250           python3 devscripts/install_deps.py --user -o --include build
 251           python3 devscripts/install_deps.py --print --include pyinstaller > requirements.txt
 252           # We need to ignore wheels otherwise we break universal2 builds
 253           python3 -m pip install -U --user --no-binary :all: -r requirements.txt
 254           # We need to fuse our own universal2 wheels for curl_cffi
 255           python3 -m pip install -U --user delocate
 256           mkdir curl_cffi_whls curl_cffi_universal2
 257           python3 devscripts/install_deps.py --print -o --include curl_cffi > requirements.txt
 258           for platform in "macosx_11_0_arm64" "macosx_11_0_x86_64"; do
 259             python3 -m pip download \
 260               --only-binary=:all: \
 261               --platform "${platform}" \
 262               --pre -d curl_cffi_whls \
 263               -r requirements.txt
 264           done
 265           python3 -m delocate.cmd.delocate_fuse curl_cffi_whls/curl_cffi*.whl -w curl_cffi_universal2
 266           python3 -m delocate.cmd.delocate_fuse curl_cffi_whls/cffi*.whl -w curl_cffi_universal2
 267           cd curl_cffi_universal2
 268           for wheel in *cffi*.whl; do mv -n -- "${wheel}" "${wheel/x86_64/universal2}"; done
 269           python3 -m pip install -U --user *cffi*.whl
 270
 271       - name: Prepare
 272         run: |
 273           python3 devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
 274           python3 devscripts/make_lazy_extractors.py
 275       - name: Build
 276         run: |
 277           python3 -m bundle.pyinstaller --target-architecture universal2 --onedir
 278           (cd ./dist/yt-dlp_macos && zip -r ../yt-dlp_macos.zip .)
 279           python3 -m bundle.pyinstaller --target-architecture universal2
 280
 281       - name: Verify --update-to
 282         if: vars.UPDATE_TO_VERIFICATION
 283         run: |
 284           chmod +x ./dist/yt-dlp_macos
 285           cp ./dist/yt-dlp_macos ./dist/yt-dlp_macos_downgraded
 286           version="$(./dist/yt-dlp_macos --version)"
 287           ./dist/yt-dlp_macos_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04
 288           downgraded_version="$(./dist/yt-dlp_macos_downgraded --version)"
 289           [[ "$version" != "$downgraded_version" ]]
 290
 291       - name: Upload artifacts
 292         uses: actions/upload-artifact@v4
 293         with:
 294           name: build-bin-${{ github.job }}
 295           path: |
 296             dist/yt-dlp_macos
 297             dist/yt-dlp_macos.zip
 298           compression-level: 0
 299
 300   macos_legacy:
 301     needs: process
 302     if: inputs.macos_legacy
 303     runs-on: macos-latest
 304
 305     steps:
 306       - uses: actions/checkout@v4
 307       - name: Install Python
 308         # We need the official Python, because the GA ones only support newer macOS versions
 309         env:
 310           PYTHON_VERSION: 3.10.5
 311           MACOSX_DEPLOYMENT_TARGET: 10.9 # Used up by the Python build tools
 312         run: |
 313           # Hack to get the latest patch version. Uncomment if needed
 314           #brew install python@3.10
 315           #export PYTHON_VERSION=$( $(brew --prefix)/opt/python@3.10/bin/python3 --version | cut -d ' ' -f 2 )
 316           curl https://www.python.org/ftp/python/${PYTHON_VERSION}/python-${PYTHON_VERSION}-macos11.pkg -o "python.pkg"
 317           sudo installer -pkg python.pkg -target /
 318           python3 --version
 319       - name: Install Requirements
 320         run: |
 321           brew install coreutils
 322           python3 devscripts/install_deps.py --user -o --include build
 323           python3 devscripts/install_deps.py --user --include pyinstaller --include curl_cffi
 324
 325       - name: Prepare
 326         run: |
 327           python3 devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
 328           python3 devscripts/make_lazy_extractors.py
 329       - name: Build
 330         run: |
 331           python3 -m bundle.pyinstaller
 332           mv dist/yt-dlp_macos dist/yt-dlp_macos_legacy
 333
 334       - name: Verify --update-to
 335         if: vars.UPDATE_TO_VERIFICATION
 336         run: |
 337           chmod +x ./dist/yt-dlp_macos_legacy
 338           cp ./dist/yt-dlp_macos_legacy ./dist/yt-dlp_macos_legacy_downgraded
 339           version="$(./dist/yt-dlp_macos_legacy --version)"
 340           ./dist/yt-dlp_macos_legacy_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04
 341           downgraded_version="$(./dist/yt-dlp_macos_legacy_downgraded --version)"
 342           [[ "$version" != "$downgraded_version" ]]
 343
 344       - name: Upload artifacts
 345         uses: actions/upload-artifact@v4
 346         with:
 347           name: build-bin-${{ github.job }}
 348           path: |
 349             dist/yt-dlp_macos_legacy
 350           compression-level: 0
 351
 352   windows:
 353     needs: process
 354     if: inputs.windows
 355     runs-on: windows-latest
 356
 357     steps:
 358       - uses: actions/checkout@v4
 359       - uses: actions/setup-python@v5
 360         with: # 3.8 is used for Win7 support
 361           python-version: "3.8"
 362       - name: Install Requirements
 363         run: | # Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds
 364           python devscripts/install_deps.py -o --include build
 365           python devscripts/install_deps.py --include py2exe --include curl_cffi
 366           python -m pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-5.8.0-py3-none-any.whl"
 367
 368       - name: Prepare
 369         run: |
 370           python devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
 371           python devscripts/make_lazy_extractors.py
 372       - name: Build
 373         run: |
 374           python -m bundle.py2exe
 375           Move-Item ./dist/yt-dlp.exe ./dist/yt-dlp_min.exe
 376           python -m bundle.pyinstaller
 377           python -m bundle.pyinstaller --onedir
 378           Compress-Archive -Path ./dist/yt-dlp/* -DestinationPath ./dist/yt-dlp_win.zip
 379
 380       - name: Verify --update-to
 381         if: vars.UPDATE_TO_VERIFICATION
 382         run: |
 383           foreach ($name in @("yt-dlp","yt-dlp_min")) {
 384             Copy-Item "./dist/${name}.exe" "./dist/${name}_downgraded.exe"
 385             $version = & "./dist/${name}.exe" --version
 386             & "./dist/${name}_downgraded.exe" -v --update-to yt-dlp/yt-dlp@2023.03.04
 387             $downgraded_version = & "./dist/${name}_downgraded.exe" --version
 388             if ($version -eq $downgraded_version) {
 389               exit 1
 390             }
 391           }
 392
 393       - name: Upload artifacts
 394         uses: actions/upload-artifact@v4
 395         with:
 396           name: build-bin-${{ github.job }}
 397           path: |
 398             dist/yt-dlp.exe
 399             dist/yt-dlp_min.exe
 400             dist/yt-dlp_win.zip
 401           compression-level: 0
 402
 403   windows32:
 404     needs: process
 405     if: inputs.windows32
 406     runs-on: windows-latest
 407
 408     steps:
 409       - uses: actions/checkout@v4
 410       - uses: actions/setup-python@v5
 411         with:
 412           python-version: "3.8"
 413           architecture: "x86"
 414       - name: Install Requirements
 415         run: |
 416           python devscripts/install_deps.py -o --include build
 417           python devscripts/install_deps.py
 418           python -m pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-5.8.0-py3-none-any.whl"
 419
 420       - name: Prepare
 421         run: |
 422           python devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
 423           python devscripts/make_lazy_extractors.py
 424       - name: Build
 425         run: |
 426           python -m bundle.pyinstaller
 427
 428       - name: Verify --update-to
 429         if: vars.UPDATE_TO_VERIFICATION
 430         run: |
 431           foreach ($name in @("yt-dlp_x86")) {
 432             Copy-Item "./dist/${name}.exe" "./dist/${name}_downgraded.exe"
 433             $version = & "./dist/${name}.exe" --version
 434             & "./dist/${name}_downgraded.exe" -v --update-to yt-dlp/yt-dlp@2023.03.04
 435             $downgraded_version = & "./dist/${name}_downgraded.exe" --version
 436             if ($version -eq $downgraded_version) {
 437               exit 1
 438             }
 439           }
 440
 441       - name: Upload artifacts
 442         uses: actions/upload-artifact@v4
 443         with:
 444           name: build-bin-${{ github.job }}
 445           path: |
 446             dist/yt-dlp_x86.exe
 447           compression-level: 0
 448
 449   meta_files:
 450     if: inputs.meta_files && always() && !cancelled()
 451     needs:
 452       - process
 453       - unix
 454       - linux_arm
 455       - macos
 456       - macos_legacy
 457       - windows
 458       - windows32
 459     runs-on: ubuntu-latest
 460     steps:
 461       - uses: actions/download-artifact@v4
 462         with:
 463           path: artifact
 464           pattern: build-bin-*
 465           merge-multiple: true
 466
 467       - name: Make SHA2-SUMS files
 468         run: |
 469           cd ./artifact/
 470           # make sure SHA sums are also printed to stdout
 471           sha256sum * | tee ../SHA2-256SUMS
 472           sha512sum * | tee ../SHA2-512SUMS
 473
 474       - name: Make Update spec
 475         run: |
 476           cat >> _update_spec << EOF
 477           # This file is used for regulating self-update
 478           lock 2022.08.18.36 .+ Python 3\.6
 479           lock 2023.11.16 (?!win_x86_exe).+ Python 3\.7
 480           lock 2023.11.16 win_x86_exe .+ Windows-(?:Vista|2008Server)
 481           lockV2 yt-dlp/yt-dlp 2022.08.18.36 .+ Python 3\.6
 482           lockV2 yt-dlp/yt-dlp 2023.11.16 (?!win_x86_exe).+ Python 3\.7
 483           lockV2 yt-dlp/yt-dlp 2023.11.16 win_x86_exe .+ Windows-(?:Vista|2008Server)
 484           lockV2 yt-dlp/yt-dlp-nightly-builds 2023.11.15.232826 (?!win_x86_exe).+ Python 3\.7
 485           lockV2 yt-dlp/yt-dlp-nightly-builds 2023.11.15.232826 win_x86_exe .+ Windows-(?:Vista|2008Server)
 486           lockV2 yt-dlp/yt-dlp-master-builds 2023.11.15.232812 (?!win_x86_exe).+ Python 3\.7
 487           lockV2 yt-dlp/yt-dlp-master-builds 2023.11.15.232812 win_x86_exe .+ Windows-(?:Vista|2008Server)
 488           EOF
 489
 490       - name: Sign checksum files
 491         env:
 492           GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }}
 493         if: env.GPG_SIGNING_KEY != ''
 494         run: |
 495           gpg --batch --import <<< "${{ secrets.GPG_SIGNING_KEY }}"
 496           for signfile in ./SHA*SUMS; do
 497             gpg --batch --detach-sign "$signfile"
 498           done
 499
 500       - name: Upload artifacts
 501         uses: actions/upload-artifact@v4
 502         with:
 503           name: build-${{ github.job }}
 504           path: |
 505             _update_spec
 506             SHA*SUMS*
 507           compression-level: 0
 508           overwrite: true