.github/workflows/build.yml

   1 name: Build Artifacts
   2 on:
   3   workflow_call:
   4     inputs:
   5       version:
   6         required: true
   7         type: string
   8       channel:
   9         required: false
  10         default: stable
  11         type: string
  12       unix:
  13         default: true
  14         type: boolean
  15       linux_arm:
  16         default: true
  17         type: boolean
  18       macos:
  19         default: true
  20         type: boolean
  21       macos_legacy:
  22         default: true
  23         type: boolean
  24       windows:
  25         default: true
  26         type: boolean
  27       windows32:
  28         default: true
  29         type: boolean
  30       meta_files:
  31         default: true
  32         type: boolean
  33       origin:
  34         required: false
  35         default: ''
  36         type: string
  37     secrets:
  38       GPG_SIGNING_KEY:
  39         required: false
  40
  41   workflow_dispatch:
  42     inputs:
  43       version:
  44         description: |
  45           VERSION: yyyy.mm.dd[.rev] or rev
  46         required: true
  47         type: string
  48       channel:
  49         description: |
  50           SOURCE of this build's updates: stable/nightly/master/<repo>
  51         required: true
  52         default: stable
  53         type: string
  54       unix:
  55         description: yt-dlp, yt-dlp.tar.gz, yt-dlp_linux, yt-dlp_linux.zip
  56         default: true
  57         type: boolean
  58       linux_arm:
  59         description: yt-dlp_linux_aarch64, yt-dlp_linux_armv7l
  60         default: true
  61         type: boolean
  62       macos:
  63         description: yt-dlp_macos, yt-dlp_macos.zip
  64         default: true
  65         type: boolean
  66       macos_legacy:
  67         description: yt-dlp_macos_legacy
  68         default: true
  69         type: boolean
  70       windows:
  71         description: yt-dlp.exe, yt-dlp_min.exe, yt-dlp_win.zip
  72         default: true
  73         type: boolean
  74       windows32:
  75         description: yt-dlp_x86.exe
  76         default: true
  77         type: boolean
  78       meta_files:
  79         description: SHA2-256SUMS, SHA2-512SUMS, _update_spec
  80         default: true
  81         type: boolean
  82       origin:
  83         description: Origin
  84         required: false
  85         default: 'current repo'
  86         type: choice
  87         options:
  88         - 'current repo'
  89
  90 permissions:
  91   contents: read
  92
  93 jobs:
  94   process:
  95     runs-on: ubuntu-latest
  96     outputs:
  97       origin: ${{ steps.process_origin.outputs.origin }}
  98     steps:
  99       - name: Process origin
 100         id: process_origin
 101         run: |
 102           echo "origin=${{ inputs.origin == 'current repo' && github.repository || inputs.origin }}" | tee "$GITHUB_OUTPUT"
 103
 104   unix:
 105     needs: process
 106     if: inputs.unix
 107     runs-on: ubuntu-latest
 108     steps:
 109       - uses: actions/checkout@v4
 110         with:
 111           fetch-depth: 0  # Needed for changelog
 112       - uses: actions/setup-python@v5
 113         with:
 114           python-version: "3.10"
 115       - uses: conda-incubator/setup-miniconda@v3
 116         with:
 117           miniforge-variant: Mambaforge
 118           use-mamba: true
 119           channels: conda-forge
 120           auto-update-conda: true
 121           activate-environment: ""
 122           auto-activate-base: false
 123       - name: Install Requirements
 124         run: |
 125           sudo apt -y install zip pandoc man sed
 126           cat > ./requirements.txt << EOF
 127           python=3.10.*
 128           brotli-python
 129           EOF
 130           python devscripts/install_deps.py --print \
 131             --exclude brotli --exclude brotlicffi \
 132             --include secretstorage --include pyinstaller >> ./requirements.txt
 133           mamba create -n build --file ./requirements.txt
 134
 135       - name: Prepare
 136         run: |
 137           python devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
 138           python devscripts/update_changelog.py -vv
 139           python devscripts/make_lazy_extractors.py
 140       - name: Build Unix platform-independent binary
 141         run: |
 142           make all tar
 143       - name: Build Unix standalone binary
 144         shell: bash -l {0}
 145         run: |
 146           unset LD_LIBRARY_PATH  # Harmful; set by setup-python
 147           conda activate build
 148           python -m bundle.pyinstaller --onedir
 149           (cd ./dist/yt-dlp_linux && zip -r ../yt-dlp_linux.zip .)
 150           python -m bundle.pyinstaller
 151           mv ./dist/yt-dlp_linux ./yt-dlp_linux
 152           mv ./dist/yt-dlp_linux.zip ./yt-dlp_linux.zip
 153
 154       - name: Verify --update-to
 155         if: vars.UPDATE_TO_VERIFICATION
 156         run: |
 157           binaries=("yt-dlp" "yt-dlp_linux")
 158           for binary in "${binaries[@]}"; do
 159             chmod +x ./${binary}
 160             cp ./${binary} ./${binary}_downgraded
 161             version="$(./${binary} --version)"
 162             ./${binary}_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04
 163             downgraded_version="$(./${binary}_downgraded --version)"
 164             [[ "$version" != "$downgraded_version" ]]
 165           done
 166
 167       - name: Upload artifacts
 168         uses: actions/upload-artifact@v4
 169         with:
 170           name: build-bin-${{ github.job }}
 171           path: |
 172             yt-dlp
 173             yt-dlp.tar.gz
 174             yt-dlp_linux
 175             yt-dlp_linux.zip
 176           compression-level: 0
 177
 178   linux_arm:
 179     needs: process
 180     if: inputs.linux_arm
 181     permissions:
 182       contents: read
 183       packages: write # for creating cache
 184     runs-on: ubuntu-latest
 185     strategy:
 186       matrix:
 187         architecture:
 188           - armv7
 189           - aarch64
 190
 191     steps:
 192       - uses: actions/checkout@v4
 193         with:
 194           path: ./repo
 195       - name: Virtualized Install, Prepare & Build
 196         uses: yt-dlp/run-on-arch-action@v2
 197         with:
 198           # Ref: https://github.com/uraimo/run-on-arch-action/issues/55
 199           env: |
 200             GITHUB_WORKFLOW: build
 201           githubToken: ${{ github.token }} # To cache image
 202           arch: ${{ matrix.architecture }}
 203           distro: ubuntu18.04 # Standalone executable should be built on minimum supported OS
 204           dockerRunArgs: --volume "${PWD}/repo:/repo"
 205           install: | # Installing Python 3.10 from the Deadsnakes repo raises errors
 206             apt update
 207             apt -y install zlib1g-dev libffi-dev python3.8 python3.8-dev python3.8-distutils python3-pip
 208             python3.8 -m pip install -U pip setuptools wheel
 209             # Cannot access any files from the repo directory at this stage
 210             python3.8 -m pip install -U Pyinstaller mutagen pycryptodomex websockets brotli certifi secretstorage cffi
 211
 212           run: |
 213             cd repo
 214             python3.8 devscripts/install_deps.py -o --include build
 215             python3.8 devscripts/install_deps.py --include pyinstaller --include secretstorage  # Cached version may be out of date
 216             python3.8 devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
 217             python3.8 devscripts/make_lazy_extractors.py
 218             python3.8 -m bundle.pyinstaller
 219
 220             if ${{ vars.UPDATE_TO_VERIFICATION && 'true' || 'false' }}; then
 221               arch="${{ (matrix.architecture == 'armv7' && 'armv7l') || matrix.architecture }}"
 222               chmod +x ./dist/yt-dlp_linux_${arch}
 223               cp ./dist/yt-dlp_linux_${arch} ./dist/yt-dlp_linux_${arch}_downgraded
 224               version="$(./dist/yt-dlp_linux_${arch} --version)"
 225               ./dist/yt-dlp_linux_${arch}_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04
 226               downgraded_version="$(./dist/yt-dlp_linux_${arch}_downgraded --version)"
 227               [[ "$version" != "$downgraded_version" ]]
 228             fi
 229
 230       - name: Upload artifacts
 231         uses: actions/upload-artifact@v4
 232         with:
 233           name: build-bin-linux_${{ matrix.architecture }}
 234           path: | # run-on-arch-action designates armv7l as armv7
 235             repo/dist/yt-dlp_linux_${{ (matrix.architecture == 'armv7' && 'armv7l') || matrix.architecture }}
 236           compression-level: 0
 237
 238   macos:
 239     needs: process
 240     if: inputs.macos
 241     runs-on: macos-11
 242
 243     steps:
 244       - uses: actions/checkout@v4
 245       # NB: Building universal2 does not work with python from actions/setup-python
 246       - name: Install Requirements
 247         run: |
 248           brew install coreutils
 249           python3 devscripts/install_deps.py --user -o --include build
 250           python3 devscripts/install_deps.py --print --include pyinstaller_macos > requirements.txt
 251           # We need to ignore wheels otherwise we break universal2 builds
 252           python3 -m pip install -U --user --no-binary :all: -r requirements.txt
 253           # We need to fuse our own universal2 wheels for curl_cffi
 254           python3 -m pip install -U --user delocate
 255           mkdir curl_cffi_whls curl_cffi_universal2
 256           python3 devscripts/install_deps.py --print -o --include curl_cffi > requirements.txt
 257           for platform in "macosx_11_0_arm64" "macosx_11_0_x86_64"; do
 258             python3 -m pip download \
 259               --only-binary=:all: \
 260               --platform "${platform}" \
 261               --pre -d curl_cffi_whls \
 262               -r requirements.txt
 263           done
 264           python3 -m delocate.cmd.delocate_fuse curl_cffi_whls/curl_cffi*.whl -w curl_cffi_universal2
 265           python3 -m delocate.cmd.delocate_fuse curl_cffi_whls/cffi*.whl -w curl_cffi_universal2
 266           cd curl_cffi_universal2
 267           for wheel in *cffi*.whl; do mv -n -- "${wheel}" "${wheel/x86_64/universal2}"; done
 268           python3 -m pip install -U --user *cffi*.whl
 269
 270       - name: Prepare
 271         run: |
 272           python3 devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
 273           python3 devscripts/make_lazy_extractors.py
 274       - name: Build
 275         run: |
 276           python3 -m bundle.pyinstaller --target-architecture universal2 --onedir
 277           (cd ./dist/yt-dlp_macos && zip -r ../yt-dlp_macos.zip .)
 278           python3 -m bundle.pyinstaller --target-architecture universal2
 279
 280       - name: Verify --update-to
 281         if: vars.UPDATE_TO_VERIFICATION
 282         run: |
 283           chmod +x ./dist/yt-dlp_macos
 284           cp ./dist/yt-dlp_macos ./dist/yt-dlp_macos_downgraded
 285           version="$(./dist/yt-dlp_macos --version)"
 286           ./dist/yt-dlp_macos_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04
 287           downgraded_version="$(./dist/yt-dlp_macos_downgraded --version)"
 288           [[ "$version" != "$downgraded_version" ]]
 289
 290       - name: Upload artifacts
 291         uses: actions/upload-artifact@v4
 292         with:
 293           name: build-bin-${{ github.job }}
 294           path: |
 295             dist/yt-dlp_macos
 296             dist/yt-dlp_macos.zip
 297           compression-level: 0
 298
 299   macos_legacy:
 300     needs: process
 301     if: inputs.macos_legacy
 302     runs-on: macos-latest
 303
 304     steps:
 305       - uses: actions/checkout@v4
 306       - name: Install Python
 307         # We need the official Python, because the GA ones only support newer macOS versions
 308         env:
 309           PYTHON_VERSION: 3.10.5
 310           MACOSX_DEPLOYMENT_TARGET: 10.9 # Used up by the Python build tools
 311         run: |
 312           # Hack to get the latest patch version. Uncomment if needed
 313           #brew install python@3.10
 314           #export PYTHON_VERSION=$( $(brew --prefix)/opt/python@3.10/bin/python3 --version | cut -d ' ' -f 2 )
 315           curl https://www.python.org/ftp/python/${PYTHON_VERSION}/python-${PYTHON_VERSION}-macos11.pkg -o "python.pkg"
 316           sudo installer -pkg python.pkg -target /
 317           python3 --version
 318       - name: Install Requirements
 319         run: |
 320           brew install coreutils
 321           python3 devscripts/install_deps.py --user -o --include build
 322           python3 devscripts/install_deps.py --user --include pyinstaller_macos --include curl_cffi
 323
 324       - name: Prepare
 325         run: |
 326           python3 devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
 327           python3 devscripts/make_lazy_extractors.py
 328       - name: Build
 329         run: |
 330           python3 -m bundle.pyinstaller
 331           mv dist/yt-dlp_macos dist/yt-dlp_macos_legacy
 332
 333       - name: Verify --update-to
 334         if: vars.UPDATE_TO_VERIFICATION
 335         run: |
 336           chmod +x ./dist/yt-dlp_macos_legacy
 337           cp ./dist/yt-dlp_macos_legacy ./dist/yt-dlp_macos_legacy_downgraded
 338           version="$(./dist/yt-dlp_macos_legacy --version)"
 339           ./dist/yt-dlp_macos_legacy_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04
 340           downgraded_version="$(./dist/yt-dlp_macos_legacy_downgraded --version)"
 341           [[ "$version" != "$downgraded_version" ]]
 342
 343       - name: Upload artifacts
 344         uses: actions/upload-artifact@v4
 345         with:
 346           name: build-bin-${{ github.job }}
 347           path: |
 348             dist/yt-dlp_macos_legacy
 349           compression-level: 0
 350
 351   windows:
 352     needs: process
 353     if: inputs.windows
 354     runs-on: windows-latest
 355
 356     steps:
 357       - uses: actions/checkout@v4
 358       - uses: actions/setup-python@v5
 359         with: # 3.8 is used for Win7 support
 360           python-version: "3.8"
 361       - name: Install Requirements
 362         run: | # Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds
 363           python devscripts/install_deps.py -o --include build
 364           python devscripts/install_deps.py --include py2exe --include curl_cffi
 365           python -m pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-5.8.0-py3-none-any.whl"
 366
 367       - name: Prepare
 368         run: |
 369           python devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
 370           python devscripts/make_lazy_extractors.py
 371       - name: Build
 372         run: |
 373           python -m bundle.py2exe
 374           Move-Item ./dist/yt-dlp.exe ./dist/yt-dlp_min.exe
 375           python -m bundle.pyinstaller
 376           python -m bundle.pyinstaller --onedir
 377           Compress-Archive -Path ./dist/yt-dlp/* -DestinationPath ./dist/yt-dlp_win.zip
 378
 379       - name: Verify --update-to
 380         if: vars.UPDATE_TO_VERIFICATION
 381         run: |
 382           foreach ($name in @("yt-dlp","yt-dlp_min")) {
 383             Copy-Item "./dist/${name}.exe" "./dist/${name}_downgraded.exe"
 384             $version = & "./dist/${name}.exe" --version
 385             & "./dist/${name}_downgraded.exe" -v --update-to yt-dlp/yt-dlp@2023.03.04
 386             $downgraded_version = & "./dist/${name}_downgraded.exe" --version
 387             if ($version -eq $downgraded_version) {
 388               exit 1
 389             }
 390           }
 391
 392       - name: Upload artifacts
 393         uses: actions/upload-artifact@v4
 394         with:
 395           name: build-bin-${{ github.job }}
 396           path: |
 397             dist/yt-dlp.exe
 398             dist/yt-dlp_min.exe
 399             dist/yt-dlp_win.zip
 400           compression-level: 0
 401
 402   windows32:
 403     needs: process
 404     if: inputs.windows32
 405     runs-on: windows-latest
 406
 407     steps:
 408       - uses: actions/checkout@v4
 409       - uses: actions/setup-python@v5
 410         with:
 411           python-version: "3.8"
 412           architecture: "x86"
 413       - name: Install Requirements
 414         run: |
 415           python devscripts/install_deps.py -o --include build
 416           python devscripts/install_deps.py
 417           python -m pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-5.8.0-py3-none-any.whl"
 418
 419       - name: Prepare
 420         run: |
 421           python devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
 422           python devscripts/make_lazy_extractors.py
 423       - name: Build
 424         run: |
 425           python -m bundle.pyinstaller
 426
 427       - name: Verify --update-to
 428         if: vars.UPDATE_TO_VERIFICATION
 429         run: |
 430           foreach ($name in @("yt-dlp_x86")) {
 431             Copy-Item "./dist/${name}.exe" "./dist/${name}_downgraded.exe"
 432             $version = & "./dist/${name}.exe" --version
 433             & "./dist/${name}_downgraded.exe" -v --update-to yt-dlp/yt-dlp@2023.03.04
 434             $downgraded_version = & "./dist/${name}_downgraded.exe" --version
 435             if ($version -eq $downgraded_version) {
 436               exit 1
 437             }
 438           }
 439
 440       - name: Upload artifacts
 441         uses: actions/upload-artifact@v4
 442         with:
 443           name: build-bin-${{ github.job }}
 444           path: |
 445             dist/yt-dlp_x86.exe
 446           compression-level: 0
 447
 448   meta_files:
 449     if: inputs.meta_files && always() && !cancelled()
 450     needs:
 451       - process
 452       - unix
 453       - linux_arm
 454       - macos
 455       - macos_legacy
 456       - windows
 457       - windows32
 458     runs-on: ubuntu-latest
 459     steps:
 460       - uses: actions/download-artifact@v4
 461         with:
 462           path: artifact
 463           pattern: build-bin-*
 464           merge-multiple: true
 465
 466       - name: Make SHA2-SUMS files
 467         run: |
 468           cd ./artifact/
 469           sha256sum * > ../SHA2-256SUMS
 470           sha512sum * > ../SHA2-512SUMS
 471
 472       - name: Make Update spec
 473         run: |
 474           cat >> _update_spec << EOF
 475           # This file is used for regulating self-update
 476           lock 2022.08.18.36 .+ Python 3\.6
 477           lock 2023.11.16 (?!win_x86_exe).+ Python 3\.7
 478           lock 2023.11.16 win_x86_exe .+ Windows-(?:Vista|2008Server)
 479           lockV2 yt-dlp/yt-dlp 2022.08.18.36 .+ Python 3\.6
 480           lockV2 yt-dlp/yt-dlp 2023.11.16 (?!win_x86_exe).+ Python 3\.7
 481           lockV2 yt-dlp/yt-dlp 2023.11.16 win_x86_exe .+ Windows-(?:Vista|2008Server)
 482           lockV2 yt-dlp/yt-dlp-nightly-builds 2023.11.15.232826 (?!win_x86_exe).+ Python 3\.7
 483           lockV2 yt-dlp/yt-dlp-nightly-builds 2023.11.15.232826 win_x86_exe .+ Windows-(?:Vista|2008Server)
 484           lockV2 yt-dlp/yt-dlp-master-builds 2023.11.15.232812 (?!win_x86_exe).+ Python 3\.7
 485           lockV2 yt-dlp/yt-dlp-master-builds 2023.11.15.232812 win_x86_exe .+ Windows-(?:Vista|2008Server)
 486           EOF
 487
 488       - name: Sign checksum files
 489         env:
 490           GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }}
 491         if: env.GPG_SIGNING_KEY != ''
 492         run: |
 493           gpg --batch --import <<< "${{ secrets.GPG_SIGNING_KEY }}"
 494           for signfile in ./SHA*SUMS; do
 495             gpg --batch --detach-sign "$signfile"
 496           done
 497
 498       - name: Upload artifacts
 499         uses: actions/upload-artifact@v4
 500         with:
 501           name: build-${{ github.job }}
 502           path: |
 503             _update_spec
 504             SHA*SUMS*
 505           compression-level: 0
 506           overwrite: true