X-Git-Url: https://jfr.im/git/yt-dlp.git/blobdiff_plain/2a86f3da0767e2109bcd78b5a9a5b72d1f2aa6c8..882e3b753c79c7799ce135c3a5edb72494b576af:/.github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 58a72c9e5..036ce4348 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,177 +1,466 @@ -name: Build - +name: Build Artifacts on: - push: - branches: - - release + workflow_call: + inputs: + version: + required: true + type: string + channel: + required: false + default: stable + type: string + unix: + default: true + type: boolean + linux_arm: + default: true + type: boolean + macos: + default: true + type: boolean + macos_legacy: + default: true + type: boolean + windows: + default: true + type: boolean + windows32: + default: true + type: boolean + meta_files: + default: true + type: boolean + origin: + required: false + default: '' + type: string + secrets: + GPG_SIGNING_KEY: + required: false + + workflow_dispatch: + inputs: + version: + description: | + VERSION: yyyy.mm.dd[.rev] or rev + required: true + type: string + channel: + description: | + SOURCE of this build's updates: stable/nightly/master/ + required: true + default: stable + type: string + unix: + description: yt-dlp, yt-dlp.tar.gz, yt-dlp_linux, yt-dlp_linux.zip + default: true + type: boolean + linux_arm: + description: yt-dlp_linux_aarch64, yt-dlp_linux_armv7l + default: true + type: boolean + macos: + description: yt-dlp_macos, yt-dlp_macos.zip + default: true + type: boolean + macos_legacy: + description: yt-dlp_macos_legacy + default: true + type: boolean + windows: + description: yt-dlp.exe, yt-dlp_min.exe, yt-dlp_win.zip + default: true + type: boolean + windows32: + description: yt-dlp_x86.exe + default: true + type: boolean + meta_files: + description: SHA2-256SUMS, SHA2-512SUMS, _update_spec + default: true + type: boolean + origin: + description: Origin + required: false + default: 'current repo' + type: choice + options: + - 'current repo' + +permissions: + contents: read jobs: - build_unix: + process: + runs-on: ubuntu-latest + outputs: + origin: ${{ steps.process_origin.outputs.origin }} + steps: + - name: Process origin + id: process_origin + run: | + echo "origin=${{ inputs.origin == 'current repo' && github.repository || inputs.origin }}" | tee "$GITHUB_OUTPUT" + unix: + needs: process + if: inputs.unix runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + - uses: conda-incubator/setup-miniconda@v2 + with: + miniforge-variant: Mambaforge + use-mamba: true + channels: conda-forge + auto-update-conda: true + activate-environment: "" + auto-activate-base: false + - name: Install Requirements + run: | + sudo apt -y install zip pandoc man sed + reqs=$(mktemp) + cat > "$reqs" << EOF + python=3.10.* + pyinstaller + cffi + brotli-python + secretstorage + EOF + sed -E '/^(brotli|secretstorage).*/d' requirements.txt >> "$reqs" + mamba create -n build --file "$reqs" - outputs: - ytdlc_version: ${{ steps.bump_version.outputs.ytdlc_version }} - upload_url: ${{ steps.create_release.outputs.upload_url }} - sha2_unix: ${{ steps.sha2_file.outputs.sha2_unix }} + - name: Prepare + run: | + python devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}" + python devscripts/make_lazy_extractors.py + - name: Build Unix platform-independent binary + run: | + make all tar + - name: Build Unix standalone binary + shell: bash -l {0} + run: | + unset LD_LIBRARY_PATH # Harmful; set by setup-python + conda activate build + python pyinst.py --onedir + (cd ./dist/yt-dlp_linux && zip -r ../yt-dlp_linux.zip .) + python pyinst.py + mv ./dist/yt-dlp_linux ./yt-dlp_linux + mv ./dist/yt-dlp_linux.zip ./yt-dlp_linux.zip + + - name: Verify --update-to + if: vars.UPDATE_TO_VERIFICATION + run: | + binaries=("yt-dlp" "yt-dlp_linux") + for binary in "${binaries[@]}"; do + chmod +x ./${binary} + cp ./${binary} ./${binary}_downgraded + version="$(./${binary} --version)" + ./${binary}_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04 + downgraded_version="$(./${binary}_downgraded --version)" + [[ "$version" != "$downgraded_version" ]] + done + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + path: | + yt-dlp + yt-dlp.tar.gz + yt-dlp_linux + yt-dlp_linux.zip + + linux_arm: + needs: process + if: inputs.linux_arm + permissions: + contents: read + packages: write # for creating cache + runs-on: ubuntu-latest + strategy: + matrix: + architecture: + - armv7 + - aarch64 steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.8' - - name: Install packages - run: sudo apt-get -y install zip pandoc man - - name: Bump version - id: bump_version - run: python devscripts/update-version.py - - name: Print version - run: echo "${{ steps.bump_version.outputs.ytdlc_version }}" - - name: Run Make - run: make - - name: Create Release - id: create_release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ steps.bump_version.outputs.ytdlc_version }} - release_name: yt-dlp ${{ steps.bump_version.outputs.ytdlc_version }} - body: | - Changelog: - PLACEHOLDER - draft: false - prerelease: false - - name: Upload youtube-dlc Unix binary - id: upload-release-asset - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ./youtube-dlc - asset_name: youtube-dlc - asset_content_type: application/octet-stream - - name: Get SHA2-256SUMS for youtube-dlc - id: sha2_file - run: echo "::set-output name=sha2_unix::$(sha256sum youtube-dlc | awk '{print $1}')" - - name: Install dependencies for pypi - env: - PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} - if: "env.PYPI_TOKEN != ''" - run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine - - name: Build and publish on pypi - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} - if: "env.TWINE_PASSWORD != ''" - run: | - rm -rf dist/* - python setup.py sdist bdist_wheel - twine upload dist/* - - build_windows: + - uses: actions/checkout@v4 + with: + path: ./repo + - name: Virtualized Install, Prepare & Build + uses: yt-dlp/run-on-arch-action@v2 + with: + # Ref: https://github.com/uraimo/run-on-arch-action/issues/55 + env: | + GITHUB_WORKFLOW: build + githubToken: ${{ github.token }} # To cache image + arch: ${{ matrix.architecture }} + distro: ubuntu18.04 # Standalone executable should be built on minimum supported OS + dockerRunArgs: --volume "${PWD}/repo:/repo" + install: | # Installing Python 3.10 from the Deadsnakes repo raises errors + apt update + apt -y install zlib1g-dev python3.8 python3.8-dev python3.8-distutils python3-pip + python3.8 -m pip install -U pip setuptools wheel + # Cannot access requirements.txt from the repo directory at this stage + python3.8 -m pip install -U Pyinstaller mutagen pycryptodomex websockets brotli certifi secretstorage - runs-on: windows-latest + run: | + cd repo + python3.8 -m pip install -U Pyinstaller secretstorage -r requirements.txt # Cached version may be out of date + python3.8 devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}" + python3.8 devscripts/make_lazy_extractors.py + python3.8 pyinst.py - outputs: - sha2_windows: ${{ steps.sha2_file_win.outputs.sha2_windows }} + if ${{ vars.UPDATE_TO_VERIFICATION && 'true' || 'false' }}; then + arch="${{ (matrix.architecture == 'armv7' && 'armv7l') || matrix.architecture }}" + chmod +x ./dist/yt-dlp_linux_${arch} + cp ./dist/yt-dlp_linux_${arch} ./dist/yt-dlp_linux_${arch}_downgraded + version="$(./dist/yt-dlp_linux_${arch} --version)" + ./dist/yt-dlp_linux_${arch}_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04 + downgraded_version="$(./dist/yt-dlp_linux_${arch}_downgraded --version)" + [[ "$version" != "$downgraded_version" ]] + fi + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + path: | # run-on-arch-action designates armv7l as armv7 + repo/dist/yt-dlp_linux_${{ (matrix.architecture == 'armv7' && 'armv7l') || matrix.architecture }} + + macos: + needs: process + if: inputs.macos + runs-on: macos-11 + + steps: + - uses: actions/checkout@v4 + # NB: Building universal2 does not work with python from actions/setup-python + - name: Install Requirements + run: | + brew install coreutils + python3 -m pip install -U --user pip setuptools wheel + # We need to ignore wheels otherwise we break universal2 builds + python3 -m pip install -U --user --no-binary :all: Pyinstaller -r requirements.txt + + - name: Prepare + run: | + python3 devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}" + python3 devscripts/make_lazy_extractors.py + - name: Build + run: | + python3 pyinst.py --target-architecture universal2 --onedir + (cd ./dist/yt-dlp_macos && zip -r ../yt-dlp_macos.zip .) + python3 pyinst.py --target-architecture universal2 + + - name: Verify --update-to + if: vars.UPDATE_TO_VERIFICATION + run: | + chmod +x ./dist/yt-dlp_macos + cp ./dist/yt-dlp_macos ./dist/yt-dlp_macos_downgraded + version="$(./dist/yt-dlp_macos --version)" + ./dist/yt-dlp_macos_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04 + downgraded_version="$(./dist/yt-dlp_macos_downgraded --version)" + [[ "$version" != "$downgraded_version" ]] - needs: build_unix + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + path: | + dist/yt-dlp_macos + dist/yt-dlp_macos.zip + + macos_legacy: + needs: process + if: inputs.macos_legacy + runs-on: macos-latest steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.8' - - name: Upgrade pip and enable wheel support - run: python -m pip install --upgrade pip setuptools wheel - - name: Install Requirements - run: pip install pyinstaller mutagen pycryptodome - - name: Bump version - id: bump_version - run: python devscripts/update-version.py - - name: Print version - run: echo "${{ steps.bump_version.outputs.ytdlc_version }}" - - name: Run PyInstaller Script - run: python pyinst.py 64 - - name: Upload youtube-dlc.exe Windows binary - id: upload-release-windows - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ needs.build_unix.outputs.upload_url }} - asset_path: ./dist/youtube-dlc.exe - asset_name: youtube-dlc.exe - asset_content_type: application/vnd.microsoft.portable-executable - - name: Get SHA2-256SUMS for youtube-dlc.exe - id: sha2_file_win - run: echo "::set-output name=sha2_windows::$((Get-FileHash dist\youtube-dlc.exe -Algorithm SHA256).Hash.ToLower())" - - build_windows32: + - uses: actions/checkout@v4 + - name: Install Python + # We need the official Python, because the GA ones only support newer macOS versions + env: + PYTHON_VERSION: 3.10.5 + MACOSX_DEPLOYMENT_TARGET: 10.9 # Used up by the Python build tools + run: | + # Hack to get the latest patch version. Uncomment if needed + #brew install python@3.10 + #export PYTHON_VERSION=$( $(brew --prefix)/opt/python@3.10/bin/python3 --version | cut -d ' ' -f 2 ) + curl https://www.python.org/ftp/python/${PYTHON_VERSION}/python-${PYTHON_VERSION}-macos11.pkg -o "python.pkg" + sudo installer -pkg python.pkg -target / + python3 --version + - name: Install Requirements + run: | + brew install coreutils + python3 -m pip install -U --user pip setuptools wheel + python3 -m pip install -U --user Pyinstaller -r requirements.txt + + - name: Prepare + run: | + python3 devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}" + python3 devscripts/make_lazy_extractors.py + - name: Build + run: | + python3 pyinst.py + mv dist/yt-dlp_macos dist/yt-dlp_macos_legacy + + - name: Verify --update-to + if: vars.UPDATE_TO_VERIFICATION + run: | + chmod +x ./dist/yt-dlp_macos_legacy + cp ./dist/yt-dlp_macos_legacy ./dist/yt-dlp_macos_legacy_downgraded + version="$(./dist/yt-dlp_macos_legacy --version)" + ./dist/yt-dlp_macos_legacy_downgraded -v --update-to yt-dlp/yt-dlp@2023.03.04 + downgraded_version="$(./dist/yt-dlp_macos_legacy_downgraded --version)" + [[ "$version" != "$downgraded_version" ]] + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + path: | + dist/yt-dlp_macos_legacy + + windows: + needs: process + if: inputs.windows runs-on: windows-latest - outputs: - sha2_windows32: ${{ steps.sha2_file_win32.outputs.sha2_windows32 }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v4 + with: # 3.8 is used for Win7 support + python-version: "3.8" + - name: Install Requirements + run: | # Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds + python -m pip install -U pip setuptools wheel py2exe + pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-5.8.0-py3-none-any.whl" -r requirements.txt + + - name: Prepare + run: | + python devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}" + python devscripts/make_lazy_extractors.py + - name: Build + run: | + python setup.py py2exe + Move-Item ./dist/yt-dlp.exe ./dist/yt-dlp_min.exe + python pyinst.py + python pyinst.py --onedir + Compress-Archive -Path ./dist/yt-dlp/* -DestinationPath ./dist/yt-dlp_win.zip + + - name: Verify --update-to + if: vars.UPDATE_TO_VERIFICATION + run: | + foreach ($name in @("yt-dlp","yt-dlp_min")) { + Copy-Item "./dist/${name}.exe" "./dist/${name}_downgraded.exe" + $version = & "./dist/${name}.exe" --version + & "./dist/${name}_downgraded.exe" -v --update-to yt-dlp/yt-dlp@2023.03.04 + $downgraded_version = & "./dist/${name}_downgraded.exe" --version + if ($version -eq $downgraded_version) { + exit 1 + } + } + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + path: | + dist/yt-dlp.exe + dist/yt-dlp_min.exe + dist/yt-dlp_win.zip + + windows32: + needs: process + if: inputs.windows32 + runs-on: windows-latest + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v4 + with: + python-version: "3.8" + architecture: "x86" + - name: Install Requirements + run: | + python -m pip install -U pip setuptools wheel + pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-5.8.0-py3-none-any.whl" -r requirements.txt + + - name: Prepare + run: | + python devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}" + python devscripts/make_lazy_extractors.py + - name: Build + run: | + python pyinst.py + + - name: Verify --update-to + if: vars.UPDATE_TO_VERIFICATION + run: | + foreach ($name in @("yt-dlp_x86")) { + Copy-Item "./dist/${name}.exe" "./dist/${name}_downgraded.exe" + $version = & "./dist/${name}.exe" --version + & "./dist/${name}_downgraded.exe" -v --update-to yt-dlp/yt-dlp@2023.03.04 + $downgraded_version = & "./dist/${name}_downgraded.exe" --version + if ($version -eq $downgraded_version) { + exit 1 + } + } - needs: [build_unix, build_windows] + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + path: | + dist/yt-dlp_x86.exe + meta_files: + if: inputs.meta_files && always() && !cancelled() + needs: + - process + - unix + - linux_arm + - macos + - macos_legacy + - windows + - windows32 + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.4.4 32-Bit - uses: actions/setup-python@v2 - with: - python-version: '3.4.4' - architecture: 'x86' - - name: Upgrade pip and enable wheel support - run: python -m pip install pip==19.1.1 setuptools==43.0.0 wheel==0.33.6 - - name: Install Requirements for 32 Bit - run: pip install pyinstaller==3.5 mutagen==1.42.0 pycryptodome==3.9.4 - - name: Bump version - id: bump_version - run: python devscripts/update-version.py - - name: Print version - run: echo "${{ steps.bump_version.outputs.ytdlc_version }}" - - name: Run PyInstaller Script for 32 Bit - run: python pyinst.py 32 - - name: Upload Executable youtube-dlc_x86.exe - id: upload-release-windows32 - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ needs.build_unix.outputs.upload_url }} - asset_path: ./dist/youtube-dlc_x86.exe - asset_name: youtube-dlc_x86.exe - asset_content_type: application/vnd.microsoft.portable-executable - - name: Get SHA2-256SUMS for youtube-dlc_x86.exe - id: sha2_file_win32 - run: echo "::set-output name=sha2_windows32::$((Get-FileHash dist\youtube-dlc_x86.exe -Algorithm SHA256).Hash.ToLower())" - - name: Make SHA2-256SUMS file - env: - SHA2_WINDOWS: ${{ needs.build_windows.outputs.sha2_windows }} - SHA2_WINDOWS32: ${{ steps.sha2_file_win32.outputs.sha2_windows32 }} - SHA2_UNIX: ${{ needs.build_unix.outputs.sha2_unix }} - YTDLC_VERSION: ${{ needs.build_unix.outputs.ytdlc_version }} - run: | - echo "version:${env:YTDLC_VERSION}" >> SHA2-256SUMS - echo "youtube-dlc.exe:${env:SHA2_WINDOWS}" >> SHA2-256SUMS - echo "youtube-dlc_x86.exe:${env:SHA2_WINDOWS32}" >> SHA2-256SUMS - echo "youtube-dlc:${env:SHA2_UNIX}" >> SHA2-256SUMS - - - name: Upload 256SUMS file - id: upload-sums - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ needs.build_unix.outputs.upload_url }} - asset_path: ./SHA2-256SUMS - asset_name: SHA2-256SUMS - asset_content_type: text/plain + - uses: actions/download-artifact@v3 + + - name: Make SHA2-SUMS files + run: | + cd ./artifact/ + sha256sum * > ../SHA2-256SUMS + sha512sum * > ../SHA2-512SUMS + + - name: Make Update spec + run: | + cat >> _update_spec << EOF + # This file is used for regulating self-update + lock 2022.08.18.36 .+ Python 3\.6 + lock 2023.11.16 (?!win_x86_exe).+ Python 3\.7 + lock 2023.11.16 win_x86_exe .+ Windows-(?:Vista|2008Server) + lockV2 yt-dlp/yt-dlp 2022.08.18.36 .+ Python 3\.6 + lockV2 yt-dlp/yt-dlp 2023.11.16 (?!win_x86_exe).+ Python 3\.7 + lockV2 yt-dlp/yt-dlp 2023.11.16 win_x86_exe .+ Windows-(?:Vista|2008Server) + lockV2 yt-dlp/yt-dlp-nightly-builds 2023.11.15.232826 (?!win_x86_exe).+ Python 3\.7 + lockV2 yt-dlp/yt-dlp-nightly-builds 2023.11.15.232826 win_x86_exe .+ Windows-(?:Vista|2008Server) + lockV2 yt-dlp/yt-dlp-master-builds 2023.11.15.232812 (?!win_x86_exe).+ Python 3\.7 + lockV2 yt-dlp/yt-dlp-master-builds 2023.11.15.232812 win_x86_exe .+ Windows-(?:Vista|2008Server) + EOF + + - name: Sign checksum files + env: + GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} + if: env.GPG_SIGNING_KEY != '' + run: | + gpg --batch --import <<< "${{ secrets.GPG_SIGNING_KEY }}" + for signfile in ./SHA*SUMS; do + gpg --batch --detach-sign "$signfile" + done + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + path: | + SHA*SUMS* + _update_spec