From 0e2bf330ca52db65b94ea0cffb912b89b6495454 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Thu, 12 Nov 2026 13:09:10 -0700 Subject: [PATCH 1/8] Move the Linux CUDA wheel workflow to OSDC runners --- .github/workflows/linux_cuda_wheel.yaml | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/.github/workflows/linux_cuda_wheel.yaml b/.github/workflows/linux_cuda_wheel.yaml index bc6677813..9ecb8bacc 100644 --- a/.github/workflows/linux_cuda_wheel.yaml +++ b/.github/workflows/linux_cuda_wheel.yaml @@ -36,6 +36,9 @@ jobs: os: linux test-infra-repository: pytorch/test-infra test-infra-ref: main + # OSDC (ARC) runners: regular on pull requests, release-isolated on + # main, nightly, release/* and v* tags. + runner-fleet: osdc with-cpu: disable with-xpu: disable with-rocm: disable @@ -68,7 +71,7 @@ jobs: build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 ENABLE_CUDA=1 python -m build --wheel -vvv --no-isolation" install-and-test: - runs-on: linux.g5.4xlarge.nvidia.gpu + runs-on: mt-l-x86aavx2-11-41-a10g strategy: fail-fast: false matrix: @@ -324,7 +327,7 @@ jobs: FAIL_WITHOUT_IMAGE_CODECS=1 python packaging/assert_gil_disabled.py --override-ini="addopts=-v" --tb=short build-docs: - runs-on: linux.g5.4xlarge.nvidia.gpu + runs-on: mt-l-x86aavx2-11-41-a10g env: PYTHON_VERSION: '3.11' CUDA_VERSION: '13.2' @@ -390,9 +393,16 @@ jobs: path: docs/build/html/ doc-preview: - runs-on: [self-hosted, linux.2xlarge] + runs-on: mt-l-x86iavx512-8-64 + container: + image: docker.io/python:3.11-slim needs: build-docs - if: github.repository == 'meta-pytorch/torchcodec' && github.event_name == 'pull_request' + # Fork PRs get no OIDC token, so they cannot assume the role the upload needs. + if: github.repository == 'meta-pytorch/torchcodec' && github.event_name == 'pull_request' && + github.event.pull_request.head.repo.fork == false + permissions: + id-token: write + contents: read steps: - uses: actions/download-artifact@v4 with: @@ -404,6 +414,13 @@ jobs: run: | find docs -name "*.html" -print0 | xargs -0 sed -i '//a \ \ '; + # The S3 write used to come from the EC2 instance role. A pod has none. + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4 + with: + role-to-assume: arn:aws:iam::308535385114:role/arc + aws-region: us-east-1 + - name: Upload docs preview uses: seemethere/upload-artifact-s3@v5 with: From 43a54d6f3377dad0a8cb6803fd11cf04f04a591d Mon Sep 17 00:00:00 2001 From: Huy Do Date: Thu, 12 Nov 2026 15:59:35 -0700 Subject: [PATCH 2/8] Fix the OSDC runner comment: no release runners in meta-pytorch --- .github/workflows/linux_cuda_wheel.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/linux_cuda_wheel.yaml b/.github/workflows/linux_cuda_wheel.yaml index 9ecb8bacc..5b9d27bbe 100644 --- a/.github/workflows/linux_cuda_wheel.yaml +++ b/.github/workflows/linux_cuda_wheel.yaml @@ -36,8 +36,7 @@ jobs: os: linux test-infra-repository: pytorch/test-infra test-infra-ref: main - # OSDC (ARC) runners: regular on pull requests, release-isolated on - # main, nightly, release/* and v* tags. + # OSDC (ARC) runners. meta-pytorch uses the regular ones on every ref. runner-fleet: osdc with-cpu: disable with-xpu: disable From 6f67c0723a67ef283324dd9d4f1a56b69762de47 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Tue, 17 Nov 2026 13:49:36 -0700 Subject: [PATCH 3/8] Drop the comment that restates runner-fleet: osdc --- .github/workflows/linux_cuda_wheel.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/linux_cuda_wheel.yaml b/.github/workflows/linux_cuda_wheel.yaml index 5b9d27bbe..5129d9d56 100644 --- a/.github/workflows/linux_cuda_wheel.yaml +++ b/.github/workflows/linux_cuda_wheel.yaml @@ -36,7 +36,6 @@ jobs: os: linux test-infra-repository: pytorch/test-infra test-infra-ref: main - # OSDC (ARC) runners. meta-pytorch uses the regular ones on every ref. runner-fleet: osdc with-cpu: disable with-xpu: disable From 896aa038a093a8ccbb0cc1d8e97c9b2b596751d1 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Tue, 17 Nov 2026 14:12:31 -0700 Subject: [PATCH 4/8] Move the docs build and preview to a docs workflow triggered by ciflow/docs --- .github/labeler.yml | 5 + .github/pytorch-probot.yml | 3 + .github/workflows/docs.yaml | 223 ++++++++++++++++++++++++ .github/workflows/linux_cuda_wheel.yaml | 151 ---------------- 4 files changed, 231 insertions(+), 151 deletions(-) create mode 100644 .github/labeler.yml create mode 100644 .github/pytorch-probot.yml create mode 100644 .github/workflows/docs.yaml diff --git a/.github/labeler.yml b/.github/labeler.yml new file mode 100644 index 000000000..abfde27e4 --- /dev/null +++ b/.github/labeler.yml @@ -0,0 +1,5 @@ +"ciflow/docs": +- "**" +# "**" skips dot paths like .github/ +- ".*" +- ".*/**" diff --git a/.github/pytorch-probot.yml b/.github/pytorch-probot.yml new file mode 100644 index 000000000..1e1dc23e9 --- /dev/null +++ b/.github/pytorch-probot.yml @@ -0,0 +1,3 @@ +ciflow_push_tags: +- ciflow/docs +labeler_config: labeler.yml diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml new file mode 100644 index 000000000..7ee579d06 --- /dev/null +++ b/.github/workflows/docs.yaml @@ -0,0 +1,223 @@ +name: Docs + +on: + push: + branches: + - main + tags: + - v[0-9]+.[0-9]+.[0-9]+ + # Pushed by pytorch-bot for the ciflow/docs label, which the labeler adds to + # every PR. This runs in this repo, so fork PRs can upload a preview too. + - ciflow/docs/* + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }} + cancel-in-progress: true + +permissions: + id-token: write + contents: read + +defaults: + run: + shell: bash -l -eo pipefail {0} + +jobs: + generate-matrix: + uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main + with: + package-type: wheel + os: linux + test-infra-repository: pytorch/test-infra + test-infra-ref: main + runner-fleet: osdc + with-cpu: disable + with-xpu: disable + with-rocm: disable + with-cuda: enable + python-versions: '["3.11"]' + + filter-matrix: + needs: generate-matrix + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.filter.outputs.matrix }} + steps: + - id: filter + env: + MATRIX: ${{ needs.generate-matrix.outputs.matrix }} + # build-docs installs only this wheel; keep in sync with its CUDA_VERSION. + run: | + echo "matrix=$(jq -c '{include: [.include[] | select(.desired_cuda == "cu132")]}' <<< "${MATRIX}")" >> "${GITHUB_OUTPUT}" + + build: + needs: filter-matrix + strategy: + fail-fast: false + name: Build wheel + uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main + with: + repository: meta-pytorch/torchcodec + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.filter-matrix.outputs.matrix }} + pre-script: packaging/pre_build_script.sh + post-script: packaging/post_build_script.sh + smoke-test-script: packaging/fake_smoke_test.py + package-name: torchcodec + trigger-event: ${{ github.event_name }} + build-platform: "python-build-package" + build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 ENABLE_CUDA=1 python -m build --wheel -vvv --no-isolation" + + build-docs: + runs-on: mt-l-x86aavx2-11-41-a10g + env: + PYTHON_VERSION: '3.11' + CUDA_VERSION: '13.2' + FFMPEG_VERSION: '7' + container: + image: "pytorch/manylinux2_28-builder:cuda13.2" # must be same as env!! + options: "--gpus all -e NVIDIA_DRIVER_CAPABILITIES=video,compute,utility" + needs: build + steps: + - name: Setup env vars + run: | + cuda_version_without_periods=$(echo "${{ env.CUDA_VERSION }}" | sed 's/\.//g') + echo cuda_version_without_periods=${cuda_version_without_periods} >> $GITHUB_ENV + + - name: Check out repo + uses: actions/checkout@v6 + + - name: Remove src/ folder + run: bash packaging/remove_src.sh + + - name: Setup miniconda using test-infra + uses: pytorch/test-infra/.github/actions/setup-miniconda@main + with: + python-version: ${{ env.PYTHON_VERSION }} + # We install conda packages at the start because otherwise conda may have conflicts with dependencies. + default-packages: "conda-forge::cuda-version=${{ env.CUDA_VERSION }} conda-forge::cuda-nvrtc conda-forge::cuda-cudart conda-forge::cuda-driver-dev conda-forge::ffmpeg=${{ env.FFMPEG_VERSION }}" + - name: Check env, set LD_LIBRARY_PATH + run: | + ${CONDA_RUN} env + ${CONDA_RUN} conda info + ${CONDA_RUN} nvidia-smi + ${CONDA_RUN} conda list + echo LD_LIBRARY_PATH=$CONDA_PREFIX/lib:/usr/local/cuda/lib64/:${LD_LIBRARY_PATH} >> $GITHUB_ENV + - name: Assert ffmpeg exists + run: | + ${CONDA_RUN} ffmpeg -buildconf + - name: Update pip + run: ${CONDA_RUN} python -m pip install --upgrade pip + - name: Install PyTorch + run: | + ${CONDA_RUN} bash packaging/install_pytorch.sh cu${{ env.cuda_version_without_periods }} "torch torchvision" + ${CONDA_RUN} python -c 'import torch; print(f"{torch.__version__}"); print(f"{torch.__file__}"); print(f"{torch.cuda.is_available()=}")' + + - uses: actions/download-artifact@v4 + with: + name: meta-pytorch_torchcodec__3.11_cu${{ env.cuda_version_without_periods }}_x86_64 + path: dist/ + + - name: Install torchcodec from the wheel + run: ${CONDA_RUN} bash packaging/install_torchcodec_wheel.sh "*cu${{ env.cuda_version_without_periods }}-py3-none*.whl" + + - name: Install doc dependencies + run: | + cd docs + ${CONDA_RUN} python -m pip install -r requirements.txt + - name: Build docs + run: | + cd docs + ${CONDA_RUN} make html + - uses: actions/upload-artifact@v4 + with: + name: Built-Docs + path: docs/build/html/ + + doc-preview: + runs-on: mt-l-x86iavx512-8-64 + container: + image: docker.io/python:3.11-slim + needs: build-docs + if: github.repository == 'meta-pytorch/torchcodec' && startsWith(github.ref, 'refs/tags/ciflow/docs/') + steps: + - uses: actions/download-artifact@v4 + with: + name: Built-Docs + path: docs + + # Update HTML to add the no-index tag so that search engines do not index these ephemeral docs + - name: Add no-index tag + run: | + find docs -name "*.html" -print0 | xargs -0 sed -i '//a \ \ '; + + - name: Get the PR number + id: pr + run: echo "number=${GITHUB_REF_NAME##*/}" >> "${GITHUB_OUTPUT}" + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4 + with: + role-to-assume: arn:aws:iam::308535385114:role/arc + aws-region: us-east-1 + + - name: Upload docs preview + uses: seemethere/upload-artifact-s3@v5 + with: + retention-days: 14 + s3-bucket: doc-previews + if-no-files-found: error + path: docs + s3-prefix: meta-pytorch/torchcodec/${{ steps.pr.outputs.number }} + + upload-docs: + # This job uploads built docs: + # - to the `main` folder of the gh-pages branch (https://meta-pytorch.org/torchcodec/main) on every commit to the `main` branch + # - to the (e.g.) `0.10` folder in the gh-pages branch whenever a corresponding tag is pushed, like `v0.10.0` (https://meta-pytorch.org/torchcodec/0.10). + + needs: build-docs + permissions: + contents: write + if: github.repository == 'meta-pytorch/torchcodec' && github.event_name == 'push' && + ((github.ref_type == 'branch' && github.ref_name == 'main') || + (github.ref_type == 'tag' && startsWith(github.ref_name, 'v') && !contains(github.ref_name, '-rc'))) + runs-on: ubuntu-latest + steps: + - name: Check out gh-pages branch + uses: actions/checkout@v6 + with: + ref: gh-pages + + - uses: actions/download-artifact@v4 + with: + name: Built-Docs + path: docs-artifact/ + + - name: Update docs and push + run: | + set -euo pipefail + + REF_TYPE=${{ github.ref_type }} + REF_NAME=${{ github.ref_name }} + + if [[ "${REF_TYPE}" == branch ]]; then + TARGET_FOLDER="${REF_NAME}" + elif [[ "${REF_TYPE}" == tag ]]; then + # Strip the leading "v" as well as the trailing patch version. For example: + # 'v0.10.2' -> '0.10' + TARGET_FOLDER=$(echo "${REF_NAME}" | sed 's/v\([0-9]\+\)\.\([0-9]\+\)\.[0-9]\+/\1.\2/') + fi + echo "Target Folder: ${TARGET_FOLDER}" + + mkdir -p "${TARGET_FOLDER}" + rm -rf "${TARGET_FOLDER}"/* + cp -r docs-artifact/* "${TARGET_FOLDER}/" + git add "${TARGET_FOLDER}" + + git config user.name 'pytorchbot' + git config user.email 'soumith+bot@pytorch.org' + git commit -m "auto-generating sphinx docs for ${TARGET_FOLDER}" || echo "No changes to commit" + git push diff --git a/.github/workflows/linux_cuda_wheel.yaml b/.github/workflows/linux_cuda_wheel.yaml index 5129d9d56..162d8d8b6 100644 --- a/.github/workflows/linux_cuda_wheel.yaml +++ b/.github/workflows/linux_cuda_wheel.yaml @@ -323,154 +323,3 @@ jobs: run: | python packaging/assert_ffmpeg_version.py ${{ env.FFMPEG_VERSION }} FAIL_WITHOUT_IMAGE_CODECS=1 python packaging/assert_gil_disabled.py --override-ini="addopts=-v" --tb=short - - build-docs: - runs-on: mt-l-x86aavx2-11-41-a10g - env: - PYTHON_VERSION: '3.11' - CUDA_VERSION: '13.2' - FFMPEG_VERSION: '7' - container: - image: "pytorch/manylinux2_28-builder:cuda13.2" # must be same as env!! - options: "--gpus all -e NVIDIA_DRIVER_CAPABILITIES=video,compute,utility" - needs: build - steps: - - name: Setup env vars - run: | - cuda_version_without_periods=$(echo "${{ env.CUDA_VERSION }}" | sed 's/\.//g') - echo cuda_version_without_periods=${cuda_version_without_periods} >> $GITHUB_ENV - - - name: Check out repo - uses: actions/checkout@v6 - - - name: Remove src/ folder - run: bash packaging/remove_src.sh - - - name: Setup miniconda using test-infra - uses: pytorch/test-infra/.github/actions/setup-miniconda@main - with: - python-version: ${{ env.PYTHON_VERSION }} - # We install conda packages at the start because otherwise conda may have conflicts with dependencies. - default-packages: "conda-forge::cuda-version=${{ env.CUDA_VERSION }} conda-forge::cuda-nvrtc conda-forge::cuda-cudart conda-forge::cuda-driver-dev conda-forge::ffmpeg=${{ env.FFMPEG_VERSION }}" - - name: Check env, set LD_LIBRARY_PATH - run: | - ${CONDA_RUN} env - ${CONDA_RUN} conda info - ${CONDA_RUN} nvidia-smi - ${CONDA_RUN} conda list - echo LD_LIBRARY_PATH=$CONDA_PREFIX/lib:/usr/local/cuda/lib64/:${LD_LIBRARY_PATH} >> $GITHUB_ENV - - name: Assert ffmpeg exists - run: | - ${CONDA_RUN} ffmpeg -buildconf - - name: Update pip - run: ${CONDA_RUN} python -m pip install --upgrade pip - - name: Install PyTorch - run: | - ${CONDA_RUN} bash packaging/install_pytorch.sh cu${{ env.cuda_version_without_periods }} "torch torchvision" - ${CONDA_RUN} python -c 'import torch; print(f"{torch.__version__}"); print(f"{torch.__file__}"); print(f"{torch.cuda.is_available()=}")' - - - uses: actions/download-artifact@v4 - with: - name: meta-pytorch_torchcodec__3.11_cu${{ env.cuda_version_without_periods }}_x86_64 - path: dist/ - - - name: Install torchcodec from the wheel - run: ${CONDA_RUN} bash packaging/install_torchcodec_wheel.sh "*cu${{ env.cuda_version_without_periods }}-py3-none*.whl" - - - name: Install doc dependencies - run: | - cd docs - ${CONDA_RUN} python -m pip install -r requirements.txt - - name: Build docs - run: | - cd docs - ${CONDA_RUN} make html - - uses: actions/upload-artifact@v4 - with: - name: Built-Docs - path: docs/build/html/ - - doc-preview: - runs-on: mt-l-x86iavx512-8-64 - container: - image: docker.io/python:3.11-slim - needs: build-docs - # Fork PRs get no OIDC token, so they cannot assume the role the upload needs. - if: github.repository == 'meta-pytorch/torchcodec' && github.event_name == 'pull_request' && - github.event.pull_request.head.repo.fork == false - permissions: - id-token: write - contents: read - steps: - - uses: actions/download-artifact@v4 - with: - name: Built-Docs - path: docs - - # Update HTML to add the no-index tag so that search engines do not index these ephemeral docs - - name: Add no-index tag - run: | - find docs -name "*.html" -print0 | xargs -0 sed -i '//a \ \ '; - - # The S3 write used to come from the EC2 instance role. A pod has none. - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4 - with: - role-to-assume: arn:aws:iam::308535385114:role/arc - aws-region: us-east-1 - - - name: Upload docs preview - uses: seemethere/upload-artifact-s3@v5 - with: - retention-days: 14 - s3-bucket: doc-previews - if-no-files-found: error - path: docs - s3-prefix: meta-pytorch/torchcodec/${{ github.event.pull_request.number }} - - upload-docs: - # This job uploads built docs: - # - to the `main` folder of the gh-pages branch (https://meta-pytorch.org/torchcodec/main) on every commit to the `main` branch - # - to the (e.g.) `0.10` folder in the gh-pages branch whenever a corresponding tag is pushed, like `v0.10.0` (https://meta-pytorch.org/torchcodec/0.10). - - needs: build-docs - if: github.repository == 'meta-pytorch/torchcodec' && github.event_name == 'push' && - ((github.ref_type == 'branch' && github.ref_name == 'main') || - (github.ref_type == 'tag' && !contains(github.ref_name, '-rc'))) - runs-on: ubuntu-latest - steps: - - name: Check out gh-pages branch - uses: actions/checkout@v6 - with: - ref: gh-pages - - - uses: actions/download-artifact@v4 - with: - name: Built-Docs - path: docs-artifact/ - - - name: Update docs and push - run: | - set -euo pipefail - - REF_TYPE=${{ github.ref_type }} - REF_NAME=${{ github.ref_name }} - - if [[ "${REF_TYPE}" == branch ]]; then - TARGET_FOLDER="${REF_NAME}" - elif [[ "${REF_TYPE}" == tag ]]; then - # Strip the leading "v" as well as the trailing patch version. For example: - # 'v0.10.2' -> '0.10' - TARGET_FOLDER=$(echo "${REF_NAME}" | sed 's/v\([0-9]\+\)\.\([0-9]\+\)\.[0-9]\+/\1.\2/') - fi - echo "Target Folder: ${TARGET_FOLDER}" - - mkdir -p "${TARGET_FOLDER}" - rm -rf "${TARGET_FOLDER}"/* - cp -r docs-artifact/* "${TARGET_FOLDER}/" - git add "${TARGET_FOLDER}" - - git config user.name 'pytorchbot' - git config user.email 'soumith+bot@pytorch.org' - git commit -m "auto-generating sphinx docs for ${TARGET_FOLDER}" || echo "No changes to commit" - git push From ee5c6d16819aca2c8fba7f8bb2502979272fd812 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Tue, 17 Nov 2026 14:50:14 -0700 Subject: [PATCH 5/8] Only label PRs touching docs/ or src/ with ciflow/docs --- .github/labeler.yml | 6 ++---- .github/workflows/docs.yaml | 3 ++- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/labeler.yml b/.github/labeler.yml index abfde27e4..de4ee94b6 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -1,5 +1,3 @@ "ciflow/docs": -- "**" -# "**" skips dot paths like .github/ -- ".*" -- ".*/**" +- docs/** +- src/** diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 7ee579d06..083ed466c 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -7,7 +7,8 @@ on: tags: - v[0-9]+.[0-9]+.[0-9]+ # Pushed by pytorch-bot for the ciflow/docs label, which the labeler adds to - # every PR. This runs in this repo, so fork PRs can upload a preview too. + # PRs touching docs/ or src/. This runs in this repo, so fork PRs can upload + # a preview too. - ciflow/docs/* workflow_dispatch: From 87f6dd23537cf98c74edc5cac92589fa660fb280 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Wed, 18 Nov 2026 11:37:25 -0700 Subject: [PATCH 6/8] Upload PR doc previews from a workflow_run workflow Keep build-docs and upload-docs in linux_cuda_wheel so docs are built on every PR. Preview uploads need OIDC, which fork pull_request runs can't get, so a workflow_run workflow uploads the Built-Docs artifact instead. This replaces the ciflow/docs workflow and labeler. --- .github/labeler.yml | 3 - .github/pytorch-probot.yml | 3 - .github/workflows/docs.yaml | 224 ------------------------ .github/workflows/docs_preview.yaml | 83 +++++++++ .github/workflows/linux_cuda_wheel.yaml | 113 ++++++++++++ 5 files changed, 196 insertions(+), 230 deletions(-) delete mode 100644 .github/labeler.yml delete mode 100644 .github/pytorch-probot.yml delete mode 100644 .github/workflows/docs.yaml create mode 100644 .github/workflows/docs_preview.yaml diff --git a/.github/labeler.yml b/.github/labeler.yml deleted file mode 100644 index de4ee94b6..000000000 --- a/.github/labeler.yml +++ /dev/null @@ -1,3 +0,0 @@ -"ciflow/docs": -- docs/** -- src/** diff --git a/.github/pytorch-probot.yml b/.github/pytorch-probot.yml deleted file mode 100644 index 1e1dc23e9..000000000 --- a/.github/pytorch-probot.yml +++ /dev/null @@ -1,3 +0,0 @@ -ciflow_push_tags: -- ciflow/docs -labeler_config: labeler.yml diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml deleted file mode 100644 index 083ed466c..000000000 --- a/.github/workflows/docs.yaml +++ /dev/null @@ -1,224 +0,0 @@ -name: Docs - -on: - push: - branches: - - main - tags: - - v[0-9]+.[0-9]+.[0-9]+ - # Pushed by pytorch-bot for the ciflow/docs label, which the labeler adds to - # PRs touching docs/ or src/. This runs in this repo, so fork PRs can upload - # a preview too. - - ciflow/docs/* - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }} - cancel-in-progress: true - -permissions: - id-token: write - contents: read - -defaults: - run: - shell: bash -l -eo pipefail {0} - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: linux - test-infra-repository: pytorch/test-infra - test-infra-ref: main - runner-fleet: osdc - with-cpu: disable - with-xpu: disable - with-rocm: disable - with-cuda: enable - python-versions: '["3.11"]' - - filter-matrix: - needs: generate-matrix - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.filter.outputs.matrix }} - steps: - - id: filter - env: - MATRIX: ${{ needs.generate-matrix.outputs.matrix }} - # build-docs installs only this wheel; keep in sync with its CUDA_VERSION. - run: | - echo "matrix=$(jq -c '{include: [.include[] | select(.desired_cuda == "cu132")]}' <<< "${MATRIX}")" >> "${GITHUB_OUTPUT}" - - build: - needs: filter-matrix - strategy: - fail-fast: false - name: Build wheel - uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main - with: - repository: meta-pytorch/torchcodec - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/fake_smoke_test.py - package-name: torchcodec - trigger-event: ${{ github.event_name }} - build-platform: "python-build-package" - build-command: "BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 ENABLE_CUDA=1 python -m build --wheel -vvv --no-isolation" - - build-docs: - runs-on: mt-l-x86aavx2-11-41-a10g - env: - PYTHON_VERSION: '3.11' - CUDA_VERSION: '13.2' - FFMPEG_VERSION: '7' - container: - image: "pytorch/manylinux2_28-builder:cuda13.2" # must be same as env!! - options: "--gpus all -e NVIDIA_DRIVER_CAPABILITIES=video,compute,utility" - needs: build - steps: - - name: Setup env vars - run: | - cuda_version_without_periods=$(echo "${{ env.CUDA_VERSION }}" | sed 's/\.//g') - echo cuda_version_without_periods=${cuda_version_without_periods} >> $GITHUB_ENV - - - name: Check out repo - uses: actions/checkout@v6 - - - name: Remove src/ folder - run: bash packaging/remove_src.sh - - - name: Setup miniconda using test-infra - uses: pytorch/test-infra/.github/actions/setup-miniconda@main - with: - python-version: ${{ env.PYTHON_VERSION }} - # We install conda packages at the start because otherwise conda may have conflicts with dependencies. - default-packages: "conda-forge::cuda-version=${{ env.CUDA_VERSION }} conda-forge::cuda-nvrtc conda-forge::cuda-cudart conda-forge::cuda-driver-dev conda-forge::ffmpeg=${{ env.FFMPEG_VERSION }}" - - name: Check env, set LD_LIBRARY_PATH - run: | - ${CONDA_RUN} env - ${CONDA_RUN} conda info - ${CONDA_RUN} nvidia-smi - ${CONDA_RUN} conda list - echo LD_LIBRARY_PATH=$CONDA_PREFIX/lib:/usr/local/cuda/lib64/:${LD_LIBRARY_PATH} >> $GITHUB_ENV - - name: Assert ffmpeg exists - run: | - ${CONDA_RUN} ffmpeg -buildconf - - name: Update pip - run: ${CONDA_RUN} python -m pip install --upgrade pip - - name: Install PyTorch - run: | - ${CONDA_RUN} bash packaging/install_pytorch.sh cu${{ env.cuda_version_without_periods }} "torch torchvision" - ${CONDA_RUN} python -c 'import torch; print(f"{torch.__version__}"); print(f"{torch.__file__}"); print(f"{torch.cuda.is_available()=}")' - - - uses: actions/download-artifact@v4 - with: - name: meta-pytorch_torchcodec__3.11_cu${{ env.cuda_version_without_periods }}_x86_64 - path: dist/ - - - name: Install torchcodec from the wheel - run: ${CONDA_RUN} bash packaging/install_torchcodec_wheel.sh "*cu${{ env.cuda_version_without_periods }}-py3-none*.whl" - - - name: Install doc dependencies - run: | - cd docs - ${CONDA_RUN} python -m pip install -r requirements.txt - - name: Build docs - run: | - cd docs - ${CONDA_RUN} make html - - uses: actions/upload-artifact@v4 - with: - name: Built-Docs - path: docs/build/html/ - - doc-preview: - runs-on: mt-l-x86iavx512-8-64 - container: - image: docker.io/python:3.11-slim - needs: build-docs - if: github.repository == 'meta-pytorch/torchcodec' && startsWith(github.ref, 'refs/tags/ciflow/docs/') - steps: - - uses: actions/download-artifact@v4 - with: - name: Built-Docs - path: docs - - # Update HTML to add the no-index tag so that search engines do not index these ephemeral docs - - name: Add no-index tag - run: | - find docs -name "*.html" -print0 | xargs -0 sed -i '//a \ \ '; - - - name: Get the PR number - id: pr - run: echo "number=${GITHUB_REF_NAME##*/}" >> "${GITHUB_OUTPUT}" - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4 - with: - role-to-assume: arn:aws:iam::308535385114:role/arc - aws-region: us-east-1 - - - name: Upload docs preview - uses: seemethere/upload-artifact-s3@v5 - with: - retention-days: 14 - s3-bucket: doc-previews - if-no-files-found: error - path: docs - s3-prefix: meta-pytorch/torchcodec/${{ steps.pr.outputs.number }} - - upload-docs: - # This job uploads built docs: - # - to the `main` folder of the gh-pages branch (https://meta-pytorch.org/torchcodec/main) on every commit to the `main` branch - # - to the (e.g.) `0.10` folder in the gh-pages branch whenever a corresponding tag is pushed, like `v0.10.0` (https://meta-pytorch.org/torchcodec/0.10). - - needs: build-docs - permissions: - contents: write - if: github.repository == 'meta-pytorch/torchcodec' && github.event_name == 'push' && - ((github.ref_type == 'branch' && github.ref_name == 'main') || - (github.ref_type == 'tag' && startsWith(github.ref_name, 'v') && !contains(github.ref_name, '-rc'))) - runs-on: ubuntu-latest - steps: - - name: Check out gh-pages branch - uses: actions/checkout@v6 - with: - ref: gh-pages - - - uses: actions/download-artifact@v4 - with: - name: Built-Docs - path: docs-artifact/ - - - name: Update docs and push - run: | - set -euo pipefail - - REF_TYPE=${{ github.ref_type }} - REF_NAME=${{ github.ref_name }} - - if [[ "${REF_TYPE}" == branch ]]; then - TARGET_FOLDER="${REF_NAME}" - elif [[ "${REF_TYPE}" == tag ]]; then - # Strip the leading "v" as well as the trailing patch version. For example: - # 'v0.10.2' -> '0.10' - TARGET_FOLDER=$(echo "${REF_NAME}" | sed 's/v\([0-9]\+\)\.\([0-9]\+\)\.[0-9]\+/\1.\2/') - fi - echo "Target Folder: ${TARGET_FOLDER}" - - mkdir -p "${TARGET_FOLDER}" - rm -rf "${TARGET_FOLDER}"/* - cp -r docs-artifact/* "${TARGET_FOLDER}/" - git add "${TARGET_FOLDER}" - - git config user.name 'pytorchbot' - git config user.email 'soumith+bot@pytorch.org' - git commit -m "auto-generating sphinx docs for ${TARGET_FOLDER}" || echo "No changes to commit" - git push diff --git a/.github/workflows/docs_preview.yaml b/.github/workflows/docs_preview.yaml new file mode 100644 index 000000000..f9a7c0d54 --- /dev/null +++ b/.github/workflows/docs_preview.yaml @@ -0,0 +1,83 @@ +name: Docs preview + +# Uploads the docs that the Linux CUDA x86 workflow built for a PR to +# docs-preview.pytorch.org. The upload needs AWS credentials from GitHub OIDC, +# which a pull_request run from a fork can't get. A workflow_run job runs from +# main in this repo, so it can, for fork and same-repo PRs alike. +on: + workflow_run: + workflows: [Linux CUDA x86] + types: [completed] + +permissions: {} + +jobs: + doc-preview: + if: github.repository == 'meta-pytorch/torchcodec' && github.event.workflow_run.event == 'pull_request' + runs-on: mt-l-x86iavx512-8-64 + container: + image: docker.io/python:3.11-slim + permissions: + actions: read + pull-requests: read + id-token: write + steps: + # The artifact comes from PR code, so we only upload it and never run it. + - name: Find the PR and the docs artifact + id: pr + uses: actions/github-script@v8 + with: + script: | + const run = context.payload.workflow_run; + // workflow_run.pull_requests is empty for fork PRs, so look the PR up from its head. + const { data: pulls } = await github.rest.pulls.list({ + ...context.repo, + state: 'all', + head: `${run.head_repository.owner.login}:${run.head_branch}`, + }); + const pr = pulls.find((p) => p.head.sha === run.head_sha); + if (!pr) { + core.notice(`No PR has head ${run.head_sha}`); + return; + } + const { data: { artifacts } } = await github.rest.actions.listWorkflowRunArtifacts({ + ...context.repo, + run_id: run.id, + name: 'Built-Docs', + }); + if (artifacts.length === 0) { + core.notice(`Run ${run.id} has no Built-Docs artifact`); + return; + } + core.setOutput('number', pr.number); + + - uses: actions/download-artifact@v4 + if: steps.pr.outputs.number + with: + name: Built-Docs + path: docs + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ github.token }} + + # Update HTML to add the no-index tag so that search engines do not index these ephemeral docs + - name: Add no-index tag + if: steps.pr.outputs.number + run: | + find docs -name "*.html" -print0 | xargs -0 sed -i '//a \ \ '; + + - name: Configure AWS credentials + if: steps.pr.outputs.number + uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4 + with: + role-to-assume: arn:aws:iam::308535385114:role/arc + aws-region: us-east-1 + + - name: Upload docs preview + if: steps.pr.outputs.number + uses: seemethere/upload-artifact-s3@v5 + with: + retention-days: 14 + s3-bucket: doc-previews + if-no-files-found: error + path: docs + s3-prefix: meta-pytorch/torchcodec/${{ steps.pr.outputs.number }} diff --git a/.github/workflows/linux_cuda_wheel.yaml b/.github/workflows/linux_cuda_wheel.yaml index 162d8d8b6..cb6088d2c 100644 --- a/.github/workflows/linux_cuda_wheel.yaml +++ b/.github/workflows/linux_cuda_wheel.yaml @@ -323,3 +323,116 @@ jobs: run: | python packaging/assert_ffmpeg_version.py ${{ env.FFMPEG_VERSION }} FAIL_WITHOUT_IMAGE_CODECS=1 python packaging/assert_gil_disabled.py --override-ini="addopts=-v" --tb=short + + build-docs: + runs-on: mt-l-x86aavx2-11-41-a10g + env: + PYTHON_VERSION: '3.11' + CUDA_VERSION: '13.2' + FFMPEG_VERSION: '7' + container: + image: "pytorch/manylinux2_28-builder:cuda13.2" # must be same as env!! + options: "--gpus all -e NVIDIA_DRIVER_CAPABILITIES=video,compute,utility" + needs: build + steps: + - name: Setup env vars + run: | + cuda_version_without_periods=$(echo "${{ env.CUDA_VERSION }}" | sed 's/\.//g') + echo cuda_version_without_periods=${cuda_version_without_periods} >> $GITHUB_ENV + + - name: Check out repo + uses: actions/checkout@v6 + + - name: Remove src/ folder + run: bash packaging/remove_src.sh + + - name: Setup miniconda using test-infra + uses: pytorch/test-infra/.github/actions/setup-miniconda@main + with: + python-version: ${{ env.PYTHON_VERSION }} + # We install conda packages at the start because otherwise conda may have conflicts with dependencies. + default-packages: "conda-forge::cuda-version=${{ env.CUDA_VERSION }} conda-forge::cuda-nvrtc conda-forge::cuda-cudart conda-forge::cuda-driver-dev conda-forge::ffmpeg=${{ env.FFMPEG_VERSION }}" + - name: Check env, set LD_LIBRARY_PATH + run: | + ${CONDA_RUN} env + ${CONDA_RUN} conda info + ${CONDA_RUN} nvidia-smi + ${CONDA_RUN} conda list + echo LD_LIBRARY_PATH=$CONDA_PREFIX/lib:/usr/local/cuda/lib64/:${LD_LIBRARY_PATH} >> $GITHUB_ENV + - name: Assert ffmpeg exists + run: | + ${CONDA_RUN} ffmpeg -buildconf + - name: Update pip + run: ${CONDA_RUN} python -m pip install --upgrade pip + - name: Install PyTorch + run: | + ${CONDA_RUN} bash packaging/install_pytorch.sh cu${{ env.cuda_version_without_periods }} "torch torchvision" + ${CONDA_RUN} python -c 'import torch; print(f"{torch.__version__}"); print(f"{torch.__file__}"); print(f"{torch.cuda.is_available()=}")' + + - uses: actions/download-artifact@v4 + with: + name: meta-pytorch_torchcodec__3.11_cu${{ env.cuda_version_without_periods }}_x86_64 + path: dist/ + + - name: Install torchcodec from the wheel + run: ${CONDA_RUN} bash packaging/install_torchcodec_wheel.sh "*cu${{ env.cuda_version_without_periods }}-py3-none*.whl" + + - name: Install doc dependencies + run: | + cd docs + ${CONDA_RUN} python -m pip install -r requirements.txt + - name: Build docs + run: | + cd docs + ${CONDA_RUN} make html + - uses: actions/upload-artifact@v4 + with: + name: Built-Docs + path: docs/build/html/ + + upload-docs: + # This job uploads built docs: + # - to the `main` folder of the gh-pages branch (https://meta-pytorch.org/torchcodec/main) on every commit to the `main` branch + # - to the (e.g.) `0.10` folder in the gh-pages branch whenever a corresponding tag is pushed, like `v0.10.0` (https://meta-pytorch.org/torchcodec/0.10). + + needs: build-docs + if: github.repository == 'meta-pytorch/torchcodec' && github.event_name == 'push' && + ((github.ref_type == 'branch' && github.ref_name == 'main') || + (github.ref_type == 'tag' && !contains(github.ref_name, '-rc'))) + runs-on: ubuntu-latest + steps: + - name: Check out gh-pages branch + uses: actions/checkout@v6 + with: + ref: gh-pages + + - uses: actions/download-artifact@v4 + with: + name: Built-Docs + path: docs-artifact/ + + - name: Update docs and push + run: | + set -euo pipefail + + REF_TYPE=${{ github.ref_type }} + REF_NAME=${{ github.ref_name }} + + if [[ "${REF_TYPE}" == branch ]]; then + TARGET_FOLDER="${REF_NAME}" + elif [[ "${REF_TYPE}" == tag ]]; then + # Strip the leading "v" as well as the trailing patch version. For example: + # 'v0.10.2' -> '0.10' + TARGET_FOLDER=$(echo "${REF_NAME}" | sed 's/v\([0-9]\+\)\.\([0-9]\+\)\.[0-9]\+/\1.\2/') + fi + echo "Target Folder: ${TARGET_FOLDER}" + + mkdir -p "${TARGET_FOLDER}" + rm -rf "${TARGET_FOLDER}"/* + cp -r docs-artifact/* "${TARGET_FOLDER}/" + git add "${TARGET_FOLDER}" + + git config user.name 'pytorchbot' + git config user.email 'soumith+bot@pytorch.org' + git commit -m "auto-generating sphinx docs for ${TARGET_FOLDER}" || echo "No changes to commit" + git push From 2149bb640f277576f259223c9b6e890a0d7461ae Mon Sep 17 00:00:00 2001 From: Huy Do Date: Wed, 18 Nov 2026 20:45:33 -0700 Subject: [PATCH 7/8] Simplify the docs preview workflow --- .github/workflows/docs_preview.yaml | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/.github/workflows/docs_preview.yaml b/.github/workflows/docs_preview.yaml index f9a7c0d54..27f5a275a 100644 --- a/.github/workflows/docs_preview.yaml +++ b/.github/workflows/docs_preview.yaml @@ -13,6 +13,7 @@ permissions: {} jobs: doc-preview: + # workflow_run.event is the event that triggered the Linux CUDA x86 run. if: github.repository == 'meta-pytorch/torchcodec' && github.event.workflow_run.event == 'pull_request' runs-on: mt-l-x86iavx512-8-64 container: @@ -23,7 +24,7 @@ jobs: id-token: write steps: # The artifact comes from PR code, so we only upload it and never run it. - - name: Find the PR and the docs artifact + - name: Find the PR id: pr uses: actions/github-script@v8 with: @@ -32,27 +33,14 @@ jobs: // workflow_run.pull_requests is empty for fork PRs, so look the PR up from its head. const { data: pulls } = await github.rest.pulls.list({ ...context.repo, - state: 'all', head: `${run.head_repository.owner.login}:${run.head_branch}`, }); const pr = pulls.find((p) => p.head.sha === run.head_sha); - if (!pr) { - core.notice(`No PR has head ${run.head_sha}`); - return; - } - const { data: { artifacts } } = await github.rest.actions.listWorkflowRunArtifacts({ - ...context.repo, - run_id: run.id, - name: 'Built-Docs', - }); - if (artifacts.length === 0) { - core.notice(`Run ${run.id} has no Built-Docs artifact`); - return; - } + if (!pr) throw new Error(`No PR has head ${run.head_sha}`); core.setOutput('number', pr.number); + # Uploaded by build-docs in linux_cuda_wheel.yaml. - uses: actions/download-artifact@v4 - if: steps.pr.outputs.number with: name: Built-Docs path: docs @@ -61,19 +49,16 @@ jobs: # Update HTML to add the no-index tag so that search engines do not index these ephemeral docs - name: Add no-index tag - if: steps.pr.outputs.number run: | find docs -name "*.html" -print0 | xargs -0 sed -i '//a \ \ '; - name: Configure AWS credentials - if: steps.pr.outputs.number uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4 with: role-to-assume: arn:aws:iam::308535385114:role/arc aws-region: us-east-1 - name: Upload docs preview - if: steps.pr.outputs.number uses: seemethere/upload-artifact-s3@v5 with: retention-days: 14 From 378abdfec485451e3e1da29a00079f3397bf5aa1 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Thu, 19 Nov 2026 15:44:31 -0700 Subject: [PATCH 8/8] Only upload the doc preview when the run built the docs --- .github/workflows/docs_preview.yaml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docs_preview.yaml b/.github/workflows/docs_preview.yaml index 27f5a275a..8828fea7f 100644 --- a/.github/workflows/docs_preview.yaml +++ b/.github/workflows/docs_preview.yaml @@ -24,12 +24,22 @@ jobs: id-token: write steps: # The artifact comes from PR code, so we only upload it and never run it. - - name: Find the PR + - name: Find the docs artifact and the PR id: pr uses: actions/github-script@v8 with: script: | const run = context.payload.workflow_run; + // A failed or cancelled run may not have built the docs. + const { data: { total_count } } = await github.rest.actions.listWorkflowRunArtifacts({ + ...context.repo, + run_id: run.id, + name: 'Built-Docs', + }); + if (total_count === 0) { + core.notice(`Run ${run.id} has no Built-Docs artifact`); + return; + } // workflow_run.pull_requests is empty for fork PRs, so look the PR up from its head. const { data: pulls } = await github.rest.pulls.list({ ...context.repo, @@ -41,6 +51,7 @@ jobs: # Uploaded by build-docs in linux_cuda_wheel.yaml. - uses: actions/download-artifact@v4 + if: steps.pr.outputs.number with: name: Built-Docs path: docs @@ -49,16 +60,19 @@ jobs: # Update HTML to add the no-index tag so that search engines do not index these ephemeral docs - name: Add no-index tag + if: steps.pr.outputs.number run: | find docs -name "*.html" -print0 | xargs -0 sed -i '//a \ \ '; - name: Configure AWS credentials + if: steps.pr.outputs.number uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4 with: role-to-assume: arn:aws:iam::308535385114:role/arc aws-region: us-east-1 - name: Upload docs preview + if: steps.pr.outputs.number uses: seemethere/upload-artifact-s3@v5 with: retention-days: 14