diff --git a/.github/AGENTS.md b/.github/AGENTS.md index f0d319bc..78a066d3 100644 --- a/.github/AGENTS.md +++ b/.github/AGENTS.md @@ -3,8 +3,7 @@ CI/CD workflows and repo automation. ## Workflows (source of truth) -- `conda-package.yml` — Intel channel conda build/test pipeline -- `conda-package-cf.yml` — conda-forge-oriented build/test pipeline +- `intel-ci.yml` — Intel CI: builds and tests the conda packages of `conda-recipe` and the wheels, for the platforms and Python versions the recipe and `pyproject.toml` declare (`ci/matrix.py`); see `ci/README.md`; its `Agent guidance` job (`ci/guidance.py`) fails a pull request that adds a broken claim to a changed guidance file - `build-with-clang.yml` — clang compatibility checks - `build_pip.yml` — editable pip build pipeline - `build-with-standard-clang.yml` — standard clang compatibility checks @@ -12,5 +11,5 @@ CI/CD workflows and repo automation. - `openssf-scorecard.yml` — security scanning ## Policy -- Treat workflow YAML as canonical for platform/Python matrices. +- Treat workflow YAML as canonical for platform/Python matrices, except `intel-ci.yml`, whose conda matrix is `conda-recipe/conda_build_config.yaml` and whose wheel matrix is the Python classifiers in `pyproject.toml`. - Avoid doc claims about CI coverage unless present in workflow config. diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index ded9e95f..6d075e54 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -35,7 +35,7 @@ Higher-precedence file overrides lower-precedence context. ## Source-of-truth files - Build/config: `pyproject.toml`, `meson.build` -- Dependencies: `pyproject.toml`, `conda-recipe/meta.yaml`, `conda-recipe-cf/meta.yaml` +- Dependencies: `pyproject.toml`, `conda-recipe/meta.yaml` - CI: `.github/workflows/*.{yml,yaml}` - API: `mkl_fft/__init__.py`, `mkl_fft/interfaces/*.py`, `mkl_fft/_pydfti.pyx` (interface wrappers are integration/patch entry points) - Tests: `mkl_fft/tests/` diff --git a/.github/workflows/conda-package-cf.yml b/.github/workflows/conda-package-cf.yml deleted file mode 100644 index 229b5f32..00000000 --- a/.github/workflows/conda-package-cf.yml +++ /dev/null @@ -1,393 +0,0 @@ -name: Conda package with conda-forge channel only - -on: - push: - branches: - - master - pull_request: - -permissions: - contents: read - -env: - PACKAGE_NAME: mkl_fft - MODULE_NAME: mkl_fft - TEST_ENV_NAME: test_mkl_fft - VER_SCRIPT1: "import json; f = open('ver.json', 'r'); j = json.load(f); f.close(); d = j['mkl_fft'][0];" - VER_SCRIPT2: "print('='.join((d[s] for s in ('version', 'build'))))" - -jobs: - build_linux: - runs-on: ubuntu-latest - strategy: - matrix: - include: - - python: "3.10" - numpy: "2.2" - - python: "3.11" - numpy: "2.3" - - python: "3.12" - numpy: "2.3" - - python: "3.13" - numpy: "2.3" - - python: "3.14" - numpy: "2.3" - python_spec: "3.14.* *_cp314" - python_tag: "3.14" - - python: "3.14" - numpy: "2.3" - python_spec: "3.14.* *_cp314t" - python_tag: "3.14t" - - python: "3.15" - numpy: "2.5" - python_spec: "3.15.* *_cp315" - python_tag: "3.15" - - steps: - - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@d07a454dad7609a92316b57b23c9ccfd4f59af66 # 0.13.1 - with: - access_token: ${{ github.token }} - - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - fetch-depth: 0 - - - name: Cache conda packages - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - env: - CACHE_NUMBER: 0 # Increase to reset cache - with: - path: ~/.conda/pkgs - key: - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python_tag || matrix.python }}-${{hashFiles('**/meta.yaml') }} - restore-keys: | - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python_tag || matrix.python }}- - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}- - - - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - miniforge-version: latest - activate-environment: "" - pkgs-dirs: ~/.conda/pkgs - - - name: Install conda-build - run: conda install conda-build - - - name: Build conda package with NumPy 2.x - run: | - CHANNELS=(-c conda-forge --override-channels) - VERSIONS=(--python "${{ matrix.python_spec || matrix.python }}" --numpy "${{ matrix.numpy }}") - TEST=(--no-test) - - conda build \ - "${TEST[@]}" \ - "${VERSIONS[@]}" \ - "${CHANNELS[@]}" \ - conda-recipe-cf - - - name: Upload artifact - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python_tag || matrix.python }} - path: ${{ env.CONDA }}/conda-bld/linux-64/${{ env.PACKAGE_NAME }}-*.conda - - test_linux: - needs: build_linux - runs-on: ${{ matrix.runner }} - - strategy: - matrix: - python: ["3.10", "3.11", "3.12", "3.13"] - numpy: ['numpy">=2"'] - experimental: [false] - runner: [ubuntu-latest] - include: - - python: "3.14" - python_spec: "3.14.* *_cp314" - python_tag: "3.14" - numpy: 'numpy">=2"' - experimental: false - runner: ubuntu-latest - - python: "3.14" - python_spec: "3.14.* *_cp314t" - python_tag: "3.14t" - numpy: 'numpy">=2"' - experimental: false - runner: ubuntu-latest - - python: "3.15" - python_spec: "3.15.* *_cp315" - python_tag: "3.15" - numpy: 'numpy">=2"' - experimental: false - runner: ubuntu-latest - continue-on-error: ${{ matrix.experimental }} - env: - CHANNELS: -c conda-forge --override-channels - - steps: - - name: Download artifact - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python_tag || matrix.python }} - - - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - miniforge-version: latest - activate-environment: "" - pkgs-dirs: ~/.conda/pkgs - - - name: Install conda-build - run: conda install conda-build - - - name: Create conda channel - run: | - mkdir -p "$GITHUB_WORKSPACE"/channel/linux-64 - mv "$PACKAGE_NAME"-*.conda "$GITHUB_WORKSPACE"/channel/linux-64 - conda index "$GITHUB_WORKSPACE"/channel - # Test channel - conda search "$PACKAGE_NAME" -c "$GITHUB_WORKSPACE"/channel --override-channels - - - name: Collect dependencies - run: | - CHANNELS=(-c "$GITHUB_WORKSPACE"/channel ${{ env.CHANNELS }}) - conda create -n ${{ env.TEST_ENV_NAME }} "$PACKAGE_NAME" "python=${{ matrix.python_spec || matrix.python }}" ${{ matrix.numpy }} "${CHANNELS[@]}" --only-deps --dry-run > lockfile - - - name: Display lockfile - run: cat lockfile - - - name: Cache conda packages - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - env: - CACHE_NUMBER: 0 # Increase to reset cache - with: - path: ~/.conda/pkgs - key: - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python_tag || matrix.python }}-${{hashFiles('lockfile') }} - restore-keys: | - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python_tag || matrix.python }}- - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}- - - - name: Install mkl_fft - run: | - CHANNELS=(-c "$GITHUB_WORKSPACE"/channel ${{ env.CHANNELS }}) - conda create -n ${{ env.TEST_ENV_NAME }} "python=${{ matrix.python_spec || matrix.python }}" ${{ matrix.numpy }} "$PACKAGE_NAME" pytest scipy mkl-service "${CHANNELS[@]}" - # Test installed packages - conda list -n ${{ env.TEST_ENV_NAME }} - - - name: Run tests - run: | - source "$CONDA"/etc/profile.d/conda.sh - conda activate ${{ env.TEST_ENV_NAME }} - pytest -v --pyargs "$MODULE_NAME" - - build_windows: - runs-on: windows-latest - defaults: - run: - shell: cmd /C CALL {0} - strategy: - matrix: - include: - - python: "3.10" - numpy: "2.2" - - python: "3.11" - numpy: "2.3" - - python: "3.12" - numpy: "2.3" - - python: "3.13" - numpy: "2.3" - - python: "3.14" - numpy: "2.3" - python_spec: "3.14.* *_cp314" - python_tag: "3.14" - - python: "3.14" - numpy: "2.3" - python_spec: "3.14.* *_cp314t" - python_tag: "3.14t" - - python: "3.15" - numpy: "2.5" - python_spec: "3.15.* *_cp315" - python_tag: "3.15" - - steps: - - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@d07a454dad7609a92316b57b23c9ccfd4f59af66 # 0.13.1 - with: - access_token: ${{ github.token }} - - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - fetch-depth: 0 - - - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - miniforge-version: latest - activate-environment: build - channels: conda-forge - python-version: ${{ matrix.python }} - - - name: Install conda-build - run: | - conda install -n base -y conda-build - conda list -n base - - - name: Cache conda packages - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - env: - CACHE_NUMBER: 3 # Increase to reset cache - with: - path: /home/runner/conda_pkgs_dir - key: - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python_tag || matrix.python }}-${{hashFiles('**/meta.yaml') }} - restore-keys: | - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python_tag || matrix.python }}- - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}- - - - name: Build conda package with NumPy 2.x - run: | - conda build --no-test --python "${{ matrix.python_spec || matrix.python }}" --numpy ${{ matrix.numpy }} -c conda-forge --override-channels conda-recipe-cf - - - name: Store conda paths as envs - shell: bash -l {0} - run: | - echo "CONDA_BLD=$CONDA/conda-bld/win-64/" | tr "\\\\" '/' >> "$GITHUB_ENV" - - - name: Upload artifact - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python_tag || matrix.python }} - path: ${{ env.CONDA_BLD }}${{ env.PACKAGE_NAME }}-*.conda - - test_windows: - needs: build_windows - runs-on: ${{ matrix.runner }} - defaults: - run: - shell: cmd /C CALL {0} - - strategy: - matrix: - python: ["3.10", "3.11", "3.12", "3.13"] - numpy: ['numpy">=2"'] - experimental: [false] - runner: [windows-latest] - include: - - python: "3.14" - python_spec: "3.14.* *_cp314" - python_tag: "3.14" - numpy: 'numpy">=2"' - experimental: false - runner: windows-latest - - python: "3.14" - python_spec: "3.14.* *_cp314t" - python_tag: "3.14t" - numpy: 'numpy">=2"' - experimental: false - runner: windows-latest - - python: "3.15" - python_spec: "3.15.* *_cp315" - python_tag: "3.15" - numpy: 'numpy">=2"' - experimental: false - runner: windows-latest - continue-on-error: ${{ matrix.experimental }} - env: - workdir: '${{ github.workspace }}' - CHANNELS: -c conda-forge --override-channels - - steps: - - name: Download artifact - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python_tag || matrix.python }} - - - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - miniforge-version: latest - channels: conda-forge - activate-environment: ${{ env.TEST_ENV_NAME }} - python-version: ${{ matrix.python }} - - - name: Create conda channel with the artifact bit - shell: cmd /C CALL {0} - run: | - echo ${{ env.workdir }} - mkdir ${{ env.workdir }}\channel\ - mkdir ${{ env.workdir }}\channel\win-64 - move ${{ env.PACKAGE_NAME }}-*.conda ${{ env.workdir }}\channel\win-64 - dir ${{ env.workdir }}\channel\win-64 - - - name: Install conda index - shell: cmd /C CALL {0} - run: conda install -n base conda-index - - - name: Index the channel - shell: cmd /C CALL {0} - run: conda index ${{ env.workdir }}\channel - - - name: Dump mkl_fft version info from created channel into ver.json - shell: cmd /C CALL {0} - run: | - conda search ${{ env.PACKAGE_NAME }} -c ${{ env.workdir }}/channel --override-channels --info --json > ${{ env.workdir }}\ver.json - - - name: Output content of produced ver.json - shell: pwsh - run: Get-Content -Path ${{ env.workdir }}\ver.json - - - name: Collect dependencies - shell: cmd /C CALL {0} - run: | - @ECHO ON - IF NOT EXIST ver.json ( - copy /Y ${{ env.workdir }}\ver.json . - ) - SET "SCRIPT=%VER_SCRIPT1% %VER_SCRIPT2%" - FOR /F "tokens=* USEBACKQ" %%F IN (`python -c "%SCRIPT%"`) DO ( - SET PACKAGE_VERSION=%%F - ) - conda install -n ${{ env.TEST_ENV_NAME }} ${{ env.PACKAGE_NAME }}=%PACKAGE_VERSION% "python=${{ matrix.python_spec || matrix.python }}" ${{ matrix.numpy }} -c ${{ env.workdir }}/channel ${{ env.CHANNELS }} --only-deps --dry-run > lockfile - - - name: Display lockfile content - shell: pwsh - run: Get-Content -Path .\lockfile - - - name: Cache conda packages - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - env: - CACHE_NUMBER: 0 # Increase to reset cache - with: - path: /home/runner/conda_pkgs_dir - key: - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python_tag || matrix.python }}-${{hashFiles('lockfile') }} - restore-keys: | - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python_tag || matrix.python }}- - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}- - - - name: Install mkl_fft - shell: cmd /C CALL {0} - run: | - @ECHO ON - IF NOT EXIST ver.json ( - copy /Y ${{ env.workdir }}\ver.json . - ) - set "SCRIPT=%VER_SCRIPT1% %VER_SCRIPT2%" - FOR /F "tokens=* USEBACKQ" %%F IN (`python -c "%SCRIPT%"`) DO ( - SET PACKAGE_VERSION=%%F - ) - SET "TEST_DEPENDENCIES=pytest scipy mkl-service" - conda install -n ${{ env.TEST_ENV_NAME }} ${{ env.PACKAGE_NAME }}=%PACKAGE_VERSION% %TEST_DEPENDENCIES% "python=${{ matrix.python_spec || matrix.python }}" ${{ matrix.numpy }} -c ${{ env.workdir }}/channel ${{ env.CHANNELS }} - - - name: Report content of test environment - shell: cmd /C CALL {0} - run: | - echo "Value of CONDA environment variable was: " %CONDA% - echo "Value of CONDA_PREFIX environment variable was: " %CONDA_PREFIX% - conda info && conda list -n ${{ env.TEST_ENV_NAME }} - - - name: Run tests - shell: cmd /C CALL {0} - run: >- - conda activate ${{ env.TEST_ENV_NAME }} && python -m pytest -v -s --pyargs ${{ env.MODULE_NAME }} diff --git a/.github/workflows/conda-package.yml b/.github/workflows/conda-package.yml deleted file mode 100644 index e5329b45..00000000 --- a/.github/workflows/conda-package.yml +++ /dev/null @@ -1,460 +0,0 @@ -name: Conda package - -on: - push: - branches: - - master - pull_request: - -permissions: - contents: read - -env: - PACKAGE_NAME: mkl_fft - MODULE_NAME: mkl_fft - TEST_ENV_NAME: test_mkl_fft - VER_SCRIPT1: "import json; f = open('ver.json', 'r'); j = json.load(f); f.close(); d = j['mkl_fft'][0];" - VER_SCRIPT2: "print('='.join((d[s] for s in ('version', 'build'))))" - CONDA_BUILD_VERSION: 26.3.0 - -jobs: - build_linux: - runs-on: ubuntu-latest - strategy: - matrix: - include: - - python: "3.10" - numpy: "2.2" - - python: "3.11" - numpy: "2.3" - - python: "3.12" - numpy: "2.3" - - python: "3.13" - numpy: "2.3" - - python: "3.14" - numpy: "2.3" - python_spec: "3.14.* *_cp314" - python_tag: "3.14" - - python: "3.14" - numpy: "2.3" - python_spec: "3.14.* *_cp314t" - python_tag: "3.14t" - - python: "3.15" - numpy: "2.5" - python_spec: "3.15.* *_cp315" - python_tag: "3.15" - steps: - - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@d07a454dad7609a92316b57b23c9ccfd4f59af66 # 0.13.1 - with: - access_token: ${{ github.token }} - - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - fetch-depth: 0 - - - name: Cache conda packages - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - env: - CACHE_NUMBER: 0 # Increase to reset cache - with: - path: ~/.conda/pkgs - key: - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python_tag || matrix.python }}-${{hashFiles('**/meta.yaml') }} - restore-keys: | - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python_tag || matrix.python }}- - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}- - - - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - miniforge-version: latest - activate-environment: "" - pkgs-dirs: ~/.conda/pkgs - - - name: Store conda paths as envs - run: | - echo "CONDA_BLD=$CONDA/conda-bld/linux-64/" >> "$GITHUB_ENV" - echo "WHEELS_OUTPUT_FOLDER=$GITHUB_WORKSPACE/" >> "$GITHUB_ENV" - - - name: Update conda - run: | - conda update -n base --all - - - name: Install conda-build - run: conda install -n base conda-build=${{ env.CONDA_BUILD_VERSION }} -c conda-forge --override-channels - - - name: Show Conda info - run: | - conda info --all - - - name: List base environment packages - run: | - conda list -n base - - - name: Build conda package - run: | - CHANNELS=(-c https://software.repos.intel.com/python/conda -c conda-forge --override-channels) - VERSIONS=(--python "${{ matrix.python_spec || matrix.python }}" --numpy "${{ matrix.numpy }}") - TEST=(--no-test) - - conda build \ - "${TEST[@]}" \ - "${VERSIONS[@]}" \ - "${CHANNELS[@]}" \ - conda-recipe - - - name: Upload artifact - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python_tag || matrix.python }} - path: ${{ env.CONDA_BLD }}${{ env.PACKAGE_NAME }}-*.conda - - - name: Upload wheels artifact - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Wheels Python ${{ matrix.python_tag || matrix.python }} - path: ${{ env.WHEELS_OUTPUT_FOLDER }}${{ env.PACKAGE_NAME }}-*.whl - - test_linux: - needs: build_linux - runs-on: ${{ matrix.runner }} - - strategy: - matrix: - python: ["3.10", "3.11", "3.12", "3.13"] - numpy: ['numpy">=2"'] - experimental: [false] - runner: [ubuntu-latest] - include: - - python: "3.14" - python_spec: "3.14.* *_cp314" - python_tag: "3.14" - numpy: 'numpy">=2"' - experimental: false - runner: ubuntu-latest - - python: "3.14" - python_spec: "3.14.* *_cp314t" - python_tag: "3.14t" - numpy: 'numpy">=2"' - experimental: false - runner: ubuntu-latest - - python: "3.15" - python_spec: "3.15.* *_cp315" - python_tag: "3.15" - numpy: 'numpy">=2"' - experimental: false - runner: ubuntu-latest - continue-on-error: ${{ matrix.experimental }} - env: - CHANNELS: -c https://software.repos.intel.com/python/conda -c conda-forge --override-channels - - steps: - - name: Download artifact - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python_tag || matrix.python }} - - - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - miniforge-version: latest - activate-environment: "" - pkgs-dirs: ~/.conda/pkgs - - - name: Update conda - run: | - conda update -n base --all - - - name: Install conda-index - run: | - conda install -n base conda-index -c conda-forge --override-channels - - - name: Show Conda info - run: | - conda info --all - - - name: List base environment packages - run: | - conda list -n base - - - name: Create conda channel - run: | - mkdir -p "$GITHUB_WORKSPACE"/channel/linux-64 - mv "$PACKAGE_NAME"-*.conda "$GITHUB_WORKSPACE"/channel/linux-64 - conda index "$GITHUB_WORKSPACE"/channel - - - name: Test conda channel - run: | - conda search "$PACKAGE_NAME" -c "$GITHUB_WORKSPACE"/channel --override-channels --info --json > "$GITHUB_WORKSPACE"/ver.json - cat "$GITHUB_WORKSPACE"/ver.json - - - name: Collect dependencies - run: | - PACKAGE_VERSION="$(python -c "${VER_SCRIPT1} ${VER_SCRIPT2}")" - export PACKAGE_VERSION - CHANNELS=(-c "$GITHUB_WORKSPACE"/channel ${{ env.CHANNELS }}) - conda create -n ${{ env.TEST_ENV_NAME }} "${PACKAGE_NAME}=${PACKAGE_VERSION}" "python=${{ matrix.python_spec || matrix.python }}" ${{ matrix.numpy }} "${CHANNELS[@]}" --only-deps --dry-run > lockfile - - - name: Display lockfile - run: cat lockfile - - - name: Cache conda packages - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - env: - CACHE_NUMBER: 0 # Increase to reset cache - with: - path: ~/.conda/pkgs - key: - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python_tag || matrix.python }}-${{hashFiles('lockfile') }} - restore-keys: | - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python_tag || matrix.python }}- - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}- - - - name: Install mkl_fft - run: | - CHANNELS=(-c "$GITHUB_WORKSPACE"/channel ${{ env.CHANNELS }}) - PACKAGE_VERSION="$(python -c "${VER_SCRIPT1} ${VER_SCRIPT2}")" - export PACKAGE_VERSION - conda create -n ${{ env.TEST_ENV_NAME }} "${PACKAGE_NAME}=${PACKAGE_VERSION}" "python=${{ matrix.python_spec || matrix.python }}" ${{ matrix.numpy }} pytest "${CHANNELS[@]}" - conda install -n ${{ env.TEST_ENV_NAME }} "scipy>=1.10" "mkl-service" "${CHANNELS[@]}" - # Test installed packages - conda list -n ${{ env.TEST_ENV_NAME }} - - - name: Run tests - run: | - source "$CONDA"/etc/profile.d/conda.sh - conda activate ${{ env.TEST_ENV_NAME }} - pytest -v --pyargs "$MODULE_NAME" - - build_windows: - runs-on: windows-latest - defaults: - run: - shell: cmd /C CALL {0} - strategy: - matrix: - include: - - python: "3.10" - numpy: "2.2" - - python: "3.11" - numpy: "2.3" - - python: "3.12" - numpy: "2.3" - - python: "3.13" - numpy: "2.3" - - python: "3.14" - numpy: "2.3" - python_spec: "3.14.* *_cp314" - python_tag: "3.14" - - python: "3.14" - numpy: "2.3" - python_spec: "3.14.* *_cp314t" - python_tag: "3.14t" - - python: "3.15" - numpy: "2.5" - python_spec: "3.15.* *_cp315" - python_tag: "3.15" - steps: - - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@d07a454dad7609a92316b57b23c9ccfd4f59af66 # 0.13.1 - with: - access_token: ${{ github.token }} - - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - fetch-depth: 0 - - - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - auto-update-conda: true - miniforge-version: latest - activate-environment: build - channels: conda-forge - python-version: ${{ matrix.python }} - conda-build-version: ${{ env.CONDA_BUILD_VERSION }} - - - name: Cache conda packages - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - env: - CACHE_NUMBER: 3 # Increase to reset cache - with: - path: /home/runner/conda_pkgs_dir - key: - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python_tag || matrix.python }}-${{hashFiles('**/meta.yaml') }} - restore-keys: | - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python_tag || matrix.python }}- - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}- - - - name: Store conda paths as envs - shell: bash -l {0} - run: | - echo "CONDA_BLD=$CONDA/conda-bld/win-64/" | tr "\\\\" '/' >> "$GITHUB_ENV" - echo "WHEELS_OUTPUT_FOLDER=$GITHUB_WORKSPACE\\" >> "$GITHUB_ENV" - - - name: Show Conda info - run: | - conda info --all - - - name: List base environment packages - run: | - conda list -n base - - - name: Build conda package - run: | - conda build --no-test --python "${{ matrix.python_spec || matrix.python }}" --numpy ${{ matrix.numpy }} -c https://software.repos.intel.com/python/conda -c conda-forge --override-channels conda-recipe - - - name: Upload artifact - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python_tag || matrix.python }} - path: ${{ env.CONDA_BLD }}${{ env.PACKAGE_NAME }}-*.conda - - - name: Upload wheels artifact - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Wheels Python ${{ matrix.python_tag || matrix.python }} - path: ${{ env.WHEELS_OUTPUT_FOLDER }}${{ env.PACKAGE_NAME }}-*.whl - - test_windows: - needs: build_windows - runs-on: ${{ matrix.runner }} - defaults: - run: - shell: cmd /C CALL {0} - - strategy: - matrix: - python: ["3.10", "3.11", "3.12", "3.13"] - numpy: ['numpy">=2"'] - experimental: [false] - runner: [windows-latest] - include: - - python: "3.14" - python_spec: "3.14.* *_cp314" - python_tag: "3.14" - numpy: 'numpy">=2"' - experimental: false - runner: windows-latest - - python: "3.14" - python_spec: "3.14.* *_cp314t" - python_tag: "3.14t" - numpy: 'numpy">=2"' - experimental: false - runner: windows-latest - - python: "3.15" - python_spec: "3.15.* *_cp315" - python_tag: "3.15" - numpy: 'numpy">=2"' - experimental: false - runner: windows-latest - continue-on-error: ${{ matrix.experimental }} - env: - workdir: '${{ github.workspace }}' - CHANNELS: -c https://software.repos.intel.com/python/conda -c conda-forge --override-channels - - steps: - - name: Download artifact - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python_tag || matrix.python }} - - - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - auto-update-conda: true - miniforge-version: latest - channels: conda-forge - activate-environment: ${{ env.TEST_ENV_NAME }} - python-version: ${{ matrix.python }} - - - name: Install conda-index - run: | - conda install -n base conda-index - - - name: Show Conda info - run: | - conda info --all - - - name: List base environment packages - run: | - conda list -n base - - - name: Create conda channel with the artifact bit - shell: cmd /C CALL {0} - run: | - echo ${{ env.workdir }} - mkdir ${{ env.workdir }}\channel\ - mkdir ${{ env.workdir }}\channel\win-64 - move ${{ env.PACKAGE_NAME }}-*.conda ${{ env.workdir }}\channel\win-64 - dir ${{ env.workdir }}\channel\win-64 - - - name: Index the channel - shell: cmd /C CALL {0} - run: conda index ${{ env.workdir }}\channel - - - name: Dump mkl_fft version info from created channel into ver.json - shell: cmd /C CALL {0} - run: | - conda search ${{ env.PACKAGE_NAME }} -c ${{ env.workdir }}/channel --override-channels --info --json > ${{ env.workdir }}\ver.json - - - name: Output content of produced ver.json - shell: pwsh - run: Get-Content -Path ${{ env.workdir }}\ver.json - - - name: Collect dependencies - shell: cmd /C CALL {0} - run: | - @ECHO ON - IF NOT EXIST ver.json ( - copy /Y ${{ env.workdir }}\ver.json . - ) - SET "SCRIPT=%VER_SCRIPT1% %VER_SCRIPT2%" - FOR /F "tokens=* USEBACKQ" %%F IN (`python -c "%SCRIPT%"`) DO ( - SET PACKAGE_VERSION=%%F - ) - conda install -n ${{ env.TEST_ENV_NAME }} ${{ env.PACKAGE_NAME }}=%PACKAGE_VERSION% "python=${{ matrix.python_spec || matrix.python }}" ${{ matrix.numpy }} -c ${{ env.workdir }}/channel ${{ env.CHANNELS }} --only-deps --dry-run > lockfile - - - name: Display lockfile content - shell: pwsh - run: Get-Content -Path .\lockfile - - - name: Cache conda packages - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - env: - CACHE_NUMBER: 0 # Increase to reset cache - with: - path: /home/runner/conda_pkgs_dir - key: - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python_tag || matrix.python }}-${{hashFiles('lockfile') }} - restore-keys: | - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python_tag || matrix.python }}- - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}- - - - name: Install mkl_fft - shell: cmd /C CALL {0} - run: | - @ECHO ON - IF NOT EXIST ver.json ( - copy /Y ${{ env.workdir }}\ver.json . - ) - set "SCRIPT=%VER_SCRIPT1% %VER_SCRIPT2%" - FOR /F "tokens=* USEBACKQ" %%F IN (`python -c "%SCRIPT%"`) DO ( - SET PACKAGE_VERSION=%%F - ) - SET "TEST_DEPENDENCIES=pytest" - conda install -n ${{ env.TEST_ENV_NAME }} ${{ env.PACKAGE_NAME }}=%PACKAGE_VERSION% %TEST_DEPENDENCIES% "python=${{ matrix.python_spec || matrix.python }}" ${{ matrix.numpy }} -c ${{ env.workdir }}/channel ${{ env.CHANNELS }} - - name: Install additional test dependencies - shell: cmd /C CALL {0} - run: | - conda install -n ${{ env.TEST_ENV_NAME }} "scipy>=1.10" "mkl-service" -c ${{ env.workdir }}/channel ${{ env.CHANNELS }} - - name: Report content of test environment - shell: cmd /C CALL {0} - run: | - echo "Value of CONDA environment variable was: " %CONDA% - echo "Value of CONDA_PREFIX environment variable was: " %CONDA_PREFIX% - conda info && conda list -n ${{ env.TEST_ENV_NAME }} - - - name: Run tests - shell: cmd /C CALL {0} - run: >- - conda activate ${{ env.TEST_ENV_NAME }} && python -m pytest -v -s --pyargs ${{ env.MODULE_NAME }} diff --git a/.github/workflows/intel-ci.yml b/.github/workflows/intel-ci.yml new file mode 100644 index 00000000..0cf3b35c --- /dev/null +++ b/.github/workflows/intel-ci.yml @@ -0,0 +1,210 @@ +# Builds and tests mkl_fft's conda packages and wheels, see ci/README.md +name: Intel CI + +on: + push: + branches: [master] + tags: ['*'] + pull_request: + +permissions: + contents: read + +# A push or tag run gets a group of its own, so a queued master push is never dropped. +concurrency: + group: intel-ci-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + matrix: + name: Platforms and Python versions + runs-on: ${{ vars.GH_CI_RUNNER || 'ubuntu-latest' }} + timeout-minutes: 10 + outputs: + matrix: ${{ steps.declared.outputs.matrix }} + wheel: ${{ steps.declared.outputs.wheel }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + + - name: Read the declared platforms and Python versions + id: declared + env: + EVENT: ${{ startsWith(github.ref, 'refs/tags/') && 'tag' || github.event_name == 'pull_request' && 'pr' || 'push' }} + run: | + python ci/matrix.py --event "$EVENT" --github-output matrix + python ci/matrix.py --system wheel --event "$EVENT" --github-output wheel + + # The check branch protection requires: new broken claims in changed agent guidance fail it. + guidance: + name: Agent guidance + runs-on: ${{ vars.GH_CI_RUNNER || 'ubuntu-latest' }} + timeout-minutes: 10 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + fetch-depth: 0 + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + + - name: Scan the changed agent guidance + run: python ci/guidance.py + + # macOS runs on Intel runners, as MKL has no Apple silicon build. + build: + name: ${{ matrix.name }} Python ${{ matrix.python }} + needs: matrix + runs-on: ${{ matrix.platform == 'windows' && (vars.GH_CI_WINDOWS_RUNNER || 'windows-2022') || matrix.platform == 'macos' && (vars.GH_CI_MACOS_RUNNER || 'macos-26-intel') || (vars.GH_CI_RUNNER || 'ubuntu-latest') }} + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix.outputs.matrix) }} + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # The recipe takes its version from git describe. + fetch-depth: 0 + + # Miniforge 26.9.4-0 fails setup-miniconda's conda init on Windows runners; 26.7.2-0 works. + - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 + with: + miniforge-version: 26.7.2-0 + activate-environment: "" + conda-remove-defaults: "true" + + - name: Install conda-build + run: conda install -y -q -n base --override-channels -c conda-forge conda-build=26.9.1 + + - name: Build and run the recipe tests + env: + PYTHON_VERSION: ${{ matrix.python }} + run: python ci/build.py --python "$PYTHON_VERSION" --output "$RUNNER_TEMP/output" + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: mkl_fft ${{ matrix.name }} Python ${{ matrix.python }} + path: ${{ runner.temp }}/output/*.conda + if-no-files-found: error + # Tags and pushes to master keep them 30 days, pull requests 3. + retention-days: ${{ github.event_name == 'push' && 30 || 3 }} + + test: + name: Test ${{ matrix.name }} Python ${{ matrix.python }} + needs: [matrix, build] + runs-on: ${{ matrix.platform == 'windows' && (vars.GH_CI_WINDOWS_RUNNER || 'windows-2022') || matrix.platform == 'macos' && (vars.GH_CI_MACOS_RUNNER || 'macos-26-intel') || (vars.GH_CI_RUNNER || 'ubuntu-latest') }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix.outputs.matrix) }} + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: mkl_fft ${{ matrix.name }} Python ${{ matrix.python }} + path: ${{ runner.temp }}/artifacts + + - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 + with: + miniforge-version: 26.7.2-0 + activate-environment: "" + conda-remove-defaults: "true" + + - name: Install the package and run ci/test.py + env: + PYTHON_VERSION: ${{ matrix.python }} + EVENT: ${{ startsWith(github.ref, 'refs/tags/') && 'tag' || github.event_name == 'pull_request' && 'pr' || 'push' }} + COMMIT: ${{ github.event.pull_request.head.sha || github.sha }} + run: >- + python ci/hooks.py --python "$PYTHON_VERSION" --artifacts "$RUNNER_TEMP/artifacts" + --work "$RUNNER_TEMP/hooks" --event "$EVENT" --commit "$COMMIT" + + build-wheel: + name: ${{ matrix.name }} Wheels Python ${{ matrix.python }} + needs: matrix + runs-on: ${{ matrix.platform == 'windows' && (vars.GH_CI_WINDOWS_RUNNER || 'windows-2022') || matrix.platform == 'macos' && (vars.GH_CI_MACOS_RUNNER || 'macos-26-intel') || (vars.GH_CI_RUNNER || 'ubuntu-latest') }} + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix.outputs.wheel) }} + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + + - name: Install cibuildwheel + run: python -m pip install --no-input cibuildwheel==4.3.5 + + - name: Build the wheel + env: + PYTHON_VERSION: ${{ matrix.python }} + run: python ci/build.py --system wheel --python "$PYTHON_VERSION" --output "$RUNNER_TEMP/output" + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: mkl_fft ${{ matrix.name }} Wheels Python ${{ matrix.python }} + path: ${{ runner.temp }}/output/*.whl + if-no-files-found: error + retention-days: ${{ github.event_name == 'push' && 30 || 3 }} + + test-wheel: + name: Test ${{ matrix.name }} Wheels Python ${{ matrix.python }} + needs: [matrix, build-wheel] + runs-on: ${{ matrix.platform == 'windows' && (vars.GH_CI_WINDOWS_RUNNER || 'windows-2022') || matrix.platform == 'macos' && (vars.GH_CI_MACOS_RUNNER || 'macos-26-intel') || (vars.GH_CI_RUNNER || 'ubuntu-latest') }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix.outputs.wheel) }} + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: mkl_fft ${{ matrix.name }} Wheels Python ${{ matrix.python }} + path: ${{ runner.temp }}/artifacts + + - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 + with: + miniforge-version: 26.7.2-0 + activate-environment: "" + conda-remove-defaults: "true" + + - name: Install the wheel and run ci/test.py + env: + PYTHON_VERSION: ${{ matrix.python }} + EVENT: ${{ startsWith(github.ref, 'refs/tags/') && 'tag' || github.event_name == 'pull_request' && 'pr' || 'push' }} + COMMIT: ${{ github.event.pull_request.head.sha || github.sha }} + run: >- + python ci/hooks.py --kind wheel --python "$PYTHON_VERSION" --artifacts "$RUNNER_TEMP/artifacts" + --work "$RUNNER_TEMP/hooks" --event "$EVENT" --commit "$COMMIT" diff --git a/AGENTS.md b/AGENTS.md index 2b9bc5e4..9c7d091c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,7 +13,8 @@ It provides accelerated FFT transforms while aiming to preserve upstream API beh - **Interface adapters:** `mkl_fft/interfaces/` - **Tests:** `mkl_fft/tests/` - **Vendored helpers:** `_vendored/` -- **Packaging:** `conda-recipe/`, `conda-recipe-cf/` +- **Packaging:** `conda-recipe/` for conda, `pyproject.toml` for wheels +- **Intel CI:** `ci/`, `.github/workflows/intel-ci.yml`; see `ci/README.md` - **Benchmarks:** `benchmarks/` ## Build/runtime basics @@ -30,7 +31,7 @@ It provides accelerated FFT transforms while aiming to preserve upstream API beh ## Where truth lives - Build/config: `pyproject.toml`, `meson.build` -- Dependencies: `pyproject.toml`, `conda-recipe*/meta.yaml` +- Dependencies: `pyproject.toml`, `conda-recipe/meta.yaml` - CI/workflows: `.github/workflows/*.{yml,yaml}` - Public API: `mkl_fft/__init__.py`, `mkl_fft/interfaces/` (including wrapper/patch adapter modules) - Tests: `mkl_fft/tests/` @@ -44,7 +45,6 @@ Use nearest local `AGENTS.md` when present: - `mkl_fft/interfaces/AGENTS.md` — NumPy/SciPy interface adapters - `mkl_fft/src/AGENTS.md` — C template backend rules - `mkl_fft/tests/AGENTS.md` — test scope/conventions -- `conda-recipe/AGENTS.md` — Intel-channel conda packaging -- `conda-recipe-cf/AGENTS.md` — conda-forge recipe context +- `conda-recipe/AGENTS.md` — conda-forge recipe context - `_vendored/AGENTS.md` — vendored tooling boundaries - `benchmarks/AGENTS.md` — ASV performance suite diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d708e7c0..ca53247e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -95,7 +95,7 @@ Style is loose, and the pre-commit hooks enforce most of it: - Keep patching reversible and observable: anything installed can be uninstalled, and `is_patched()` reports the truth. - Cite the source-of-truth file for mutable details: `pyproject.toml`, - `meson.build`, `conda-recipe*/meta.yaml`, `.github/workflows/`. + `meson.build`, `conda-recipe/meta.yaml`, `.github/workflows/`. - Give benchmark numbers reproducible context — hardware, versions, and the command you ran. diff --git a/README.md b/README.md index de9810dc..a279abec 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,5 @@ -[![Conda package](https://github.com/IntelPython/mkl_fft/actions/workflows/conda-package.yml/badge.svg)](https://github.com/IntelPython/mkl_fft/actions/workflows/conda-package.yml) [![Editable build using pip and pre-release NumPy](https://github.com/IntelPython/mkl_fft/actions/workflows/build_pip.yml/badge.svg)](https://github.com/IntelPython/mkl_fft/actions/workflows/build_pip.yml) -[![Conda package with conda-forge channel only](https://github.com/IntelPython/mkl_fft/actions/workflows/conda-package-cf.yml/badge.svg)](https://github.com/IntelPython/mkl_fft/actions/workflows/conda-package-cf.yml) +[![Intel CI](https://github.com/IntelPython/mkl_fft/actions/workflows/intel-ci.yml/badge.svg)](https://github.com/IntelPython/mkl_fft/actions/workflows/intel-ci.yml) [![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/IntelPython/mkl_fft/badge)](https://securityscorecards.dev/viewer/?uri=github.com/IntelPython/mkl_fft) [![Coverity Scan Build Status](https://scan.coverity.com/projects/33202/badge.svg)](https://scan.coverity.com/projects/intelpython-mkl_fft) diff --git a/ci/README.md b/ci/README.md new file mode 100644 index 00000000..173c3f34 --- /dev/null +++ b/ci/README.md @@ -0,0 +1,36 @@ +# How Intel CI tests mkl_fft + +`.github/workflows/intel-ci.yml` builds mkl_fft's conda packages and wheels in GitHub Actions and tests each one. It needs this directory, the recipe, and `pyproject.toml`. + +| File | Role | +|---|---| +| `../conda-recipe/` | The conda recipe. `conda_build_config.yaml` lists the Python versions. The recipe builds on Linux, Windows and Intel macOS unless a platform-only skip in `meta.yaml`, such as `skip: true # [win]`, rules one out. | +| `../pyproject.toml` | What the wheels are built from. Its Python classifiers list the wheels' Python versions, and its Operating System classifiers their platforms. `[tool.cibuildwheel]` holds the wheel build settings, and `[tool.gh-ci]` holds the run-time requirements the wheels need on top of `[project]`'s, which the conda packages get from the recipe: `pin-compatible` requires at least the `mkl-devel` version the build used, below its next major. | +| `../.github/workflows/intel-ci.yml` | One job reads both matrices. One job per platform and Python version builds and uploads `mkl_fft Python `, and one builds `mkl_fft Wheels Python `. Then one job per package tests it. | +| `matrix.py` | Reads the platforms and Python versions from the recipe, or with `--system wheel` from `pyproject.toml`. Only a tag gets every version: a pull request or a push gets the oldest and the newest regular versions and the newest free-threaded one. | +| `build.py` | Builds one Python version: with conda-build, which runs the recipe's own tests, or with `--system wheel` with cibuildwheel, in a manylinux container on Linux. The ELF files in a wheel keep only their `$ORIGIN` RPATH and RUNPATH entries, and must have full RELRO. | +| `hooks.py` | The workflow's test jobs: installs one package into a clean environment with the requirements in `config.yml`, and runs `test.py`. A conda package's environment comes from conda-forge; a wheel goes into a venv made from conda-forge's Python, with everything else from PyPI. | +| `guidance.py` | The workflow's `Agent guidance` job, the check branch protection requires. A pull request that changes no agent guidance file passes. Otherwise it scans the merge base and the head with dokimi, installed with the hashes in `guidance-requirements.txt`, and fails on a claim that is new in a changed file and fails at the head: a broken link, path, script or identifier. Claims that already fail do not fail it, but a repeated copy of one does. The hashes cover Linux x86_64 and CPython 3.11 to 3.14, so `GH_CI_RUNNER` must be such a runner or the hashes need extending. | +| `config.yml` | Names the workflow that builds the packages, its artifact names, and what the tests and benchmarks need. | +| `test.py` | Runs the installed package's test suite with pytest. | +| `benchmark.py` | Runs the asv suite in `../benchmarks/` against the installed package. | + +The scripts need the Python standard library and the tools each one's docstring names. macOS packages build and test on Intel runners, as MKL has no Apple silicon build. + +Nothing in `intel-ci.yml` reads `workflow`, `artifacts` or `benchmark` from `config.yml`, runs `benchmark.py`, or keeps the JUnit reports `test.py` writes; they are there to run the same build, tests and benchmarks outside this workflow. + +**A pull request or a push to `master`.** Actions builds the oldest and the newest regular Python versions and the newest free-threaded one on Linux, Windows and macOS, as a conda package and, where `pyproject.toml` declares the platform, as a wheel, and tests each package. + +**A tag.** Actions builds and tests every declared version. + +To support another Python, add it to `conda-recipe/conda_build_config.yaml` for the conda packages and as a classifier in `pyproject.toml` for the wheels; nothing else lists versions. Try it locally from a conda base environment with conda-build, or with cibuildwheel for wheels: + +```bash +python ci/matrix.py # every platform and Python version +python ci/matrix.py --event pr # what a pull request or a push builds +python ci/build.py --python VERSION --output /tmp/out # VERSION: one matrix.py lists +python ci/hooks.py --python VERSION --artifacts /tmp/out --work /tmp/hooks +python ci/build.py --system wheel --python VERSION --output /tmp/wheels +python ci/hooks.py --kind wheel --python VERSION --artifacts /tmp/wheels --work /tmp/wheel-hooks +python ci/guidance.py --base REV --head REV # the Agent guidance job, for a pull request +``` diff --git a/ci/benchmark.py b/ci/benchmark.py new file mode 100644 index 00000000..6cb356bb --- /dev/null +++ b/ci/benchmark.py @@ -0,0 +1,84 @@ +""" +Benchmarks the installed package. + +Runs the asv suite in benchmarks/ against the package installed in this +environment, from an empty working directory, without a build of its own, and +records the results under the commit being tested, in $GH_CI_RESULTS. A pull +request samples each benchmark once, enough to show that every one still runs; +a tag measures them properly. A benchmark that fails fails the run; a slower one +does not, as nothing compares the timings with a baseline yet. +""" + +import json +import os +import pathlib +import subprocess +import sys + +# asv's own dependency, which reads the comments in its configuration. +import json5 + +ASV_CONFIG = pathlib.Path("benchmarks", "asv.conf.json") + + +def asv_config(source: pathlib.Path, results: pathlib.Path) -> pathlib.Path: + """Writes this run's asv configuration into the working directory. + + It is the repository's own, with absolute paths, since asv reads them from + where the configuration is, and with results under GH_CI_RESULTS. It lists + one branch, HEAD: the checkout is the commit under test, detached, and asv + cannot resolve the repository's branches there. + """ + path = source / ASV_CONFIG + config = json5.loads(path.read_text(encoding="utf-8")) + work = pathlib.Path.cwd() + config.update( + repo=str((path.parent / config.get("repo", "..")).resolve()), + benchmark_dir=str( + (path.parent / config.get("benchmark_dir", "benchmarks")).resolve() + ), + branches=["HEAD"], + results_dir=str(results / "asv"), + html_dir=str(work / "html"), + env_dir=str(work / "env"), + ) + written = work / "asv.conf.json" + written.write_text(json.dumps(config, indent=2), encoding="utf-8") + return written + + +def main() -> int: + """Runs the asv suite against the installed package.""" + config = asv_config( + pathlib.Path(os.environ["GH_CI_SOURCE"]), + pathlib.Path(os.environ["GH_CI_RESULTS"]), + ) + asv = [sys.executable, "-m", "asv"] + # asv keeps its machine description in ~/.asv-machine.json; this one stays + # with the run rather than with the machine's user. + home = pathlib.Path.cwd() / "home" + home.mkdir(exist_ok=True) + env = dict(os.environ, HOME=str(home)) + described = subprocess.run( + asv + ["machine", "--yes", "--config", str(config)], env=env + ) + if described.returncode: + return described.returncode + run = [ + "run", + "--config", + str(config), + f"--environment=existing:{sys.executable}", + f"--set-commit-hash={os.environ['GH_CI_COMMIT']}", + "--no-pull", + "--show-stderr", + "--launch-method=spawn", + ] + if os.environ.get("GH_CI_EVENT") != "tag": + run.append("--quick") + # asv exits 2 when a benchmark failed. + return subprocess.run(asv + run, env=env).returncode + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/build.py b/ci/build.py new file mode 100644 index 00000000..3ceb895a --- /dev/null +++ b/ci/build.py @@ -0,0 +1,568 @@ +""" +Builds this repository's packages for one Python version. + +For the intel-ci.yml GitHub Actions workflow. + +This file and ci/matrix.py work out the build system from the files present, +and the Python versions from that build system's own declaration: + + rattler-build .conda recipe.yaml or recipe/recipe.yaml, versions in + variants.yaml + conda-build .conda conda-recipe/meta.yaml or meta.yaml, versions in + conda_build_config.yaml + cibuildwheel .whl pyproject.toml, versions from its classifiers + +Usage: + python ci/build.py --python 3.14 --output OUT [--system rattler-build] + [--channel conda-forge] + +--python must be one of the declared versions; `python ci/matrix.py` lists them. + +Packages land directly in OUT, ready for actions/upload-artifact. A conda +recipe's own tests run during the build. Needs only the Python standard library. +rattler-build is used from PATH, or downloaded at a pinned version for Linux or +Windows and checked against its published sha256; conda-build, or cibuildwheel +for wheels, must already be installed for this interpreter, so run it with the +conda base install's Python to build with conda-build. + +Wheels build with cibuildwheel, in a manylinux container on Linux, as +pyproject.toml's [tool.cibuildwheel] says. Run-time requirements that +pyproject.toml cannot declare, because the conda package gets them from its +recipe instead, such as MKL's libraries, go in [tool.gh-ci] wheel-requires: +build.py adds them to each wheel's metadata. [tool.gh-ci] pin-compatible maps a +run-time requirement to the build requirement it follows, like conda's +pin_compatible: build.py requires >= the version python -m build installed for +the build, < its next major. It also cuts the RPATH and +RUNPATH of each ELF file in a wheel to their $ORIGIN entries, dropping absolute +ones such as the isolated build environment's lib directory, which meson-python +keeps, and refuses a file that would have none left, or that is not linked +with full RELRO. +""" + +import argparse +import base64 +import hashlib +import os +import platform +import re +import shlex +import shutil +import struct +import subprocess +import sys +import tempfile +import urllib.request +import zipfile +from pathlib import Path + +import matrix + +ROOT = matrix.ROOT +PACKAGE_SUFFIXES = (".conda", ".tar.bz2", ".whl") +# cibuildwheel's platform part of a build identifier, per system. +WHEEL_PLATFORMS = { + "Linux": "manylinux_x86_64", + "Windows": "win_amd64", + "Darwin": "macosx_x86_64", +} +# The ELF program header and dynamic section values the RUNPATH check reads. +PT_LOAD, PT_DYNAMIC, PT_GNU_RELRO = 1, 2, 0x6474E552 +DT_NULL, DT_STRTAB, DT_RPATH, DT_RUNPATH = 0, 5, 15, 29 +DT_BIND_NOW, DT_FLAGS, DT_FLAGS_1 = 24, 30, 0x6FFFFFFB +DF_BIND_NOW, DF_1_NOW = 0x8, 0x1 +ORIGIN = ("$ORIGIN", "${ORIGIN}") +ANSI_RE = re.compile(r"\x1b\[[0-9;]*m") + +# The rattler-build release fetched when none is on PATH, per system, with the +# file name it goes under and the sha256 its release publishes. +RATTLER_BUILD_VERSION = "v0.77.0" +RATTLER_BUILD_RELEASES = { + "Linux": ( + "rattler-build-x86_64-unknown-linux-musl", + "rattler-build", + "5a5e8150c703b21e88a01f9130ea49a7a680198e9ac9414170c0a293fed7e5af", + ), + "Windows": ( + "rattler-build-x86_64-pc-windows-msvc.exe", + "rattler-build.exe", + "e144334f782cee9aa23e26eb8d095c143e1696fbbec7ba7534a156663d4abb2c", + ), +} + + +class BuildError(Exception): + """Raised when the build cannot start or produced nothing.""" + + +def ensure_rattler_build(tools: Path) -> None: + """ + Puts rattler-build on PATH, downloading the pinned release when none is + there. + """ + if shutil.which("rattler-build"): + return + release = RATTLER_BUILD_RELEASES.get(platform.system()) + if release is None or platform.machine().lower() not in ("x86_64", "amd64"): + raise BuildError( + "no rattler-build on PATH, and only the Linux and Windows x86_64 " + "releases are pinned here" + ) + asset, name, sha256 = release + print(f"build: fetching rattler-build {RATTLER_BUILD_VERSION}", flush=True) + url = ( + "https://github.com/prefix-dev/rattler-build/releases/download/" + f"{RATTLER_BUILD_VERSION}/{asset}" + ) + with urllib.request.urlopen(url, timeout=300) as reply: + binary = reply.read() + if hashlib.sha256(binary).hexdigest() != sha256: + raise BuildError( + f"rattler-build {RATTLER_BUILD_VERSION} does not match its pinned " + "sha256" + ) + tools.mkdir(parents=True, exist_ok=True) + target = tools / name + target.write_bytes(binary) + target.chmod(0o755) + os.environ["PATH"] = f'{tools}{os.pathsep}{os.environ.get("PATH", "")}' + + +def cibuildwheel_identifier(python: matrix.Python) -> str: + """The one build cibuildwheel runs, such as cp314t-manylinux_x86_64.""" + tag = WHEEL_PLATFORMS.get(platform.system()) + if tag is None: + raise BuildError( + f"wheels build on Linux, Windows or macOS, not {platform.system()}" + ) + minor, threaded = matrix.version_key(python.version) + return f'cp3{minor}{"t" if threaded else ""}-{tag}' + + +def wheel_settings() -> tuple[list[str], dict[str, str]]: + """[tool.gh-ci] wheel-requires and pin-compatible from pyproject.toml.""" + if matrix.TOMLLIB is None: + raise BuildError("reading pyproject.toml needs Python 3.11 or newer") + text = (ROOT / "pyproject.toml").read_text(encoding="utf-8") + tool = matrix.TOMLLIB.loads(text).get("tool", {}).get("gh-ci", {}) + requires = tool.get("wheel-requires", []) + if not isinstance(requires, list) or not all( + isinstance(r, str) for r in requires + ): + raise BuildError("[tool.gh-ci] wheel-requires must be a list of specs") + pins = tool.get("pin-compatible", {}) + if not isinstance(pins, dict) or not all( + isinstance(k, str) and isinstance(v, str) for k, v in pins.items() + ): + raise BuildError( + "[tool.gh-ci] pin-compatible must map a run-time requirement to " + "a build requirement" + ) + return requires, pins + + +def canonical(name: str) -> str: + """A distribution name as PEP 503 compares it.""" + return re.sub(r"[-_.]+", "-", name).lower() + + +def build_versions(log: list[str]) -> dict[str, str]: + """ + The version of each build requirement the isolated build environment + installed, from the list python -m build logs. + """ + versions, listing = {}, False + for line in log: + text = ANSI_RE.sub("", line).strip() + if text.endswith("Installed build dependency versions:"): + listing = True + elif listing and (match := re.fullmatch(r"- ([\w.-]+)==(\S+)", text)): + versions[canonical(match[1])] = match[2] + else: + listing = False + return versions + + +def pinned(pins: dict[str, str], versions: dict[str, str]) -> list[str]: + """Each pinned run-time requirement: >= its build version, < next major.""" + specs = [] + for runtime, build in pins.items(): + version = versions.get(canonical(build)) + major = version.split(".")[0] if version else "" + if not major.isdigit(): + raise BuildError( + f"the build logged no installed {build} version, which " + f"[tool.gh-ci] pin-compatible pins {runtime} to" + ) + specs.append(f"{runtime} >={version},<{int(major) + 1}") + return specs + + +def run_logged(command: list[str]) -> list[str]: + """Runs a command, passing its output through, and returns that output.""" + lines = [] + sys.stdout.flush() + with subprocess.Popen( + command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT + ) as process: + for line in process.stdout: + sys.stdout.buffer.write(line) + sys.stdout.buffer.flush() + lines.append(line.decode("utf-8", "replace")) + if process.returncode: + raise subprocess.CalledProcessError(process.returncode, command) + return lines + + +def record_hash(data: bytes) -> str: + """A file's hash as a wheel's RECORD spells it.""" + digest = base64.urlsafe_b64encode(hashlib.sha256(data).digest()) + return "sha256=" + digest.rstrip(b"=").decode("ascii") + + +def elf_dynamic( + data: bytes, name: str +) -> tuple[list[tuple[int, int, int, int]], list[tuple[int, int]]] | None: + """ + The program headers, as (p_type, p_offset, p_vaddr, p_filesz), and the + dynamic entries of an ELF file; None for a file that is not ELF. + """ + if data[:4] != b"\x7fELF": + return None + if data[4:6] != b"\x02\x01": + raise BuildError(f"{name}: only 64-bit little-endian ELF is read") + (phoff,) = struct.unpack_from(" list[tuple[str, int, int, str]]: + """ + The RPATH and RUNPATH strings of an ELF file, as (RPATH or RUNPATH, file + offset, length, value); none for a file that is not ELF. + """ + if (parsed := elf_dynamic(data, name)) is None: + return [] + segments, entries = parsed + strtab = dict(entries).get(DT_STRTAB) + if strtab is None: + return [] + base = next( + offset + strtab - vaddr + for kind, offset, vaddr, size in segments + if kind == PT_LOAD and vaddr <= strtab < vaddr + size + ) + found = [] + for tag, value in entries: + if tag in (DT_RPATH, DT_RUNPATH): + start = base + value + end = data.index(b"\0", start) + kind = "RPATH" if tag == DT_RPATH else "RUNPATH" + found.append((kind, start, end - start, data[start:end].decode())) + return found + + +def full_relro(data: bytes, name: str) -> bool: + """Whether an ELF file has full RELRO; True for a file that is not ELF.""" + if (parsed := elf_dynamic(data, name)) is None: + return True + segments, entries = parsed + flags = dict(entries) + return any(kind == PT_GNU_RELRO for kind, *_ in segments) and ( + DT_BIND_NOW in flags + or bool(flags.get(DT_FLAGS, 0) & DF_BIND_NOW) + or bool(flags.get(DT_FLAGS_1, 0) & DF_1_NOW) + ) + + +def origin_relative(path: str) -> bool: + """Whether a search path entry is $ORIGIN or a directory under it.""" + return any( + path == token or path.startswith(token + "/") for token in ORIGIN + ) + + +def origin_only(data: bytes, name: str) -> tuple[bytes, list[str]]: + """ + An ELF file with its RPATH and RUNPATH cut to their $ORIGIN entries, in + place as patchelf does, and the entries it dropped. + """ + edited, dropped = bytearray(data), [] + for kind, start, length, value in elf_search_paths(data, name): + paths = value.split(":") + kept = [path for path in paths if origin_relative(path)] + if kept == paths: + continue + # An empty search path would mean the working directory + if not kept: + raise BuildError(f"{name}: no $ORIGIN entry in {kind} {value!r}") + edited[start : start + length] = ( + ":".join(kept).encode().ljust(length, b"\0") + ) + dropped += [ + f"{kind} entry {path}" for path in paths if path not in kept + ] + return bytes(edited), dropped + + +def with_requirements(metadata: bytes, requirements: list[str]) -> bytes: + """A wheel's METADATA with Requires-Dist lines added.""" + text = metadata.decode("utf-8") + # Some Windows build backends write the metadata with CRLF. Take the line + # ending from the headers, as the description can keep the README's. + headers = re.split(r"\r?\n\r?\n", text, maxsplit=1)[0] + eol = "\r\n" if "\r\n" in headers else "\n" + head, blank, body = text.partition(eol * 2) + head = head.rstrip("\r\n") + "".join( + f"{eol}Requires-Dist: {r}" for r in requirements + ) + return (head + (eol * 2 + body if blank else eol)).encode("utf-8") + + +def rewrite_wheel(wheel: Path, requirements: list[str]) -> list[str]: + """ + Adds Requires-Dist lines to a built wheel and cuts its ELF files' RPATH + and RUNPATH to their $ORIGIN entries, updating its RECORD; lists the + entries it dropped. Refuses an ELF file without full RELRO. + """ + with zipfile.ZipFile(wheel) as source: + members = [(info, source.read(info)) for info in source.infolist()] + names = [info.filename for info, _ in members] + metadata = next(n for n in names if n.endswith(".dist-info/METADATA")) + record = metadata.removesuffix("METADATA") + "RECORD" + changed, dropped = {}, [] + for index, (info, data) in enumerate(members): + if info.filename == metadata and requirements: + new = with_requirements(data, requirements) + else: + new, paths = origin_only(data, info.filename) + dropped += [f"{path} from {info.filename}" for path in paths] + if not full_relro(new, info.filename): + raise BuildError( + f"{info.filename} has no full RELRO; link it with " + "-Wl,-z,relro -Wl,-z,now" + ) + if new != data: + members[index] = (info, new) + changed[info.filename] = new + if not changed: + return dropped + for index, (info, data) in enumerate(members): + if info.filename == record: + lines = [ + ( + f"{name},{record_hash(changed[name])},{len(changed[name])}" + if (name := line.split(",")[0]) in changed + else line + ) + for line in data.decode("utf-8").splitlines() + ] + members[index] = (info, ("\n".join(lines) + "\n").encode()) + rewritten = wheel.with_suffix(".tmp") + with zipfile.ZipFile(rewritten, "w", zipfile.ZIP_DEFLATED) as target: + for info, data in members: + target.writestr(info, data) + rewritten.replace(wheel) + return dropped + + +def commands( + system: str, + selector: Path, + python: matrix.Python, + channels: list[str], + scratch: Path, +) -> list[list[str]]: + """ + The commands that build one Python version's packages into scratch. + + Each build system gets the version back exactly as its own file declares it, + and keeps every other variant setting from that file. + """ + channel_args = [arg for channel in channels for arg in ("-c", channel)] + # The commit a conda package came from, in its info/about.json + github = [ + os.environ.get(name, "") + for name in ("GITHUB_SERVER_URL", "GITHUB_REPOSITORY", "GITHUB_SHA") + ] + extra_meta = ( + [ + "--extra-meta", + f"remote_url={github[0]}/{github[1]}", + f"sha={github[2]}", + ] + if all(github) + else [] + ) + if system == "rattler-build": + return [ + [ + "rattler-build", + "build", + "-r", + str(selector), + "--variant", + f"python={python.spec}", + "--output-dir", + str(scratch / "conda"), + *channel_args, + ] + ] + if system == "conda-build": + # conda runs as a module of this interpreter, so Windows needs no + # conda.bat on PATH. + return [ + [ + sys.executable, + "-m", + "conda", + "build", + str(selector.parent), + "--python", + python.spec, + "--no-anaconda-upload", + "--output-folder", + str(scratch / "conda"), + *extra_meta, + "--override-channels", + *channel_args, + ] + ] + return [ + [ + sys.executable, + "-m", + "cibuildwheel", + "--only", + cibuildwheel_identifier(python), + "--output-dir", + str(scratch / "wheel"), + str(ROOT), + ] + ] + + +def collect(scratch: Path, output: Path) -> list[Path]: + """ + Moves built packages into output, flat, and lists the ones it moved. + Two packages of one name are an error: flat, one would overwrite the other. + """ + output.mkdir(parents=True, exist_ok=True) + built = [] + sources: dict[str, Path] = {} + for package in sorted(scratch.rglob("*")): + if ( + package.is_file() + and package.name.endswith(PACKAGE_SUFFIXES) + and "broken" not in package.parts + ): + if package.name in sources: + raise BuildError( + f"two packages named {package.name}: " + f"{sources[package.name]} and {package}" + ) + sources[package.name] = package + built.append(output / package.name) + shutil.move(str(package), built[-1]) + if not built: + raise BuildError(f"the build left no package in {output}") + return built + + +def main(argv: list[str] | None = None) -> int: + """Builds one Python version.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[1]) + parser.add_argument( + "--python", required=True, help="a declared version, such as 3.14t" + ) + parser.add_argument( + "--output", required=True, type=Path, help="where the packages go" + ) + parser.add_argument( + "--system", + default="", + choices=("", *matrix.SYSTEMS), + help="build system to use instead of the detected one", + ) + parser.add_argument( + "--channel", + action="append", + default=None, + help="conda channel to build against, repeatable; default conda-forge", + ) + default_tools = ( + Path(os.environ.get("RUNNER_TEMP", tempfile.gettempdir())) / "ci-tools" + ) + parser.add_argument( + "--tools", + type=Path, + default=default_tools, + help="where a downloaded rattler-build goes", + ) + args = parser.parse_args(argv) + + system, selector = matrix.detect(ROOT, args.system) + declared = { + python.version: python for python in matrix.declared(system, selector) + } + if args.python not in declared: + raise BuildError( + f"Python {args.python} is not declared for {system}; " + f'declared: {", ".join(declared)}' + ) + python = declared[args.python] + print( + f"build: {system} from {selector.relative_to(ROOT)} " + f"for Python {python.version} ({python.spec})", + flush=True, + ) + if system == "rattler-build": + ensure_rattler_build(args.tools) + requires, pins = wheel_settings() if system == "wheel" else ([], {}) + output = args.output.resolve() + log: list[str] = [] + with tempfile.TemporaryDirectory() as scratch: + for command in commands( + system, + selector, + python, + args.channel or ["conda-forge"], + Path(scratch), + ): + print(f"+ {shlex.join(command)}", flush=True) + log += run_logged(command) + # Rewritten in scratch, so a refused wheel never reaches the output + if system == "wheel": + requires = requires + pinned(pins, build_versions(log)) + for wheel in sorted(Path(scratch).rglob("*.whl")): + for dropped in rewrite_wheel(wheel, requires): + print(f"build: dropped {dropped}") + built = collect(Path(scratch), output) + if requires: + print("build: added Requires-Dist " + ", ".join(requires)) + print("build: " + ", ".join(path.name for path in built)) + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except ( + BuildError, + matrix.DeclarationError, + subprocess.CalledProcessError, + ) as error: + print(f"build: {error}", file=sys.stderr) + sys.exit(1) diff --git a/ci/config.yml b/ci/config.yml new file mode 100644 index 00000000..58b221a1 --- /dev/null +++ b/ci/config.yml @@ -0,0 +1,15 @@ +# Intel CI settings for mkl_fft, see ci/README.md +version: 1 +workflow: intel-ci.yml +artifacts: + conda: "mkl_fft {platform} Python {python}" + wheel: "mkl_fft {platform} Wheels Python {python}" +test: + requires: [pytest, "scipy >=1.10", mkl-service] + # PyPI has no mkl-service for Python 3.15 yet + wheel_requires: [pytest, "scipy >=1.10", "mkl-service; python_version < '3.15'"] +benchmark: + # PyYAML, which asv needs, has no PyPI wheel for every Python the recipe declares. + requires: [psutil, "scipy >=1.10", mkl-service, pyyaml] + # conda-forge lacks asv for some Python the recipe declares; the abi3 wheel on PyPI fits all. + pip_requires: [asv] diff --git a/ci/guidance-requirements.txt b/ci/guidance-requirements.txt new file mode 100644 index 00000000..92020aab --- /dev/null +++ b/ci/guidance-requirements.txt @@ -0,0 +1,32 @@ +# Hash-pinned install for ci/guidance.py: dokimi 0.1.0 and what `dokimi repo scan` imports. +# Wheels only: pure Python, or CPython 3.11-3.14 on Linux x86_64 (manylinux). +attrs==26.1.0 \ + --hash=sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309 +dokimi==0.1.0 \ + --hash=sha256:4e288728f3e9d666850b4b76847a0b79e2fabbc570b538e4056dfa1d71f63579 +jsonschema==4.26.0 \ + --hash=sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce +jsonschema-specifications==2025.9.1 \ + --hash=sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 +referencing==0.37.0 \ + --hash=sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231 +regex==2026.9.29 \ + --hash=sha256:0def9fb6abac55492d6d51cddb7225d07d6f279e774e0adc08569a54a5fc8d46 \ + --hash=sha256:2f7f7aa47b229f2b39a2ae2596d2ad5625d77b5eb9856fac2dab3eb506cdd0a0 \ + --hash=sha256:39ab5894d971f9ac68baa6eca5c50387db579cfcacf36ae8df3feceb1815e6d0 \ + --hash=sha256:612b709381c0355b70d89cdb51b7f670591ed5cbbc0e3b5337488019dc667b65 \ + --hash=sha256:d49c18f1ea294cf4adde2e5ac256e98c82ea9d708462ce4bf799dffa7cfe8a2c +rpds-py==2026.9.1 \ + --hash=sha256:0da298fb372dc192610a4b9ecbc68a0cd8b675bbbd1fc519d01b41cfd658333e \ + --hash=sha256:136a1c3fe4402b7008bc81cb62ee538481795b61a7e83df88dff3b3f02b726ff \ + --hash=sha256:7868b85224291c6cb6759f9b5adb9745f486d226f62b16a614dd5a2a5ab2b35b \ + --hash=sha256:addeda51556dac7c1a2f14cda62db8b621cd12afba3091d03a96c72932387eab \ + --hash=sha256:eac2f5dbafd585dfe31f86a23ebf0d3ba480a9d49ebc87947267b5608d4ea0cd +typing_extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 diff --git a/ci/guidance.py b/ci/guidance.py new file mode 100644 index 00000000..ca417324 --- /dev/null +++ b/ci/guidance.py @@ -0,0 +1,316 @@ +""" +Fails a pull request that adds a broken claim to the agent guidance files. + +For the intel-ci.yml GitHub Actions workflow's "Agent guidance" job. + +Guidance files are AGENTS.md and CLAUDE.md at any depth, +.github/copilot-instructions.md, .github/instructions/, SKILL.md, +.claude/skills/ and skills/. + +Usage: + python ci/guidance.py [--event pull_request|push] [--base REV] [--head REV] + [--work DIR] + +On a pull_request, with no guidance file changed between the merge base of the +pull request and its head, the job passes. Otherwise it installs dokimi with +hashes, scans the merge base and the head with `dokimi repo scan` (no network, +no model), and fails on a claim that fails at the head, in a changed guidance +file, and did not fail in that file at the merge base. A claim is its file, +kind and text, never its line. Claims that already fail do not fail it, and +neither do untouched files. It also fails when the merge base had guidance and +the head has none. A repeated copy of a failing claim counts as new. On a +push it scans the head, reports, and always passes. A summary goes +to $GITHUB_STEP_SUMMARY. The event, --base and --head default to the +workflow's. Needs only the Python standard library, and Python 3.11 or newer +for dokimi. +""" + +import argparse +import json +import os +import re +import subprocess +import sys +import tempfile +import venv +from collections import Counter +from pathlib import Path, PurePosixPath + +ROOT = Path(__file__).resolve().parent.parent +REQUIREMENTS = Path(__file__).resolve().parent / "guidance-requirements.txt" +SCHEMA = "repo_readiness.v1" +# dokimi's default globs, plus skills +GLOBS = [ + "**/AGENTS.md", + "**/CLAUDE.md", + ".github/copilot-instructions.md", + ".github/instructions/**/*.md", + "**/SKILL.md", + ".claude/skills/**/*.md", + "skills/**/*.md", +] +MAX_TEXT = 200 + + +class GuidanceError(Exception): + """Raised when the files or the scan cannot be trusted.""" + + +def is_guidance(path: str) -> bool: + """Whether a repository path is a guidance file.""" + name = PurePosixPath(path).name + return ( + name in ("AGENTS.md", "CLAUDE.md", "SKILL.md") + or path == ".github/copilot-instructions.md" + or ( + path.endswith(".md") + and path.startswith( + (".github/instructions/", ".claude/skills/", "skills/") + ) + ) + ) + + +def git(*args: str) -> str: + """Output of git run in this repository.""" + done = subprocess.run( + ["git", "-C", str(ROOT), *args], + check=False, + capture_output=True, + text=True, + ) + if done.returncode: + raise GuidanceError( + f"git {' '.join(args)} failed: {done.stderr.strip()}; " + "the checkout needs fetch-depth: 0" + ) + return done.stdout + + +def has_file(rev: str, name: str) -> bool: + """Whether the tree at rev holds a file.""" + return ( + subprocess.run( + ["git", "-C", str(ROOT), "cat-file", "-e", f"{rev}:{name}"], + check=False, + capture_output=True, + ).returncode + == 0 + ) + + +def touched_files(base: str, head: str) -> list[str]: + """Guidance files that differ between the merge base and the head.""" + names = git("diff", "--name-only", "--no-renames", "-z", base, head) + return sorted(path for path in names.split("\0") if is_guidance(path)) + + +def install_dokimi(work: Path) -> Path: + """Installs the pinned dokimi into a new environment; its executable.""" + env = work / "venv" + venv.create(env, with_pip=True) + bindir = env / ("Scripts" if os.name == "nt" else "bin") + command = [ + str(bindir / "python"), + "-m", + "pip", + "install", + "--quiet", + "--require-hashes", + "--no-deps", + "--only-binary", + ":all:", + "--requirement", + str(REQUIREMENTS), + ] + print(f"+ {' '.join(command)}", flush=True) + subprocess.run(command, check=True) + return bindir / "dokimi" + + +def write_manifest(work: Path, base: str) -> Path: + """The guidance-only manifest for both scans, from the base tree.""" + guidance = {"globs": GLOBS} + if has_file(base, "AGENTS.md"): + guidance["also_scan"] = ["AGENTS.md"] + manifest = { + "schema": "repo_manifest.v1", + "id": "guidance", + "guidance": guidance, + } + path = work / "manifest.yaml" + path.write_text(json.dumps(manifest), encoding="utf-8") + return path + + +def scan( + dokimi: Path, work: Path, manifest: Path, label: str, rev: str +) -> dict: + """The dokimi report of the repository at one revision.""" + out = work / f"{label}.json" + command = [ + str(dokimi), + "repo", + "scan", + f"{ROOT}@{rev}", + "--manifest-file", + str(manifest), + "--cache-dir", + str(work / "cache"), + "--out", + str(out), + ] + print(f"+ {' '.join(command)}", flush=True) + subprocess.run(command, check=True) + report = json.loads(out.read_text(encoding="utf-8")) + if report.get("schema_version") != SCHEMA: + raise GuidanceError( + f"dokimi wrote {report.get('schema_version')!r}, not {SCHEMA}" + ) + return report + + +def failing(report: dict) -> list[dict]: + """The claims that fail in a report.""" + return [c for c in report["claims"] if c["status"] == "fail"] + + +def key(claim: dict) -> tuple[str, str, str]: + """What identifies a claim across revisions.""" + return claim["file"], claim["kind"], claim["claim"] + + +def new_failures(base: dict, head: dict, touched: list[str]) -> list[dict]: + """Failing head claims in touched files that did not fail at the base.""" + old = Counter(key(claim) for claim in failing(base)) + found = [] + for claim in failing(head): + if claim["file"] not in touched: + continue + if old[key(claim)] > 0: + old[key(claim)] -= 1 + else: + found.append(claim) + return found + + +def code(text: object) -> str: + """Untrusted text as one Markdown code span: no links, no table breaks.""" + text = re.sub(r"\s+", " ", str(text)).strip() + if len(text) > MAX_TEXT: + text = text[: MAX_TEXT - 3] + "..." + text = text.replace("`", "'").replace("|", "\\|") + return f"`{text}`" + + +def plain(text: object) -> str: + """Untrusted text for the log, which cannot start a workflow command.""" + text = re.sub(r"[^\x20-\x7e\n]", "?", str(text)) + return text.replace("::", ": :") + + +def claim_counts(report: dict) -> str: + """One line of a report's counts.""" + counts = report["counts"] + return ( + f"Claims: {counts['total']} " + f"(ok {counts['ok']}, unresolved {counts['unresolved']}, " + f"fail {counts['fail']})." + ) + + +def summarize( + touched: list[str], head: dict, found: list[dict], problem: str +) -> str: + """The Markdown job summary of a scanned pull request.""" + lines = [ + "## Agent guidance", + "", + f"Guidance files changed: {len(touched)}.", + claim_counts(head), + f"New failing claims: {len(found)}.", + ] + if problem: + lines.extend(["", problem]) + if found: + lines.extend( + ["", "| Location | Kind | Claim | Why |", "|---|---|---|---|"] + ) + for claim in found: + location = f"{claim['file']}:{claim['line']}" + lines.append( + f"| {code(location)} | {code(claim['kind'])} " + f"| {code(claim['claim'])} | {code(claim['why'])} |" + ) + return "\n".join([*lines, ""]) + + +def pull_request_range(args: argparse.Namespace) -> tuple[str, str]: + """The pull request's base and head revisions.""" + base, head = args.base, args.head + if not (base and head): + with open(os.environ["GITHUB_EVENT_PATH"], encoding="utf-8") as file: + event = json.load(file)["pull_request"] + base = base or event["base"]["sha"] + head = head or event["head"]["sha"] + return git("merge-base", base, head).strip(), head + + +def check(args: argparse.Namespace, work: Path) -> tuple[str, int]: + """The summary and exit status for one event.""" + if args.event == "push": + head = args.head or os.environ["GITHUB_SHA"] + try: + dokimi = install_dokimi(work) + manifest = write_manifest(work, head) + report = scan(dokimi, work, manifest, "head", head) + except (GuidanceError, subprocess.CalledProcessError) as error: + text = f"Report only, the scan failed: {plain(error)}" + return f"## Agent guidance\n\n{text}\n", 0 + text = f"## Agent guidance\n\nReport only.\n{claim_counts(report)}\n" + return text, 0 + base, head = pull_request_range(args) + touched = touched_files(base, head) + if not touched: + return "## Agent guidance\n\nNo guidance files changed.\n", 0 + dokimi = install_dokimi(work) + manifest = write_manifest(work, base) + old = scan(dokimi, work, manifest, "base", base) + new = scan(dokimi, work, manifest, "head", head) + found = new_failures(old, new, touched) + problem = "" + if new["incomplete"] and not old["incomplete"]: + problem = "The head has no guidance left to scan, the base had some." + return summarize(touched, new, found, problem), int(bool(found or problem)) + + +def main(argv: list[str] | None = None) -> int: + """Runs the gate.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[1]) + parser.add_argument( + "--event", + default=os.environ.get("GITHUB_EVENT_NAME", "pull_request"), + choices=("pull_request", "push"), + ) + parser.add_argument("--base", help="the pull request's base revision") + parser.add_argument("--head", help="the pull request's head revision") + parser.add_argument( + "--work", type=Path, help="scratch directory; default is a new one" + ) + args = parser.parse_args(argv) + with tempfile.TemporaryDirectory() as scratch: + summary, status = check(args, args.work or Path(scratch)) + print(plain(summary)) + target = os.environ.get("GITHUB_STEP_SUMMARY") + if target: + with open(target, "a", encoding="utf-8") as file: + file.write(summary) + return status + + +if __name__ == "__main__": + try: + sys.exit(main()) + except (GuidanceError, subprocess.CalledProcessError) as error: + print(f"guidance: {plain(error)}", file=sys.stderr) + sys.exit(1) diff --git a/ci/hooks.py b/ci/hooks.py new file mode 100644 index 00000000..c6897e2f --- /dev/null +++ b/ci/hooks.py @@ -0,0 +1,452 @@ +""" +Runs ci/test.py against one built artifact. + +For the GitHub Actions test job. + +Each Python version's artifact is installed into a fresh conda environment from +a local channel plus conda-forge, pinned to its exact build, with the test +requirements ci/config.yml lists, and the test runs with that environment's +Python from an empty working directory, on Linux or Windows. A wheel goes into +a venv made from conda-forge's Python, with the test requirements, by pip, as +the wheel's users install it. The test gets the variables ci/test.py reads, and +nothing else from the runner. + +Usage, from any Python 3.10 or newer; it runs itself again with the conda base +install's Python, from $CONDA, which setup-miniconda sets, or the conda on PATH: + python ci/hooks.py --python 3.14 --artifacts DIR --work DIR [--kind wheel] + +Needs conda, conda-package-handling and ruamel.yaml, which every conda base has, +and installs nothing into it. +""" + +import argparse +import hashlib +import json +import os +import re +import shlex +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +import matrix + +# Dependencies of conda itself; YAML is read with PyYAML, else ruamel.yaml +try: + from conda_package_handling import api as CPH_API +except ModuleNotFoundError: + CPH_API = None +try: + import yaml as PYYAML +except ModuleNotFoundError: + PYYAML = None +try: + from ruamel.yaml import YAML as RUAMEL_YAML +except ModuleNotFoundError: + RUAMEL_YAML = None + +ROOT = matrix.ROOT +IS_WINDOWS = os.name == "nt" +if IS_WINDOWS: + PLATFORM, HOST_SUBDIR = "windows", "win-64" +elif sys.platform == "darwin": + PLATFORM, HOST_SUBDIR = "macos", "osx-64" +else: + PLATFORM, HOST_SUBDIR = "linux", "linux-64" +CONDA_SUFFIXES = (".conda", ".tar.bz2") +# All the test sees of the runner's environment, so no tokens reach it +KEEP = ( + "PATH", + "HOME", + "USER", + "LANG", + "TERM", + "TMPDIR", + "TZ", + "http_proxy", + "https_proxy", + "no_proxy", + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "SSL_CERT_FILE", + "REQUESTS_CA_BUNDLE", + "CONDA_PKGS_DIRS", + "SYSTEMROOT", + "SYSTEMDRIVE", + "WINDIR", + "COMSPEC", + "PATHEXT", + "TEMP", + "TMP", + "USERPROFILE", + "APPDATA", + "LOCALAPPDATA", + "PROGRAMDATA", + "NUMBER_OF_PROCESSORS", + "PROCESSOR_ARCHITECTURE", +) + + +class HookError(Exception): + """Raised when the artifact cannot be installed or ci/test.py is missing.""" + + +def run(command: list[str], **options: object) -> None: + """Echoes and runs a command, never through a shell.""" + print(f"+ {shlex.join(command)}", flush=True) + subprocess.run(command, check=True, **options) + + +def conda(*args: str) -> None: + """Runs conda from the base environment running this script.""" + run([sys.executable, "-m", "conda", *args]) + + +def empty_repodata(subdir: str) -> dict: + """The repodata.json of a channel subdir with no packages.""" + return { + "info": {"subdir": subdir}, + "packages": {}, + "packages.conda": {}, + "repodata_version": 1, + } + + +def package_record(package: Path) -> tuple[str, dict]: + """A conda package's subdir and repodata record, from its index.json.""" + if CPH_API is None: + raise HookError( + "run this with a conda base Python, " + "which has conda-package-handling" + ) + with tempfile.TemporaryDirectory() as scratch: + CPH_API.extract(str(package), scratch, components="info") + record = json.loads( + (Path(scratch) / "info" / "index.json").read_text(encoding="utf-8") + ) + data = package.read_bytes() + record.update( + { + "md5": hashlib.md5(data, usedforsecurity=False).hexdigest(), + "sha256": hashlib.sha256(data).hexdigest(), + "size": len(data), + } + ) + subdir = str(record.get("subdir") or "noarch") + if not re.fullmatch(r"[a-z0-9]+(-[a-z0-9_]+)?", subdir): + raise HookError( + f"{package.name} declares an unusable subdir {subdir!r}" + ) + return subdir, record + + +def stage_channel(packages: list[Path], channel: Path) -> None: + """ + Lays the packages out as a local channel, by the subdir each one declares, + and writes its repodata.json itself, without conda-index. + """ + repodata = { + subdir: empty_repodata(subdir) for subdir in ("noarch", HOST_SUBDIR) + } + for package in packages: + subdir, record = package_record(package) + (channel / subdir).mkdir(parents=True, exist_ok=True) + shutil.copy2(package, channel / subdir / package.name) + section = ( + "packages.conda" if package.name.endswith(".conda") else "packages" + ) + repodata.setdefault(subdir, empty_repodata(subdir))[section][ + package.name + ] = record + for subdir, data in repodata.items(): + (channel / subdir).mkdir(parents=True, exist_ok=True) + (channel / subdir / "repodata.json").write_text( + json.dumps(data, indent=2) + "\n", encoding="utf-8" + ) + + +def test_python(version: str) -> str: + """The conda spec for the test environment's interpreter.""" + minor, threaded = matrix.version_key(version) + # The .* matters: conda reads a bare 3.11 as exactly 3.11.0 + if minor < matrix.FREE_THREADED_MINOR: + return f"3.{minor}.*" + # conda-forge marks the ABI from 3.13, so 3.14 can't resolve to 3.14t + return f'3.{minor}.* *_cp3{minor}{"t" if threaded else ""}' + + +def test_requirements() -> tuple[list[str], list[str], list[str] | None]: + """ + The conda-forge and pip packages ci/config.yml installs next to the + artifact, and a wheel's PyPI requirements, None when it gives none. + """ + text = (ROOT / "ci" / "config.yml").read_text(encoding="utf-8") + if PYYAML is not None: + config = PYYAML.safe_load(text) + elif RUAMEL_YAML is not None: + config = RUAMEL_YAML(typ="safe", pure=True).load(text) + else: + raise HookError("reading ci/config.yml needs PyYAML or ruamel.yaml") + test = (config or {}).get("test") or {} + lists = [] + for key in ("requires", "pip_requires", "wheel_requires"): + values = test.get(key) + if values is not None and ( + not isinstance(values, list) + or not all(isinstance(v, str) for v in values) + ): + raise HookError( + f"ci/config.yml: test.{key} must be a list of package specs" + ) + lists.append(values) + return lists[0] or [], lists[1] or [], lists[2] + + +def exact_spec(package: Path) -> str: + """name=version=build from a conda package file name.""" + stem = package.name + for suffix in CONDA_SUFFIXES: + stem = stem.removesuffix(suffix) + name, version, build = stem.rsplit("-", 2) + return f"{name}={version}={build}" + + +def env_python(env_dir: Path, kind: str = "conda") -> Path: + """The interpreter of a conda environment, or of a wheel's venv.""" + if not IS_WINDOWS: + return env_dir / "bin" / "python" + if kind == "wheel": + return env_dir / "Scripts" / "python.exe" + return env_dir / "python.exe" + + +def env_path(env_dir: Path, kind: str) -> list[Path]: + """ + The directories activating the conda environment, or a wheel's venv, would + put first on PATH. + """ + if not IS_WINDOWS: + return [env_dir / "bin"] + if kind == "wheel": + return [env_dir / "Scripts"] + library = env_dir / "Library" + return [ + env_dir, + library / "mingw-w64" / "bin", + library / "usr" / "bin", + library / "bin", + env_dir / "Scripts", + env_dir / "bin", + ] + + +def test_env( + env_dir: Path, artifacts: Path, results: Path, args: argparse.Namespace +) -> dict: + """The variables ci/test.py reads, and no runner tokens.""" + env = {key: value for key, value in os.environ.items() if key in KEEP} + env["PATH"] = os.pathsep.join( + [*map(str, env_path(env_dir, args.kind)), env.get("PATH", "")] + ) + prefix = "VIRTUAL_ENV" if args.kind == "wheel" else "CONDA_PREFIX" + env.update( + { + prefix: str(env_dir), + "GH_CI_ARTIFACTS": str(artifacts), + "GH_CI_SOURCE": str(ROOT), + "GH_CI_RESULTS": str(results), + "GH_CI_KIND": args.kind, + "GH_CI_PYTHON": args.python, + "GH_CI_PLATFORM": PLATFORM, + "GH_CI_EVENT": args.event, + "GH_CI_COMMIT": args.commit, + } + ) + return env + + +def summarize(lines: list[str]) -> None: + """Adds the results to the job summary page, in GitHub Actions.""" + summary = os.environ.get("GITHUB_STEP_SUMMARY") + if summary: + with open(summary, "a", encoding="utf-8") as page: + page.write("\n".join(lines) + "\n") + + +def install(args: argparse.Namespace) -> Path: + """ + Creates the test environment with the artifact, pinned to its exact build; + returns it. + """ + system, selector = matrix.detect( + ROOT, "wheel" if args.kind == "wheel" else "" + ) + declared = { + python.version: python for python in matrix.declared(system, selector) + } + if args.python not in declared: + raise HookError( + f"Python {args.python} is not declared; " + f'declared: {", ".join(declared)}' + ) + suffixes = (".whl",) if args.kind == "wheel" else CONDA_SUFFIXES + packages = sorted( + path + for path in args.artifacts.rglob("*") + if path.name.endswith(suffixes) + ) + if not packages: + raise HookError(f"{args.artifacts} holds no {args.kind} package") + requires, pip_requires, wheel_requires = test_requirements() + env_dir = args.work / "env" + pip = [ + str(env_python(env_dir, args.kind)), + "-m", + "pip", + "install", + "--no-input", + ] + python = f"python {test_python(args.python)}" + if args.kind == "wheel": + base = args.work / "python" + conda_create(base, python) + run([str(env_python(base)), "-m", "venv", str(env_dir)]) + wheel_specs = requires if wheel_requires is None else wheel_requires + run([*pip, *map(str, packages), *wheel_specs]) + else: + channel = args.work / "channel" + stage_channel(packages, channel) + conda_create( + env_dir, + python, + *requires, + *map(exact_spec, packages), + channels=(channel.as_uri(),), + ) + if pip_requires: + run([*pip, *pip_requires]) + return env_dir + + +def conda_create( + env_dir: Path, *specs: str, channels: tuple[str, ...] = () +) -> None: + """Creates a conda environment from conda-forge and channels.""" + conda( + "create", + "-y", + "-q", + "-p", + str(env_dir), + "--override-channels", + *(arg for channel in channels for arg in ("-c", channel)), + "-c", + "conda-forge", + *specs, + ) + + +def run_test(env_dir: Path, args: argparse.Namespace) -> int: + """Runs ci/test.py with the environment's Python; returns its status.""" + script = ROOT / "ci" / "test.py" + if not script.is_file(): + raise HookError("ci/test.py is required and missing") + results, scratch = ( + args.work / "results" / "test", + args.work / "scratch" / "test", + ) + for directory in (results, scratch): + directory.mkdir(parents=True, exist_ok=True) + command = [str(env_python(env_dir, args.kind)), str(script)] + print(f"+ {shlex.join(command)}", flush=True) + status = subprocess.run( + command, + cwd=scratch, + check=False, + env=test_env(env_dir, args.artifacts, results, args), + ).returncode + result = "passed" if status == 0 else f"failed, exit {status}" + summarize( + [ + f"### {matrix.PLATFORMS[PLATFORM]} Python {args.python}" + f" {args.kind}", + "", + "| Test | Result |", + "|---|---|", + f"| ci/test.py | {result} |", + ] + ) + return status + + +def conda_base_python() -> Path: + """The conda base install's Python: from $CONDA, else the conda on PATH.""" + root = os.environ.get("CONDA", "") + if not root: + found = shutil.which("conda") + if not found: + raise HookError( + "no conda base install: set CONDA or put conda on PATH" + ) + # conda lives in bin, Scripts or condabin, one level below the install. + root = str(Path(found).resolve().parent.parent) + python = env_python(Path(root)) + if not python.is_file(): + raise HookError( + f"{root} is not a conda base install; it has no {python.name}" + ) + return python + + +def main(argv: list[str] | None = None) -> int: + """Installs the artifact, then runs ci/test.py against it.""" + if CPH_API is None: + # Re-run with base's Python, which has conda-package-handling + command = [ + str(conda_base_python()), + str(Path(__file__).resolve()), + *(sys.argv[1:] if argv is None else argv), + ] + return subprocess.run(command, check=False).returncode + parser = argparse.ArgumentParser(description=__doc__.splitlines()[1]) + parser.add_argument( + "--python", required=True, help="a declared version, such as 3.14t" + ) + parser.add_argument( + "--artifacts", required=True, type=Path, help="the downloaded artifact" + ) + parser.add_argument( + "--work", required=True, type=Path, help="an empty scratch directory" + ) + parser.add_argument( + "--event", default="push", choices=("pr", "tag", "push") + ) + parser.add_argument("--commit", default="") + parser.add_argument( + "--kind", + default="conda", + choices=("conda", "wheel"), + help="what the artifact holds", + ) + args = parser.parse_args(argv) + args.artifacts, args.work = args.artifacts.resolve(), args.work.resolve() + if args.work.exists() and ( + not args.work.is_dir() or any(args.work.iterdir()) + ): + raise HookError(f"--work {args.work} must be an empty directory") + return run_test(install(args), args) + + +if __name__ == "__main__": + try: + sys.exit(main()) + except ( + HookError, + matrix.DeclarationError, + subprocess.CalledProcessError, + ) as error: + print(f"hooks: {error}", file=sys.stderr) + sys.exit(1) diff --git a/ci/matrix.py b/ci/matrix.py new file mode 100644 index 00000000..76fad674 --- /dev/null +++ b/ci/matrix.py @@ -0,0 +1,698 @@ +""" +The platforms and Python versions this repository builds. + +Both are read from the repository's own build files. + +Every build system already has a standard place for both, so each is written +once, where the build reads it, and the GitHub Actions matrix follows it: + + rattler-build, selected by recipe.yaml + Python versions python: in variants.yaml next to the recipe + platforms build: skip: in the recipe + conda-build, selected by meta.yaml + Python versions python: in conda_build_config.yaml next to the recipe + platforms skip: true # [selector] lines in the recipe + wheel, selected by pyproject.toml + Python versions the Programming Language :: Python :: 3.N classifiers; + any Free Threading classifier adds 3.Nt for each from 3.14 + platforms the Operating System :: classifiers + +A recipe builds on every platform in PLATFORMS, Linux, Windows and macOS on +Intel, unless a skip that depends on the platform alone rules one out, such as +`skip: [win]` or `skip: true # [osx]`; `osx and arm64` rules out none, as +macOS here is Intel. A skip that also depends on something else, such as +`win and py < 311`, or a skip: line that cannot be read, is refused, as the +matrix would still list the builds it skips. A pyproject.toml without Operating +System classifiers, or with OS Independent, builds on all three. Of the +declared platforms, the matrix holds those GitHub Actions has runners for, +BUILT_PLATFORMS. + +meson-python, scikit-build-core and every other PEP 517 backend declare these in +pyproject.toml too, so the wheel row covers them. Another convention, such as +pixi.toml, is one more reader in READERS and PLATFORM_READERS and, if it is a +new build system, one more entry in SYSTEMS. + +Usage: + python ci/matrix.py [--system SYSTEM] [--event EVENT] [--github-output NAME] + +Prints the build matrix as JSON for strategy.matrix: {"include": [{"platform": +"linux", "name": "Linux", "python": "3.14"}, ...]}, every platform with every +Python version. Only a tag builds them all: for --event pr or push only the +oldest and the newest regular versions and the newest free-threaded one are +listed. --github-output also writes NAME= to $GITHUB_OUTPUT, for later +jobs. Standard library only; reading pyproject.toml needs Python 3.11 or newer +for tomllib. +""" + +import argparse +import ast +import json +import os +import re +import sys +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path + +# Only the wheel reader needs it, and only it needs Python 3.11 or newer. +try: + import tomllib as TOMLLIB +except ModuleNotFoundError: + TOMLLIB = None + +ROOT = Path(__file__).resolve().parent.parent + +# Build system -> files that select it, in the order they are looked for. +SYSTEMS = { + "rattler-build": ("recipe.yaml", "recipe/recipe.yaml"), + "conda-build": ( + "meta.yaml", + "conda-recipe/meta.yaml", + "recipe/meta.yaml", + ), + "wheel": ("pyproject.toml",), +} + +# The platforms CI builds and tests, and how artifact names spell them. +PLATFORMS = {"linux": "Linux", "windows": "Windows", "macos": "macOS"} +# The platforms the workflow can build on: GitHub-hosted runners have all +# three. A declared platform missing here is reported and left out. +BUILT_PLATFORMS = ("linux", "windows", "macos") +# What a platform-only selector means on each platform, as rattler-build defines +# it; CONDA_BUILD_SELECTORS holds where conda-build reads it differently. +SELECTORS = { + "linux": { + "linux": True, + "linux64": True, + "unix": True, + "win": False, + "win64": False, + "win32": False, + "osx": False, + "osx64": False, + "x86_64": True, + "x86": False, + "aarch64": False, + "arm64": False, + "ppc64le": False, + "s390x": False, + }, + "windows": { + "linux": False, + "linux64": False, + "unix": False, + "win": True, + "win64": True, + "win32": False, + "osx": False, + "osx64": False, + "x86_64": True, + "x86": False, + "aarch64": False, + "arm64": False, + "ppc64le": False, + "s390x": False, + }, + # Intel Macs only, as MKL has no Apple silicon build. + "macos": { + "linux": False, + "linux64": False, + "unix": True, + "win": False, + "win64": False, + "win32": False, + "osx": True, + "osx64": True, + "x86_64": True, + "x86": False, + "aarch64": False, + "arm64": False, + "ppc64le": False, + "s390x": False, + }, +} +# conda-build reads x86 as any x86 CPU and has no osx64; None refuses a skip +# that uses it. +CONDA_BUILD_SELECTORS = {"x86": True, "osx64": None} +CONDA_SKIP_RE = re.compile(r"\s*skip\s*:") +# conda-build's own selector pattern, which lets a comment follow the selector +CONDA_SELECTOR_RE = re.compile(r"(.+?)\s*(#.*)?\[([^\[\]]+)\](?(2)[^\(\)]*)$") +OS_CLASSIFIERS = { + "Operating System :: OS Independent": tuple(PLATFORMS), + "Operating System :: POSIX": ("linux",), + "Operating System :: Unix": ("linux",), + "Operating System :: POSIX :: Linux": ("linux",), + "Operating System :: Microsoft :: Windows": ("windows",), + "Operating System :: MacOS": ("macos",), +} + +# Events that build only some of the declared Python versions +SUBSET_EVENTS = ("pr", "push") +OLDEST_MINOR = 10 +FREE_THREADED_MINOR = 13 +# A Free Threading classifier adds 3.Nt from 3.14, the first supported one, +# which is also the oldest cibuildwheel builds. +CLASSIFIED_FREE_THREADED_MINOR = 14 + +# A conda python variant with its ABI: 3.12.* *_cpython, 3.13.* *_cp313 or +# 3.14.* *_cp314t; from 3.13 it must be cp3N or cp3Nt. +CONDA_SPEC_RE = re.compile(r"3\.(\d+)\.\*\s+\*_(?:cpython|cp3(\d+)(t?))") +CLASSIFIER_RE = re.compile(r"Programming Language :: Python :: 3\.(\d+)") +FREE_THREADING_CLASSIFIER = "Programming Language :: Python :: Free Threading" +LOWER_BOUND_RE = re.compile(r">=\s*3\.(\d+)") + + +class DeclarationError(Exception): + """ + Raised when the build files declare no Python versions, or ones this cannot + read. + """ + + +@dataclass(frozen=True) +class Python: + """One declared Python version.""" + + version: str # as artifact names spell it: 3.14 or 3.14t + spec: str # as the build system spells it, passed back to it unchanged + + +def version_key(version: str) -> tuple[int, bool]: + """(minor, free-threaded); also the sort order.""" + return int(version.rstrip("t").split(".")[1]), version.endswith("t") + + +def checked(minor: int, threaded: bool, where: str) -> str: + """The version string, once it is one CI can test.""" + if minor < OLDEST_MINOR or (threaded and minor < FREE_THREADED_MINOR): + raise DeclarationError( + f'{where}: 3.{minor}{"t" if threaded else ""} is not supported; ' + f"the oldest is 3.{OLDEST_MINOR}, free-threaded from " + f"3.{FREE_THREADED_MINOR}" + ) + return f'3.{minor}{"t" if threaded else ""}' + + +def _strip_comment(line: str, where: str) -> str: + """ + The line without a trailing comment; conda-build selectors are refused, not + guessed at. + """ + in_quote = "" + for index, char in enumerate(line): + if char in "\"'" and in_quote in ("", char): + in_quote = "" if in_quote else char + elif ( + char == "#" + and not in_quote + and (index == 0 or line[index - 1].isspace()) + ): + if line[index:].lstrip("# ").startswith("["): + raise DeclarationError( + f"{where}: selectors in the python list are not supported" + ) + return line[:index].rstrip() + return line.rstrip() + + +def _unquote(value: str, where: str) -> str: + """A plain or quoted YAML scalar.""" + value = value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'": + return value[1:-1] + if not value or value[0] in "{[&*!|>" or ":" in value: + raise DeclarationError( + f"{where}: write each python entry as a plain value, not {value!r}" + ) + return value + + +def yaml_python_list(text: str, where: str) -> list[str]: + """ + The top-level python: list of a variants file, without a YAML library. + + Accepts a block list or a one-line [a, b] list of plain or quoted values. + Anything richer, such as if/then selectors or templating, is refused so the + matrix is never guessed. + """ + lines = text.splitlines() + for number, line in enumerate(lines): + if not re.match(r"python\s*:", line): + continue + rest = _strip_comment(line.split(":", 1)[1], where).strip() + if rest.startswith("[") and rest.endswith("]"): + return [ + _unquote(item, where) + for item in rest[1:-1].split(",") + if item.strip() + ] + if rest: + raise DeclarationError(f"{where}: python must be a list") + values = [] + for item in lines[number + 1 :]: + # Checked before the comment, as the next key may have a selector + if item[:1] not in ("", " ", "\t", "-", "#"): + break # the next top-level key + entry = _strip_comment(item, where).lstrip() + if not entry: + continue + if not entry.startswith("- "): + raise DeclarationError( + f"{where}: python must be a plain list of versions" + ) + values.append(_unquote(entry[2:], where)) + return values + raise DeclarationError(f"{where} has no top-level python: list") + + +def from_conda_variants(path: Path) -> list[Python]: + """The python: list of a variants.yaml or a conda_build_config.yaml.""" + where = ( + path.relative_to(ROOT).as_posix() + if path.is_relative_to(ROOT) + else str(path) + ) + try: + text = path.read_text(encoding="utf-8") + except FileNotFoundError as ex: + raise DeclarationError( + f"{where} is missing; list the Python versions to build there" + ) from ex + pythons = [] + for spec in yaml_python_list(text, where): + match = CONDA_SPEC_RE.fullmatch(spec) + if ( + not match + or (match[2] and match[2] != match[1]) + or (not match[2] and int(match[1]) >= FREE_THREADED_MINOR) + ): + raise DeclarationError( + f"{where}: {spec!r} is not a CPython variant such as " + '"3.12.* *_cpython", "3.13.* *_cp313" or "3.14.* *_cp314t"' + ) + pythons.append( + Python(checked(int(match[1]), bool(match[3]), where), spec) + ) + return pythons + + +def from_pyproject(path: Path) -> list[Python]: + """ + The 3.N classifiers in pyproject.toml, plus 3.Nt from 3.14 if free-threading + is declared. + """ + if TOMLLIB is None: + raise DeclarationError( + "reading pyproject.toml needs Python 3.11 or newer" + ) + project = TOMLLIB.loads(path.read_text(encoding="utf-8")).get("project", {}) + classifiers = project.get("classifiers", []) + minors = sorted( + { + int(match[1]) + for classifier in classifiers + if (match := CLASSIFIER_RE.fullmatch(classifier)) + } + ) + if not minors: + raise DeclarationError( + "pyproject.toml lists no Programming Language :: Python :: 3.N " + "classifiers" + ) + bound = LOWER_BOUND_RE.search(project.get("requires-python", "")) + if bound and minors[0] < int(bound[1]): + raise DeclarationError( + f"pyproject.toml classifies 3.{minors[0]}, below requires-python " + f'{project["requires-python"]}' + ) + threaded = any( + classifier.startswith(FREE_THREADING_CLASSIFIER) + for classifier in classifiers + ) + versions = [checked(minor, False, "pyproject.toml") for minor in minors] + versions += [ + f"3.{minor}t" + for minor in minors + if threaded and minor >= CLASSIFIED_FREE_THREADED_MINOR + ] + return [Python(version, version) for version in versions] + + +def skips_platform(expression: str, platform: str, system: str) -> bool | None: + """ + Whether a skip expression rules out a whole platform, as the build system + reads it: True or False when it depends on the platform alone, None when it + also depends on anything else, such as the Python version. + """ + names = SELECTORS[platform] | ( + CONDA_BUILD_SELECTORS if system == "conda-build" else {} + ) + try: + tree = ast.parse( + expression.strip().removeprefix("${{").removesuffix("}}").strip(), + mode="eval", + ).body + except SyntaxError: + return None + + def value(node: ast.AST) -> bool: + if isinstance(node, ast.BoolOp): + values = [value(item) for item in node.values] + return all(values) if isinstance(node.op, ast.And) else any(values) + if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.Not): + return not value(node.operand) + if isinstance(node, ast.Name) and names.get(node.id) is not None: + return names[node.id] + if isinstance(node, ast.Constant) and isinstance(node.value, bool): + return node.value + raise ValueError(ast.dump(node)) + + try: + return value(tree) + except ValueError: + return None + + +def platforms_without(skips: list[str], where: str, system: str) -> list[str]: + """ + Every platform in PLATFORMS, but those a skip rules out; a skip that + depends on more than the platform, or on a name the build system doesn't + define, is refused. + """ + for skip in skips: + if any( + skips_platform(skip, platform, system) is None + for platform in PLATFORMS + ): + raise DeclarationError( + f"{where}: cannot tell which platforms the skip {skip!r} rules " + "out, so the matrix would still list the builds it skips" + ) + kept = [ + platform + for platform in PLATFORMS + if not any(skips_platform(skip, platform, system) for skip in skips) + ] + if not kept: + raise DeclarationError( + f"{where} skips every platform CI builds: " + + ", ".join(PLATFORMS.values()) + ) + return kept + + +def yaml_build_skips(text: str, where: str) -> list[str]: + """ + The skip: entries directly under a rattler-build recipe's top-level build:, + as a list of plain or quoted values, a one-line [a, b] list, or one value; + none when there is no skip. + """ + lines = text.splitlines() + starts = [ + number + for number, line in enumerate(lines) + if re.match(r"build\s*:\s*(#.*)?$", line) + ] + if not starts: + return [] + child = "" + for number in range(starts[0] + 1, len(lines)): + line = ( + _strip_comment(lines[number], where) + if "#" in lines[number] + else lines[number] + ) + if not line.strip(): + continue + if not line[0].isspace(): + return [] # the next top-level key + indent = line[: len(line) - len(line.lstrip())] + child = child or indent + match = ( + re.match(r"skip\s*:(.*)", line.lstrip()) + if indent == child + else None + ) + if not match: + continue + rest = match[1].strip() + if rest.startswith("[") and rest.endswith("]"): + return [ + _unquote(item, where) + for item in rest[1:-1].split(",") + if item.strip() + ] + if rest: + return [_unquote(rest, where)] + values = [] + for item in lines[number + 1 :]: + entry = item.strip() + if not entry or entry.startswith("#"): + continue + if not entry.startswith("- "): + break + values.append(_unquote(entry[2:].split(" #")[0], where)) + return values + return [] + + +def from_rattler_recipe(recipe: Path) -> list[str]: + """The platforms a rattler-build recipe builds on, from its build: skip:.""" + where = ( + recipe.relative_to(ROOT).as_posix() + if recipe.is_relative_to(ROOT) + else str(recipe) + ) + return platforms_without( + yaml_build_skips(recipe.read_text(encoding="utf-8"), where), + where, + "rattler-build", + ) + + +def from_conda_recipe(recipe: Path) -> list[str]: + """ + The platforms a conda-build meta.yaml builds on, from its skip: true # + [selector] lines; a skip: line it cannot read is refused. + """ + where = ( + recipe.relative_to(ROOT).as_posix() + if recipe.is_relative_to(ROOT) + else str(recipe) + ) + skips = [] + for line in recipe.read_text(encoding="utf-8").splitlines(): + if not CONDA_SKIP_RE.match(line): + continue + selected = CONDA_SELECTOR_RE.match(line) + key, selector = ( + (selected[1], selected[3]) + if selected + else (line.split(" #")[0], "True") + ) + value = key.split(":", 1)[1].strip() + if value in ("false", "False", "no"): + continue + if value not in ("true", "True", "yes"): + raise DeclarationError( + f"{where}: cannot read {line.strip()!r}; write " + "skip: true # [selector]" + ) + skips.append(selector) + return platforms_without(skips, where, "conda-build") + + +def from_os_classifiers(path: Path) -> list[str]: + """ + The platforms pyproject.toml's Operating System classifiers name; all of + them when it names none. + """ + if TOMLLIB is None: + raise DeclarationError( + "reading pyproject.toml needs Python 3.11 or newer" + ) + classifiers = ( + TOMLLIB.loads(path.read_text(encoding="utf-8")) + .get("project", {}) + .get("classifiers", []) + ) + named = [ + classifier + for classifier in classifiers + if classifier.startswith("Operating System ::") + ] + if not named: + return list(PLATFORMS) + found = { + platform + for classifier in named + for prefix, platforms in OS_CLASSIFIERS.items() + if classifier == prefix or classifier.startswith(prefix + " ::") + for platform in platforms + } + if not found: + raise DeclarationError( + "pyproject.toml names no Operating System CI builds: " + + ", ".join(PLATFORMS.values()) + ) + return [platform for platform in PLATFORMS if platform in found] + + +# Build system -> where it declares Python versions, given the file that +# selected it. +READERS: dict[str, Callable[[Path], list[Python]]] = { + "rattler-build": lambda recipe: from_conda_variants( + recipe.parent / "variants.yaml" + ), + "conda-build": lambda recipe: from_conda_variants( + recipe.parent / "conda_build_config.yaml" + ), + "wheel": from_pyproject, +} + +# Build system -> where it declares the platforms it builds on, given the file +# that selected it. +PLATFORM_READERS: dict[str, Callable[[Path], list[str]]] = { + "rattler-build": from_rattler_recipe, + "conda-build": from_conda_recipe, + "wheel": from_os_classifiers, +} + + +def detect(root: Path, system: str = "") -> tuple[str, Path]: + """The build system and the file that selects it.""" + for name, candidates in SYSTEMS.items(): + if system and name != system: + continue + for candidate in candidates: + if (root / candidate).is_file(): + return name, root / candidate + wanted = system or " or ".join(SYSTEMS) + raise DeclarationError( + f"no {wanted} input found; looked for " + + ", ".join(c for name, cs in SYSTEMS.items() for c in cs) + ) + + +def declared(system: str, selector: Path) -> list[Python]: + """The build system's Python versions, sorted, each listed once.""" + pythons = READERS[system](selector) + versions = [python.version for python in pythons] + if not pythons: + raise DeclarationError(f"{system} declares no Python versions") + if len(set(versions)) != len(versions): + raise DeclarationError( + f"{system} declares a Python version twice: {versions}" + ) + return sorted(pythons, key=lambda python: version_key(python.version)) + + +def declared_platforms(system: str, selector: Path) -> list[str]: + """The platforms the build system builds on, in PLATFORMS order.""" + return PLATFORM_READERS[system](selector) + + +def for_event(pythons: list[Python], event: str) -> list[Python]: + """ + All of them for a tag; for a pull request or a push the oldest and newest + regular ones and the newest free-threaded one. + """ + if event not in SUBSET_EVENTS: + return pythons + regular = [python for python in pythons if not python.version.endswith("t")] + threaded = [python for python in pythons if python.version.endswith("t")] + regular = regular or pythons + return list(dict.fromkeys(regular[:1] + regular[-1:] + threaded[-1:])) + + +def build_matrix(system: str, selector: Path, event: str) -> dict: + """ + Every declared platform Actions can build on, with every Python version the + event builds. + """ + pythons = for_event(declared(system, selector), event) + platforms = declared_platforms(system, selector) + waiting = [ + PLATFORMS[platform] + for platform in platforms + if platform not in BUILT_PLATFORMS + ] + if waiting: + print( + f'matrix: {", ".join(waiting)} declared but not built: ' + "no runners for it yet", + file=sys.stderr, + ) + built = [platform for platform in platforms if platform in BUILT_PLATFORMS] + if not built: + raise DeclarationError( + "the build files declare only platforms there are no runners " + "for yet: " + ", ".join(waiting) + ) + return { + "include": [ + { + "platform": platform, + "name": PLATFORMS[platform], + "python": python.version, + } + for platform in built + for python in pythons + ] + } + + +def main(argv: list[str] | None = None) -> int: + """Prints the matrix to build; writes it for GitHub Actions if asked.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[1]) + parser.add_argument( + "--system", + default="", + choices=("", *SYSTEMS), + help="build system to read instead of the detected one", + ) + parser.add_argument( + "--event", + default="tag", + choices=("pr", "tag", "push"), + help="what the build is for; only a tag builds every version", + ) + parser.add_argument( + "--github-output", + default="", + metavar="NAME", + help="also write NAME= to $GITHUB_OUTPUT", + ) + args = parser.parse_args(argv) + if args.github_output: + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_-]*", args.github_output): + raise DeclarationError( + f"{args.github_output!r} is not an output name" + ) + if not os.environ.get("GITHUB_OUTPUT"): + raise DeclarationError( + "--github-output needs $GITHUB_OUTPUT, which Actions sets" + ) + system, selector = detect(ROOT, args.system) + matrix = build_matrix(system, selector, args.event) + cells = ", ".join( + f'{cell["name"]} {cell["python"]}' for cell in matrix["include"] + ) + print( + f"matrix: {system} from {selector.relative_to(ROOT)}: {cells}", + file=sys.stderr, + ) + text = json.dumps(matrix) + print(text) + if args.github_output: + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: + output.write(f"{args.github_output}={text}\n") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except DeclarationError as error: + print(f"matrix: {error}", file=sys.stderr) + sys.exit(1) diff --git a/ci/test.py b/ci/test.py new file mode 100644 index 00000000..80729ce4 --- /dev/null +++ b/ci/test.py @@ -0,0 +1,69 @@ +""" +Tests mkl_fft's installed package. + +Runs with one built conda package or wheel installed in a clean environment, as +ci/hooks.py sets up, from an empty working directory, so the tests import the +installed mkl_fft rather than the source tree. The test requirements come from +ci/config.yml. Writes a JUnit report to $GH_CI_RESULTS. +""" + +import importlib +import importlib.util +import os +import pathlib +import sys +import sysconfig + +import pytest + +PACKAGE = "mkl_fft" + + +def running_python() -> str: + """This interpreter's version as artifact names spell it, such as 3.14t.""" + version = f"{sys.version_info.major}.{sys.version_info.minor}" + threaded = sysconfig.get_config_var("Py_GIL_DISABLED") + return version + ("t" if threaded else "") + + +def scipy_interface_error() -> str | None: + """Why mkl_fft.interfaces.scipy_fft fails to import with SciPy and mkl-service installed.""" + if not all(importlib.util.find_spec(name) for name in ("scipy", "mkl")): + return None + try: + importlib.import_module("mkl_fft.interfaces.scipy_fft") + except ImportError as error: + return f"{type(error).__name__}: {error}" + return None + + +def main() -> int: + """Runs the installed package's test suite.""" + expected = os.environ["GH_CI_PYTHON"] + if running_python() != expected: + print(f"running Python {running_python()}, expected {expected}") + return 1 + # Its tests skip without the interface, which would hide a broken import + error = scipy_interface_error() + if error: + print(f"mkl_fft.interfaces.scipy_fft does not import: {error}") + return 1 + # The suite name says which platform and Python ran. + cell = f"{os.environ.get('GH_CI_PLATFORM', 'linux')}.py{expected}" + report = pathlib.Path(os.environ["GH_CI_RESULTS"]) / f"{PACKAGE}.xml" + return pytest.main( + [ + "-v", + "-p", + "no:cacheprovider", + f"--junitxml={report}", + "-o", + f"junit_suite_name={PACKAGE} {cell}", + "--pyargs", + PACKAGE, + ] + ) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/conda-recipe-cf/AGENTS.md b/conda-recipe-cf/AGENTS.md deleted file mode 100644 index 9d8655e2..00000000 --- a/conda-recipe-cf/AGENTS.md +++ /dev/null @@ -1,10 +0,0 @@ -# AGENTS.md — conda-recipe-cf/ - -Conda-forge recipe context for `mkl_fft`. - -## Scope -- conda-forge specific recipe files under this directory. - -## Guardrails -- Keep conda-forge recipe semantics separate from Intel-channel recipe. -- Align changes with `conda-package-cf.yml` workflow behavior. diff --git a/conda-recipe-cf/bld.bat b/conda-recipe-cf/bld.bat deleted file mode 100644 index a08c27bb..00000000 --- a/conda-recipe-cf/bld.bat +++ /dev/null @@ -1,2 +0,0 @@ -%PYTHON% -m pip install --no-build-isolation --no-deps . -if errorlevel 1 exit 1 diff --git a/conda-recipe-cf/build.sh b/conda-recipe-cf/build.sh deleted file mode 100644 index 72e5e4e6..00000000 --- a/conda-recipe-cf/build.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -ex - -$PYTHON -m pip install --no-build-isolation --no-deps . diff --git a/conda-recipe-cf/conda_build_config.yaml b/conda-recipe-cf/conda_build_config.yaml deleted file mode 100644 index d4b34a3b..00000000 --- a/conda-recipe-cf/conda_build_config.yaml +++ /dev/null @@ -1,16 +0,0 @@ -c_compiler: # [linux] - - gcc # [linux] -cxx_compiler: # [linux] - - gxx # [linux] -cxx_compiler_version: # [linux] - - '14' # [linux] -c_stdlib: # [linux] - - sysroot # [linux] -c_stdlib_version: # [linux] - - '2.28' # [linux] -c_stdlib: # [win] - - vs # [win] -cxx_compiler: # [win] - - vs2022 # [win] -c_compiler: # [win] - - vs2022 # [win] diff --git a/conda-recipe-cf/meta.yaml b/conda-recipe-cf/meta.yaml deleted file mode 100644 index 47382240..00000000 --- a/conda-recipe-cf/meta.yaml +++ /dev/null @@ -1,59 +0,0 @@ -package: - name: mkl_fft - version: {{ GIT_DESCRIBE_TAG }} - -source: - path: ../ - -build: - number: {{ GIT_DESCRIBE_NUMBER }} - ignore_run_exports: - - blas - - numpy - -requirements: - build: - - {{ compiler('c') }} - - {{ stdlib('c') }} - host: - - meson-python >=0.13.0 - - cmake - - ninja - - python - - pip - - mkl-devel - - cython >=3.1.0 - - numpy - - wheel >=0.41.3 - run: - - python - - {{ pin_compatible('mkl') }} - - numpy - -test: - commands: - - pip check - - pytest -v --pyargs mkl_fft - requires: - - pip - - pytest - - scipy >=1.10 - - mkl-service - imports: - - mkl_fft - - mkl_fft.interfaces - -about: - home: http://github.com/IntelPython/mkl_fft - license: BSD-3-Clause - license_file: LICENSE.txt - summary: NumPy-based implementation of Fast Fourier Transform using Intel® oneAPI Math Kernel Library (oneMKL) - description: | - LEGAL NOTICE: Use of this software package is subject to the - software license agreement (as set forth above, in the license section of - the installed Conda package and/or the README file) and all notices, - disclaimers or license terms for third party or open source software - included in or with the software. -

- EULA: BSD-3-Clause -

diff --git a/conda-recipe/AGENTS.md b/conda-recipe/AGENTS.md index 7c7cef4b..4a1bbd17 100644 --- a/conda-recipe/AGENTS.md +++ b/conda-recipe/AGENTS.md @@ -1,11 +1,11 @@ # AGENTS.md — conda-recipe/ -Intel-channel conda packaging context. +The conda recipe for `mkl_fft`, built against conda-forge only. ## Scope -- `meta.yaml` — package metadata and dependency pins -- `build.sh` / `bld.bat` — platform build scripts +- conda-forge specific recipe files under this directory. ## Guardrails -- Treat recipe files as canonical for packaging intent. -- Keep recipe updates synchronized with CI workflow behavior. +- Align changes with how `intel-ci.yml` builds and tests the recipe (`ci/build.py`, `ci/test.py`). +- `conda_build_config.yaml`'s `python:` list is the Intel CI conda matrix, read by `ci/matrix.py`: one version per line, no selectors. +- Wheels don't come from this recipe: `intel-ci.yml` builds them from `pyproject.toml` with `ci/build.py`. diff --git a/conda-recipe/bld.bat b/conda-recipe/bld.bat index 34f84f2d..b7fd9299 100644 --- a/conda-recipe/bld.bat +++ b/conda-recipe/bld.bat @@ -1,22 +1,2 @@ -:: -wnx flags mean: --wheel --no-isolation --skip-dependency-check -:: -Ccompile-args=-v makes ninja print full compiler commands (verbose build) -%PYTHON% -m build -w -n -x -Ccompile-args=-v -if %ERRORLEVEL% neq 0 exit 1 - -:: wheel file was renamed -for /f %%f in ('dir /b /S .\dist') do ( - %PYTHON% -m pip install %%f ^ - --no-build-isolation ^ - --no-deps ^ - --only-binary :all: ^ - --no-index ^ - --prefix %PREFIX% ^ - -vv - if %ERRORLEVEL% neq 0 exit 1 -) - -:: Copy wheel package -if NOT "%WHEELS_OUTPUT_FOLDER%"=="" ( - copy dist\mkl_fft*.whl %WHEELS_OUTPUT_FOLDER% - if %ERRORLEVEL% neq 0 exit 1 -) +%PYTHON% -m pip install --no-build-isolation --no-deps -v -Ccompile-args=-v . +if errorlevel 1 exit 1 diff --git a/conda-recipe/build.sh b/conda-recipe/build.sh index 4f94516d..a966047b 100644 --- a/conda-recipe/build.sh +++ b/conda-recipe/build.sh @@ -1,29 +1,3 @@ #!/bin/bash -ex -if [ -d "build" ]; then - rm -rf build -fi - -read -r GLIBC_MAJOR GLIBC_MINOR <<<"$(conda list '^sysroot_linux-64$' \ - | tail -n 1 | awk '{print $2}' | grep -oP '\d+' | head -n 2 | tr '\n' ' ')" - -# -wnx flags mean: --wheel --no-isolation --skip-dependency-check -# -Ccompile-args=-v makes ninja print full compiler commands (verbose build) -${PYTHON} -m build -w -n -x -Ccompile-args=-v - -${PYTHON} -m wheel tags --remove \ - --platform-tag "manylinux_${GLIBC_MAJOR}_${GLIBC_MINOR}_x86_64" \ - dist/mkl_fft*.whl - -${PYTHON} -m pip install dist/mkl_fft*.whl \ - --no-build-isolation \ - --no-deps \ - --only-binary :all: \ - --no-index \ - --prefix "${PREFIX}" \ - -vv - -# Copy wheel package -if [[ -d "${WHEELS_OUTPUT_FOLDER}" ]]; then - cp dist/mkl_fft*.whl "${WHEELS_OUTPUT_FOLDER[@]}" -fi +$PYTHON -m pip install --no-build-isolation --no-deps -v -Ccompile-args=-v . diff --git a/conda-recipe/conda_build_config.yaml b/conda-recipe/conda_build_config.yaml index d4b34a3b..b229309b 100644 --- a/conda-recipe/conda_build_config.yaml +++ b/conda-recipe/conda_build_config.yaml @@ -1,16 +1,33 @@ +# Python versions for ci/matrix.py, one per line, without selectors +python: + - "3.11.* *_cpython" + - "3.12.* *_cpython" + - "3.13.* *_cp313" + - "3.14.* *_cp314" + - "3.14.* *_cp314t" + - "3.15.* *_cp315" + # No 3.15t: conda-forge has no SciPy for it yet, which the recipe test needs +numpy: + - '2' c_compiler: # [linux] - gcc # [linux] -cxx_compiler: # [linux] - - gxx # [linux] -cxx_compiler_version: # [linux] - - '14' # [linux] +c_compiler_version: # [linux] + - '15' # [linux] c_stdlib: # [linux] - sysroot # [linux] c_stdlib_version: # [linux] - '2.28' # [linux] c_stdlib: # [win] - vs # [win] -cxx_compiler: # [win] - - vs2022 # [win] c_compiler: # [win] - vs2022 # [win] +c_compiler: # [osx] + - clang # [osx] +c_compiler_version: # [osx] + - '21' # [osx] +c_stdlib: # [osx] + - macosx_deployment_target # [osx] +c_stdlib_version: # [osx] + - '11.0' # [osx] +MACOSX_DEPLOYMENT_TARGET: # [osx] + - '11.0' # [osx] diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index 50a45378..770f326a 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -7,11 +7,8 @@ source: build: number: {{ GIT_DESCRIBE_NUMBER }} - script_env: - - WHEELS_OUTPUT_FOLDER ignore_run_exports: - blas - - numpy requirements: build: @@ -19,7 +16,6 @@ requirements: - {{ stdlib('c') }} host: - meson-python >=0.13.0 - - python-build >=1.2.2 - cmake - ninja - python @@ -35,8 +31,10 @@ requirements: test: commands: + - pip check - pytest -v --pyargs mkl_fft requires: + - pip - pytest - scipy >=1.10 - mkl-service diff --git a/meson.build b/meson.build index b7b5b932..df712b6f 100644 --- a/meson.build +++ b/meson.build @@ -42,12 +42,19 @@ if host_machine.system() != 'windows' ] endif +# mkl-devel's CMake config is under \Library on Windows, which CMake only +# finds through a Library\bin PATH entry: a conda env has one, a venv doesn't +mkl_cmake_prefix = [] +if host_machine.system() == 'windows' + mkl_cmake_prefix += '-DCMAKE_PREFIX_PATH=' + (py.get_path('data') / 'Library') +endif + mkl_dep = dependency('MKL', method: 'cmake', modules: ['MKL::MKL'], cmake_args: [ '-DMKL_ARCH=intel64', '-DMKL_LINK=sdl', - ], + ] + mkl_cmake_prefix, required: true ) diff --git a/mkl_fft/tests/test_interfaces.py b/mkl_fft/tests/test_interfaces.py index 226d4625..14b8c4a0 100644 --- a/mkl_fft/tests/test_interfaces.py +++ b/mkl_fft/tests/test_interfaces.py @@ -43,12 +43,18 @@ ids.append("numpy") +def require_scipy_fft(): + """Skips the test unless mkl_fft has its SciPy interface.""" + if scipy_fft is None: + pytest.skip("requires scipy and mkl-service") + + @pytest.mark.parametrize("norm", [None, "forward", "backward", "ortho"]) @pytest.mark.parametrize( "dtype", [np.float32, np.float64, np.complex64, np.complex128] ) def test_scipy_fft(norm, dtype): - pytest.importorskip("scipy", reason="requires scipy") + require_scipy_fft() x = np.ones(511, dtype=dtype) w = mfi.scipy_fft.fft(x, norm=norm, workers=None, plan=None) xx = mfi.scipy_fft.ifft(w, norm=norm, workers=None, plan=None) @@ -71,7 +77,7 @@ def test_numpy_fft(norm, dtype): @pytest.mark.parametrize("norm", [None, "forward", "backward", "ortho"]) @pytest.mark.parametrize("dtype", [np.float32, np.float64]) def test_scipy_rfft(norm, dtype): - pytest.importorskip("scipy", reason="requires scipy") + require_scipy_fft() x = np.ones(511, dtype=dtype) w = mfi.scipy_fft.rfft(x, norm=norm, workers=None, plan=None) xx = mfi.scipy_fft.irfft( @@ -102,7 +108,7 @@ def test_numpy_rfft(norm, dtype): "dtype", [np.float32, np.float64, np.complex64, np.complex128] ) def test_scipy_fftn(norm, dtype): - pytest.importorskip("scipy", reason="requires scipy") + require_scipy_fft() x = np.ones((37, 83), dtype=dtype) w = mfi.scipy_fft.fftn(x, norm=norm, workers=None, plan=None) xx = mfi.scipy_fft.ifftn(w, norm=norm, workers=None, plan=None) @@ -125,7 +131,7 @@ def test_numpy_fftn(norm, dtype): @pytest.mark.parametrize("norm", [None, "forward", "backward", "ortho"]) @pytest.mark.parametrize("dtype", [np.float32, np.float64]) def test_scipy_rfftn(norm, dtype): - pytest.importorskip("scipy", reason="requires scipy") + require_scipy_fft() x = np.ones((37, 83), dtype=dtype) w = mfi.scipy_fft.rfftn(x, norm=norm, workers=None, plan=None) xx = mfi.scipy_fft.irfftn(w, s=x.shape, norm=norm, workers=None, plan=None) @@ -160,13 +166,13 @@ def _get_blacklisted_dtypes(): @pytest.mark.parametrize("dtype", _get_blacklisted_dtypes()) def test_scipy_no_support_for(dtype): - pytest.importorskip("scipy", reason="requires scipy") + require_scipy_fft() x = np.ones(16, dtype=dtype) assert_raises(NotImplementedError, mfi.scipy_fft.ifft, x) def test_scipy_fft_arg_validate(): - pytest.importorskip("scipy", reason="requires scipy") + require_scipy_fft() with pytest.raises(ValueError): mfi.scipy_fft.fft([1, 2, 3, 4], norm=b"invalid") diff --git a/mkl_fft/tests/third_party/scipy/test_basic.py b/mkl_fft/tests/third_party/scipy/test_basic.py index 25cd3d61..a8e5025f 100644 --- a/mkl_fft/tests/third_party/scipy/test_basic.py +++ b/mkl_fft/tests/third_party/scipy/test_basic.py @@ -32,7 +32,10 @@ else: from numpy import permute_dims -import mkl_fft.interfaces.scipy_fft as fft +try: + import mkl_fft.interfaces.scipy_fft as fft +except ImportError: + pytest.skip("This test file needs mkl-service", allow_module_level=True) skip_xp_backends = pytest.mark.skip_xp_backends diff --git a/mkl_fft/tests/third_party/scipy/test_multithreading.py b/mkl_fft/tests/third_party/scipy/test_multithreading.py index f948e221..414c3902 100644 --- a/mkl_fft/tests/third_party/scipy/test_multithreading.py +++ b/mkl_fft/tests/third_party/scipy/test_multithreading.py @@ -3,15 +3,18 @@ import multiprocessing -import mkl import numpy as np import pytest from numpy.testing import assert_allclose try: + import mkl + import mkl_fft.interfaces.scipy_fft as fft except ImportError: - pytest.skip("This test file needs scipy", allow_module_level=True) + pytest.skip( + "This test file needs scipy and mkl-service", allow_module_level=True + ) @pytest.fixture(scope="module") diff --git a/pyproject.toml b/pyproject.toml index 635c7e46..f9ba6064 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,10 +80,32 @@ extend-exclude = "(^|/)_vendored/" force-exclude = "(^|/)_vendored/" line-length = 80 +[tool.cibuildwheel] +build-verbosity = 1 + +[tool.cibuildwheel.linux] +config-settings = { compile-args = "-v" } +# The hardening flags conda-forge's compilers set for the conda packages +environment = { CFLAGS = "-fstack-protector-strong", CPPFLAGS = "-D_FORTIFY_SOURCE=2", LDFLAGS = "-Wl,-z,relro -Wl,-z,now -Wl,--disable-new-dtags" } +# MKL comes from the mkl wheel at run time, see [tool.gh-ci] +# Tagged as the mkl wheel is, as auditwheel doesn't check the excluded library +repair-wheel-command = "auditwheel repair --plat manylinux_2_28_x86_64 --only-plat --exclude 'libmkl_rt.so.*' -w {dest_dir} {wheel}" + +[tool.cibuildwheel.windows] +# MSVC, as python.org builds CPython with, not the runner's MinGW +config-settings = { setup-args = "--vsenv", compile-args = "-v" } +# MKL comes from the mkl wheel at run time, see [tool.gh-ci] +repair-wheel-command = "delvewheel repair --no-dll mkl_rt.*.dll -w {dest_dir} {wheel}" + [tool.cython-lint] ignore = ['E722'] # do not use bare 'except' max-line-length = 80 +[tool.gh-ci] +# What the wheels need at run time, which conda packages get from the recipe: +# mkl >= the mkl-devel version the build used, < its next major +pin-compatible = { mkl = "mkl-devel" } + [tool.isort] ensure_newline_before_comments = true force_grid_wrap = 0