Move macOS and Windows CI from Azure to GHA

Author: jonashaag

.circleci/config.yml

@@ -13,7 +13,7 @@ jobs:
       PANDAS_CI: "1"
     steps:
       - checkout
-      - run: ci/setup_env.sh
+      - run: .circleci/setup_env.sh
       - run: PATH=$HOME/miniconda3/envs/pandas-dev/bin:$HOME/miniconda3/condabin:$PATH ci/run_tests.sh
 
 workflows:

ci/setup_env.sh renamed to .circleci/setup_env.sh

@@ -3,15 +3,25 @@ description: Rebuilds the C extensions and installs pandas
 runs:
   using: composite
   steps:
-
     - name: Environment Detail
       run: |
-        conda info
-        conda list
+        if which mamba 2>/dev/null; then
+          mamba info
+          mamba list
+          mamba info | grep -Ei 'environment.+:' | grep -qEiv 'environment.+:.+none'
+        fi
+        pip list
+        python --version
       shell: bash -el {0}
 
     - name: Build Pandas
       run: |
-        python setup.py build_ext -j 2
-        python -m pip install -e . --no-build-isolation --no-use-pep517 --no-index
+        which python
+        which pip
+        time python setup.py build_ext -v -j 3
+        pip install -v -e . --no-build-isolation --no-use-pep517 --no-index
+      shell: bash -el {0}
+
+    - name: Build Version
+      run: pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd
       shell: bash -el {0}

.github/actions/build_pandas/action.yml

@@ -0,0 +1,34 @@
+name: Run tests and report results
+inputs:
+  check-pyarrow-version:
+    required: false
+runs:
+  using: composite
+  steps:
+    - name: Check PyArrow version
+      run: |
+        # Double check that we have the expected PyArrow
+        pushd /tmp
+        python -c "import pandas; pandas.show_versions()" | egrep -i "pyarrow.+: ${{ matrix.check-pyarrow-version }}"
+        popd
+      shell: bash -el {0}
+      if: ${{ inputs.check-pyarrow-version }}
+
+    - name: Test
+      run: ci/run_tests.sh
+      shell: bash -el {0}
+
+    - name: Publish test results
+      uses: actions/upload-artifact@v2
+      with:
+        name: Test results
+        path: test-data.xml
+      if: failure()
+
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v2
+      with:
+        flags: unittests
+        name: codecov-pandas
+        fail_ci_if_error: false
+      if: failure()

.github/actions/run-tests/action.yml

@@ -1,12 +1,58 @@
 name: Set up pandas
 description: Runs all the setup steps required to have a built pandas ready to use
+inputs:
+  environment-file:
+    default: environment.yml
+  pyarrow-version:
+      required: false
+  is-pypy:
+    default: false
+  environment-name:
+    default: pandas-dev
+  python-version:
+    required: false
 runs:
   using: composite
   steps:
-    - name: Setting conda path
-      run: echo "${HOME}/miniconda3/bin" >> $GITHUB_PATH
-      shell: bash -el {0}
+    - name: Set Arrow version in ${{ inputs.environment-file }} to ${{ inputs.pyarrow-version }}
+      run: |
+        grep -q '  - pyarrow' ${{ inputs.environment-file }}
+        sed -i"" -e "s/  - pyarrow/  - pyarrow=${{ inputs.pyarrow-version }}/" ${{ inputs.environment-file }}
+        cat ${{ inputs.environment-file }}
+      shell: bash
+      if: ${{ inputs.pyarrow-version }}
 
-    - name: Setup environment and build pandas
-      run: ci/setup_env.sh
-      shell: bash -el {0}
+    - name: Set Python version in ${{ inputs.environment-file }} to ${{ inputs.python-version }}
+      run: |
+        echo "  - python=${{ inputs.pyarrow-version }}" >> ${{ inputs.environment-file }}
+        cat ${{ inputs.environment-file }}
+      shell: bash
+      if: ${{ inputs.python-version }}
+
+    - name: Pin setuptools (GH#44980)
+      run: |
+        echo '  - setuptools <60' >> ${{ inputs.environment-file }}
+      shell: bash
+
+    - name: Install ${{ inputs.environment-file }} (Python ${{ inputs.python-version }})
+      uses: conda-incubator/setup-miniconda@v2
+      with:
+        activate-environment: ${{ inputs.environment-name }}
+        environment-file: ${{ inputs.environment-file }}
+        channel-priority: strict
+        channels: conda-forge
+        mamba-version: "0.22"
+        use-mamba: true
+      if: ${{ inputs.is-pypy == 'false' }} # No pypy3.8 support
+
+    - name: Setup PyPy
+      uses: actions/setup-python@v3
+      with:
+        python-version: "pypy-3.8"
+      if: ${{ inputs.is-pypy == 'true' }}
+
+    - name: Setup PyPy dependencies
+      # TODO: re-enable cov, its slowing the tests down though
+      run: pip install Cython numpy python-dateutil pytz pytest>=6.0 pytest-xdist>=1.31.0 hypothesis>=5.5.3 pytest-asyncio>=0.17
+      shell: bash
+      if: ${{ inputs.is-pypy == 'true' }}

.github/actions/setup/action.yml

@@ -0,0 +1,62 @@
+name: macOS and Windows
+
+on:
+  push:
+    branches:
+      - main
+      - 1.4.x
+  pull_request:
+    branches:
+      - main
+      - 1.4.x
+    paths-ignore:
+      - "doc/**"
+
+env:
+  PANDAS_CI: 1
+  PYTEST_TARGET: pandas
+  PYTEST_WORKERS: auto
+  PATTERN: "not slow and not db and not network and not single_cpu"
+
+jobs:
+  pytest:
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        shell: bash -el {0}
+    timeout-minutes: 120
+    strategy:
+      matrix:
+        os: [macos-10.15, windows-2019]
+        env_file: [actions-38.yaml, actions-39.yaml, actions-310.yaml]
+        pyarrow_version: [6]
+      fail-fast: false
+    name: ${{ matrix.name || format('{0} {1} pyarrow={2}', matrix.os, matrix.env_file, matrix.pyarrow_version) }}
+    concurrency:
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.os }}-${{ matrix.env_file }}-${{ matrix.pyarrow_version }}-mac-win
+      cancel-in-progress: true
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+
+    - name: wmic.exe
+      run: wmic.exe cpu get caption, deviceid, name, numberofcores, maxclockspeed
+      if: ${{ runner.os == 'Windows' }}
+
+    - name: Set up Conda (${{ matrix.env_file }}, Arrow ${{ matrix.pyarrow_version}})
+      uses: ./.github/actions/setup
+      with:
+        environment-file: ci/deps/${{ matrix.env_file }}
+        pyarrow-version: ${{ matrix.pyarrow_version }}
+
+    - name: Build pandas
+      uses: ./.github/actions/build_pandas
+
+    - name: Run tests
+      uses: ./.github/actions/run-tests
+      with:
+        check-pyarrow-version: ${{ matrix.pyarrow_version }}

.github/workflows/macos-windows.yml

@@ -22,16 +22,6 @@ variables:
   PANDAS_CI: 1
 
 jobs:
-- template: ci/azure/posix.yml
-  parameters:
-    name: macOS
-    vmImage: macOS-10.15
-
-- template: ci/azure/windows.yml
-  parameters:
-    name: Windows
-    vmImage: windows-2019
-
 - job: py38_32bit
   pool:
     vmImage: ubuntu-18.04

azure-pipelines.yml

@@ -30,6 +30,10 @@ if [[ "$PATTERN" ]]; then
   PYTEST_CMD="$PYTEST_CMD -m \"$PATTERN\""
 fi
 
+which pytest
+which python
+pytest -VV || true
+python -VV || true
 echo $PYTEST_CMD
 sh -c "$PYTEST_CMD"
 

ci/azure/posix.yml

@@ -334,7 +334,7 @@ def run(self):
         extra_compile_args.append("/Z7")
         extra_link_args.append("/DEBUG")
 else:
-    # PANDAS_CI=1 is set by ci/setup_env.sh
+    # PANDAS_CI=1 is set in CI
     if os.environ.get("PANDAS_CI", "0") == "1":
         extra_compile_args.append("-Werror")
     if debugging_symbols_requested: