python-telegram-bot/.github/workflows/test.yml

name: Unit Tests
on:
  pull_request:
    branches:
      - master
  push:
    branches:
      - master
  schedule:
    # Run monday and friday morning at 03:07 - odd time to spread load on GitHub Actions
    - cron: '7 3 * * 1,5'

jobs:
  pytest:
    name: pytest
    runs-on: ${{matrix.os}}
    strategy:
      matrix:
        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12.0-beta.3']
        os: [ubuntu-latest, windows-latest, macos-latest]
      fail-fast: False
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
          cache: 'pip'
          cache-dependency-path: '**/requirements*.txt'
      - name: Install dependencies
        run: |
          python -W ignore -m pip install --upgrade pip
          python -W ignore -m pip install -U pytest-cov
          python -W ignore -m pip install -r requirements.txt
          python -W ignore -m pip install -r requirements-dev.txt
          python -W ignore -m pip install pytest-xdist[psutil]          

      - name: Test with pytest
        # We run 4 different suites here
        # 1. Test just utils.datetime.py without pytz being installed
        # 2. Test just test_no_passport.py without passport dependencies being installed
        # 3. Test just test_rate_limiter.py without passport dependencies being installed
        # 4. Test everything else
        # The first & second one are achieved by mocking the corresponding import
        # See test_helpers.py & test_no_passport.py for details
        run: |
          # We test without optional dependencies first. This includes:
          # - without pytz
          # - without jobqueue
          # - without ratelimiter
          # - without webhooks
          # - without arbitrary callback data
          # - without socks support
          # - without http2 support
          TO_TEST="test_no_passport.py or test_datetime.py or test_defaults.py or test_jobqueue.py or test_applicationbuilder.py or test_ratelimiter.py or test_updater.py or test_callbackdatacache.py or test_request.py"
          pytest -v --cov -k "${TO_TEST}"
          # Rerun only failed tests (--lf), and don't run any tests if none failed (--lfnf=none)
          pytest -v --cov --cov-append -k "${TO_TEST}" --lf --lfnf=none --junit-xml=.test_report_no_optionals.xml
          # No tests were selected, convert returned status code to 0
          opt_dep_status=$(( $? == 5 ? 0 : $? ))

          # Test the rest
          export TEST_WITH_OPT_DEPS='true'
          pip install -r requirements-opts.txt
          # `-n auto --dist loadfile` uses pytest-xdist to run each test file on a different CPU
          # worker. Increasing number of workers has little effect on test duration, but it seems
          # to increase flakyness, specially on python 3.7 with --dist=loadgroup.
          pytest -v --cov --cov-append -n auto --dist loadfile
          pytest -v --cov --cov-append -n auto --dist loadfile --lf --lfnf=none --junit-xml=.test_report_optionals.xml
          main_status=$(( $? == 5 ? 0 : $? ))
          # exit with non-zero status if any of the two pytest runs failed
          exit $(( ${opt_dep_status} || ${main_status} ))          
        env:
          JOB_INDEX: ${{ strategy.job-index }}
          BOTS: W3sidG9rZW4iOiAiNjk2MTg4NzMyOkFBR1Z3RUtmSEhsTmpzY3hFRE5LQXdraEdzdFpfa28xbUMwIiwgInBheW1lbnRfcHJvdmlkZXJfdG9rZW4iOiAiMjg0Njg1MDYzOlRFU1Q6WldGaU1UUmxNbVF5TnpNeSIsICJuYW1lIjogIlBUQiB0ZXN0cyBvbiBUcmF2aXMgdXNpbmcgQ1B5dGhvbiAyLjciLCAic3VwZXJfZ3JvdXBfaWQiOiAiLTEwMDEzOTA5ODM5OTciLCAidXNlcm5hbWUiOiAiQHB0Yl90cmF2aXNfY3B5dGhvbl8yN19ib3QiLCAiZm9ydW1fZ3JvdXBfaWQiOiAiLTEwMDE3MTA4NTA4MjIifSwgeyJ0b2tlbiI6ICI2NzE0Njg4ODY6QUFHUEdmY2lSSUJVTkZlODI0dUlWZHE3SmUzX1luQVROR3ciLCAicGF5bWVudF9wcm92aWRlcl90b2tlbiI6ICIyODQ2ODUwNjM6VEVTVDpaR1l3T1Rsa016TXhOMlkyIiwgIm5hbWUiOiAiUFRCIHRlc3RzIG9uIFRyYXZpcyB1c2luZyBDUHl0aG9uIDMuNCIsICJzdXBlcl9ncm91cF9pZCI6ICItMTAwMTQ0NjAyMjUyMiIsICJ1c2VybmFtZSI6ICJAcHRiX3RyYXZpc19jcHl0aG9uXzM0X2JvdCIsICJmb3J1bV9ncm91cF9pZCI6ICItMTAwMTg5MTQ0MTc5MSJ9LCB7InRva2VuIjogIjYyOTMyNjUzODpBQUZSclpKckI3b0IzbXV6R3NHSlhVdkdFNUNRek01Q1U0byIsICJwYXltZW50X3Byb3ZpZGVyX3Rva2VuIjogIjI4NDY4NTA2MzpURVNUOk1tTTVZV0poWXpreE0yVTEiLCAibmFtZSI6ICJQVEIgdGVzdHMgb24gVHJhdmlzIHVzaW5nIENQeXRob24gMy41IiwgInN1cGVyX2dyb3VwX2lkIjogIi0xMDAxNDk2OTE3NzUwIiwgInVzZXJuYW1lIjogIkBwdGJfdHJhdmlzX2NweXRob25fMzVfYm90IiwgImZvcnVtX2dyb3VwX2lkIjogIi0xMDAxNTc3NTA0Nzg3In0sIHsidG9rZW4iOiAiNjQwMjA4OTQzOkFBRmhCalFwOXFtM1JUeFN6VXBZekJRakNsZS1Kano1aGNrIiwgInBheW1lbnRfcHJvdmlkZXJfdG9rZW4iOiAiMjg0Njg1MDYzOlRFU1Q6WXpoa1pUZzFOamMxWXpWbCIsICJuYW1lIjogIlBUQiB0ZXN0cyBvbiBUcmF2aXMgdXNpbmcgQ1B5dGhvbiAzLjYiLCAic3VwZXJfZ3JvdXBfaWQiOiAiLTEwMDEzMzM4NzE0NjEiLCAidXNlcm5hbWUiOiAiQHB0Yl90cmF2aXNfY3B5dGhvbl8zNl9ib3QiLCAiZm9ydW1fZ3JvdXBfaWQiOiAiLTEwMDE4Njc5MDExNzIifSwgeyJ0b2tlbiI6ICI2OTUxMDQwODg6QUFIZnp5bElPalNJSVMtZU9uSTIweTJFMjBIb2RIc2Z6LTAiLCAicGF5bWVudF9wcm92aWRlcl90b2tlbiI6ICIyODQ2ODUwNjM6VEVTVDpPR1ExTURnd1pqSXdaakZsIiwgIm5hbWUiOiAiUFRCIHRlc3RzIG9uIFRyYXZpcyB1c2luZyBDUHl0aG9uIDMuNyIsICJzdXBlcl9ncm91cF9pZCI6ICItMTAwMTQ3ODI5MzcxNCIsICJ1c2VybmFtZSI6ICJAcHRiX3RyYXZpc19jcHl0aG9uXzM3X2JvdCIsICJmb3J1bV9ncm91cF9pZCI6ICItMTAwMTg2NDA1NDg3OSJ9LCB7InRva2VuIjogIjY5MTQyMzU1NDpBQUY4V2tqQ1pibkhxUF9pNkdoVFlpckZFbFpyR2FZT2hYMCIsICJwYXltZW50X3Byb3ZpZGVyX3Rva2VuIjogIjI4NDY4NTA2MzpURVNUOllqYzVOVGhpTW1ReU1XVmgiLCAibmFtZSI6ICJQVEIgdGVzdHMgb24gVHJhdmlzIHVzaW5nIFB5UHkgMi43IiwgInN1cGVyX2dyb3VwX2lkIjogIi0xMDAxMzYzOTMyNTczIiwgInVzZXJuYW1lIjogIkBwdGJfdHJhdmlzX3B5cHlfMjdfYm90IiwgImZvcnVtX2dyb3VwX2lkIjogIi0xMDAxODY3ODU1OTM2In0sIHsidG9rZW4iOiAiNjg0MzM5OTg0OkFBRk1nRUVqcDAxcjVyQjAwN3lDZFZOc2c4QWxOc2FVLWNjIiwgInBheW1lbnRfcHJvdmlkZXJfdG9rZW4iOiAiMjg0Njg1MDYzOlRFU1Q6TVRBek1UWTNNR1V5TmpnMCIsICJuYW1lIjogIlBUQiB0ZXN0cyBvbiBUcmF2aXMgdXNpbmcgUHlQeSAzLjUiLCAic3VwZXJfZ3JvdXBfaWQiOiAiLTEwMDE0MDc4MzY2MDUiLCAidXNlcm5hbWUiOiAiQHB0Yl90cmF2aXNfcHlweV8zNV9ib3QiLCAiZm9ydW1fZ3JvdXBfaWQiOiAiLTEwMDE1NTg5OTAyODIifSwgeyJ0b2tlbiI6ICI2OTAwOTEzNDc6QUFGTG1SNXBBQjVZY3BlX21PaDd6TTRKRkJPaDB6M1QwVG8iLCAicGF5bWVudF9wcm92aWRlcl90b2tlbiI6ICIyODQ2ODUwNjM6VEVTVDpaRGhsTnpFNU1Ea3dZV0ppIiwgIm5hbWUiOiAiUFRCIHRlc3RzIG9uIEFwcFZleW9yIHVzaW5nIENQeXRob24gMy40IiwgInN1cGVyX2dyb3VwX2lkIjogIi0xMDAxMjc5NjAwMDI2IiwgInVzZXJuYW1lIjogIkBwdGJfYXBwdmV5b3JfY3B5dGhvbl8zNF9ib3QiLCAiZm9ydW1fZ3JvdXBfaWQiOiAiLTEwMDE3MjU2OTEzODcifSwgeyJ0b2tlbiI6ICI2OTQzMDgwNTI6QUFFQjJfc29uQ2s1NUxZOUJHOUFPLUg4anhpUFM1NW9vQkEiLCAicGF5bWVudF9wcm92aWRlcl90b2tlbiI6ICIyODQ2ODUwNjM6VEVTVDpZbVppWVdabU1qSmhaR015IiwgIm5hbWUiOiAiUFRCIHRlc3RzIG9uIEFwcFZleW9yIHVzaW5nIENQeXRob24gMi43IiwgInN1cGVyX2dyb3VwX2lkIjogIi0xMDAxMjkzMDc5MTY1IiwgInVzZXJuYW1lIjogIkBwdGJfYXBwdmV5b3JfY3B5dGhvbl8yN19ib3QiLCAiZm9ydW1fZ3JvdXBfaWQiOiAiLTEwMDE1NjU4NTU5ODcifSwgeyJ0b2tlbiI6ICIxMDU1Mzk3NDcxOkFBRzE4bkJfUzJXQXd1SjNnN29oS0JWZ1hYY2VNbklPeVNjIiwgInBheW1lbnRfcHJvdmlkZXJfdG9rZW4iOiAiMjg0Njg1MDYzOlRFU1Q6TmpBd056QXpZalZpTkdOayIsICJuYW1lIjogIlBUQiB0ZXN0cyBbMF0iLCAic3VwZXJfZ3JvdXBfaWQiOiAiLTEwMDExODU1MDk2MzYiLCAidXNlcm5hbWUiOiAicHRiXzBfYm90IiwgImZvcnVtX2dyb3VwX2lkIjogIi0xMDAxODE5MDM3MzExIn0sIHsidG9rZW4iOiAiMTA0NzMyNjc3MTpBQUY4bk90ODFGcFg4bGJidno4VWV3UVF2UmZUYkZmQnZ1SSIsICJwYXltZW50X3Byb3ZpZGVyX3Rva2VuIjogIjI4NDY4NTA2MzpURVNUOllUVTFOVEk0WkdSallqbGkiLCAibmFtZSI6ICJQVEIgdGVzdHMgWzFdIiwgInN1cGVyX2dyb3VwX2lkIjogIi0xMDAxNDg0Nzk3NjEyIiwgInVzZXJuYW1lIjogInB0Yl8xX2JvdCIsICJmb3J1bV9ncm91cF9pZCI6ICItMTAwMTc5NzMwODQ0NCJ9LCB7InRva2VuIjogIjk3MTk5Mjc0NTpBQUdPa09hVzBOSGpnSXY1LTlqUWJPajR2R3FkaFNGLVV1cyIsICJwYXltZW50X3Byb3ZpZGVyX3Rva2VuIjogIjI4NDY4NTA2MzpURVNUOk5XWmtNV1ZoWWpsallqVTUiLCAibmFtZSI6ICJQVEIgdGVzdHMgWzJdIiwgInN1cGVyX2dyb3VwX2lkIjogIi0xMDAxNDAyMjU1MDcwIiwgInVzZXJuYW1lIjogInB0Yl8yX2JvdCIsICJmb3J1bV9ncm91cF9pZCI6ICItMTAwMTUyMzU3NTA3MiJ9LCB7InRva2VuIjogIjU1MTg2NDU0MTE6QUFHdzBxaEs3ZTRHbmoxWjJjc1BBQzdaYWtvTWs1NkVKZmsiLCAicGF5bWVudF9wcm92aWRlcl90b2tlbiI6ICIyODQ2ODUwNjM6VEVTVDpNRE0wT1RCbE9UUXpNVEU1IiwgIm5hbWUiOiAiUFRCIFRlc3QgQm90IFszXSIsICJzdXBlcl9ncm91cF9pZCI6ICItMTAwMTgwMzgxMDE5NiIsICJ1c2VybmFtZSI6ICJwdGJfdGVzdF8wM19ib3QiLCAiZm9ydW1fZ3JvdXBfaWQiOiAiLTEwMDE2MTk2NzMyNjEifSwgeyJ0b2tlbiI6ICI1NzM3MDE4MzU2OkFBSDEzOFN1aUtRRjBMRENXc2ZnV2VYZmpKNWQ2M2tDV0xBIiwgInBheW1lbnRfcHJvdmlkZXJfdG9rZW4iOiAiMjg0Njg1MDYzOlRFU1Q6TjJWaVpqUmxaak01TlRNdyIsICJuYW1lIjogIlBUQiBUZXN0IEJvdCBbNF0iLCAidXNlcm5hbWUiOiAicHRiX3Rlc3RfMDRfYm90IiwgInN1cGVyX2dyb3VwX2lkIjogIi0xMDAxODQyNDM5NjQxIiwgImZvcnVtX2dyb3VwX2lkIjogIi0xMDAxODQyOTk2MTk5In0sIHsidG9rZW4iOiAiNTc0NDY0NDUyMjpBQUVBZHNyRjBoQzZwNkhVTzBQMDFROGJfakNoVTUyWEctTSIsICJwYXltZW50X3Byb3ZpZGVyX3Rva2VuIjogIjI4NDY4NTA2MzpURVNUOlpqSmtZVGd5TmpnMlpHRTAiLCAibmFtZSI6ICJQVEIgVGVzdCBCb3QgWzVdIiwgInVzZXJuYW1lIjogInB0Yl90ZXN0XzA1X2JvdCIsICJzdXBlcl9ncm91cF9pZCI6ICItMTAwMTg1NTM2MDk4NiIsICJmb3J1bV9ncm91cF9pZCI6ICItMTAwMTY0NDM2NjkwMiJ9XQ==
          TEST_WITH_OPT_DEPS : "false"
          TEST_BUILD: "true"
        shell: bash --noprofile --norc {0}

      - name: Test Summary
        id: test_summary
        uses: test-summary/action@v2.1
        if: always()  # always run, even if tests fail
        with:
          paths: |
            .test_report_no_optionals.xml
            .test_report_optionals.xml            

      - name: Submit coverage
        uses: codecov/codecov-action@v3
        with:
          env_vars: OS,PYTHON
          name: ${{ matrix.os }}-${{ matrix.python-version }}
          fail_ci_if_error: true

  test_official:
    name: test-official
    runs-on: ${{matrix.os}}
    strategy:
      matrix:
        python-version: [3.11]
        os: [ubuntu-latest]
      fail-fast: False
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          python -W ignore -m pip install --upgrade pip
          python -W ignore -m pip install -r requirements.txt
          python -W ignore -m pip install -r requirements-opts.txt
          python -W ignore -m pip install -r requirements-dev.txt          
      - name: Compare to official api
        run: |
          pytest -v tests/test_official.py --junit-xml=.test_report_official.xml
          exit $?          
        env:
          TEST_OFFICIAL: "true"
        shell: bash --noprofile --norc {0}

      - name: Test Summary
        id: test_summary
        uses: test-summary/action@v2.1
        if: always()  # always run, even if tests fail
        with:
          paths: .test_report_official.xml