From 7e0ed2235ebbd19e277e7a60efacbc03a11693ff Mon Sep 17 00:00:00 2001
From: Harshil <37377066+harshil21@users.noreply.github.com>
Date: Sun, 2 Apr 2023 02:05:20 +0530
Subject: [PATCH] Stabilize CI by Rerunning Failed Tests (#3631)

Co-authored-by: Bibo-Joshi <22366557+Bibo-Joshi@users.noreply.github.com>
---
 .github/workflows/test.yml | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index ef75cf16a..cea40da81 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -54,7 +54,10 @@ jobs:
           # - without http2 support
           TO_TEST="test_no_passport.py or test_datetime.py or test_defaults.py or test_jobqueue.py or test_applicationbuilder.py or test_ratelimiter.py or test_updater.py or test_callbackdatacache.py or test_request.py"
           pytest -v --cov -k "${TO_TEST}"
-          status=$?
+          # Rerun only failed tests (--lf), and don't run any tests if none failed (--lfnf=none)
+          pytest -v --cov --cov-append -k "${TO_TEST}" --lf --lfnf=none
+          # No tests were selected, convert returned status code to 0
+          opt_dep_status=$(( $? == 5 ? 0 : $? ))
 
           # Test the rest
           export TEST_WITH_OPT_DEPS='true'
@@ -63,8 +66,10 @@ jobs:
           # worker. Increasing number of workers has little effect on test duration, but it seems
           # to increase flakyness, specially on python 3.7 with --dist=loadgroup.
           pytest -v --cov --cov-append -n auto --dist loadfile
-          status=$(( $? > status ? $? : status))
-          exit ${status}
+          pytest -v --cov --cov-append -n auto --dist loadfile --lf --lfnf=none
+          main_status=$(( $? == 5 ? 0 : $? ))
+          # exit with non-zero status if any of the two pytest runs failed
+          exit $(( ${opt_dep_status} || ${main_status} ))
         env:
           JOB_INDEX: ${{ strategy.job-index }}
           BOTS: W3sidG9rZW4iOiAiNjk2MTg4NzMyOkFBR1Z3RUtmSEhsTmpzY3hFRE5LQXdraEdzdFpfa28xbUMwIiwgInBheW1lbnRfcHJvdmlkZXJfdG9rZW4iOiAiMjg0Njg1MDYzOlRFU1Q6WldGaU1UUmxNbVF5TnpNeSIsICJuYW1lIjogIlBUQiB0ZXN0cyBvbiBUcmF2aXMgdXNpbmcgQ1B5dGhvbiAyLjciLCAic3VwZXJfZ3JvdXBfaWQiOiAiLTEwMDEzOTA5ODM5OTciLCAidXNlcm5hbWUiOiAiQHB0Yl90cmF2aXNfY3B5dGhvbl8yN19ib3QiLCAiZm9ydW1fZ3JvdXBfaWQiOiAiLTEwMDE3MTA4NTA4MjIifSwgeyJ0b2tlbiI6ICI2NzE0Njg4ODY6QUFHUEdmY2lSSUJVTkZlODI0dUlWZHE3SmUzX1luQVROR3ciLCAicGF5bWVudF9wcm92aWRlcl90b2tlbiI6ICIyODQ2ODUwNjM6VEVTVDpaR1l3T1Rsa016TXhOMlkyIiwgIm5hbWUiOiAiUFRCIHRlc3RzIG9uIFRyYXZpcyB1c2luZyBDUHl0aG9uIDMuNCIsICJzdXBlcl9ncm91cF9pZCI6ICItMTAwMTQ0NjAyMjUyMiIsICJ1c2VybmFtZSI6ICJAcHRiX3RyYXZpc19jcHl0aG9uXzM0X2JvdCIsICJmb3J1bV9ncm91cF9pZCI6ICItMTAwMTg5MTQ0MTc5MSJ9LCB7InRva2VuIjogIjYyOTMyNjUzODpBQUZSclpKckI3b0IzbXV6R3NHSlhVdkdFNUNRek01Q1U0byIsICJwYXltZW50X3Byb3ZpZGVyX3Rva2VuIjogIjI4NDY4NTA2MzpURVNUOk1tTTVZV0poWXpreE0yVTEiLCAibmFtZSI6ICJQVEIgdGVzdHMgb24gVHJhdmlzIHVzaW5nIENQeXRob24gMy41IiwgInN1cGVyX2dyb3VwX2lkIjogIi0xMDAxNDk2OTE3NzUwIiwgInVzZXJuYW1lIjogIkBwdGJfdHJhdmlzX2NweXRob25fMzVfYm90IiwgImZvcnVtX2dyb3VwX2lkIjogIi0xMDAxNTc3NTA0Nzg3In0sIHsidG9rZW4iOiAiNjQwMjA4OTQzOkFBRmhCalFwOXFtM1JUeFN6VXBZekJRakNsZS1Kano1aGNrIiwgInBheW1lbnRfcHJvdmlkZXJfdG9rZW4iOiAiMjg0Njg1MDYzOlRFU1Q6WXpoa1pUZzFOamMxWXpWbCIsICJuYW1lIjogIlBUQiB0ZXN0cyBvbiBUcmF2aXMgdXNpbmcgQ1B5dGhvbiAzLjYiLCAic3VwZXJfZ3JvdXBfaWQiOiAiLTEwMDEzMzM4NzE0NjEiLCAidXNlcm5hbWUiOiAiQHB0Yl90cmF2aXNfY3B5dGhvbl8zNl9ib3QiLCAiZm9ydW1fZ3JvdXBfaWQiOiAiLTEwMDE4Njc5MDExNzIifSwgeyJ0b2tlbiI6ICI2OTUxMDQwODg6QUFIZnp5bElPalNJSVMtZU9uSTIweTJFMjBIb2RIc2Z6LTAiLCAicGF5bWVudF9wcm92aWRlcl90b2tlbiI6ICIyODQ2ODUwNjM6VEVTVDpPR1ExTURnd1pqSXdaakZsIiwgIm5hbWUiOiAiUFRCIHRlc3RzIG9uIFRyYXZpcyB1c2luZyBDUHl0aG9uIDMuNyIsICJzdXBlcl9ncm91cF9pZCI6ICItMTAwMTQ3ODI5MzcxNCIsICJ1c2VybmFtZSI6ICJAcHRiX3RyYXZpc19jcHl0aG9uXzM3X2JvdCIsICJmb3J1bV9ncm91cF9pZCI6ICItMTAwMTg2NDA1NDg3OSJ9LCB7InRva2VuIjogIjY5MTQyMzU1NDpBQUY4V2tqQ1pibkhxUF9pNkdoVFlpckZFbFpyR2FZT2hYMCIsICJwYXltZW50X3Byb3ZpZGVyX3Rva2VuIjogIjI4NDY4NTA2MzpURVNUOllqYzVOVGhpTW1ReU1XVmgiLCAibmFtZSI6ICJQVEIgdGVzdHMgb24gVHJhdmlzIHVzaW5nIFB5UHkgMi43IiwgInN1cGVyX2dyb3VwX2lkIjogIi0xMDAxMzYzOTMyNTczIiwgInVzZXJuYW1lIjogIkBwdGJfdHJhdmlzX3B5cHlfMjdfYm90IiwgImZvcnVtX2dyb3VwX2lkIjogIi0xMDAxODY3ODU1OTM2In0sIHsidG9rZW4iOiAiNjg0MzM5OTg0OkFBRk1nRUVqcDAxcjVyQjAwN3lDZFZOc2c4QWxOc2FVLWNjIiwgInBheW1lbnRfcHJvdmlkZXJfdG9rZW4iOiAiMjg0Njg1MDYzOlRFU1Q6TVRBek1UWTNNR1V5TmpnMCIsICJuYW1lIjogIlBUQiB0ZXN0cyBvbiBUcmF2aXMgdXNpbmcgUHlQeSAzLjUiLCAic3VwZXJfZ3JvdXBfaWQiOiAiLTEwMDE0MDc4MzY2MDUiLCAidXNlcm5hbWUiOiAiQHB0Yl90cmF2aXNfcHlweV8zNV9ib3QiLCAiZm9ydW1fZ3JvdXBfaWQiOiAiLTEwMDE1NTg5OTAyODIifSwgeyJ0b2tlbiI6ICI2OTAwOTEzNDc6QUFGTG1SNXBBQjVZY3BlX21PaDd6TTRKRkJPaDB6M1QwVG8iLCAicGF5bWVudF9wcm92aWRlcl90b2tlbiI6ICIyODQ2ODUwNjM6VEVTVDpaRGhsTnpFNU1Ea3dZV0ppIiwgIm5hbWUiOiAiUFRCIHRlc3RzIG9uIEFwcFZleW9yIHVzaW5nIENQeXRob24gMy40IiwgInN1cGVyX2dyb3VwX2lkIjogIi0xMDAxMjc5NjAwMDI2IiwgInVzZXJuYW1lIjogIkBwdGJfYXBwdmV5b3JfY3B5dGhvbl8zNF9ib3QiLCAiZm9ydW1fZ3JvdXBfaWQiOiAiLTEwMDE3MjU2OTEzODcifSwgeyJ0b2tlbiI6ICI2OTQzMDgwNTI6QUFFQjJfc29uQ2s1NUxZOUJHOUFPLUg4anhpUFM1NW9vQkEiLCAicGF5bWVudF9wcm92aWRlcl90b2tlbiI6ICIyODQ2ODUwNjM6VEVTVDpZbVppWVdabU1qSmhaR015IiwgIm5hbWUiOiAiUFRCIHRlc3RzIG9uIEFwcFZleW9yIHVzaW5nIENQeXRob24gMi43IiwgInN1cGVyX2dyb3VwX2lkIjogIi0xMDAxMjkzMDc5MTY1IiwgInVzZXJuYW1lIjogIkBwdGJfYXBwdmV5b3JfY3B5dGhvbl8yN19ib3QiLCAiZm9ydW1fZ3JvdXBfaWQiOiAiLTEwMDE1NjU4NTU5ODcifSwgeyJ0b2tlbiI6ICIxMDU1Mzk3NDcxOkFBRzE4bkJfUzJXQXd1SjNnN29oS0JWZ1hYY2VNbklPeVNjIiwgInBheW1lbnRfcHJvdmlkZXJfdG9rZW4iOiAiMjg0Njg1MDYzOlRFU1Q6TmpBd056QXpZalZpTkdOayIsICJuYW1lIjogIlBUQiB0ZXN0cyBbMF0iLCAic3VwZXJfZ3JvdXBfaWQiOiAiLTEwMDExODU1MDk2MzYiLCAidXNlcm5hbWUiOiAicHRiXzBfYm90IiwgImZvcnVtX2dyb3VwX2lkIjogIi0xMDAxODE5MDM3MzExIn0sIHsidG9rZW4iOiAiMTA0NzMyNjc3MTpBQUY4bk90ODFGcFg4bGJidno4VWV3UVF2UmZUYkZmQnZ1SSIsICJwYXltZW50X3Byb3ZpZGVyX3Rva2VuIjogIjI4NDY4NTA2MzpURVNUOllUVTFOVEk0WkdSallqbGkiLCAibmFtZSI6ICJQVEIgdGVzdHMgWzFdIiwgInN1cGVyX2dyb3VwX2lkIjogIi0xMDAxNDg0Nzk3NjEyIiwgInVzZXJuYW1lIjogInB0Yl8xX2JvdCIsICJmb3J1bV9ncm91cF9pZCI6ICItMTAwMTc5NzMwODQ0NCJ9LCB7InRva2VuIjogIjk3MTk5Mjc0NTpBQUdPa09hVzBOSGpnSXY1LTlqUWJPajR2R3FkaFNGLVV1cyIsICJwYXltZW50X3Byb3ZpZGVyX3Rva2VuIjogIjI4NDY4NTA2MzpURVNUOk5XWmtNV1ZoWWpsallqVTUiLCAibmFtZSI6ICJQVEIgdGVzdHMgWzJdIiwgInN1cGVyX2dyb3VwX2lkIjogIi0xMDAxNDAyMjU1MDcwIiwgInVzZXJuYW1lIjogInB0Yl8yX2JvdCIsICJmb3J1bV9ncm91cF9pZCI6ICItMTAwMTUyMzU3NTA3MiJ9LCB7InRva2VuIjogIjU1MTg2NDU0MTE6QUFHdzBxaEs3ZTRHbmoxWjJjc1BBQzdaYWtvTWs1NkVKZmsiLCAicGF5bWVudF9wcm92aWRlcl90b2tlbiI6ICIyODQ2ODUwNjM6VEVTVDpNRE0wT1RCbE9UUXpNVEU1IiwgIm5hbWUiOiAiUFRCIFRlc3QgQm90IFszXSIsICJzdXBlcl9ncm91cF9pZCI6ICItMTAwMTgwMzgxMDE5NiIsICJ1c2VybmFtZSI6ICJwdGJfdGVzdF8wM19ib3QiLCAiZm9ydW1fZ3JvdXBfaWQiOiAiLTEwMDE2MTk2NzMyNjEifSwgeyJ0b2tlbiI6ICI1NzM3MDE4MzU2OkFBSDEzOFN1aUtRRjBMRENXc2ZnV2VYZmpKNWQ2M2tDV0xBIiwgInBheW1lbnRfcHJvdmlkZXJfdG9rZW4iOiAiMjg0Njg1MDYzOlRFU1Q6TjJWaVpqUmxaak01TlRNdyIsICJuYW1lIjogIlBUQiBUZXN0IEJvdCBbNF0iLCAidXNlcm5hbWUiOiAicHRiX3Rlc3RfMDRfYm90IiwgInN1cGVyX2dyb3VwX2lkIjogIi0xMDAxODQyNDM5NjQxIiwgImZvcnVtX2dyb3VwX2lkIjogIi0xMDAxODQyOTk2MTk5In0sIHsidG9rZW4iOiAiNTc0NDY0NDUyMjpBQUVBZHNyRjBoQzZwNkhVTzBQMDFROGJfakNoVTUyWEctTSIsICJwYXltZW50X3Byb3ZpZGVyX3Rva2VuIjogIjI4NDY4NTA2MzpURVNUOlpqSmtZVGd5TmpnMlpHRTAiLCAibmFtZSI6ICJQVEIgVGVzdCBCb3QgWzVdIiwgInVzZXJuYW1lIjogInB0Yl90ZXN0XzA1X2JvdCIsICJzdXBlcl9ncm91cF9pZCI6ICItMTAwMTg1NTM2MDk4NiIsICJmb3J1bV9ncm91cF9pZCI6ICItMTAwMTY0NDM2NjkwMiJ9XQ==