Stabilize CI (#2522)

* asking the test what is wrong with them

* fix botscore_not_modified tests for good

* xfail game tests due to race conditions

* address review (add a comment)

* fix xfail marker

* address review

* simplify expression
This commit is contained in:
Harshil 2021-05-19 17:03:41 +05:30 committed by GitHub
parent cd69f69b28
commit 5ff3b76e18
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 63 additions and 55 deletions

View file

@ -49,14 +49,10 @@ from telegram import (
from telegram.constants import MAX_INLINE_QUERY_RESULTS
from telegram.error import BadRequest, InvalidToken, NetworkError, RetryAfter
from telegram.utils.helpers import from_timestamp, escape_markdown, to_timestamp
from tests.conftest import expect_bad_request, check_defaults_handling
from tests.conftest import expect_bad_request, check_defaults_handling, GITHUB_ACTION
from tests.bots import FALLBACKS
BASE_TIME = time.time()
HIGHSCORE_DELTA = 1450000000
@pytest.fixture(scope='class')
def message(bot, chat_id):
to_reply_to = bot.send_message(
@ -98,6 +94,15 @@ def inline_results():
return inline_results_callback()
BASE_GAME_SCORE = 60 # Base game score for game tests
xfail = pytest.mark.xfail(
bool(GITHUB_ACTION), # This condition is only relevant for github actions game tests.
reason='Can fail due to race conditions when multiple test suites '
'with the same bot token are run at the same time',
)
class TestBot:
@pytest.mark.parametrize(
'token',
@ -1273,35 +1278,33 @@ class TestBot:
chat_id, game_short_name, reply_to_message_id=reply_to_message.message_id
)
@flaky(3, 1)
@xfail
def test_set_game_score_1(self, bot, chat_id):
# NOTE: numbering of methods assures proper order between test_set_game_scoreX methods
def func():
game_short_name = 'test_game'
game = bot.send_game(chat_id, game_short_name)
message = bot.set_game_score(
user_id=chat_id,
score=int(BASE_TIME) - HIGHSCORE_DELTA,
chat_id=game.chat_id,
message_id=game.message_id,
)
assert message.game.description == game.game.description
assert message.game.animation.file_id == game.game.animation.file_id
assert message.game.photo[0].file_size == game.game.photo[0].file_size
assert message.game.text != game.game.text
expect_bad_request(func, 'Bot_score_not_modified', 'This test is a diva for some reason.')
@flaky(3, 1)
def test_set_game_score_2(self, bot, chat_id):
# NOTE: numbering of methods assures proper order between test_set_game_scoreX methods
# First, test setting a score.
game_short_name = 'test_game'
game = bot.send_game(chat_id, game_short_name)
score = int(BASE_TIME) - HIGHSCORE_DELTA + 1
message = bot.set_game_score(
user_id=chat_id,
score=BASE_GAME_SCORE, # Score value is relevant for other set_game_score_* tests!
chat_id=game.chat_id,
message_id=game.message_id,
)
assert message.game.description == game.game.description
assert message.game.photo[0].file_size == game.game.photo[0].file_size
assert message.game.animation.file_unique_id == game.game.animation.file_unique_id
assert message.game.text != game.game.text
@xfail
def test_set_game_score_2(self, bot, chat_id):
# NOTE: numbering of methods assures proper order between test_set_game_scoreX methods
# Test setting a score higher than previous
game_short_name = 'test_game'
game = bot.send_game(chat_id, game_short_name)
score = BASE_GAME_SCORE + 1
message = bot.set_game_score(
user_id=chat_id,
@ -1312,30 +1315,33 @@ class TestBot:
)
assert message.game.description == game.game.description
assert message.game.animation.file_id == game.game.animation.file_id
assert message.game.photo[0].file_size == game.game.photo[0].file_size
assert message.game.animation.file_unique_id == game.game.animation.file_unique_id
assert message.game.text == game.game.text
@flaky(3, 1)
@xfail
def test_set_game_score_3(self, bot, chat_id):
# NOTE: numbering of methods assures proper order between test_set_game_scoreX methods
# Test setting a score lower than previous (should raise error)
game_short_name = 'test_game'
game = bot.send_game(chat_id, game_short_name)
score = int(BASE_TIME) - HIGHSCORE_DELTA - 1
score = BASE_GAME_SCORE # Even a score equal to previous raises an error.
with pytest.raises(BadRequest, match='Bot_score_not_modified'):
bot.set_game_score(
user_id=chat_id, score=score, chat_id=game.chat_id, message_id=game.message_id
)
@flaky(3, 1)
@xfail
def test_set_game_score_4(self, bot, chat_id):
# NOTE: numbering of methods assures proper order between test_set_game_scoreX methods
# Test force setting a lower score
game_short_name = 'test_game'
game = bot.send_game(chat_id, game_short_name)
time.sleep(2)
score = int(BASE_TIME) - HIGHSCORE_DELTA - 2
score = BASE_GAME_SCORE - 10
message = bot.set_game_score(
user_id=chat_id,
@ -1346,37 +1352,26 @@ class TestBot:
)
assert message.game.description == game.game.description
assert message.game.animation.file_id == game.game.animation.file_id
assert message.game.photo[0].file_size == game.game.photo[0].file_size
assert message.game.animation.file_unique_id == game.game.animation.file_unique_id
# For some reason the returned message does not contain the updated score. need to fetch
# the game again...
# For some reason the returned message doesn't contain the updated score. need to fetch
# the game again... (the service message is also absent when running the test suite)
game2 = bot.send_game(chat_id, game_short_name)
assert str(score) in game2.game.text
@flaky(3, 1)
def test_set_game_score_too_low_score(self, bot, chat_id):
# We need a game to set the score for
game_short_name = 'test_game'
game = bot.send_game(chat_id, game_short_name)
with pytest.raises(BadRequest):
bot.set_game_score(
user_id=chat_id, score=100, chat_id=game.chat_id, message_id=game.message_id
)
@flaky(3, 1)
@xfail
def test_get_game_high_scores(self, bot, chat_id):
# We need a game to get the scores for
game_short_name = 'test_game'
game = bot.send_game(chat_id, game_short_name)
high_scores = bot.get_game_high_scores(chat_id, game.chat_id, game.message_id)
# We assume that the other game score tests ran within 20 sec
assert pytest.approx(high_scores[0].score, abs=20) == int(BASE_TIME) - HIGHSCORE_DELTA
assert high_scores[0].score == BASE_GAME_SCORE - 10
# send_invoice is tested in test_invoice
# TODO: Needs improvement. Need incoming shippping queries to test
# TODO: Needs improvement. Need incoming shipping queries to test
def test_answer_shipping_query_ok(self, monkeypatch, bot):
# For now just test that our internals pass the correct data
def test(url, data, *args, **kwargs):

View file

@ -305,7 +305,9 @@ def sticker_set(bot):
try:
for i in range(1, 50):
bot.delete_sticker_from_set(ss.stickers[-i].file_id)
except BadRequest:
except BadRequest as e:
if e.message == 'Stickerset_not_modified':
return ss
raise Exception('stickerset is growing too large.')
return ss
@ -317,7 +319,9 @@ def animated_sticker_set(bot):
try:
for i in range(1, 50):
bot.delete_sticker_from_set(ss.stickers[-i].file_id)
except BadRequest:
except BadRequest as e:
if e.message == 'Stickerset_not_modified':
return ss
raise Exception('stickerset is growing too large.')
return ss

View file

@ -44,7 +44,7 @@ from telegram.ext.utils.webhookhandler import WebhookServer
signalskip = pytest.mark.skipif(
sys.platform == 'win32',
reason='Can\'t send signals without stopping ' 'whole process on windows',
reason="Can't send signals without stopping whole process on windows",
)
@ -366,6 +366,10 @@ class TestUpdater:
port = randrange(1024, 49152) # Select random port
updater.start_webhook(ip, port, clean=True, force_event_loop=False)
updater.stop()
for warning in recwarn.list:
print(warning.message)
assert len(recwarn) == 3
assert str(recwarn[0].message).startswith('Old Handler API')
assert str(recwarn[1].message).startswith('The argument `clean` of')
@ -380,9 +384,9 @@ class TestUpdater:
updater.start_polling(clean=True)
updater.stop()
assert len(recwarn) == 2
for msg in recwarn:
print(msg)
assert len(recwarn) == 2
assert str(recwarn[0].message).startswith('Old Handler API')
assert str(recwarn[1].message).startswith('The argument `clean` of')
@ -478,6 +482,11 @@ class TestUpdater:
with caplog.at_level(logging.INFO):
updater.idle()
# There is a chance of a conflict when getting updates since there can be many tests
# (bots) running simultaneously while testing in github actions.
if caplog.records[-1].getMessage().startswith('Error while getting Updates: Conflict'):
caplog.records.pop() # For stability
rec = caplog.records[-2]
assert rec.getMessage().startswith(f'Received signal {signal.SIGTERM}')
assert rec.levelname == 'INFO'