flaky tests: try up to 3 times, need to succeed once

This commit is contained in:
Noam Meltzer 2016-02-22 01:52:31 +02:00
parent ec13a36bdd
commit e0cf0abb1b
8 changed files with 59 additions and 60 deletions

View file

@ -24,7 +24,7 @@ import sys
import signal
import traceback
from nose.tools import make_decorator, TimeExpired
from nose.tools import make_decorator
sys.path.append('.')
@ -64,8 +64,7 @@ class BaseTest(object):
class TestTimedOut(AssertionError):
def __init__(self, time_limit, frame):
super(TestTimedOut, self).__init__('time_limit={0}\n{1}'.format(
time_limit, ''.join(traceback.format_stack(frame))))
super(TestTimedOut, self).__init__('time_limit={0}'.format(time_limit))
self.time_limit = time_limit
self.frame = frame

View file

@ -52,7 +52,7 @@ class AudioTest(BaseTest, unittest.TestCase):
'file_size': self.file_size
}
@flaky
@flaky(3, 1)
@timeout(10)
def test_send_audio_required_args_only(self):
"""Test telegram.Bot sendAudio method"""
@ -71,7 +71,7 @@ class AudioTest(BaseTest, unittest.TestCase):
self.assertEqual(audio.mime_type, self.mime_type)
self.assertEqual(audio.file_size, self.file_size)
@flaky
@flaky(3, 1)
@timeout(10)
def test_send_audio_all_args(self):
"""Test telegram.Bot sendAudio method"""
@ -95,7 +95,7 @@ class AudioTest(BaseTest, unittest.TestCase):
self.assertEqual(audio.mime_type, self.mime_type)
self.assertEqual(audio.file_size, self.file_size)
@flaky
@flaky(3, 1)
@timeout(10)
def test_send_audio_mp3_file(self):
"""Test telegram.Bot sendAudio method"""
@ -117,7 +117,7 @@ class AudioTest(BaseTest, unittest.TestCase):
self.assertEqual(audio.mime_type, self.mime_type)
self.assertEqual(audio.file_size, self.file_size)
@flaky
@flaky(3, 1)
@timeout(10)
def test_send_audio_mp3_file_custom_filename(self):
"""Test telegram.Bot sendAudio method"""
@ -140,7 +140,7 @@ class AudioTest(BaseTest, unittest.TestCase):
self.assertEqual(audio.mime_type, self.mime_type)
self.assertEqual(audio.file_size, self.file_size)
@flaky
@flaky(3, 1)
@timeout(10)
def test_send_audio_mp3_url_file(self):
"""Test telegram.Bot sendAudio method"""
@ -162,7 +162,7 @@ class AudioTest(BaseTest, unittest.TestCase):
self.assertEqual(audio.mime_type, self.mime_type)
self.assertEqual(audio.file_size, self.file_size)
@flaky
@flaky(3, 1)
@timeout(10)
def test_send_audio_resend(self):
"""Test telegram.Bot sendAudio method"""
@ -217,7 +217,7 @@ class AudioTest(BaseTest, unittest.TestCase):
self.assertEqual(audio['mime_type'], self.mime_type)
self.assertEqual(audio['file_size'], self.file_size)
@flaky
@flaky(3, 1)
@timeout(10)
def test_error_send_audio_empty_file(self):
print('Testing bot.sendAudio - Null file')
@ -231,7 +231,7 @@ class AudioTest(BaseTest, unittest.TestCase):
lambda: self._bot.sendAudio(chat_id=self._chat_id,
**json_dict))
@flaky
@flaky(3, 1)
@timeout(10)
def test_error_send_audio_empty_file_id(self):
print('Testing bot.sendAudio - Empty file_id')
@ -245,7 +245,7 @@ class AudioTest(BaseTest, unittest.TestCase):
lambda: self._bot.sendAudio(chat_id=self._chat_id,
**json_dict))
@flaky
@flaky(3, 1)
@timeout(10)
def test_error_audio_without_required_args(self):
print('Testing bot.sendAudio - Without required arguments')

View file

@ -39,7 +39,7 @@ from tests.base import BaseTest, timeout
class BotTest(BaseTest, unittest.TestCase):
"""This object represents Tests for Telegram Bot."""
@flaky
@flaky(3, 1)
@timeout(10)
def testGetMe(self):
'''Test the telegram.Bot getMe method'''
@ -53,7 +53,7 @@ class BotTest(BaseTest, unittest.TestCase):
self.assertEqual(bot.username, 'PythonTelegramBot')
self.assertEqual(bot.name, '@PythonTelegramBot')
@flaky
@flaky(3, 1)
@timeout(10)
def testSendMessage(self):
'''Test the telegram.Bot sendMessage method'''
@ -65,7 +65,7 @@ class BotTest(BaseTest, unittest.TestCase):
self.assertEqual(message.text, u'Моё судно на воздушной подушке полно угрей')
self.assertTrue(isinstance(message.date, datetime))
@flaky
@flaky(3, 1)
@timeout(10)
def testGetUpdates(self):
'''Test the telegram.Bot getUpdates method'''
@ -76,7 +76,7 @@ class BotTest(BaseTest, unittest.TestCase):
self.assertTrue(self.is_json(updates[0].to_json()))
self.assertTrue(isinstance(updates[0], telegram.Update))
@flaky
@flaky(3, 1)
@timeout(10)
def testForwardMessage(self):
'''Test the telegram.Bot forwardMessage method'''
@ -90,7 +90,7 @@ class BotTest(BaseTest, unittest.TestCase):
self.assertEqual(message.forward_from.username, 'leandrotoledo')
self.assertTrue(isinstance(message.forward_date, datetime))
@flaky
@flaky(3, 1)
@timeout(10)
def testSendPhoto(self):
'''Test the telegram.Bot sendPhoto method'''
@ -103,7 +103,7 @@ class BotTest(BaseTest, unittest.TestCase):
self.assertEqual(message.photo[0].file_size, 1451)
self.assertEqual(message.caption, 'testSendPhoto')
@flaky
@flaky(3, 1)
@timeout(10)
def testResendPhoto(self):
'''Test the telegram.Bot sendPhoto method'''
@ -114,7 +114,7 @@ class BotTest(BaseTest, unittest.TestCase):
self.assertTrue(self.is_json(message.to_json()))
self.assertEqual(message.photo[0].file_id, 'AgADAQADyKcxGx8j9Qdp6d-gpUsw4Gja1i8ABEVJsVqQk8LfJ3wAAgI')
@flaky
@flaky(3, 1)
@timeout(10)
def testSendJPGURLPhoto(self):
'''Test the telegram.Bot sendPhoto method'''
@ -125,7 +125,7 @@ class BotTest(BaseTest, unittest.TestCase):
self.assertTrue(self.is_json(message.to_json()))
self.assertEqual(message.photo[0].file_size, 822)
@flaky
@flaky(3, 1)
@timeout(10)
def testSendPNGURLPhoto(self):
'''Test the telegram.Bot sendPhoto method'''
@ -136,7 +136,7 @@ class BotTest(BaseTest, unittest.TestCase):
self.assertTrue(self.is_json(message.to_json()))
self.assertEqual(message.photo[0].file_size, 684)
@flaky
@flaky(3, 1)
@timeout(10)
def testSendGIFURLPhoto(self):
'''Test the telegram.Bot sendPhoto method'''
@ -147,7 +147,7 @@ class BotTest(BaseTest, unittest.TestCase):
self.assertTrue(self.is_json(message.to_json()))
self.assertEqual(message.photo[0].file_size, 684)
@flaky
@flaky(3, 1)
@timeout(10)
def testSendChatAction(self):
'''Test the telegram.Bot sendChatAction method'''
@ -156,7 +156,7 @@ class BotTest(BaseTest, unittest.TestCase):
self._bot.sendChatAction(action=telegram.ChatAction.TYPING,
chat_id=self._chat_id)
@flaky
@flaky(3, 1)
@timeout(10)
def testGetUserProfilePhotos(self):
'''Test the telegram.Bot getUserProfilePhotos method'''

View file

@ -53,7 +53,7 @@ class DocumentTest(BaseTest, unittest.TestCase):
'file_size': self.file_size
}
@flaky
@flaky(3, 1)
@timeout(10)
def test_send_document_png_file(self):
"""Test telegram.Bot sendDocument method"""
@ -71,7 +71,7 @@ class DocumentTest(BaseTest, unittest.TestCase):
self.assertEqual(document.mime_type, self.mime_type)
self.assertEqual(document.file_size, self.file_size)
@flaky
@flaky(3, 1)
@timeout(10)
def test_send_document_png_file_with_custom_file_name(self):
"""Test telegram.Bot sendDocument method"""
@ -90,7 +90,7 @@ class DocumentTest(BaseTest, unittest.TestCase):
self.assertEqual(document.mime_type, self.mime_type)
self.assertEqual(document.file_size, self.file_size)
@flaky
@flaky(3, 1)
@timeout(10)
def test_send_document_url_gif_file(self):
"""Test telegram.Bot sendDocument method"""
@ -108,7 +108,7 @@ class DocumentTest(BaseTest, unittest.TestCase):
self.assertEqual(document.mime_type, 'image/gif')
self.assertEqual(document.file_size, 3878)
@flaky
@flaky(3, 1)
@timeout(10)
def test_send_document_resend(self):
"""Test telegram.Bot sendDocument method"""
@ -157,7 +157,7 @@ class DocumentTest(BaseTest, unittest.TestCase):
self.assertEqual(document['mime_type'], self.mime_type)
self.assertEqual(document['file_size'], self.file_size)
@flaky
@flaky(3, 1)
@timeout(10)
def test_error_send_document_empty_file(self):
print('Testing bot.sendDocument - Null file')
@ -171,7 +171,7 @@ class DocumentTest(BaseTest, unittest.TestCase):
lambda: self._bot.sendDocument(chat_id=self._chat_id,
**json_dict))
@flaky
@flaky(3, 1)
@timeout(10)
def test_error_send_document_empty_file_id(self):
print('Testing bot.sendDocument - Empty file_id')
@ -185,7 +185,7 @@ class DocumentTest(BaseTest, unittest.TestCase):
lambda: self._bot.sendDocument(chat_id=self._chat_id,
**json_dict))
@flaky
@flaky(3, 1)
@timeout(10)
def test_error_document_without_required_args(self):
print('Testing bot.sendDocument - Without required arguments')

View file

@ -55,7 +55,7 @@ class PhotoTest(BaseTest, unittest.TestCase):
'file_size': self.file_size
}
@flaky
@flaky(3, 1)
@timeout(10)
def test_sendphotoo_all_args(self):
"""Test telegram.Bot sendAudio method"""
@ -83,7 +83,7 @@ class PhotoTest(BaseTest, unittest.TestCase):
self.assertEqual(message.caption, self.caption)
@flaky
@flaky(3, 1)
@timeout(10)
def test_send_photo_jpg_file(self):
"""Test telegram.Bot sendPhoto method"""
@ -108,7 +108,7 @@ class PhotoTest(BaseTest, unittest.TestCase):
self.assertEqual(photo.height, self.height)
self.assertEqual(photo.file_size, self.file_size)
@flaky
@flaky(3, 1)
@timeout(10)
def test_send_photo_url_jpg_file(self):
"""Test telegram.Bot sendPhoto method"""
@ -133,7 +133,7 @@ class PhotoTest(BaseTest, unittest.TestCase):
self.assertEqual(photo.height, self.height)
self.assertEqual(photo.file_size, self.file_size)
@flaky
@flaky(3, 1)
@timeout(10)
def test_send_photo_resend(self):
"""Test telegram.Bot sendPhoto method"""
@ -187,7 +187,7 @@ class PhotoTest(BaseTest, unittest.TestCase):
self.assertEqual(photo['height'], self.height)
self.assertEqual(photo['file_size'], self.file_size)
@flaky
@flaky(3, 1)
@timeout(10)
def test_error_send_photo_empty_file(self):
print('Testing bot.sendPhoto - Null file')
@ -201,7 +201,7 @@ class PhotoTest(BaseTest, unittest.TestCase):
lambda: self._bot.sendPhoto(chat_id=self._chat_id,
**json_dict))
@flaky
@flaky(3, 1)
@timeout(10)
def test_error_send_photo_empty_file_id(self):
print('Testing bot.sendPhoto - Empty file_id')
@ -215,7 +215,7 @@ class PhotoTest(BaseTest, unittest.TestCase):
lambda: self._bot.sendPhoto(chat_id=self._chat_id,
**json_dict))
@flaky
@flaky(3, 1)
@timeout(10)
def test_error_photo_without_required_args(self):
print('Testing bot.sendPhoto - Without required arguments')

View file

@ -51,12 +51,12 @@ class StickerTest(BaseTest, unittest.TestCase):
'file_size': self.file_size
}
@flaky
@flaky(3, 1)
@timeout(10)
def test_send_sticker_file(self):
pass
@flaky
@flaky(3, 1)
@timeout(10)
def test_send_sticker_resend(self):
"""Test telegram.Bot sendSticker method"""
@ -105,7 +105,7 @@ class StickerTest(BaseTest, unittest.TestCase):
self.assertTrue(isinstance(sticker['thumb'], telegram.PhotoSize))
self.assertEqual(sticker['file_size'], self.file_size)
@flaky
@flaky(3, 1)
@timeout(10)
def test_error_send_sticker_empty_file(self):
print('Testing bot.sendSticker - Null file')
@ -119,7 +119,7 @@ class StickerTest(BaseTest, unittest.TestCase):
lambda: self._bot.sendSticker(chat_id=self._chat_id,
**json_dict))
@flaky
@flaky(3, 1)
@timeout(10)
def test_error_send_sticker_empty_file_id(self):
print('Testing bot.sendSticker - Empty file_id')
@ -133,7 +133,7 @@ class StickerTest(BaseTest, unittest.TestCase):
lambda: self._bot.sendSticker(chat_id=self._chat_id,
**json_dict))
@flaky
@flaky(3, 1)
@timeout(10)
def test_error_sticker_without_required_args(self):
print('Testing bot.sendSticker - Without required arguments')

View file

@ -57,7 +57,7 @@ class VideoTest(BaseTest, unittest.TestCase):
'file_size': self.file_size
}
@flaky
@flaky(3, 1)
@timeout(10)
def test_send_video_required_args_only(self):
"""Test telegram.Bot sendVideo method"""
@ -77,7 +77,7 @@ class VideoTest(BaseTest, unittest.TestCase):
self.assertEqual(video.mime_type, '')
self.assertEqual(video.file_size, self.file_size)
@flaky
@flaky(3, 1)
@timeout(10)
def test_send_video_all_args(self):
"""Test telegram.Bot sendAudio method"""
@ -101,7 +101,7 @@ class VideoTest(BaseTest, unittest.TestCase):
self.assertEqual(message.caption, self.caption)
@flaky
@flaky(3, 1)
@timeout(10)
def test_send_video_mp4_file(self):
"""Test telegram.Bot sendVideo method"""
@ -125,7 +125,7 @@ class VideoTest(BaseTest, unittest.TestCase):
self.assertEqual(message.caption, self.caption)
@flaky
@flaky(3, 1)
@timeout(10)
def test_send_video_mp4_file_with_custom_filename(self):
"""Test telegram.Bot sendVideo method"""
@ -150,7 +150,7 @@ class VideoTest(BaseTest, unittest.TestCase):
self.assertEqual(message.caption, self.caption)
@flaky
@flaky(3, 1)
@timeout(10)
def test_send_video_mp4_file_url(self):
"""Test telegram.Bot sendVideo method"""
@ -174,7 +174,7 @@ class VideoTest(BaseTest, unittest.TestCase):
self.assertEqual(message.caption, self.caption)
@flaky
@flaky(3, 1)
@timeout(10)
def test_send_video_resend(self):
"""Test telegram.Bot sendVideo method"""
@ -230,7 +230,7 @@ class VideoTest(BaseTest, unittest.TestCase):
self.assertEqual(video['mime_type'], self.mime_type)
self.assertEqual(video['file_size'], self.file_size)
@flaky
@flaky(3, 1)
@timeout(10)
def test_error_send_video_empty_file(self):
print('Testing bot.sendVideo - Null file')
@ -244,7 +244,7 @@ class VideoTest(BaseTest, unittest.TestCase):
lambda: self._bot.sendVideo(chat_id=self._chat_id,
**json_dict))
@flaky
@flaky(3, 1)
@timeout(10)
def test_error_send_video_empty_file_id(self):
print('Testing bot.sendVideo - Empty file_id')
@ -258,7 +258,7 @@ class VideoTest(BaseTest, unittest.TestCase):
lambda: self._bot.sendVideo(chat_id=self._chat_id,
**json_dict))
@flaky
@flaky(3, 1)
@timeout(10)
def test_error_video_without_required_args(self):
print('Testing bot.sendVideo - Without required arguments')

View file

@ -48,7 +48,7 @@ class VoiceTest(BaseTest, unittest.TestCase):
'file_size': self.file_size
}
@flaky
@flaky(3, 1)
@timeout(10)
def test_send_voice_required_args_only(self):
"""Test telegram.Bot sendVoice method"""
@ -65,7 +65,7 @@ class VoiceTest(BaseTest, unittest.TestCase):
self.assertEqual(voice.mime_type, self.mime_type)
self.assertEqual(voice.file_size, self.file_size)
@flaky
@flaky(3, 1)
@timeout(10)
def test_send_voice_all_args(self):
"""Test telegram.Bot sendAudio method"""
@ -85,7 +85,7 @@ class VoiceTest(BaseTest, unittest.TestCase):
self.assertEqual(voice.mime_type, self.mime_type)
self.assertEqual(voice.file_size, self.file_size)
@flaky
@flaky(3, 1)
@timeout(10)
def test_send_voice_ogg_file(self):
"""Test telegram.Bot sendVoice method"""
@ -103,7 +103,7 @@ class VoiceTest(BaseTest, unittest.TestCase):
self.assertEqual(voice.mime_type, self.mime_type)
self.assertEqual(voice.file_size, self.file_size)
@flaky
@flaky(3, 1)
@timeout(10)
def test_send_voice_ogg_file_with_custom_filename(self):
"""Test telegram.Bot sendVoice method"""
@ -122,7 +122,7 @@ class VoiceTest(BaseTest, unittest.TestCase):
self.assertEqual(voice.mime_type, self.mime_type)
self.assertEqual(voice.file_size, self.file_size)
@flaky
@flaky(3, 1)
@timeout(10)
def test_send_voice_ogg_url_file(self):
"""Test telegram.Bot sendVoice method"""
@ -140,7 +140,7 @@ class VoiceTest(BaseTest, unittest.TestCase):
self.assertEqual(voice.mime_type, self.mime_type)
self.assertEqual(voice.file_size, self.file_size)
@flaky
@flaky(3, 1)
@timeout(10)
def test_send_voice_resend(self):
"""Test telegram.Bot sendVoice method"""
@ -187,7 +187,7 @@ class VoiceTest(BaseTest, unittest.TestCase):
self.assertEqual(voice['mime_type'], self.mime_type)
self.assertEqual(voice['file_size'], self.file_size)
@flaky
@flaky(3, 1)
@timeout(10)
def test_error_send_voice_empty_file(self):
print('Testing bot.sendVoice - Null file')
@ -201,7 +201,7 @@ class VoiceTest(BaseTest, unittest.TestCase):
lambda: self._bot.sendVoice(chat_id=self._chat_id,
**json_dict))
@flaky
@flaky(3, 1)
@timeout(10)
def test_error_send_voice_empty_file_id(self):
print('Testing bot.sendVoice - Empty file_id')
@ -215,7 +215,7 @@ class VoiceTest(BaseTest, unittest.TestCase):
lambda: self._bot.sendVoice(chat_id=self._chat_id,
**json_dict))
@flaky
@flaky(3, 1)
@timeout(10)
def test_error_voice_without_required_args(self):
print('Testing bot.sendVoice - Without required arguments')