# This is a list of known failures (=bugs). # The tests listed there must fail (or testrunner.py will report error) unless they are prefixed with FLAKY # in which cases the result of them is simply ignored from __future__ import print_function import sys import struct from gevent.testing import sysinfo class Condition(object): __slots__ = () def __and__(self, other): return AndCondition(self, other) def __or__(self, other): return OrCondition(self, other) def __nonzero__(self): return self.__bool__() def __bool__(self): raise NotImplementedError class AbstractBinaryCondition(Condition): # pylint:disable=abstract-method __slots__ = ( 'lhs', 'rhs', ) OP = None def __init__(self, lhs, rhs): self.lhs = lhs self.rhs = rhs def __repr__(self): return "(%r %s %r)" % ( self.lhs, self.OP, self.rhs ) class OrCondition(AbstractBinaryCondition): __slots__ = () OP = '|' def __bool__(self): return bool(self.lhs) or bool(self.rhs) class AndCondition(AbstractBinaryCondition): __slots__ = () OP = '&' def __bool__(self): return bool(self.lhs) and bool(self.rhs) class ConstantCondition(Condition): __slots__ = ( 'value', '__name__', ) def __init__(self, value, name=None): self.value = bool(value) self.__name__ = name or str(value) def __bool__(self): return self.value def __repr__(self): return self.__name__ ALWAYS = ConstantCondition(True) NEVER = ConstantCondition(False) class _AttrCondition(ConstantCondition): __slots__ = ( ) def __init__(self, name): ConstantCondition.__init__(self, getattr(sysinfo, name), name) PYPY = _AttrCondition('PYPY') PY3 = _AttrCondition('PY3') PY2 = _AttrCondition('PY2') OSX = _AttrCondition('OSX') LIBUV = _AttrCondition('LIBUV') WIN = _AttrCondition('WIN') APPVEYOR = _AttrCondition('RUNNING_ON_APPVEYOR') TRAVIS = _AttrCondition('RUNNING_ON_TRAVIS') CI = _AttrCondition('RUNNING_ON_CI') LEAKTEST = _AttrCondition('RUN_LEAKCHECKS') COVERAGE = _AttrCondition('RUN_COVERAGE') RESOLVER_NOT_SYSTEM = _AttrCondition('RESOLVER_NOT_SYSTEM') BIT_64 = ConstantCondition(struct.calcsize('P') * 8 == 64, 'BIT_64') PY380_EXACTLY = ConstantCondition(sys.version_info[:3] == (3, 8, 0), 'PY380_EXACTLY') class _Definition(object): __slots__ = ( '__name__', # When does the class of this condition apply? 'when', # When should this test be run alone, if it's run? 'run_alone', # Should this test be ignored during coverage measurement? 'ignore_coverage', # {name: (Condition, value)} 'options', ) def __init__(self, when, run_alone, ignore_coverage, options): assert isinstance(when, Condition) assert isinstance(run_alone, Condition) assert isinstance(ignore_coverage, Condition) self.when = when self.__name__ = None # pylint:disable=non-str-assignment-to-dunder-name self.run_alone = run_alone self.ignore_coverage = ignore_coverage if options: for v in options.values(): assert isinstance(v, tuple) and len(v) == 2 assert isinstance(v[0], Condition) self.options = options def __set_name__(self, owner, name): self.__name__ = name def __repr__(self): return '<%s for %s when=%r=%s run_alone=%r=%s>' % ( type(self).__name__, self.__name__, self.when, bool(self.when), self.run_alone, bool(self.run_alone) ) class _Action(_Definition): __slots__ = ( 'reason', ) def __init__(self, reason='', when=ALWAYS, run_alone=NEVER, ignore_coverage=NEVER, options=None): _Definition.__init__(self, when, run_alone, ignore_coverage, options) self.reason = reason class RunAlone(_Action): __slots__ = () def __init__(self, reason='', when=ALWAYS, ignore_coverage=NEVER): _Action.__init__(self, reason, run_alone=when, ignore_coverage=ignore_coverage) class Failing(_Action): __slots__ = () class Flaky(Failing): __slots__ = () class Ignored(_Action): __slots__ = () class Multi(object): def __init__(self): self._conds = [] def flaky(self, reason='', when=True): self._conds.append(Flaky(reason, when)) return self def ignored(self, reason='', when=True): self._conds.append(Ignored(reason, when)) return self def __set_name__(self, owner, name): for c in self._conds: c.__set_name__(owner, name) class DefinitionsMeta(type): # a metaclass on Python 3 that makes sure we only set attributes once. pylint doesn't # warn about that. @classmethod def __prepare__(cls, name, bases): # pylint:disable=unused-argument return SetOnceMapping() class SetOnceMapping(dict): def __setitem__(self, name, value): if name in self: raise AttributeError(name) dict.__setitem__(self, name, value) som = SetOnceMapping() som[1] = 1 try: som[1] = 2 except AttributeError: del som else: raise AssertionError("SetOnceMapping is broken") DefinitionsBase = DefinitionsMeta('DefinitionsBase', (object,), {}) class Definitions(DefinitionsBase): test__issue6 = Flaky( """test__issue6 (see comments in test file) is really flaky on both Travis and Appveyor; on Travis we could just run the test again (but that gets old fast), but on appveyor we don't have that option without a new commit---and sometimes we really need a build to succeed in order to get a release wheel""" ) test__core_fork = Ignored( """fork watchers don't get called on windows because fork is not a concept windows has. See this file for a detailed explanation.""", when=WIN ) test__greenletset = Flaky( when=WIN, ignore_coverage=PYPY ) test__example_udp_client = test__example_udp_server = Flaky( """ These both run on port 9000 and can step on each other...seems like the appveyor containers aren't fully port safe? Or it takes longer for the processes to shut down? Or we run them in a different order in the process pool than we do other places? On PyPy on Travis, this fails to get the correct results, sometimes. I can't reproduce locally """, when=APPVEYOR | (PYPY & TRAVIS) ) # This one sometimes randomly closes connections, but no indication # of a server crash, only a client side close. test__server_pywsgi = Flaky(when=APPVEYOR) test_threading = Multi().ignored( """ This one seems to just stop right after patching is done. It passes on a local win 10 vm, and the main test_threading_2.py does as well. Based on the printouts we added, it appears to not even finish importing: https://ci.appveyor.com/project/denik/gevent/build/1.0.1277/job/tpvhesij5gldjxqw#L1190 Ignored because it takes two minutes to time out. """, when=APPVEYOR & LIBUV & PYPY ).flaky( """ test_set_and_clear in Py3 relies on 5 threads all starting and coming to an Event wait point while a sixth thread sleeps for a half second. The sixth thread then does something and checks that the 5 threads were all at the wait point. But the timing is sometimes too tight for appveyor. This happens even if Event isn't monkey-patched """, when=APPVEYOR & PY3 ) test__socketpair = Ignored( """ Py35 added socket.socketpair, all other releases are missing it. No reason to even test it. """, when=WIN & PY2 ) test_ftplib = Flaky( r""" could be a problem of appveyor - not sure ====================================================================== ERROR: test_af (__main__.TestIPv6Environment) ---------------------------------------------------------------------- File "C:\Python27-x64\lib\ftplib.py", line 135, in connect self.sock = socket.create_connection((self.host, self.port), self.timeout) File "c:\projects\gevent\gevent\socket.py", line 73, in create_connection raise err error: [Errno 10049] [Error 10049] The requested address is not valid in its context. XXX: On Jan 3 2016 this suddenly started passing on Py27/64; no idea why, the python version was 2.7.11 before and after. """, when=APPVEYOR & BIT_64 ) test__backdoor = Flaky(when=LEAKTEST | PYPY) test__socket_errors = Flaky(when=LEAKTEST) test_signal = Flaky( "On Travis, this very frequently fails due to timing", when=TRAVIS & LEAKTEST, # Partial workaround for the _testcapi issue on PyPy, # but also because signal delivery can sometimes be slow, and this # spawn processes of its own run_alone=APPVEYOR, ) test__monkey_sigchld_2 = Ignored( """ This hangs for no apparent reason when run by the testrunner, even wher maked standalone when run standalone from the command line, it's fine. Issue in pypy2 6.0? """, when=PYPY & LIBUV ) test_ssl = Ignored( """ PyPy 7.0 and 7.1 on Travis with Ubunto Xenial 16.04 can't allocate SSL Context objects, either in Python 2.7 or 3.6. There must be some library incompatibility. No point even running them. XXX: Remember to turn this back on. """, when=PYPY & TRAVIS ) test__pywsgi = Ignored( """ XXX: Re-enable this when we can investigate more. This has started crashing with a SystemError. I cannot reproduce with the same version on macOS and I cannot reproduce with the same version in a Linux vm. Commenting out individual tests just moves the crash around. https://bitbucket.org/pypy/pypy/issues/2769/systemerror-unexpected-internal-exception On Appveyor 3.8.0, for some reason this takes *way* too long, about 100s, which often goes just over the default timeout of 100s. This makes no sense. But it also takes nearly that long in 3.7. 3.6 and earlier are much faster. """, when=(PYPY & TRAVIS & LIBUV) | PY380_EXACTLY, # https://bitbucket.org/pypy/pypy/issues/2769/systemerror-unexpected-internal-exception run_alone=(CI & LEAKTEST & PY3) | (PYPY & LIBUV), ) test_subprocess = Flaky( "Unknown, can't reproduce locally; times out one test", when=PYPY & PY3 & TRAVIS, ignore_coverage=ALWAYS, ) test__threadpool = Ignored( """ XXX: Re-enable these when we have more time to investigate. This test, which normally takes ~60s, sometimes hangs forever after running several tests. I cannot reproduce, it seems highly load dependent. Observed with both libev and libuv. """, when=TRAVIS & (PYPY | OSX), # This often takes much longer on PyPy on CI. options={'timeout': (CI & PYPY, 180)}, ) test__threading_2 = Ignored( """ This test, which normally takes 4-5s, sometimes hangs forever after running two tests. I cannot reproduce, it seems highly load dependent. Observed with both libev and libuv. """, when=TRAVIS & (PYPY | OSX), # This often takes much longer on PyPy on CI. options={'timeout': (CI & PYPY, 180)}, ) test__issue230 = Ignored( """ This rarely hangs for unknown reasons. I cannot reproduce locally. """, when=TRAVIS & OSX ) test_selectors = Flaky( """ Timing issues on appveyor. """, when=PY3 & APPVEYOR, ignore_coverage=ALWAYS, ) test__example_portforwarder = Flaky( """ This one sometimes times out, often after output "The process with PID XXX could not be terminated. Reason: There is no running instance of the task.", """, when=APPVEYOR | COVERAGE ) test__issue302monkey = test__threading_vs_settrace = Flaky( """ The gevent concurrency plugin tends to slow things down and get us past our default timeout value. These tests in particular are sensitive to it. So in fact we just turn them off. """, when=COVERAGE, ignore_coverage=ALWAYS ) test__hub_join_timeout = Ignored( r""" This sometimes times out. It appears to happen when the times take too long and a test raises a FlakyTestTimeout error, aka a unittest.SkipTest error. This probably indicates that we're not cleaning something up correctly: .....ss GEVENTTEST_USE_RESOURCES=-network C:\Python38-x64\python.exe -u \ -mgevent.tests.test__hub_join_timeout [code TIMEOUT] [took 100.4s] """, when=APPVEYOR ) test__example_wsgiserver = test__example_webproxy = RunAlone( """ These share the same port, which means they can conflict between concurrent test runs too XXX: Fix this by dynamically picking a port. """, ) test__pool = RunAlone( """ On a heavily loaded box, these can all take upwards of 200s. """, when=CI & LEAKTEST ) test_socket = RunAlone( "Sometimes has unexpected timeouts", when=CI & PYPY & PY3, ignore_coverage=ALWAYS, # times out ) test__refcount = Ignored( "Sometimes fails to connect for no reason", when=(CI & OSX) | (CI & PYPY) | APPVEYOR, ignore_coverage=PYPY ) test__doctests = Ignored( "Sometimes times out during/after gevent._config.Config", when=CI & OSX ) # tests that can't be run when coverage is enabled # TODO: Now that we have this declarative, we could eliminate this list, # just add them to the main IGNORED_TESTS list. IGNORE_COVERAGE = [ ] # A mapping from test file basename to a dictionary of # options that will be applied on top of the DEFAULT_RUN_OPTIONS. TEST_FILE_OPTIONS = { } FAILING_TESTS = [] IGNORED_TESTS = [] # tests that don't do well when run on busy box # or that are mutually exclusive RUN_ALONE = [ ] def populate(): # pylint:disable=too-many-branches # TODO: Maybe move to the metaclass. # TODO: This could be better. for k, v in Definitions.__dict__.items(): if isinstance(v, Multi): actions = v._conds else: actions = (v,) test_name = k + '.py' del k, v for action in actions: if not isinstance(action, _Action): continue if action.run_alone: RUN_ALONE.append(test_name) if action.ignore_coverage: IGNORE_COVERAGE.append(test_name) if action.options: for opt_name, (condition, value) in action.options.items(): # TODO: Verify that this doesn't match more than once. if condition: TEST_FILE_OPTIONS.setdefault(test_name, {})[opt_name] = value if action.when: if isinstance(action, Ignored): IGNORED_TESTS.append(test_name) elif isinstance(action, Flaky): FAILING_TESTS.append('FLAKY ' + test_name) elif isinstance(action, Failing): FAILING_TESTS.append(test_name) FAILING_TESTS.sort() IGNORED_TESTS.sort() RUN_ALONE.sort() populate() if __name__ == '__main__': print('known_failures:\n', FAILING_TESTS) print('ignored tests:\n', IGNORED_TESTS) print('run alone:\n', RUN_ALONE) print('options:\n', TEST_FILE_OPTIONS) print("ignore during coverage:\n", IGNORE_COVERAGE)