From ced2ff8aa75f1cca4e5711dac5727485bfcaf2f9 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Wed, 22 May 2019 14:42:35 +0200 Subject: [PATCH] pytest: Mark repeat offenders as flaky It's always the same few tests that fail, so I'm marking them as flaky and will do a cleanup round later to individually address them. I've been tracking the failures on the master branch for a few months now and these are the worst offenders: |-----------------------------------------|--------|--------|-------| | test_name | state | branch | count | |-----------------------------------------|--------|--------|-------| | test_pay_direct | FAILED | master | 31 | | test_reconnect_gossiping | FAILED | master | 20 | | test_htlc_send_timeout | FAILED | master | 15 | | test_pay_limits | FAILED | master | 13 | | test_permfail | FAILED | master | 13 | | test_funding_reorg_private | FAILED | master | 12 | | test_invoice | FAILED | master | 12 | | test_invoice_preimage | FAILED | master | 12 | | test_shutdown | FAILED | master | 12 | | test_withdraw | FAILED | master | 12 | | test_gossip_store_load_v3 | FAILED | master | 11 | | test_onchain_multihtlc_their_unilateral | FAILED | master | 10 | | test_opening_tiny_channel | FAILED | master | 10 | | test_channel_reenable | FAILED | master | 9 | | test_crashlog | FAILED | master | 9 | | test_gossip_weirdalias | FAILED | master | 9 | | test_invoice_expiry | FAILED | master | 9 | | test_onchain_multihtlc_our_unilateral | FAILED | master | 9 | | test_peerinfo | FAILED | master | 9 | | test_private_channel | FAILED | master | 9 | | test_shutdown_reconnect | FAILED | master | 9 | | test_closing | FAILED | master | 8 | | test_closing_different_fees | FAILED | master | 8 | | test_closing_while_disconnected | FAILED | master | 8 | | test_gossip_timestamp_filter | FAILED | master | 8 | | test_gossipwith | FAILED | master | 8 | | test_invoice_routeboost | FAILED | master | 8 | | test_onchain_middleman | FAILED | master | 8 | | test_plugin_notifications | FAILED | master | 8 | | test_reconnect_channel_peers | FAILED | master | 8 | | test_bad_onion | FAILED | master | 7 | | test_closing_torture | FAILED | master | 7 | | test_fulfill_incoming_first | FAILED | master | 7 | | test_funding_reorg_disagree_scid_route | FAILED | master | 7 | | test_pay_disconnect | FAILED | master | 7 | | test_balance | FAILED | master | 6 | | test_check_command | FAILED | master | 6 | | test_closing_id | FAILED | master | 6 | | test_gossip_addresses | FAILED | master | 6 | | test_gossip_notices_close | FAILED | master | 6 | | test_setchannelfee_usage | FAILED | master | 6 | | test_announce_address | FAILED | master | 5 | | test_connect_by_gossip | FAILED | master | 5 | | test_gossip_badsig | FAILED | master | 5 | | test_gossip_disable_channels | FAILED | master | 5 | | test_gossip_jsonrpc | FAILED | master | 5 | | test_gossip_persistence | FAILED | master | 5 | | test_gossip_pruning | FAILED | master | 5 | | test_onchain_different_fees | FAILED | master | 5 | | test_onchain_dust_out | FAILED | master | 5 | |-----------------------------------------|--------|--------|-------| --- tests/test_connection.py | 2 ++ tests/test_misc.py | 2 +- tests/test_pay.py | 2 ++ tests/test_plugin.py | 1 + 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_connection.py b/tests/test_connection.py index 58ffe9c0f..96810d90a 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -1,5 +1,6 @@ from collections import namedtuple from fixtures import * # noqa: F401,F403 +from flaky import flaky # noqa: F401 from lightning import RpcError from utils import DEVELOPER, only_one, wait_for, sync_blockheight, VALGRIND @@ -565,6 +566,7 @@ def test_reconnect_receiver_fulfill(node_factory): assert only_one(l2.rpc.listinvoices('testpayment2')['invoices'])['status'] == 'paid' +@flaky @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") def test_shutdown_reconnect(node_factory): disconnects = ['-WIRE_SHUTDOWN', diff --git a/tests/test_misc.py b/tests/test_misc.py index 9fba052bf..b53bc7b98 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -1,6 +1,6 @@ from decimal import Decimal from fixtures import * # noqa: F401,F403 -from flaky import flaky +from flaky import flaky # noqa: F401 from lightning import RpcError from utils import DEVELOPER, VALGRIND, sync_blockheight, only_one, wait_for, TailableProc from ephemeral_port_reserve import reserve diff --git a/tests/test_pay.py b/tests/test_pay.py index e0918ad1c..87c6b1e22 100644 --- a/tests/test_pay.py +++ b/tests/test_pay.py @@ -1,4 +1,5 @@ from fixtures import * # noqa: F401,F403 +from flaky import flaky # noqa: F401 from lightning import RpcError, Millisatoshi from utils import DEVELOPER, wait_for, only_one, sync_blockheight, SLOW_MACHINE @@ -1614,6 +1615,7 @@ def test_pay_routeboost(node_factory, bitcoind): assert [h['channel'] for h in attempts[2]['routehint']] == [r['short_channel_id'] for r in routel3l5] +@flaky @unittest.skipIf(not DEVELOPER, "gossip without DEVELOPER=1 is slow") def test_pay_direct(node_factory, bitcoind): """Check that we prefer the direct route. diff --git a/tests/test_plugin.py b/tests/test_plugin.py index ae9d1a3c2..7eef9e3fd 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -1,5 +1,6 @@ from collections import OrderedDict from fixtures import * # noqa: F401,F403 +from flaky import flaky # noqa: F401 from lightning import RpcError, Millisatoshi from utils import only_one, wait_for