From 696dc6b597536503ef44928042ae5fe35fe0b5c3 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Tue, 4 Jun 2019 03:38:25 +0930 Subject: [PATCH] gossipd: disable gossip_store upgrade. We're about to bump version again, and the code to upgrade it was quite hairy (and buggy!). It's not worthwhile for such a poorly-tested path: I will just add code to limit how much incoming gossip we get to avoid flooding when we upgrade, however. I also use a modern gossip_store version in our test_gossip_store_load test, instead of relying on the upgrade path. Signed-off-by: Rusty Russell --- gossipd/gossip_store.c | 103 ------------------------- gossipd/gossip_store.csv | 18 ----- gossipd/test/run-bench-find_route.c | 12 --- gossipd/test/run-find_route-specific.c | 12 --- gossipd/test/run-find_route.c | 12 --- gossipd/test/run-overlong.c | 12 --- tests/test_gossip.py | 31 ++++---- 7 files changed, 17 insertions(+), 183 deletions(-) diff --git a/gossipd/gossip_store.c b/gossipd/gossip_store.c index 6171d9a2d..8f75dbe64 100644 --- a/gossipd/gossip_store.c +++ b/gossipd/gossip_store.c @@ -63,106 +63,6 @@ static bool append_msg(int fd, const u8 *msg, u64 *len) write(fd, msg, msglen) == msglen); } -static bool upgrade_gs(struct gossip_store *gs) -{ - beint32_t hdr[2]; - size_t off = gs->len; - int newfd; - const u8 newversion = GOSSIP_STORE_VERSION; - - if (gs->version != 3) - return false; - - newfd = open(GOSSIP_STORE_TEMP_FILENAME, - O_RDWR|O_APPEND|O_CREAT|O_TRUNC, - 0600); - if (newfd < 0) { - status_broken("gossip_store: can't create temp %s: %s", - GOSSIP_STORE_TEMP_FILENAME, strerror(errno)); - return false; - } - - if (!write_all(newfd, &newversion, sizeof(newversion))) { - status_broken("gossip_store: can't write header to %s: %s", - GOSSIP_STORE_TEMP_FILENAME, strerror(errno)); - close(newfd); - return false; - } - - while (pread(gs->fd, hdr, sizeof(hdr), off) == sizeof(hdr)) { - u32 msglen, checksum; - u8 *msg, *gossip_msg; - struct amount_sat satoshis; - - msglen = be32_to_cpu(hdr[0]); - checksum = be32_to_cpu(hdr[1]); - msg = tal_arr(tmpctx, u8, msglen); - - if (pread(gs->fd, msg, msglen, off+sizeof(hdr)) != msglen) { - status_unusual("gossip_store: truncated file @%zu?", - off + sizeof(hdr)); - goto fail; - } - - if (checksum != crc32c(0, msg, msglen)) { - status_unusual("gossip_store: checksum failed"); - goto fail; - } - - /* These need to be appended with channel size */ - if (fromwire_gossip_store_v3_channel_announcement(msg, msg, - &gossip_msg, - &satoshis)) { - u8 *amt = towire_gossip_store_channel_amount(msg, - satoshis); - if (!append_msg(newfd, gossip_msg, NULL)) - goto write_fail; - if (!append_msg(newfd, amt, NULL)) - goto write_fail; - /* These are extracted and copied verbatim */ - } else if (fromwire_gossip_store_v3_channel_update(msg, msg, - &gossip_msg) - || fromwire_gossip_store_v3_node_announcement(msg, - msg, - &gossip_msg) - || fromwire_gossip_store_v3_local_add_channel(msg, - msg, - &gossip_msg)) { - if (!append_msg(newfd, gossip_msg, NULL)) - goto write_fail; - } else { - /* Just copy into new store. */ - if (write(newfd, hdr, sizeof(hdr)) != sizeof(hdr) - || write(newfd, msg, tal_bytelen(msg)) != - tal_bytelen(msg)) - goto write_fail; - } - off += sizeof(hdr) + msglen; - clean_tmpctx(); - } - - if (rename(GOSSIP_STORE_TEMP_FILENAME, GOSSIP_STORE_FILENAME) == -1) { - status_broken( - "Error swapping compacted gossip_store into place: %s", - strerror(errno)); - goto fail; - } - - status_info("Upgraded gossip_store from version %u to %u", - gs->version, newversion); - close(gs->fd); - gs->fd = newfd; - gs->version = newversion; - return true; - -write_fail: - status_unusual("gossip_store: write failed for upgrade: %s", - strerror(errno)); -fail: - close(newfd); - return false; -} - struct gossip_store *gossip_store_new(struct routing_state *rstate) { struct gossip_store *gs = tal(rstate, struct gossip_store); @@ -183,9 +83,6 @@ struct gossip_store *gossip_store_new(struct routing_state *rstate) if (gs->version == GOSSIP_STORE_VERSION) return gs; - if (upgrade_gs(gs)) - return gs; - status_unusual("Gossip store version %u not %u: removing", gs->version, GOSSIP_STORE_VERSION); if (ftruncate(gs->fd, 0) != 0) diff --git a/gossipd/gossip_store.csv b/gossipd/gossip_store.csv index fec90f853..9d893955f 100644 --- a/gossipd/gossip_store.csv +++ b/gossipd/gossip_store.csv @@ -12,21 +12,3 @@ gossip_store_channel_delete,,short_channel_id,struct short_channel_id gossip_store_private_update,4102 gossip_store_private_update,,len,u16 gossip_store_private_update,,update,len*u8 - -### Older v3 messages -gossip_store_v3_channel_announcement,4096 -gossip_store_v3_channel_announcement,,len,u16 -gossip_store_v3_channel_announcement,,announcement,len*u8 -gossip_store_v3_channel_announcement,,satoshis,struct amount_sat - -gossip_store_v3_channel_update,4097 -gossip_store_v3_channel_update,,len,u16 -gossip_store_v3_channel_update,,update,len*u8 - -gossip_store_v3_node_announcement,4098 -gossip_store_v3_node_announcement,,len,u16 -gossip_store_v3_node_announcement,,announcement,len*u8 - -gossip_store_v3_local_add_channel,4100 -gossip_store_v3_local_add_channel,,len,u16 -gossip_store_v3_local_add_channel,,local_add,len*u8 diff --git a/gossipd/test/run-bench-find_route.c b/gossipd/test/run-bench-find_route.c index e83feb4e0..fe22b7d16 100644 --- a/gossipd/test/run-bench-find_route.c +++ b/gossipd/test/run-bench-find_route.c @@ -48,18 +48,6 @@ bool fromwire_gossip_store_channel_delete(const void *p UNNEEDED, struct short_c /* Generated stub for fromwire_gossip_store_private_update */ bool fromwire_gossip_store_private_update(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, u8 **update UNNEEDED) { fprintf(stderr, "fromwire_gossip_store_private_update called!\n"); abort(); } -/* Generated stub for fromwire_gossip_store_v3_channel_announcement */ -bool fromwire_gossip_store_v3_channel_announcement(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, u8 **announcement UNNEEDED, struct amount_sat *satoshis UNNEEDED) -{ fprintf(stderr, "fromwire_gossip_store_v3_channel_announcement called!\n"); abort(); } -/* Generated stub for fromwire_gossip_store_v3_channel_update */ -bool fromwire_gossip_store_v3_channel_update(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, u8 **update UNNEEDED) -{ fprintf(stderr, "fromwire_gossip_store_v3_channel_update called!\n"); abort(); } -/* Generated stub for fromwire_gossip_store_v3_local_add_channel */ -bool fromwire_gossip_store_v3_local_add_channel(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, u8 **local_add UNNEEDED) -{ fprintf(stderr, "fromwire_gossip_store_v3_local_add_channel called!\n"); abort(); } -/* Generated stub for fromwire_gossip_store_v3_node_announcement */ -bool fromwire_gossip_store_v3_node_announcement(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, u8 **announcement UNNEEDED) -{ fprintf(stderr, "fromwire_gossip_store_v3_node_announcement called!\n"); abort(); } /* Generated stub for fromwire_node_announcement */ bool fromwire_node_announcement(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, secp256k1_ecdsa_signature *signature UNNEEDED, u8 **features UNNEEDED, u32 *timestamp UNNEEDED, struct node_id *node_id UNNEEDED, u8 rgb_color[3] UNNEEDED, u8 alias[32] UNNEEDED, u8 **addresses UNNEEDED) { fprintf(stderr, "fromwire_node_announcement called!\n"); abort(); } diff --git a/gossipd/test/run-find_route-specific.c b/gossipd/test/run-find_route-specific.c index bbd745f3a..c3be41e0c 100644 --- a/gossipd/test/run-find_route-specific.c +++ b/gossipd/test/run-find_route-specific.c @@ -37,18 +37,6 @@ bool fromwire_gossip_store_channel_delete(const void *p UNNEEDED, struct short_c /* Generated stub for fromwire_gossip_store_private_update */ bool fromwire_gossip_store_private_update(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, u8 **update UNNEEDED) { fprintf(stderr, "fromwire_gossip_store_private_update called!\n"); abort(); } -/* Generated stub for fromwire_gossip_store_v3_channel_announcement */ -bool fromwire_gossip_store_v3_channel_announcement(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, u8 **announcement UNNEEDED, struct amount_sat *satoshis UNNEEDED) -{ fprintf(stderr, "fromwire_gossip_store_v3_channel_announcement called!\n"); abort(); } -/* Generated stub for fromwire_gossip_store_v3_channel_update */ -bool fromwire_gossip_store_v3_channel_update(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, u8 **update UNNEEDED) -{ fprintf(stderr, "fromwire_gossip_store_v3_channel_update called!\n"); abort(); } -/* Generated stub for fromwire_gossip_store_v3_local_add_channel */ -bool fromwire_gossip_store_v3_local_add_channel(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, u8 **local_add UNNEEDED) -{ fprintf(stderr, "fromwire_gossip_store_v3_local_add_channel called!\n"); abort(); } -/* Generated stub for fromwire_gossip_store_v3_node_announcement */ -bool fromwire_gossip_store_v3_node_announcement(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, u8 **announcement UNNEEDED) -{ fprintf(stderr, "fromwire_gossip_store_v3_node_announcement called!\n"); abort(); } /* Generated stub for fromwire_node_announcement */ bool fromwire_node_announcement(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, secp256k1_ecdsa_signature *signature UNNEEDED, u8 **features UNNEEDED, u32 *timestamp UNNEEDED, struct node_id *node_id UNNEEDED, u8 rgb_color[3] UNNEEDED, u8 alias[32] UNNEEDED, u8 **addresses UNNEEDED) { fprintf(stderr, "fromwire_node_announcement called!\n"); abort(); } diff --git a/gossipd/test/run-find_route.c b/gossipd/test/run-find_route.c index 226ca0865..698983b19 100644 --- a/gossipd/test/run-find_route.c +++ b/gossipd/test/run-find_route.c @@ -35,18 +35,6 @@ bool fromwire_gossip_store_channel_delete(const void *p UNNEEDED, struct short_c /* Generated stub for fromwire_gossip_store_private_update */ bool fromwire_gossip_store_private_update(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, u8 **update UNNEEDED) { fprintf(stderr, "fromwire_gossip_store_private_update called!\n"); abort(); } -/* Generated stub for fromwire_gossip_store_v3_channel_announcement */ -bool fromwire_gossip_store_v3_channel_announcement(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, u8 **announcement UNNEEDED, struct amount_sat *satoshis UNNEEDED) -{ fprintf(stderr, "fromwire_gossip_store_v3_channel_announcement called!\n"); abort(); } -/* Generated stub for fromwire_gossip_store_v3_channel_update */ -bool fromwire_gossip_store_v3_channel_update(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, u8 **update UNNEEDED) -{ fprintf(stderr, "fromwire_gossip_store_v3_channel_update called!\n"); abort(); } -/* Generated stub for fromwire_gossip_store_v3_local_add_channel */ -bool fromwire_gossip_store_v3_local_add_channel(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, u8 **local_add UNNEEDED) -{ fprintf(stderr, "fromwire_gossip_store_v3_local_add_channel called!\n"); abort(); } -/* Generated stub for fromwire_gossip_store_v3_node_announcement */ -bool fromwire_gossip_store_v3_node_announcement(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, u8 **announcement UNNEEDED) -{ fprintf(stderr, "fromwire_gossip_store_v3_node_announcement called!\n"); abort(); } /* Generated stub for fromwire_node_announcement */ bool fromwire_node_announcement(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, secp256k1_ecdsa_signature *signature UNNEEDED, u8 **features UNNEEDED, u32 *timestamp UNNEEDED, struct node_id *node_id UNNEEDED, u8 rgb_color[3] UNNEEDED, u8 alias[32] UNNEEDED, u8 **addresses UNNEEDED) { fprintf(stderr, "fromwire_node_announcement called!\n"); abort(); } diff --git a/gossipd/test/run-overlong.c b/gossipd/test/run-overlong.c index d31910e5a..297551099 100644 --- a/gossipd/test/run-overlong.c +++ b/gossipd/test/run-overlong.c @@ -35,18 +35,6 @@ bool fromwire_gossip_store_channel_delete(const void *p UNNEEDED, struct short_c /* Generated stub for fromwire_gossip_store_private_update */ bool fromwire_gossip_store_private_update(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, u8 **update UNNEEDED) { fprintf(stderr, "fromwire_gossip_store_private_update called!\n"); abort(); } -/* Generated stub for fromwire_gossip_store_v3_channel_announcement */ -bool fromwire_gossip_store_v3_channel_announcement(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, u8 **announcement UNNEEDED, struct amount_sat *satoshis UNNEEDED) -{ fprintf(stderr, "fromwire_gossip_store_v3_channel_announcement called!\n"); abort(); } -/* Generated stub for fromwire_gossip_store_v3_channel_update */ -bool fromwire_gossip_store_v3_channel_update(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, u8 **update UNNEEDED) -{ fprintf(stderr, "fromwire_gossip_store_v3_channel_update called!\n"); abort(); } -/* Generated stub for fromwire_gossip_store_v3_local_add_channel */ -bool fromwire_gossip_store_v3_local_add_channel(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, u8 **local_add UNNEEDED) -{ fprintf(stderr, "fromwire_gossip_store_v3_local_add_channel called!\n"); abort(); } -/* Generated stub for fromwire_gossip_store_v3_node_announcement */ -bool fromwire_gossip_store_v3_node_announcement(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, u8 **announcement UNNEEDED) -{ fprintf(stderr, "fromwire_gossip_store_v3_node_announcement called!\n"); abort(); } /* Generated stub for fromwire_node_announcement */ bool fromwire_node_announcement(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, secp256k1_ecdsa_signature *signature UNNEEDED, u8 **features UNNEEDED, u32 *timestamp UNNEEDED, struct node_id *node_id UNNEEDED, u8 rgb_color[3] UNNEEDED, u8 alias[32] UNNEEDED, u8 **addresses UNNEEDED) { fprintf(stderr, "fromwire_node_announcement called!\n"); abort(); } diff --git a/tests/test_gossip.py b/tests/test_gossip.py index 4dc967700..4afd0812a 100644 --- a/tests/test_gossip.py +++ b/tests/test_gossip.py @@ -861,25 +861,28 @@ def test_gossip_store_load(node_factory): """Make sure we can read canned gossip store""" l1 = node_factory.get_node(start=False) with open(os.path.join(l1.daemon.lightning_dir, 'gossip_store'), 'wb') as f: - f.write(bytearray.fromhex("03" # GOSSIP_VERSION - "000001bc" # len - "521ef598" # csum - "1000" # WIRE_GOSSIP_STORE_CHANNEL_ANNOUNCEMENT - "01b00100bb8d7b6998cca3c2b3ce12a6bd73a8872c808bb48de2a30c5ad9cdf835905d1e27505755087e675fb517bbac6beb227629b694ea68f49d357458327138978ebfd7adfde1c69d0d2f497154256f6d5567a5cf2317c589e0046c0cc2b3e986cf9b6d3b44742bd57bce32d72cd1180a7f657795976130b20508b239976d3d4cdc4d0d6e6fbb9ab6471f664a662972e406f519eab8bce87a8c0365646df5acbc04c91540b4c7c518cec680a4a6af14dae1aca0fd5525220f7f0e96fcd2adef3c803ac9427fe71034b55a50536638820ef21903d09ccddd38396675b598587fa886ca711415c813fc6d69f46552b9a0a539c18f265debd0e2e286980a118ba349c216000043497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea33090000000013a63c0000b50001021bf3de4e84e3d52f9a3e36fbdcd2c4e8dbf203b9ce4fc07c2f03be6c21d0c67503f113414ebdc6c1fb0f33c99cd5a1d09dd79e7fdf2468cf1fe1af6674361695d203801fd8ab98032f11cc9e4916dd940417082727077609d5c7f8cc6e9a3ad25dd102517164b97ab46cee3826160841a36c46a2b7b9c74da37bdc070ed41ba172033a0000000001000000" - "00000086" # len - "88c703c8" # csum - "1001" # WIRE_GOSSIP_STORE_CHANNEL_UPDATE - "008201021ea7c2eadf8a29eb8690511a519b5656e29aa0a853771c4e38e65c5abf43d907295a915e69e451f4c7a0c3dc13dd943cfbe3ae88c0b96667cd7d58955dbfedcf43497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea33090000000013a63c0000b500015b8d9b440000009000000000000003e8000003e800000001" - "00000099" # len - "12abbbba" # csum - "1002" # WIRE_GOSSIP_STORE_NODE_ANNOUNCEMENT - "00950101cf5d870bc7ecabcb7cd16898ef66891e5f0c6c5851bd85b670f03d325bc44d7544d367cd852e18ec03f7f4ff369b06860a3b12b07b29f36fb318ca11348bf8ec00005aab817c03f113414ebdc6c1fb0f33c99cd5a1d09dd79e7fdf2468cf1fe1af6674361695d23974b250757a7a6c6549544300000000000000000000000000000000000000000000000007010566933e2607")) + f.write(bytearray.fromhex("04" # GOSSIP_STORE_VERSION + "000001b0" # len + "697dac9f" # csum + "0100" # WIRE_CHANNEL_ANNOUNCEMENT + "bb8d7b6998cca3c2b3ce12a6bd73a8872c808bb48de2a30c5ad9cdf835905d1e27505755087e675fb517bbac6beb227629b694ea68f49d357458327138978ebfd7adfde1c69d0d2f497154256f6d5567a5cf2317c589e0046c0cc2b3e986cf9b6d3b44742bd57bce32d72cd1180a7f657795976130b20508b239976d3d4cdc4d0d6e6fbb9ab6471f664a662972e406f519eab8bce87a8c0365646df5acbc04c91540b4c7c518cec680a4a6af14dae1aca0fd5525220f7f0e96fcd2adef3c803ac9427fe71034b55a50536638820ef21903d09ccddd38396675b598587fa886ca711415c813fc6d69f46552b9a0a539c18f265debd0e2e286980a118ba349c216000043497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea33090000000013a63c0000b50001021bf3de4e84e3d52f9a3e36fbdcd2c4e8dbf203b9ce4fc07c2f03be6c21d0c67503f113414ebdc6c1fb0f33c99cd5a1d09dd79e7fdf2468cf1fe1af6674361695d203801fd8ab98032f11cc9e4916dd940417082727077609d5c7f8cc6e9a3ad25dd102517164b97ab46cee3826160841a36c46a2b7b9c74da37bdc070ed41ba172033a" + "0000000a" # len + "7a0168df" # csum + "1005" # WIRE_GOSSIP_STORE_CHANNEL_AMOUNT + "0000000001000000" + "00000082" # len + "f56ae7ee" # csum + "0102" # WIRE_CHANNEL_UPDATE + "1ea7c2eadf8a29eb8690511a519b5656e29aa0a853771c4e38e65c5abf43d907295a915e69e451f4c7a0c3dc13dd943cfbe3ae88c0b96667cd7d58955dbfedcf43497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea33090000000013a63c0000b500015b8d9b440000009000000000000003e8000003e800000001" + "00000095" # len + "3d934473" # csum + "0101" # WIRE_NODE_ANNOUNCEMENT + "cf5d870bc7ecabcb7cd16898ef66891e5f0c6c5851bd85b670f03d325bc44d7544d367cd852e18ec03f7f4ff369b06860a3b12b07b29f36fb318ca11348bf8ec00005aab817c03f113414ebdc6c1fb0f33c99cd5a1d09dd79e7fdf2468cf1fe1af6674361695d23974b250757a7a6c6549544300000000000000000000000000000000000000000000000007010566933e2607")) l1.start() # May preceed the Started msg waited for in 'start'. wait_for(lambda: l1.daemon.is_in_log('gossip_store: Read 1/1/1/0 cannounce/cupdate/nannounce/cdelete from store in 754 bytes')) assert not l1.daemon.is_in_log('gossip_store.*truncating') - assert l1.daemon.is_in_log('Upgraded gossip_store from version 3 to 4') @unittest.skipIf(not DEVELOPER, "Needs fast gossip propagation")