Makefiles: simplify dependencies.

Gather all binaries and objects and make the depend on external
requirements and common headers.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

Makefile

@@ -23,26 +23,6 @@ endif
 # This is where we add new features as bitcoin adds them.
 FEATURES :=
 
-TEST_PROGRAMS :=				\
-	test/test_protocol			\
-	test/test_sphinx
-
-BITCOIN_SRC :=					\
-	bitcoin/base58.c			\
-	bitcoin/block.c				\
-	bitcoin/chainparams.c			\
-	bitcoin/locktime.c			\
-	bitcoin/pubkey.c			\
-	bitcoin/pullpush.c			\
-	bitcoin/script.c			\
-	bitcoin/shadouble.c			\
-	bitcoin/short_channel_id.c		\
-	bitcoin/signature.c			\
-	bitcoin/tx.c				\
-	bitcoin/varint.c
-
-BITCOIN_OBJS := $(BITCOIN_SRC:.c=.o)
-
 CCAN_OBJS :=					\
 	ccan-asort.o				\
 	ccan-autodata.o				\
@@ -142,61 +122,36 @@ CCAN_HEADERS :=						\
 	$(CCANDIR)/ccan/timer/timer.h			\
 	$(CCANDIR)/ccan/typesafe_cb/typesafe_cb.h
 
-BITCOIN_HEADERS := bitcoin/address.h		\
-	bitcoin/base58.h			\
-	bitcoin/block.h				\
-	bitcoin/chainparams.h			\
-	bitcoin/locktime.h			\
-	bitcoin/preimage.h			\
-	bitcoin/privkey.h			\
-	bitcoin/pubkey.h			\
-	bitcoin/pullpush.h			\
-	bitcoin/script.h			\
-	bitcoin/shadouble.h			\
-	bitcoin/short_channel_id.h		\
-	bitcoin/signature.h			\
-	bitcoin/tx.h				\
-	bitcoin/varint.h
-
 GEN_HEADERS := 	gen_version.h
 
 CDUMP_OBJS := ccan-cdump.o ccan-strmap.o
 
 WIRE_GEN := tools/generate-wire.py
 
-PROGRAMS += $(TEST_PROGRAMS)
+ALL_PROGRAMS =
 
 CWARNFLAGS := -Werror -Wall -Wundef -Wmissing-prototypes -Wmissing-declarations -Wstrict-prototypes -Wold-style-definition
 CDEBUGFLAGS := -std=gnu11 -g -fstack-protector
 CFLAGS = $(CWARNFLAGS) $(CDEBUGFLAGS) -I $(CCANDIR) $(EXTERNAL_INCLUDE_FLAGS) -I . $(FEATURES) $(COVFLAGS) -DSHACHAIN_BITS=48
-LDFLAGS := -Lexternal
 
-LDLIBS = -lgmp -lsqlite3 $(COVFLAGS) $(EXTERNAL_LDLIBS)
+LDLIBS = -lgmp -lsqlite3 $(COVFLAGS)
 
-default: $(TEST_PROGRAMS) doc-all
+default: $(ALL_TEST_PROGRAMS) doc-all
 
 include external/Makefile
-include common/Makefile
-include doc/Makefile
 include bitcoin/Makefile
+include common/Makefile
 include wire/Makefile
 include wallet/Makefile
 include lightningd/Makefile
 include cli/Makefile
+include test/Makefile
+include doc/Makefile
 
 # Git doesn't maintain timestamps, so we only regen if git says we should.
 CHANGED_FROM_GIT = [ x"`git log $@ | head -n1`" != x"`git log $< | head -n1`" -o x"`git diff $<`" != x"" ]
 
-# Everything depends on the CCAN headers.
+check:
-$(CCAN_OBJS) $(CDUMP_OBJS) $(HELPER_OBJS) $(BITCOIN_OBJS) $(TEST_PROGRAMS:=.o) ccan/ccan/cdump/tools/cdump-enumstr.o: $(CCAN_HEADERS)
-
-# Except for CCAN, everything depends on bitcoin, ccan, library and common headers.
-$(HELPER_OBJS) $(COMMON_OBJS) $(BITCOIN_OBJS) $(LIBBASE58_OBJS) $(WIRE_OBJS) $(WALLET_LIB_OBJS) $(TEST_PROGRAMS:=.o): $(BITCOIN_HEADERS) $(COMMON_HEADERS) $(CCAN_HEADERS) $(GEN_HEADERS) $(EXTERNAL_HEADERS)
-
-test-protocol: test/test_protocol
-	set -e; TMP=`mktemp`; for f in test/commits/*.script; do if ! $(VALGRIND) test/test_protocol < $$f > $$TMP; then echo "test/test_protocol < $$f FAILED" >&2; exit 1; fi; diff -u $$TMP $$f.expected; done; rm $$TMP
-
-check: test-protocol
 	$(MAKE) pytest
 
 pytest: cli/lightning-cli lightningd-all
@@ -222,7 +177,7 @@ bolt-check/%: % bolt-precheck tools/check-bolt
 bolt-precheck:
 	@rm -rf .tmp.lightningrfc; if [ ! -d $(BOLTDIR) ]; then echo Not checking BOLT references: BOLTDIR $(BOLTDIR) does not exist >&2; exit 0; fi; set -e; if [ -n "$(BOLTVERSION)" ]; then git clone -q -b $(BOLTVERSION) $(BOLTDIR) .tmp.lightningrfc; else cp -a $(BOLTDIR) .tmp.lightningrfc; fi
 
-check-source-bolt: $(TEST_PROGRAMS:%=bolt-check/%.c)
+check-source-bolt: $(ALL_TEST_PROGRAMS:%=bolt-check/%.c)
 
 tools/check-bolt: tools/check-bolt.o $(CCAN_OBJS)
 
@@ -231,15 +186,13 @@ tools/check-bolt.o: $(CCAN_HEADERS)
 check-whitespace/%: %
 	@if grep -Hn '[ 	]$$' $<; then echo Extraneous whitespace found >&2; exit 1; fi
 
-check-whitespace: check-whitespace/Makefile check-whitespace/tools/check-bolt.c
+check-whitespace: check-whitespace/Makefile check-whitespace/tools/check-bolt.c $(ALL_TEST_PROGRAMS:%=check-whitespace/%.c)
 
-check-source: check-makefile check-source-bolt check-whitespace	\
+check-source: check-makefile check-source-bolt check-whitespace
-	$(BITCOIN_SRC:%=check-src-include-order/%)		\
-	$(BITCOIN_HEADERS:%=check-hdr-include-order/%)
 
-full-check: check $(TEST_PROGRAMS) check-source
+full-check: check check-source
 
-coverage/coverage.info: check $(TEST_PROGRAMS) pytest
+coverage/coverage.info: check pytest
 	mkdir coverage || true
 	lcov --capture --directory . --output-file coverage/coverage.info
 
@@ -251,11 +204,11 @@ TAGS: FORCE
 	$(RM) TAGS; find * -name test -type d -prune -o -name '*.[ch]' -print | xargs etags --append
 FORCE::
 
-ccan/ccan/cdump/tools/cdump-enumstr: ccan/ccan/cdump/tools/cdump-enumstr.o $(CDUMP_OBJS) $(CCAN_OBJS) $(EXTERNAL_LIBS)
+ccan/ccan/cdump/tools/cdump-enumstr: ccan/ccan/cdump/tools/cdump-enumstr.o $(CDUMP_OBJS) $(CCAN_OBJS)
 
-PROGRAMS += ccan/ccan/cdump/tools/cdump-enumstr
+ALL_PROGRAMS += ccan/ccan/cdump/tools/cdump-enumstr
-
+# Can't add to ALL_OBJS, as that makes a circular dep.
-$(TEST_PROGRAMS): % : %.o $(BITCOIN_OBJS) $(WIRE_OBJS) $(CCAN_OBJS) common/sphinx.o common/utils.o
+ccan/ccan/cdump/tools/cdump-enumstr.o: $(CCAN_HEADERS)
 
 ccan/config.h: ccan/tools/configurator/configurator
 	if $< > $@.new; then mv $@.new $@; else rm $@.new; exit 1; fi
@@ -264,10 +217,24 @@ gen_version.h: FORCE
 	@(echo "#define VERSION \"`git describe --always --dirty`\"" && echo "#define VERSION_NAME \"$(NAME)\"" && echo "#define BUILD_FEATURES \"$(FEATURES)\"") > $@.new
 	@if cmp $@.new $@ >/dev/null 2>&2; then rm -f $@.new; else mv $@.new $@; echo Version updated; fi
 
-version.o: gen_version.h
+# All binaries require the external libs, ccan
+$(ALL_PROGRAMS) $(ALL_TEST_PROGRAMS): $(EXTERNAL_LIBS) $(CCAN_OBJS)
+
+# Each test program depends on its own object.
+$(ALL_TEST_PROGRAMS): %: %.o
 
-# All binaries require the external libs
+# Without this rule, the (built-in) link line contains
-$(PROGRAMS): $(EXTERNAL_LIBS)
+# external/libwallycore.a directly, which causes a symbol clash (it
+# uses some ccan modules internally).  We want to rely on -lwallycore etc.
+# (as per EXTERNAL_LDLIBS) so we filter them out here.
+$(ALL_PROGRAMS) $(ALL_TEST_PROGRAMS):
+	$(LINK.o) $(filter-out %.a,$^) $(LOADLIBES) $(EXTERNAL_LDLIBS) $(LDLIBS) -o $@
+
+# Everything depends on the CCAN headers.
+$(CCAN_OBJS) $(CDUMP_OBJS): $(CCAN_HEADERS)
+
+# Except for CCAN, we treat everything else as dependent on external/ bitcoin/ common/ wire/ and generated version headers.
+$(ALL_OBJS): $(BITCOIN_HEADERS) $(COMMON_HEADERS) $(CCAN_HEADERS) $(WIRE_HEADERS) $(GEN_HEADERS) $(EXTERNAL_HEADERS)
 
 update-ccan:
 	mv ccan ccan.old
@@ -294,8 +261,9 @@ maintainer-clean: distclean
 	@echo 'deletes files that may need special tools to rebuild.'
 
 clean: wire-clean
-	$(RM) $(TEST_PROGRAMS)
+	$(RM) $(CCAN_OBJS) $(CDUMP_OBJS) $(ALL_OBJS)
-	$(RM) bitcoin/*.o *.o $(TEST_PROGRAMS:=.o) $(CCAN_OBJS)
+	$(RM) $(ALL_PROGRAMS) $(ALL_PROGRAMS:=.o)
+	$(RM) $(ALL_TEST_PROGRAMS) $(ALL_TEST_PROGRAMS:=.o)
 	$(RM) ccan/config.h gen_*.h
 	$(RM) ccan/ccan/cdump/tools/cdump-enumstr.o
 	$(RM) check-bolt tools/check-bolt tools/*.o
@@ -317,8 +285,6 @@ update-mocks/%: %
 	  tail -n +$$END $< >> $$BASE.new; mv $$BASE.new $<; \
 	fi
 
-test/test_sphinx: libsodium.a
-
 unittest/%: %
 	$(VALGRIND) $(VALGRIND_TEST_ARGS) $*
 

bitcoin/Makefile

@@ -1,5 +1,43 @@
 # Included for one dir up.
 
+BITCOIN_SRC :=					\
+	bitcoin/base58.c			\
+	bitcoin/block.c				\
+	bitcoin/chainparams.c			\
+	bitcoin/locktime.c			\
+	bitcoin/pubkey.c			\
+	bitcoin/pullpush.c			\
+	bitcoin/script.c			\
+	bitcoin/shadouble.c			\
+	bitcoin/short_channel_id.c		\
+	bitcoin/signature.c			\
+	bitcoin/tx.c				\
+	bitcoin/varint.c
+
+BITCOIN_OBJS := $(BITCOIN_SRC:.c=.o)
+
+BITCOIN_HEADERS := bitcoin/address.h		\
+	bitcoin/base58.h			\
+	bitcoin/block.h				\
+	bitcoin/chainparams.h			\
+	bitcoin/locktime.h			\
+	bitcoin/preimage.h			\
+	bitcoin/privkey.h			\
+	bitcoin/pubkey.h			\
+	bitcoin/pullpush.h			\
+	bitcoin/script.h			\
+	bitcoin/shadouble.h			\
+	bitcoin/short_channel_id.h		\
+	bitcoin/signature.h			\
+	bitcoin/tx.h				\
+	bitcoin/varint.h
+
+check-source: $(BITCOIN_SRC:%=check-src-include-order/%)		\
+	$(BITCOIN_HEADERS:%=check-hdr-include-order/%)
+
+# Bitcoin objects depends on bitcoin/ external/ and ccan
+$(BITCOIN_OBJS): $(CCAN_HEADERS) $(BITCOIN_HEADERS) $(EXTERNAL_HEADERS)
+
 # Note that these actually #include everything they need, except ccan/.
 # That allows for unit testing of statics, and special effects.
 

cli/Makefile

@@ -14,6 +14,10 @@ $(LIGHTNING_CLI_OBJS) $(JSMN_OBJS): $(JSMN_HEADERS) $(COMMON_HEADERS) $(CCAN_HEA
 
 cli/lightning-cli: $(LIGHTNING_CLI_OBJS) $(LIGHTNING_CLI_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS)
 
+# Make sure these depend on everything.
+ALL_PROGRAMS += cli/lightning-cli
+ALL_OBJS += $(LIGHTNING_CLI_OBJS)
+
 check-source: $(LIGHTNING_CLI_SRC:%=check-src-include-order/%)
 
 clean: lightning-cli-clean

common/Makefile

@@ -38,6 +38,9 @@ COMMON_HEADERS_GEN := common/gen_htlc_state_names.h
 COMMON_HEADERS := $(COMMON_HEADERS_GEN) $(COMMON_HEADERS_NOGEN)
 COMMON_OBJS := $(COMMON_SRC:.c=.o)
 
+# Common objects depends on bitcoin/ external/ and ccan
+$(COMMON_OBJS): $(CCAN_HEADERS) $(BITCOIN_HEADERS) $(EXTERNAL_HEADERS) $(COMMON_HEADERS_GEN)
+
 common/gen_htlc_state_names.h: common/htlc_state.h ccan/ccan/cdump/tools/cdump-enumstr
 	ccan/ccan/cdump/tools/cdump-enumstr common/htlc_state.h > $@
 
@@ -51,3 +54,5 @@ check-whitespace: $(COMMON_SRC:%=check-whitespace/%) $(COMMON_HEADERS:%=check-wh
 
 check-source: $(COMMON_SRC:%=check-src-include-order/%)			\
 	$(COMMON_HEADERS_NOGEN:%=check-hdr-include-order/%)
+
+include common/test/Makefile

common/initial_commit_tx.h

@@ -4,7 +4,6 @@
 #include "config.h"
 #include <bitcoin/pubkey.h>
 #include <common/htlc.h>
-#include <lightningd/channel/channeld_htlc.h>
 
 struct keyset;
 struct sha256_double;

common/test/Makefile

@@ -0,0 +1,13 @@
+# FIXME: Move to common/test
+COMMON_TEST_PROGRAMS :=					\
+	common/test/test_sphinx
+
+COMMON_TEST_COMMON_OBJS :=				\
+	common/sphinx.o					\
+	common/utils.o
+
+$(COMMON_TEST_PROGRAMS): $(TEST_COMMON_OBJS) $(BITCOIN_OBJS) $(WIRE_OBJS)
+
+ALL_TEST_PROGRAMS += $(COMMON_TEST_PROGRAMS)
+ALL_OBJS += $(COMMON_TEST_PROGRAMS:=.o)
+

external/Makefile

@@ -16,7 +16,8 @@ EXTERNAL_INCLUDE_FLAGS :=					\
 	-I external/libwally-core/src/secp256k1/include/	\
 	-I external/jsmn/					\
 	-I external/libbase58/
-EXTERNAL_LDLIBS := -Lexternal $(patsubst lib%.a,-l%, $(notdir $(EXTERNAL_LIBS)))
+
+EXTERNAL_LDLIBS := -Lexternal $(patsubst lib%.a,-l%,$(notdir $(EXTERNAL_LIBS)))
 
 # We build libsodium, since Ubuntu xenial has one too old.
 external/libsodium.a: external/libsodium/src/libsodium/libsodium.la

lightningd/Makefile

@@ -6,7 +6,7 @@ lightningd-wrongdir:
 
 LIGHTNINGD_BINS := lightningd/lightningd lightningd/lightningd_hsm lightningd/lightningd_handshake lightningd/lightningd_gossip lightningd/lightningd_opening lightningd/lightningd_channel lightningd/lightningd_closing lightningd/lightningd_onchain
 
-PROGRAMS += $(LIGHTNINGD_BINS)
+ALL_PROGRAMS += $(LIGHTNINGD_BINS)
 
 lightningd-all: $(LIGHTNINGD_BINS)
 
@@ -64,24 +64,20 @@ LIGHTNINGD_SRC_NOHDR :=				\
 
 LIGHTNINGD_OBJS := $(LIGHTNINGD_SRC:.c=.o) $(LIGHTNINGD_SRC_NOHDR:.c=.o)
 
+# Make sure these depend on everything.
+ALL_OBJS += $(LIGHTNINGD_OBJS)
+
 # We accumulate all lightningd/ headers in these three:
 LIGHTNINGD_HEADERS_NOGEN =			\
 	$(LIGHTNINGD_SRC:.c=.h)			\
-	lightningd/peer_state.h			\
+	lightningd/peer_state.h
-	$(WIRE_HEADERS)				\
-	$(BITCOIN_HEADERS)			\
-	$(COMMON_HEADERS_NOGEN)			\
-	$(WALLET_LIB_HEADERS)
 
 # Generated headers
 LIGHTNINGD_HEADERS_GEN =			\
-	lightningd/gen_peer_state_names.h	\
+	lightningd/gen_peer_state_names.h
-	$(COMMON_HEADERS_GEN)			\
-	$(WIRE_GEN_HEADERS)			\
-	$(GEN_HEADERS)
 
 # All together in one convenient var
-LIGHTNINGD_HEADERS = $(LIGHTNINGD_HEADERS_NOGEN) $(LIGHTNINGD_HEADERS_GEN) $(EXTERNAL_HEADERS)
+LIGHTNINGD_HEADERS = $(LIGHTNINGD_HEADERS_NOGEN) $(LIGHTNINGD_HEADERS_GEN) $(EXTERNAL_HEADERS) $(WIRE_HEADERS) $(BITCOIN_HEADERS) $(COMMON_HEADERS) $(WALLET_LIB_HEADERS)
 
 # These included makefiles add their headers to the LIGHTNINGD_HEADERS
 # variable so the include must preceed any actual use of the variable.

lightningd/channel/Makefile

@@ -25,6 +25,9 @@ LIGHTNINGD_CHANNEL_SRC := lightningd/channel/channel.c	\
 	$(LIGHTNINGD_CHANNEL_HEADERS_GEN:.h=.c)
 LIGHTNINGD_CHANNEL_OBJS := $(LIGHTNINGD_CHANNEL_SRC:.c=.o)
 
+# Make sure these depend on everything.
+ALL_OBJS += $(LIGHTNINGD_CHANNEL_OBJS)
+
 # Common source we use.
 CHANNELD_COMMON_OBJS :=				\
 	common/channel_config.o			\
@@ -76,8 +79,7 @@ lightningd/channel/gen_channel_wire.c: $(WIRE_GEN) lightningd/channel/channel_wi
 
 LIGHTNINGD_CHANNEL_OBJS := $(LIGHTNINGD_CHANNEL_SRC:.c=.o) $(LIGHTNINGD_CHANNEL_GEN_SRC:.c=.o)
 
-lightningd/lightningd_channel: $(LIGHTNINGD_CHANNEL_OBJS) $(WIRE_ONION_OBJS) $(CHANNELD_COMMON_OBJS) $(WIRE_OBJS) $(BITCOIN_OBJS) $(CCAN_OBJS) $(LIGHTNINGD_HSM_CLIENT_OBJS) $(LIBBASE58_OBJS) libsecp256k1.a libsodium.a libwallycore.a
+lightningd/lightningd_channel: $(LIGHTNINGD_CHANNEL_OBJS) $(WIRE_ONION_OBJS) $(CHANNELD_COMMON_OBJS) $(WIRE_OBJS) $(BITCOIN_OBJS) $(LIGHTNINGD_HSM_CLIENT_OBJS)
-	$(CC) $(CFLAGS) -o $@ $^ $(LDLIBS)
 
 check-source: $(LIGHTNINGD_CHANNEL_SRC_NOGEN:%=check-src-include-order/%)
 check-source-bolt: $(LIGHTNINGD_CHANNEL_SRC:%=bolt-check/%) $(LIGHTNINGD_CHANNEL_HEADERS:%=bolt-check/%)

lightningd/closing/Makefile

@@ -20,6 +20,9 @@ LIGHTNINGD_CLOSING_SRC := lightningd/closing/closing.c	\
 	$(LIGHTNINGD_CLOSING_HEADERS:.h=.c)
 LIGHTNINGD_CLOSING_OBJS := $(LIGHTNINGD_CLOSING_SRC:.c=.o)
 
+# Make sure these depend on everything.
+ALL_OBJS += $(LIGHTNINGD_CLOSING_OBJS)
+
 # Control daemon uses this:
 LIGHTNINGD_CLOSING_CONTROL_HEADERS := $(LIGHTNINGD_CLOSING_HEADERS)
 LIGHTNINGD_CLOSING_CONTROL_SRC := $(LIGHTNINGD_CLOSING_HEADERS:.h=.c)
@@ -60,8 +63,7 @@ lightningd/closing/gen_closing_wire.c: $(WIRE_GEN) lightningd/closing/closing_wi
 
 LIGHTNINGD_CLOSING_OBJS := $(LIGHTNINGD_CLOSING_SRC:.c=.o) $(LIGHTNINGD_CLOSING_GEN_SRC:.c=.o)
 
-lightningd/lightningd_closing: $(LIGHTNINGD_CLOSING_OBJS) $(WIRE_ONION_OBJS) $(CLOSINGD_COMMON_OBJS) $(WIRE_OBJS) $(BITCOIN_OBJS) $(CCAN_OBJS) $(LIGHTNINGD_HSM_CLIENT_OBJS) $(LIBBASE58_OBJS) libsecp256k1.a libsodium.a libwallycore.a
+lightningd/lightningd_closing: $(LIGHTNINGD_CLOSING_OBJS) $(WIRE_ONION_OBJS) $(CLOSINGD_COMMON_OBJS) $(WIRE_OBJS) $(BITCOIN_OBJS)
-	$(CC) $(CFLAGS) -o $@ $^ $(LDLIBS)
 
 check-source: $(LIGHTNINGD_CLOSING_SRC_NOGEN:%=check-src-include-order/%)
 check-source-bolt: $(LIGHTNINGD_CLOSING_SRC:%=bolt-check/%) $(LIGHTNINGD_CLOSING_HEADERS:%=bolt-check/%)

lightningd/gossip/Makefile

@@ -19,6 +19,9 @@ LIGHTNINGD_GOSSIP_SRC := lightningd/gossip/gossip.c	\
 	$(LIGHTNINGD_GOSSIP_HEADERS:.h=.c)
 LIGHTNINGD_GOSSIP_OBJS := $(LIGHTNINGD_GOSSIP_SRC:.c=.o)
 
+# Make sure these depend on everything.
+ALL_OBJS += $(LIGHTNINGD_GOSSIP_OBJS)
+
 # For checking
 LIGHTNINGD_GOSSIP_ALLSRC_NOGEN := $(filter-out lightningd/gossip/gen_%, $(LIGHTNINGD_GOSSIP_CLIENT_SRC) $(LIGHTNINGD_GOSSIP_SRC))
 LIGHTNINGD_GOSSIP_ALLHEADERS_NOGEN := $(filter-out lightningd/gossip/gen_%, $(LIGHTNINGD_GOSSIP_CLIENT_HEADERS) $(LIGHTNINGD_GOSSIP_HEADERS))
@@ -49,8 +52,7 @@ $(LIGHTNINGD_GOSSIP_CONTROL_OBJS) : $(LIGHTNINGD_GOSSIP_CONTROL_HEADERS)
 
 lightningd/gossip-all: lightningd/lightningd_gossip $(LIGHTNINGD_GOSSIP_CLIENT_OBJS)
 
-lightningd/lightningd_gossip: $(LIGHTNINGD_GOSSIP_OBJS) $(GOSSIPD_COMMON_OBJS) $(BITCOIN_OBJS) $(WIRE_OBJS) $(CCAN_OBJS) $(LIBBASE58_OBJS) libsecp256k1.a libsodium.a libwallycore.a
+lightningd/lightningd_gossip: $(LIGHTNINGD_GOSSIP_OBJS) $(GOSSIPD_COMMON_OBJS) $(BITCOIN_OBJS) $(WIRE_OBJS)
-	$(CC) $(CFLAGS) -o $@ $^ $(LDLIBS)
 
 lightningd/gossip/gen_gossip_wire.h: $(WIRE_GEN) lightningd/gossip/gossip_wire.csv
 	$(WIRE_GEN) --header $@ gossip_wire_type < lightningd/gossip/gossip_wire.csv > $@

lightningd/handshake/Makefile

@@ -15,6 +15,9 @@ LIGHTNINGD_HANDSHAKE_SRC := lightningd/handshake/handshake.c	\
 	$(LIGHTNINGD_HANDSHAKE_HEADERS:.h=.c)
 LIGHTNINGD_HANDSHAKE_OBJS := $(LIGHTNINGD_HANDSHAKE_SRC:.c=.o)
 
+# Make sure these depend on everything.
+ALL_OBJS += $(LIGHTNINGD_HANDSHAKE_OBJS)
+
 # Control daemon uses this:
 LIGHTNINGD_HANDSHAKE_CONTROL_HEADERS := $(LIGHTNINGD_HANDSHAKE_HEADERS)
 LIGHTNINGD_HANDSHAKE_CONTROL_SRC := $(LIGHTNINGD_HANDSHAKE_HEADERS:.h=.c)
@@ -50,8 +53,7 @@ lightningd/handshake/gen_handshake_wire.c: $(WIRE_GEN) lightningd/handshake/hand
 
 LIGHTNINGD_HANDSHAKE_OBJS := $(LIGHTNINGD_HANDSHAKE_SRC:.c=.o) $(LIGHTNINGD_HANDSHAKE_GEN_SRC:.c=.o)
 
-lightningd/lightningd_handshake: $(LIGHTNINGD_OLD_LIB_OBJS) $(LIGHTNINGD_LIB_OBJS) $(LIGHTNINGD_HANDSHAKE_OBJS) $(HANDSHAKED_COMMON_OBJS) $(WIRE_OBJS) $(BITCOIN_OBJS) $(CCAN_OBJS) $(LIGHTNINGD_HSM_CLIENT_OBJS) $(LIBBASE58_OBJS) libsecp256k1.a libsodium.a libwallycore.a
+lightningd/lightningd_handshake: $(LIGHTNINGD_OLD_LIB_OBJS) $(LIGHTNINGD_LIB_OBJS) $(LIGHTNINGD_HANDSHAKE_OBJS) $(HANDSHAKED_COMMON_OBJS) $(WIRE_OBJS) $(BITCOIN_OBJS) $(LIGHTNINGD_HSM_CLIENT_OBJS)
-	$(CC) $(CFLAGS) -o $@ $^ $(LDLIBS)
 
 check-source: $(LIGHTNINGD_HANDSHAKE_SRC_NOGEN:%=check-src-include-order/%)
 check-source-bolt: $(LIGHTNINGD_HANDSHAKE_SRC:%=bolt-check/%) $(LIGHTNINGD_HANDSHAKE_HEADERS:%=bolt-check/%)

lightningd/handshake/test/Makefile

@@ -19,9 +19,12 @@ LIGHTNINGD_HANDSHAKE_TEST_COMMON_OBJS :=	\
 
 update-mocks: $(LIGHTNINGD_HANDSHAKE_TEST_SRC:%=update-mocks/%)
 
-$(LIGHTNINGD_HANDSHAKE_TEST_PROGRAMS): $(CCAN_OBJS) $(BITCOIN_OBJS) $(WIRE_OBJS) $(LIBBASE58_OBJS) libsecp256k1.a libsodium.a $(LIGHTNINGD_HANDSHAKE_TEST_COMMON_OBJS) $(LIGHTNINGD_HANDSHAKE_GEN_SRC:.c=.o) libwallycore.a
+$(LIGHTNINGD_HANDSHAKE_TEST_PROGRAMS): $(LIGHTNINGD_HANDSHAKE_TEST_COMMON_OBJS) $(LIGHTNINGD_HANDSHAKE_GEN_SRC:.c=.o) $(BITCOIN_OBJS) $(WIRE_OBJS)
 
-$(LIGHTNINGD_HANDSHAKE_TEST_OBJS): $(LIGHTNINGD_HANDSHAKE_HEADERS) $(BITCOIN_HEADERS) $(GEN_HEADERS) $(WIRE_HEADERS) $(CCAN_HEADERS) $(LIBBASE58_HEADERS) $(LIBSODIUM_HEADERS)
+$(LIGHTNINGD_HANDSHAKE_TEST_OBJS): $(LIGHTNINGD_HANDSHAKE_HEADERS)
+
+ALL_OBJS += $(LIGHTNINGD_HANDSHAKE_TEST_OBJS)
+ALL_TEST_PROGRAMS += $(LIGHTNINGD_HANDSHAKE_TEST_PROGRAMS)
 
 lightningd/handshake-tests: $(LIGHTNINGD_HANDSHAKE_TEST_PROGRAMS:%=unittest/%)
 

lightningd/hsm/Makefile

@@ -48,12 +48,12 @@ $(LIGHTNINGD_HSM_OBJS) $(LIGHTNINGD_HSM_CLIENT_OBJS): $(LIGHTNINGD_HEADERS)
 
 $(LIGHTNINGD_HSM_CONTROL_OBJS) : $(LIGHTNINGD_HSM_CONTROL_HEADERS)
 
-$(LIGHTNINGD_HSM_OBJS): $(COMMON_HEADERS)
+# Make sure these depend on everything.
+ALL_OBJS += $(LIGHTNINGD_HSM_OBJS)
 
 lightningd/hsm-all: lightningd/lightningd_hsm $(LIGHTNINGD_HSM_CLIENT_OBJS)
 
-lightningd/lightningd_hsm: $(LIGHTNINGD_HSM_OBJS) $(LIGHTNINGD_OLD_LIB_OBJS) $(LIGHTNINGD_LIB_OBJS) $(HSMD_COMMON_OBJS) $(BITCOIN_OBJS) $(WIRE_OBJS) $(CCAN_OBJS) $(LIBBASE58_OBJS) libsecp256k1.a libsodium.a libwallycore.a
+lightningd/lightningd_hsm: $(LIGHTNINGD_HSM_OBJS) $(LIGHTNINGD_OLD_LIB_OBJS) $(LIGHTNINGD_LIB_OBJS) $(HSMD_COMMON_OBJS) $(BITCOIN_OBJS) $(WIRE_OBJS)
-	$(CC) $(CFLAGS) -o $@ $^ $(LDLIBS)
 
 lightningd/hsm/gen_hsm_client_wire.h: $(WIRE_GEN) lightningd/hsm/hsm_client_wire_csv
 	$(WIRE_GEN) --header $@ hsm_client_wire_type < lightningd/hsm/hsm_client_wire_csv > $@

lightningd/onchain/Makefile

@@ -71,8 +71,10 @@ lightningd/onchain/gen_onchain_wire.c: $(WIRE_GEN) lightningd/onchain/onchain_wi
 
 LIGHTNINGD_ONCHAIN_OBJS := $(LIGHTNINGD_ONCHAIN_SRC:.c=.o) $(LIGHTNINGD_ONCHAIN_GEN_SRC:.c=.o)
 
-lightningd/lightningd_onchain: $(LIGHTNINGD_ONCHAIN_OBJS) $(WIRE_ONION_OBJS) $(ONCHAIND_COMMON_OBJS) $(WIRE_OBJS) $(BITCOIN_OBJS) $(CCAN_OBJS) $(LIGHTNINGD_HSM_CLIENT_OBJS) $(LIBBASE58_OBJS) libsecp256k1.a libsodium.a libwallycore.a
+# Make sure these depend on everything.
-	$(CC) $(CFLAGS) -o $@ $^ $(LDLIBS)
+ALL_OBJS += $(LIGHTNINGD_ONCHAIN_OBJS)
+
+lightningd/lightningd_onchain: $(LIGHTNINGD_ONCHAIN_OBJS) $(WIRE_ONION_OBJS) $(ONCHAIND_COMMON_OBJS) $(WIRE_OBJS) $(BITCOIN_OBJS)
 
 check-source: $(LIGHTNINGD_ONCHAIN_SRC_NOGEN:%=check-src-include-order/%)
 check-source-bolt: $(LIGHTNINGD_ONCHAIN_SRC:%=bolt-check/%) $(LIGHTNINGD_ONCHAIN_HEADERS:%=bolt-check/%)

lightningd/opening/Makefile

@@ -70,8 +70,10 @@ lightningd/opening/gen_opening_wire.c: $(WIRE_GEN) lightningd/opening/opening_wi
 
 LIGHTNINGD_OPENING_OBJS := $(LIGHTNINGD_OPENING_SRC:.c=.o) $(LIGHTNINGD_OPENING_GEN_SRC:.c=.o)
 
-lightningd/lightningd_opening: $(LIGHTNINGD_OPENING_OBJS) $(OPENINGD_COMMON_OBJS) $(WIRE_OBJS) $(BITCOIN_OBJS) $(CCAN_OBJS) $(LIGHTNINGD_HSM_CLIENT_OBJS) $(LIBBASE58_OBJS) libsecp256k1.a libsodium.a libwallycore.a
+# Make sure these depend on everything.
-	$(CC) $(CFLAGS) -o $@ $^ $(LDLIBS)
+ALL_OBJS += $(LIGHTNINGD_OPENING_OBJS)
+
+lightningd/lightningd_opening: $(LIGHTNINGD_OPENING_OBJS) $(OPENINGD_COMMON_OBJS) $(WIRE_OBJS) $(BITCOIN_OBJS) $(LIGHTNINGD_HSM_CLIENT_OBJS)
 
 check-source: $(LIGHTNINGD_OPENING_SRC_NOGEN:%=check-src-include-order/%)
 check-source-bolt: $(LIGHTNINGD_OPENING_SRC:%=bolt-check/%) $(LIGHTNINGD_OPENING_HEADERS:%=bolt-check/%)

lightningd/test/Makefile

@@ -6,6 +6,8 @@ LIGHTNINGD_TEST_SRC := $(wildcard lightningd/test/run-*.c)
 LIGHTNINGD_TEST_OBJS := $(LIGHTNINGD_TEST_SRC:.c=.o)
 LIGHTNINGD_TEST_PROGRAMS := $(LIGHTNINGD_TEST_OBJS:.o=)
 
+ALL_TEST_PROGRAMS += $(LIGHTNINGD_TEST_PROGRAMS)
+
 LIGHTNINGD_TEST_COMMON_OBJS :=			\
 	common/daemon_conn.o			\
 	common/htlc_state.o			\
@@ -17,7 +19,7 @@ LIGHTNINGD_TEST_COMMON_OBJS :=			\
 
 update-mocks: $(LIGHTNINGD_TEST_SRC:%=update-mocks/%)
 
-$(LIGHTNINGD_TEST_PROGRAMS): $(CCAN_OBJS) $(BITCOIN_OBJS) $(WIRE_OBJS) $(LIBBASE58_OBJS) $(LIGHTNINGD_OLD_LIB_OBJS) $(LIGHTNINGD_TEST_COMMON_OBJS) libsecp256k1.a libsodium.a libwallycore.a
+$(LIGHTNINGD_TEST_PROGRAMS): $(CCAN_OBJS) $(BITCOIN_OBJS) $(WIRE_OBJS) $(LIBBASE58_OBJS) $(LIGHTNINGD_OLD_LIB_OBJS) $(LIGHTNINGD_TEST_COMMON_OBJS)
 
 $(LIGHTNINGD_TEST_OBJS): $(LIGHTNINGD_HEADERS) $(LIGHTNINGD_SRC)
 

test/Makefile

@@ -0,0 +1,19 @@
+TEST_PROGRAMS :=				\
+	test/test_protocol
+
+TEST_COMMON_OBJS :=					\
+	common/utils.o
+
+$(TEST_PROGRAMS): $(TEST_COMMON_OBJS) $(BITCOIN_OBJS) $(WIRE_OBJS)
+
+ALL_TEST_PROGRAMS += $(TEST_PROGRAMS)
+ALL_OBJS += $(TEST_PROGRAMS:=.o)
+
+test-protocol: test/test_protocol
+	@set -e; TMP=`mktemp`; for f in test/commits/*.script; do if ! $(VALGRIND) test/test_protocol < $$f > $$TMP; then echo "test/test_protocol < $$f FAILED" >&2; exit 1; fi; diff -u $$TMP $$f.expected; done; rm $$TMP
+
+check: test-protocol
+clean: test-clean
+
+test-clean:
+	$(RM) $(TEST_PROGRAMS) $(TEST_PROGRAMS:=.o)

wallet/Makefile

@@ -14,23 +14,27 @@ WALLET_LIB_SRC :=		\
 WALLET_LIB_OBJS := $(WALLET_LIB_SRC:.c=.o)
 WALLET_LIB_HEADERS := $(WALLET_LIB_SRC:.c=.h)
 
-$(WALLET_LIB_OBJS): $(CCAN_OBJS)
+WALLET_TEST_COMMON_OBJS :=			\
+	lightningd/log.o			\
+	common/type_to_string.o			\
+	common/pseudorand.o			\
+	common/utils.o
 
 WALLET_TEST_SRC := $(wildcard wallet/*_tests.c)
 WALLET_TEST_OBJS := $(WALLET_TEST_SRC:.c=.o)
 WALLET_TEST_PROGRAMS := $(WALLET_TEST_OBJS:.o=)
 
-$(WALLET_TEST_OBJS): $(WALLET_LIB_OBJS)
+# Make sure these depend on everything.
+ALL_TEST_PROGRAMS += $(WALLET_TEST_PROGRAMS)
+ALL_OBJS += $(WALLET_LIB_OBJS) $(WALLET_TEST_OBJS)
 
-$(WALLET_TEST_PROGRAMS): $(BITCOIN_OBJS) $(CCAN_OBJS) $(LIBBASE58_OBJS) lightningd/log.o common/type_to_string.o common/pseudorand.o common/utils.o libwallycore.a libsecp256k1.a libsodium.a
+$(WALLET_TEST_PROGRAMS): $(BITCOIN_OBJS) $(WALLET_TEST_COMMON_OBJS)
+$(WALLET_TEST_OBJS): $(WALLET_LIB_HEADERS)
 
-$(WALLET_TEST_OBJS): $(CCAN_HEADERS)
 wallet/tests: $(WALLET_TEST_PROGRAMS:%=unittest/%)
 
 check-whitespace: $(WALLET_LIB_SRC:%=check-whitespace/%) $(WALLET_LIB_HEADERS:%=check-whitespace/%)
 
-check-makefile: check-lightningd-makefile
-
 clean: wallet-clean
 
 wallet-clean:

wire/Makefile

@@ -4,7 +4,7 @@
 wire-wrongdir:
 	$(MAKE) -C .. wire-all
 
-WIRE_HEADERS := wire/peer_wire.h		\
+WIRE_HEADERS_NOGEN := wire/peer_wire.h		\
 		wire/wire.h			\
 		wire/wire_sync.h		\
 		wire/wire_io.h
@@ -16,10 +16,12 @@ WIRE_SRC := wire/wire_sync.c			\
 		wire/fromwire.c			\
 		wire/peer_wire.c		\
 		wire/towire.c
+WIRE_HEADERS := $(WIRE_HEADERS_NOGEN) $(WIRE_GEN_HEADERS)
 
 WIRE_OBJS := $(WIRE_SRC:.c=.o) $(WIRE_GEN_SRC:.c=.o)
 WIRE_ONION_OBJS := $(WIRE_GEN_ONION_SRC:.c=.o)
-$(WIRE_OBJS): $(CCAN_HEADERS) $(WIRE_HEADERS) $(WIRE_GEN_HEADERS)
+
+ALL_OBJS += $(WIRE_OBJS) $(WIRE_ONION_OBJS)
 
 # They may not have the bolts.
 BOLT_EXTRACT=$(BOLTDIR)/tools/extract-formats.py
@@ -42,14 +44,11 @@ wire/gen_onion_wire.h: $(WIRE_GEN) wire/gen_onion_wire_csv
 wire/gen_onion_wire.c: $(WIRE_GEN) wire/gen_onion_wire_csv
 	$(WIRE_GEN) ${@:.c=.h} onion_type < wire/gen_onion_wire_csv > $@
 
-wire/gen_peer_wire.o: wire/gen_peer_wire.h $(CCAN_HEADERS)
+check-source: $(WIRE_SRC:%=check-src-include-order/%) $(WIRE_HEADERS_NOGEN:%=check-hdr-include-order/%)
-wire/gen_onion_wire.o: wire/gen_onion_wire.h $(CCAN_HEADERS)
-
-check-source: $(WIRE_SRC:%=check-src-include-order/%) $(WIRE_HEADERS:%=check-hdr-include-order/%)
 
-check-source-bolt: $(WIRE_SRC:%=bolt-check/%) $(WIRE_HEADERS:%=bolt-check/%)
+check-source-bolt: $(WIRE_SRC:%=bolt-check/%) $(WIRE_HEADERS_NOGEN:%=bolt-check/%)
 
-check-whitespace: $(WIRE_SRC:%=check-whitespace/%) $(WIRE_HEADERS:%=check-whitespace/%) check-whitespace/wire/Makefile
+check-whitespace: $(WIRE_SRC:%=check-whitespace/%) $(WIRE_HEADERS_NOGEN:%=check-whitespace/%) check-whitespace/wire/Makefile
 
 clean: wire-clean
 

wire/test/Makefile

@@ -6,11 +6,18 @@ WIRE_TEST_SRC := $(wildcard wire/test/run-*.c)
 WIRE_TEST_OBJS := $(WIRE_TEST_SRC:.c=.o)
 WIRE_TEST_PROGRAMS := $(WIRE_TEST_OBJS:.o=)
 
+WIRE_TEST_COMMON_OBJS :=		\
+	common/utils.o
+
 update-mocks: $(WIRE_TEST_SRC:%=update-mocks/%)
 
-$(WIRE_TEST_PROGRAMS): $(CCAN_OBJS) $(BITCOIN_OBJS) $(LIBBASE58_OBJS) libsecp256k1.a common/utils.o libsodium.a
+$(WIRE_TEST_PROGRAMS): $(WIRE_TEST_COMMON_OBJS) $(BITCOIN_OBJS)
+
+# Test objects require source to be generated, since they include ..
+$(WIRE_TEST_OBJS): $(WIRE_GEN_SRC)
 
-$(WIRE_TEST_OBJS): $(WIRE_HEADERS) $(BITCOIN_HEADERS) $(CORE_HEADERS) $(GEN_HEADERS) $(WIRE_GEN_HEADERS) $(WIRE_GEN_SRC) $(CCAN_HEADERS) $(LIBSODIUM_HEADERS)
+ALL_OBJS += $(WIRE_TEST_OBJS)
+ALL_TEST_PROGRAMS += $(WIRE_TEST_PROGRAMS)
 
 wire-tests: $(WIRE_TEST_PROGRAMS:%=unittest/%)