From 76dd1577bdc073477c0bfc6911f5420a5a3edd2a Mon Sep 17 00:00:00 2001
From: Rusty Russell <rusty@rustcorp.com.au>
Date: Wed, 6 Jun 2018 13:03:27 +0930
Subject: [PATCH] external: fix submodule handling for parallel builds,
 submodule URL changes.

If we change an upstream URL, all submodules break.  Users would need
to run 'git submodule sync'.  Note that the libbacktrace fix was merged
upstream so this is no longer necessary, but it's good for future changes.

Also, stress-testing reveals that git submodule fails locking
'.git/config' when run in paralell.  It also segfaults and other
problems.

This is my final attempt to fix submodules; I've wasted far too many
days on obscure problems it creates: I've already lost one copy of my
repo to apparently unfixable submodule preoblems.  The next "fix" will
be to simply import the source code so it works properly.

Reported-by: @jsarenik
Fixes: #1543
Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
---
 external/Makefile           | 22 ++++++++++++++--------
 tools/refresh-submodules.sh | 30 ++++++++++++++++++++++++++++++
 2 files changed, 44 insertions(+), 8 deletions(-)
 create mode 100755 tools/refresh-submodules.sh

diff --git a/external/Makefile b/external/Makefile
index 48bca5479..71b9d81ef 100644
--- a/external/Makefile
+++ b/external/Makefile
@@ -1,3 +1,10 @@
+SUBMODULES =					\
+	external/libsodium			\
+	external/libwally-core			\
+	external/jsmn				\
+	external/libbase58			\
+	external/libbacktrace
+
 LIBSODIUM_HEADERS := external/libsodium/src/libsodium/include/sodium.h
 LIBWALLY_HEADERS := external/libwally-core/include/wally_bip32.h		\
 			external/libwally-core/include/wally_core.h	\
@@ -21,20 +28,19 @@ EXTERNAL_INCLUDE_FLAGS :=					\
 
 EXTERNAL_LDLIBS := -Lexternal $(patsubst lib%.a,-l%,$(notdir $(EXTERNAL_LIBS)))
 
-# Might exist, but need updating.  Nuke and rebuild.
-submodcheck-%: FORCE
-	if git submodule status external/$* | grep -q '^[-+]'; then rm -rf external/$*; git submodule update --init external/$*; fi
+submodcheck: FORCE
+	@tools/refresh-submodules.sh $(SUBMODULES)
 
 # We build libsodium, since Ubuntu xenial has one too old.
 external/libsodium.a: external/libsodium/src/libsodium/libsodium.la
 	$(MAKE) -C external/libsodium DESTDIR=$$(pwd)/external install-exec
 
-external/libsodium/src/libsodium/include/sodium.h: submodcheck-libsodium
+external/libsodium/src/libsodium/include/sodium.h: submodcheck
 
 external/libsodium/src/libsodium/libsodium.la: external/libsodium/src/libsodium/include/sodium.h
 	cd external/libsodium && ./autogen.sh && ./configure CC="$(CC)" --enable-static=yes --host="$(MAKE_HOST)" --build="$(BUILD)" --enable-shared=no --enable-tests=no --prefix=/ --libdir=/ && $(MAKE)
 
-$(LIBWALLY_HEADERS) $(LIBSECP_HEADERS): submodcheck-libwally-core
+$(LIBWALLY_HEADERS) $(LIBSECP_HEADERS): submodcheck
 
 # libsecp included in libwally.
 # Wildcards here are magic.  See http://stackoverflow.com/questions/2973445/gnu-makefile-rule-generating-a-few-targets-from-a-single-source-file
@@ -44,7 +50,7 @@ external/libsecp256k1.% external/libwallycore.%: external/libwally-core/src/secp
 external/libwally-core/src/libwallycore.% external/libwally-core/src/secp256k1/libsecp256k1.%: $(LIBWALLY_HEADERS) $(LIBSECP_HEADERS)
 	cd external/libwally-core && ./tools/autogen.sh && ./configure CC="$(CC)" --enable-static=yes --host="$(MAKE_HOST)" --build="$(BUILD)" --enable-module-recovery --enable-shared=no --prefix=/ --libdir=/ && $(MAKE)
 
-external/jsmn/jsmn.h: submodcheck-jsmn
+external/jsmn/jsmn.h: submodcheck
 
 # If we tell Make that the above builds both, it runs it twice in
 # parallel.  So we lie :(
@@ -61,7 +67,7 @@ LIBBASE58_SRC := external/libbase58/base58.c
 
 $(LIBBASE58_SRC): $(LIBBASE58_HEADERS)
 
-$(LIBBASE58_HEADERS): submodcheck-libbase58
+$(LIBBASE58_HEADERS): submodcheck
 
 # Can't be inside submodule, as that makes git think it's dirty.
 external/base58.o: $(LIBBASE58_SRC) Makefile
@@ -70,7 +76,7 @@ external/base58.o: $(LIBBASE58_SRC) Makefile
 external/libbase58.a: external/base58.o
 	$(AR) rc $@ $<
 
-external/libbacktrace/backtrace.h: submodcheck-libbacktrace
+external/libbacktrace/backtrace.h: submodcheck
 
 # Need separate build dir: changes inside submodule make git think it's dirty.
 external/libbacktrace.a: external/libbacktrace/backtrace.h
diff --git a/tools/refresh-submodules.sh b/tools/refresh-submodules.sh
new file mode 100755
index 000000000..c34784c32
--- /dev/null
+++ b/tools/refresh-submodules.sh
@@ -0,0 +1,30 @@
+#! /bin/sh
+
+if [ $# = 0 ]; then
+    echo "Usage: $0 <submoduledir1>..." >&2
+    exit 1
+fi
+
+# git submodule can't run in parallel.  Really.
+echo $$ > .refresh-submodules.$$
+if ! mv -n .refresh-submodules.$$ .refresh-submodules; then
+    rm -f .refresh-submodules.$$
+    exit 0
+fi
+trap "rm -f .refresh-submodules" EXIT
+
+# Be a little careful here, since we do rm -rf!
+for m in "$@"; do
+    if ! grep -q "path = $m\$" .gitmodules; then
+	echo "$m is not a submodule!" >&2
+	exit 1
+    fi
+done
+
+# git submodule can segfault.  Really.
+if [ "$(git submodule status "$@" | grep -c '^ ')" != $# ]; then
+    echo Reinitializing submodules "$@" ...
+    git submodule sync "$@"
+    rm -rf "$@"
+    git submodule update --init "$@"
+fi