Merge branch 'Kukks-feature/traefik'

.gitignore

@@ -297,3 +297,8 @@ Production/.env
 .vscode/
 *docker-compose.generated.yml
 
+Generated/acme.json
+Generated/traefik_logs/
+Generated/error
+
+

Generated/.gitignore

@@ -1,2 +1,4 @@
 *.yml
 *.tmpl
+*.toml
+*.json

Production/README.md

@@ -1,6 +1,6 @@
 # How to use docker-compose with NGinx
 
-NGinx acts as a reverse proxy, and take care of renewing HTTPS certificates for you.
+NGinx acts as a reverse proxy, and takes care of renewing HTTPS certificates for you.
 BTCPay Server deployment using NGinx are typically composed of:
 
 1. One full node per supported cryptocurrency (bitcoind/litecoind)
@@ -27,8 +27,6 @@ The relevant environment variables are:
 
 If `BTCPAY_HOST` is `btcpay.example.com` and `BTCPAY_ROOTPATH` is `/btcpay`, then you can access the site via `https://btcpay.example.com/btcpay`
 
-Use `docker-compose.btc-ltc.yml` for bitcoin and litecoin support, or `docker-compose.btc.yml` for only bitcoin.
-
 Any unset or empty environment variable will be set for a `regtest` deployment.
 
 The ports mapped on the host are:

README.md

@@ -115,7 +115,7 @@ You can read [the article](https://medium.com/@BtcpayServer/hosting-btcpay-serve
 * `BTCPAYGEN_CRYPTO1`: First supported crypto currency (eg. `btc`, `ltc`. Default: `btc`)
 * `BTCPAYGEN_CRYPTO2`: Second supported crypto currency (eg. `btc`, `ltc`. Default: `(empty)`)
 * `BTCPAYGEN_CRYPTON`: N'th supported crypto currency where N is 9 at maximum. (eg. `btc`, `ltc`. Default: `(empty)`)
-* `BTCPAYGEN_REVERSEPROXY`: Specify reverse proxy to use; NGinx has HTTPS support. (eg. `nginx`, `(empty)`. Default: `nginx`)
+* `BTCPAYGEN_REVERSEPROXY`: Specify reverse proxy to use; NGinx has HTTPS support. (eg. `nginx`, `traefik`,  `(empty)`. Default: `nginx`)
 * `BTCPAYGEN_LIGHTNING`: Lightning network implementation to use (eg. `clightning`, `(empty)`)
 * `BTCPAYGEN_SUBNAME`: The subname of the generated docker-compose file, where the full name is `Generated/docker-compose.SUBNAME.yml` (Default: `generated`)
 * `BTCPAYGEN_ADDITIONAL_FRAGMENTS`: Semicolon-separated list of additional fragments you want to use (eg. `opt-save-storage`)
@@ -123,6 +123,7 @@ You can read [the article](https://medium.com/@BtcpayServer/hosting-btcpay-serve
 * `ACME_CA_URI`: The API endpoint to ask for HTTPS certificate (Default: `https://acme-v01.api.letsencrypt.org/directory`)
 * `BTCPAY_HOST_SSHKEYFILE`: Optional, SSH private key that BTCPay can use to connect to this VM's SSH server. This key will be copied to BTCPay's data directory
 * `BTCPAY_SSHTRUSTEDFINGERPRINTS`: Optional, BTCPay will ensure that it is connecting to the expected SSH server by checking the host's public key against these fingerprints
+* `BTCPAYGEN_DOCKER_IMAGE`: Optional, Specify which generator image to use if you have customized the C# generator. Set to `btcpayserver/docker-compose-generator:local` to build the generator locally at runtime.
 
 # Tooling
 

Traefik/README.md

@@ -0,0 +1,17 @@
+# How to use docker-compose with Traefik
+
+Traefik is a modern reverse proxy aimed towards applications running through container orchestrators. 
+
+Some of the benefits of using Traefik over NGinx are:
+* Real-time configuration changes - no need to reload the proxy
+* Auto discovery and configuration of services through a vast amount of container orchestrators.
+* Built-in official support for Let's Encrypt SSL with certificate auto-renewal
+
+## Traefik Specific Environment Variables
+
+* `BTCPAYGEN_REVERSEPROXY` to `traefik`.
+* `LETSENCRYPT_EMAIL`: Optional, The email Let's Encrypt will use to notify you about certificate expiration.
+* `BTCPAYGEN_ADDITIONAL_FRAGMENTS`: In the case that you have an already deployed traefik container, you can use the fragment `traefik-labels` which will tag the btcpayserver service with the needed labels to be discovered.
+
+
+![Architecture](Production.png)

Traefik/traefik.toml

@@ -0,0 +1,34 @@
+defaultEntryPoints = ["https","http"]
+
+logLevel = "ERROR"
+
+[entryPoints]
+  [entryPoints.http]
+  address = ":80"
+    [entryPoints.http.redirect]
+    entryPoint = "https"
+  [entryPoints.https]
+  address = ":443"
+  [entryPoints.https.tls]
+
+[retry]
+
+[docker]
+endpoint = "unix:///var/run/docker.sock"
+watch = true
+exposedByDefault = false
+
+[acme]
+storage = "acme.json"
+entryPoint = "https"
+onHostRule = true
+[acme.httpChallenge]
+entryPoint = "http"
+
+[traefikLog]
+  filePath = "/traefik_logs/traefik.log"
+  format   = "json"
+
+[accessLog]
+  filePath = "/traefik_logs/access.log"
+  format = "json"

btcpay-setup.sh

@@ -55,12 +55,12 @@ Environment variables:
     BTCPAYGEN_CRYPTO1: First supported crypto currency (eg. btc, ltc, btg, grs, ftc, via, none. Default: btc)
     BTCPAYGEN_CRYPTO2: Second supported crypto currency (Default: empty)
     BTCPAYGEN_CRYPTON: N th supported crypto currency where N is maximum at maximum 9. (Default: none)
-    BTCPAYGEN_REVERSEPROXY: Whether to use or not a reverse proxy. NGinx setup HTTPS for you. (eg. nginx, none. Default: nginx)
+    BTCPAYGEN_REVERSEPROXY: Whether to use or not a reverse proxy. NGinx setup HTTPS for you. (eg. nginx, traefik, none. Default: nginx)
     BTCPAYGEN_LIGHTNING: Lightning network implementation to use (eg. clightning, lnd, none)
     BTCPAYGEN_ADDITIONAL_FRAGMENTS: Semi colon separated list of additional fragments you want to use (eg. opt-save-storage)
     ACME_CA_URI: The API endpoint to ask for HTTPS certificate (default: https://acme-v01.api.letsencrypt.org/directory)
     BTCPAY_HOST_SSHKEYFILE: Optional, SSH private key that BTCPay can use to connect to this VM's SSH server. This key will be copied on BTCPay's data directory
-
+    BTCPAYGEN_DOCKER_IMAGE: Allows you to specify a custom docker image for the generator (Default: btcpayserver/docker-compose-generator)
 END
 }
 

build.ps1

@@ -28,3 +28,9 @@ docker run -v "$(Get-Location)\Generated:/app/Generated" `
 If ($BTCPAYGEN_REVERSEPROXY -eq "nginx") {
     Copy-Item ".\Production\nginx.tmpl" -Destination ".\Generated"
 }
+
+If ($BTCPAYGEN_REVERSEPROXY -eq "traefik") {
+    Copy-Item ".\Traefik\traefik.toml" -Destination ".\Generated"
+    
+    New-Item  ".\Generated\acme.json" -type file
+}

build.sh

@@ -29,3 +29,9 @@ docker run -v "$(pwd)/Generated:/app/Generated" \
 if [ "$BTCPAYGEN_REVERSEPROXY" == "nginx" ]; then
     cp Production/nginx.tmpl Generated/nginx.tmpl
 fi
+
+if [ "$BTCPAYGEN_REVERSEPROXY" == "traefik" ]; then
+    cp Traefik/traefik.toml Generated/traefik.toml
+    :> Generated/acme.json
+    chmod 600 Generated/acme.json
+fi

docker-compose-generator/docker-fragments/btcpayserver-nginx.yml

@@ -0,0 +1,15 @@
+version: "3"
+
+services:
+
+  btcpayserver:
+    environment:
+      # NGINX settings
+      VIRTUAL_NETWORK: nginx-proxy
+      VIRTUAL_PORT: 49392
+      VIRTUAL_HOST: ${BTCPAY_HOST}
+      SSL_POLICY: Mozilla-Modern 
+
+      # Let's encrypt settings
+      LETSENCRYPT_HOST: ${BTCPAY_HOST}
+      LETSENCRYPT_EMAIL: ${LETSENCRYPT_EMAIL:-<no value>}

docker-compose-generator/docker-fragments/btcpayserver.yml

@@ -17,16 +17,6 @@ services:
       BTCPAY_SSHTRUSTEDFINGERPRINTS: ${BTCPAY_SSHTRUSTEDFINGERPRINTS}
       BTCPAY_SSHKEYFILE: ${BTCPAY_SSHKEYFILE}
       
-      # NGINX settings
-      VIRTUAL_NETWORK: nginx-proxy
-      VIRTUAL_PORT: 49392
-      VIRTUAL_HOST: ${BTCPAY_HOST}
-      SSL_POLICY: Mozilla-Modern 
-
-      # Let's encrypt settings
-      LETSENCRYPT_HOST: ${BTCPAY_HOST}
-      LETSENCRYPT_EMAIL: ${LETSENCRYPT_EMAIL:-<no value>}
-      
     links:
       - nbxplorer
       - postgres
@@ -34,24 +24,5 @@ services:
      - "btcpay_datadir:/datadir"
      - "nbxplorer_datadir:/root/.nbxplorer"
 
-  nbxplorer:
-    restart: unless-stopped
-    image: nicolasdorier/nbxplorer:1.0.2.31
-    expose: 
-      - "32838"
-    environment:
-      NBXPLORER_NETWORK: ${NBITCOIN_NETWORK:-regtest}
-      NBXPLORER_BIND: 0.0.0.0:32838
-    volumes:
-      - "nbxplorer_datadir:/datadir"
-
-  postgres:
-    restart: unless-stopped
-    image:  postgres:9.6.5
-    volumes:
-      - "postgres_datadir:/var/lib/postgresql/data"
-
 volumes:
-    postgres_datadir:
     btcpay_datadir:
-    nbxplorer_datadir:

docker-compose-generator/docker-fragments/nbxplorer.yml

@@ -0,0 +1,17 @@
+version: "3"
+
+services:
+
+  nbxplorer:
+    restart: unless-stopped
+    image: nicolasdorier/nbxplorer:1.0.2.31
+    expose: 
+      - "32838"
+    environment:
+      NBXPLORER_NETWORK: ${NBITCOIN_NETWORK:-regtest}
+      NBXPLORER_BIND: 0.0.0.0:32838
+    volumes:
+      - "nbxplorer_datadir:/datadir"
+
+volumes:
+    nbxplorer_datadir:

docker-compose-generator/docker-fragments/postgres.yml

@@ -0,0 +1,11 @@
+version: "3"
+
+services:
+  postgres:
+    restart: unless-stopped
+    image:  postgres:9.6.5
+    volumes:
+      - "postgres_datadir:/var/lib/postgresql/data"
+
+volumes:
+    postgres_datadir:

docker-compose-generator/docker-fragments/traefik-labels.yml

@@ -0,0 +1,12 @@
+version: "3"
+
+services:
+  btcpayserver:
+    labels:
+      - "traefik.backend=btcpayserver"
+      - "traefik.backend.loadbalancer.sticky=true"
+      - "traefik.enable=true"
+      - "traefik.frontend.rule=Host:${BTCPAY_HOST}"
+      - "traefik.port.rule=49392"
+      - "traefik.acme.domains=${BTCPAY_HOST},www.${BTCPAY_HOST}"
+      - "traefik.acme.email=${LETSENCRYPT_EMAIL}"

docker-compose-generator/docker-fragments/traefik.yml

@@ -0,0 +1,22 @@
+version: "3"
+
+services:
+  traefik:
+    restart: unless-stopped
+    image: traefik
+    container_name: traefik
+    ports:
+      - "80:80"
+      - "443:443"
+    volumes:
+      - "/var/run/docker.sock:/var/run/docker.sock"
+      - "./traefik.toml:/traefik.toml"
+      - "./acme.json:/acme.json:ro"
+      - "./servers.toml:/servers.toml"
+      - "./traefik_logs:/traefik_logs"
+      
+    links:
+      - btcpayserver
+
+volumes:
+  traefik_logs:

docker-compose-generator/src/DockerComposeDefinition.cs

@@ -8,146 +8,160 @@ using System.IO;
 
 namespace DockerGenerator
 {
-    public class DockerComposeDefinition
-    {
-        public List<string> Fragments
-        {
-            get; set;
-        }
-        private string _Name;
+	public class DockerComposeDefinition
+	{
+		public List<string> Fragments
+		{
+			get; set;
+		}
+		private string _Name;
 
-        public DockerComposeDefinition(string name, List<string> fragments)
-        {
-            Fragments = fragments;
-            _Name = name;
-        }
+		public DockerComposeDefinition(string name, List<string> fragments)
+		{
+			Fragments = fragments;
+			_Name = name;
+		}
 
-        public string FragmentLocation
-        {
-            get; set;
-        }
-        public string BuildOutputDirectory
-        {
-            get; set;
-        }
+		public string FragmentLocation
+		{
+			get; set;
+		}
+		public string BuildOutputDirectory
+		{
+			get; set;
+		}
 
-        public string GetFilePath()
-        {
-            return Path.Combine(BuildOutputDirectory, $"docker-compose.{_Name}.yml");
-        }
-        public void Build()
-        {
-            Console.WriteLine($"Generating {GetFilePath()}");
-            var deserializer = new DeserializerBuilder().Build();
-            var serializer = new SerializerBuilder().Build();
+		public string GetFilePath()
+		{
+			return Path.Combine(BuildOutputDirectory, $"docker-compose.{_Name}.yml");
+		}
+		public void Build()
+		{
+			Console.WriteLine($"Generating {GetFilePath()}");
+			var deserializer = new DeserializerBuilder().Build();
+			var serializer = new SerializerBuilder().Build();
 
-            Console.WriteLine($"With fragments:");
-            foreach(var fragment in Fragments)
-            {
-                Console.WriteLine($"\t{fragment}");
-            }
-            var services = new List<KeyValuePair<YamlNode, YamlNode>>();
-            var volumes = new List<KeyValuePair<YamlNode, YamlNode>>();
+			Console.WriteLine($"With fragments:");
+			foreach (var fragment in Fragments.ToList())
+			{
+				var fragmentPath = GetFragmentLocation(fragment);
+				if (!File.Exists(fragmentPath))
+				{
+					Console.WriteLine($"\t{fragment} not found in {fragmentPath}, ignoring...");
+					Fragments.Remove(fragment);
+				}
+				else
+				{
+					Console.WriteLine($"\t{fragment}");
+				}
+			}
+			var services = new List<KeyValuePair<YamlNode, YamlNode>>();
+			var volumes = new List<KeyValuePair<YamlNode, YamlNode>>();
 
-            foreach(var doc in Fragments.Select(f => ParseDocument(f)))
-            {
-                if(doc.Children.ContainsKey("services") && doc.Children["services"] is YamlMappingNode fragmentServicesRoot)
-                {
-                    services.AddRange(fragmentServicesRoot.Children);
-                }
+			foreach (var doc in Fragments.Select(f => ParseDocument(f)))
+			{
+				if (doc.Children.ContainsKey("services") && doc.Children["services"] is YamlMappingNode fragmentServicesRoot)
+				{
+					services.AddRange(fragmentServicesRoot.Children);
+				}
 
-                if(doc.Children.ContainsKey("volumes") && doc.Children["volumes"] is YamlMappingNode fragmentVolumesRoot)
-                {
-                    volumes.AddRange(fragmentVolumesRoot.Children);
-                }
-            }
+				if (doc.Children.ContainsKey("volumes") && doc.Children["volumes"] is YamlMappingNode fragmentVolumesRoot)
+				{
+					volumes.AddRange(fragmentVolumesRoot.Children);
+				}
+			}
 
 
-            YamlMappingNode output = new YamlMappingNode();
-            output.Add("version", new YamlScalarNode("3") { Style = YamlDotNet.Core.ScalarStyle.DoubleQuoted });
-            output.Add("services", new YamlMappingNode(Merge(services)));
-            output.Add("volumes", new YamlMappingNode(volumes));
-            var result = serializer.Serialize(output);
-            var outputFile = GetFilePath();
-            File.WriteAllText(outputFile, result.Replace("''", ""));
-            Console.WriteLine($"Generated {outputFile}");
-            Console.WriteLine();
-        }
+			YamlMappingNode output = new YamlMappingNode();
+			output.Add("version", new YamlScalarNode("3") { Style = YamlDotNet.Core.ScalarStyle.DoubleQuoted });
+			output.Add("services", new YamlMappingNode(Merge(services)));
+			output.Add("volumes", new YamlMappingNode(volumes));
+			var result = serializer.Serialize(output);
+			var outputFile = GetFilePath();
+			File.WriteAllText(outputFile, result.Replace("''", ""));
+			Console.WriteLine($"Generated {outputFile}");
+			Console.WriteLine();
+		}
 
-        private KeyValuePair<YamlNode, YamlNode>[] Merge(List<KeyValuePair<YamlNode, YamlNode>> services)
-        {
-            return services
-                .GroupBy(s => s.Key.ToString(), s => s.Value)
-                .Select(group =>
-                    (GroupName: group.Key,
-                     MainNode: group.OfType<YamlMappingNode>().SingleOrDefault(n => n.Children.ContainsKey("image")),
-                     MergedNodes: group.OfType<YamlMappingNode>().Where(n => !n.Children.ContainsKey("image"))))
-                .Where(_ => _.MainNode != null)
-                .Select(_ =>
-                {
-                    foreach(var node in _.MergedNodes)
-                    {
-                        foreach(var child in node)
-                        {
-                            var childValue = child.Value;
-                            if(!_.MainNode.Children.TryGetValue(child.Key, out var mainChildValue))
-                            {
-                                mainChildValue = child.Value;
-                                _.MainNode.Add(child.Key, child.Value);
-                            }
-                            else if(childValue is YamlMappingNode childMapping && mainChildValue is YamlMappingNode mainChildMapping)
-                            {
-                                foreach(var leaf in childMapping)
-                                {
-                                    if(mainChildMapping.Children.TryGetValue(leaf.Key, out var mainLeaf))
-                                    {
-                                        if(leaf.Value is YamlScalarNode leafScalar && mainLeaf is YamlScalarNode leafMainScalar)
-                                        {
-                                            var eof = EOF(leafMainScalar.Value) ?? EOF(leaf.Value.ToString());
-                                            if(eof != null)
-                                            {
-                                                leafMainScalar.Value = leafMainScalar.Value + eof + leaf.Value;
-                                            }
-                                            else
-                                            {
-                                                leafMainScalar.Value = leafMainScalar.Value + "," + leaf.Value;
-                                            }
-                                        }
-                                    }
-                                    else
-                                    {
-                                        mainChildMapping.Add(leaf.Key, leaf.Value);
-                                    }
-                                }
-                            }
-                            else if(childValue is YamlSequenceNode childSequence && mainChildValue is YamlSequenceNode mainSequence)
-                            {
-                                foreach(var c in childSequence.Children)
-                                {
-                                    mainSequence.Add(c);
-                                }
-                            }
-                        }
-                    }
-                    return new KeyValuePair<YamlNode, YamlNode>(_.GroupName, _.MainNode);
-                }).ToArray();
-        }
+		private KeyValuePair<YamlNode, YamlNode>[] Merge(List<KeyValuePair<YamlNode, YamlNode>> services)
+		{
+			return services
+				.GroupBy(s => s.Key.ToString(), s => s.Value)
+				.Select(group =>
+					(GroupName: group.Key,
+					 MainNode: group.OfType<YamlMappingNode>().SingleOrDefault(n => n.Children.ContainsKey("image")),
+					 MergedNodes: group.OfType<YamlMappingNode>().Where(n => !n.Children.ContainsKey("image"))))
+				.Where(_ => _.MainNode != null)
+				.Select(_ =>
+				{
+					foreach (var node in _.MergedNodes)
+					{
+						foreach (var child in node)
+						{
+							var childValue = child.Value;
+							if (!_.MainNode.Children.TryGetValue(child.Key, out var mainChildValue))
+							{
+								mainChildValue = child.Value;
+								_.MainNode.Add(child.Key, child.Value);
+							}
+							else if (childValue is YamlMappingNode childMapping && mainChildValue is YamlMappingNode mainChildMapping)
+							{
+								foreach (var leaf in childMapping)
+								{
+									if (mainChildMapping.Children.TryGetValue(leaf.Key, out var mainLeaf))
+									{
+										if (leaf.Value is YamlScalarNode leafScalar && mainLeaf is YamlScalarNode leafMainScalar)
+										{
+											var eof = EOF(leafMainScalar.Value) ?? EOF(leaf.Value.ToString());
+											if (eof != null)
+											{
+												leafMainScalar.Value = leafMainScalar.Value + eof + leaf.Value;
+											}
+											else
+											{
+												leafMainScalar.Value = leafMainScalar.Value + "," + leaf.Value;
+											}
+										}
+									}
+									else
+									{
+										mainChildMapping.Add(leaf.Key, leaf.Value);
+									}
+								}
+							}
+							else if (childValue is YamlSequenceNode childSequence && mainChildValue is YamlSequenceNode mainSequence)
+							{
+								foreach (var c in childSequence.Children)
+								{
+									mainSequence.Add(c);
+								}
+							}
+						}
+					}
+					return new KeyValuePair<YamlNode, YamlNode>(_.GroupName, _.MainNode);
+				}).ToArray();
+		}
 
-        private string EOF(string value)
-        {
-            if(value.Contains("\r\n", StringComparison.OrdinalIgnoreCase))
-                return "\r\n";
-            if(value.Contains("\n", StringComparison.OrdinalIgnoreCase))
-                return "\n";
-            return null;
-        }
+		private string EOF(string value)
+		{
+			if (value.Contains("\r\n", StringComparison.OrdinalIgnoreCase))
+				return "\r\n";
+			if (value.Contains("\n", StringComparison.OrdinalIgnoreCase))
+				return "\n";
+			return null;
+		}
 
-        private YamlMappingNode ParseDocument(string fragment)
-        {
-            var input = new StringReader(File.ReadAllText(Path.Combine(FragmentLocation, $"{fragment}.yml")));
-            YamlStream stream = new YamlStream();
-            stream.Load(input);
-            return (YamlMappingNode)stream.Documents[0].RootNode;
-        }
-    }
+		private YamlMappingNode ParseDocument(string fragment)
+		{
+			var input = new StringReader(File.ReadAllText(GetFragmentLocation(fragment)));
+			YamlStream stream = new YamlStream();
+			stream.Load(input);
+			return (YamlMappingNode)stream.Documents[0].RootNode;
+		}
+
+		private string GetFragmentLocation(string fragment)
+		{
+			return Path.Combine(FragmentLocation, $"{fragment}.yml");
+		}
+	}
 }

docker-compose-generator/src/Program.cs

@@ -32,15 +32,24 @@ namespace DockerGenerator
 			fragmentLocation = Path.GetFullPath(Path.Combine(fragmentLocation, "docker-fragments"));
 
 			var fragments = new List<string>();
-			if (composition.SelectedProxy == "nginx")
+			switch (composition.SelectedProxy)
 			{
-				fragments.Add("nginx");
-			}
-			else
-			{
-				fragments.Add("btcpayserver-noreverseproxy");
+				case "nginx":
+
+					fragments.Add("nginx");
+					fragments.Add("btcpayserver-nginx");
+					break;
+				case "traefik":
+					fragments.Add("traefik");
+					fragments.Add("traefik-labels");
+					break;
+				case "no-reverseproxy":
+					fragments.Add("btcpayserver-noreverseproxy");
+					break;
 			}
 			fragments.Add("btcpayserver");
+			fragments.Add("nbxplorer");
+			fragments.Add("postgres");
 			foreach (var crypto in CryptoDefinition.GetDefinitions())
 			{
 				if (!composition.SelectedCryptos.Contains(crypto.Crypto))

docker-compose-generator/src/docker-compose-generator.csproj

@@ -9,5 +9,4 @@
   <ItemGroup>
 	  <PackageReference Include="YamlDotNet" Version="4.3.1" />
   </ItemGroup>
-
 </Project>