Merge branch 'new/generator'

build.sh

@@ -30,7 +30,7 @@ docker run -v "$(pwd)/Generated:/app/Generated" \
            -e "BTCPAYGEN_LIGHTNING=$BTCPAYGEN_LIGHTNING" \
            -e "BTCPAYGEN_SUBNAME=$BTCPAYGEN_SUBNAME" \
            -e "BTCPAY_HOST_SSHAUTHORIZEDKEYS=$BTCPAY_HOST_SSHAUTHORIZEDKEYS" \
-           --rm $BTCPAYGEN_DOCKER_IMAGE
+           --rm $BTCPAYGEN_DOCKER_IMAGE || echo "Building the fragment failed"
 
 if [ "$BTCPAYGEN_REVERSEPROXY" == "nginx" ]; then
     cp Production/nginx.tmpl Generated/nginx.tmpl

docker-compose-generator/docker-fragments/btcpayserver-noreverseproxy.yml

@@ -4,3 +4,6 @@ services:
   btcpayserver:
       ports:
       - "80:49392"
+
+exclusive:
+  - proxy

docker-compose-generator/docker-fragments/btcpayserver.yml

@@ -26,4 +26,11 @@ services:
      - "$<BTCPAY_HOST_SSHAUTHORIZEDKEYS>?:${BTCPAY_SSHAUTHORIZEDKEYS}"
 
 volumes:
-    btcpay_datadir:
+  btcpay_datadir:
+
+required:
+  - "postgres"
+  - "nbxplorer"
+  - "btcpayserver"
+recommended:
+  - "opt-add-tor"

docker-compose-generator/docker-fragments/nginx.yml

@@ -34,4 +34,11 @@ volumes:
     nginx_conf:
     nginx_vhost:
     nginx_html:
-    nginx_certs:
+    nginx_certs:
+
+exclusive:
+  - proxy
+required:
+  - "btcpayserver-nginx"
+recommended:
+  - "nginx-https"

docker-compose-generator/docker-fragments/opt-more-memory.yml

@@ -30,3 +30,5 @@ services:
     environment:
       BITCOIN_EXTRA_ARGS: |
         dbcache=1024
+exclusive:
+  - pruning

docker-compose-generator/docker-fragments/opt-save-memory.yml

@@ -37,3 +37,5 @@ services:
       BITCOIN_EXTRA_ARGS: |
         dbcache=50
         maxmempool=50
+exclusive:
+  - pruning

docker-compose-generator/docker-fragments/opt-save-storage-s.yml

@@ -30,3 +30,5 @@ services:
   monacoind:
     environment:
       BITCOIN_EXTRA_ARGS: prune=50000
+exclusive:
+  - pruning

docker-compose-generator/docker-fragments/opt-save-storage-xs.yml

@@ -30,3 +30,5 @@ services:
   monacoind:
     environment:
       BITCOIN_EXTRA_ARGS: prune=25000
+exclusive:
+  - pruning

docker-compose-generator/docker-fragments/opt-save-storage-xxs.yml

@@ -30,3 +30,5 @@ services:
   monacoind:
     environment:
       BITCOIN_EXTRA_ARGS: prune=5000
+exclusive:
+  - pruning

docker-compose-generator/docker-fragments/opt-save-storage.yml

@@ -30,3 +30,5 @@ services:
   monacoind:
     environment:
       BITCOIN_EXTRA_ARGS: prune=100000
+exclusive:
+  - pruning

docker-compose-generator/docker-fragments/opt-txindex.yml

@@ -30,3 +30,5 @@ services:
     environment:
       BITCOIN_EXTRA_ARGS: |
         txindex=1
+exclusive:
+  - pruning

docker-compose-generator/docker-fragments/traefik.yml

@@ -19,4 +19,9 @@ services:
       - btcpayserver
 
 volumes:
-  traefik_logs:
+  traefik_logs:
+
+exclusive:
+  - proxy
+required:
+  - "traefik"

docker-compose-generator/src/ConsoleUtils.cs

@@ -0,0 +1,17 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace DockerGenerator
+{
+	public static class ConsoleUtils
+	{
+		public static void WriteLine(string message, ConsoleColor color)
+		{
+			var old = Console.ForegroundColor;
+			Console.ForegroundColor = color;
+			Console.WriteLine(message);
+			Console.ForegroundColor = old;
+		}
+	}
+}

docker-compose-generator/src/DockerComposeDefinition.cs

@@ -10,13 +10,13 @@ namespace DockerGenerator
 {
 	public class DockerComposeDefinition
 	{
-		public HashSet<string> Fragments
+		public HashSet<FragmentName> Fragments
 		{
 			get; set;
 		}
 		private string _Name;
 
-		public DockerComposeDefinition(string name, HashSet<string> fragments)
+		public DockerComposeDefinition(string name, HashSet<FragmentName> fragments)
 		{
 			Fragments = fragments;
 			_Name = name;
@@ -30,6 +30,7 @@ namespace DockerGenerator
 		{
 			get; set;
 		}
+		public HashSet<FragmentName> ExcludeFragments { get; internal set; }
 
 		public string GetFilePath()
 		{
@@ -44,28 +45,33 @@ namespace DockerGenerator
 			Console.WriteLine($"Generating {GetFilePath()}");
 			var deserializer = new DeserializerBuilder().Build();
 			var serializer = new SerializerBuilder().Build();
+			var fragmentsNotFound = new HashSet<FragmentName>();
+			var requiredFragments = new HashSet<FragmentName>();
+			var recommendedFragments = new HashSet<FragmentName>();
+			var processedFragments = new HashSet<FragmentName>();
+			var unprocessedFragments = new HashSet<FragmentName>();
+			var services = new List<KeyValuePair<YamlNode, YamlNode>>();
+			var volumes = new List<KeyValuePair<YamlNode, YamlNode>>();
+			var networks = new List<KeyValuePair<YamlNode, YamlNode>>();
+			var exclusives = new List<(FragmentName FragmentName, string Exclusivity)>();
 
-			Console.WriteLine($"With fragments:");
-			foreach (var fragment in Fragments.ToList())
+			foreach (var fragment in Fragments.Where(NotExcluded))
+			{
+				unprocessedFragments.Add(fragment);
+			}
+		reprocessFragments:
+			foreach (var fragment in unprocessedFragments)
 			{
 				var fragmentPath = GetFragmentLocation(fragment);
 				if (!File.Exists(fragmentPath))
 				{
-					Console.WriteLine($"\t{fragment} not found in {fragmentPath}, ignoring...");
-					Fragments.Remove(fragment);
-				}
-				else
-				{
-					Console.WriteLine($"\t{fragment}");
+					fragmentsNotFound.Add(fragment);
 				}
 			}
-
-			var services = new List<KeyValuePair<YamlNode, YamlNode>>();
-			var volumes = new List<KeyValuePair<YamlNode, YamlNode>>();
-			var networks = new List<KeyValuePair<YamlNode, YamlNode>>();
-
-			foreach (var doc in Fragments.Select(f => ParseDocument(f)))
+			foreach (var o in unprocessedFragments.Select(f => (f, ParseDocument(f))).ToList())
 			{
+				var doc = o.Item2;
+				var fragment = o.f;
 				if (doc.Children.ContainsKey("services") && doc.Children["services"] is YamlMappingNode fragmentServicesRoot)
 				{
 					services.AddRange(fragmentServicesRoot.Children);
@@ -79,8 +85,57 @@ namespace DockerGenerator
 				{
 					networks.AddRange(fragmentNetworksRoot.Children);
 				}
+				if (doc.Children.ContainsKey("exclusive") && doc.Children["exclusive"] is YamlSequenceNode fragmentExclusiveRoot)
+				{
+					foreach (var node in fragmentExclusiveRoot)
+					{
+						exclusives.Add((fragment, node.ToString()));
+					}
+				}
+				if (doc.Children.ContainsKey("required") && doc.Children["required"] is YamlSequenceNode fragmentRequireRoot)
+				{
+					foreach (var node in fragmentRequireRoot)
+					{
+						if (ExcludeFragments.Contains(new FragmentName(node.ToString())))
+							throw new YamlBuildException($"You excluded fragment {new FragmentName(node.ToString())} but it is required by {fragment}");
+						requiredFragments.Add(new FragmentName(node.ToString()));
+					}
+				}
+				if (doc.Children.ContainsKey("recommended") && doc.Children["recommended"] is YamlSequenceNode fragmentRecommendedRoot)
+				{
+					foreach (var node in fragmentRecommendedRoot)
+					{
+						if (!ExcludeFragments.Contains(new FragmentName(node.ToString())))
+							recommendedFragments.Add(new FragmentName(node.ToString()));
+					}
+				}
+				processedFragments.Add(fragment);
+				unprocessedFragments.Remove(fragment);
+			}
+
+			foreach (var fragment in requiredFragments.Concat(recommendedFragments).Where(f => !processedFragments.Contains(f)))
+			{
+				unprocessedFragments.Add(fragment);
 			}
+			if (unprocessedFragments.Count != 0)
+				goto reprocessFragments;
 
+			var exclusiveConflict = exclusives.GroupBy(e => e.Exclusivity)
+			.Where(e => e.Count() != 1)
+			.FirstOrDefault();
+			if (exclusiveConflict != null)
+				throw new YamlBuildException($"The fragments {String.Join(", ", exclusiveConflict.Select(e => e.FragmentName))} can't be used simultaneously (group '{exclusiveConflict.Key}')");
+
+			Console.WriteLine($"Selected fragments:");
+			foreach (var fragment in processedFragments)
+			{
+				Console.WriteLine($"\t{fragment}");
+			}
+			foreach (var fragment in fragmentsNotFound)
+			{
+				var fragmentPath = GetFragmentLocation(fragment);
+				ConsoleUtils.WriteLine($"\t{fragment} not found in {fragmentPath}, ignoring...", ConsoleColor.Yellow);
+			}
 
 			YamlMappingNode output = new YamlMappingNode();
 			output.Add("version", new YamlScalarNode("3") { Style = YamlDotNet.Core.ScalarStyle.DoubleQuoted });
@@ -119,6 +174,11 @@ namespace DockerGenerator
 			Console.WriteLine();
 		}
 
+		private bool NotExcluded(FragmentName arg)
+		{
+			return !ExcludeFragments.Contains(arg);
+		}
+
 		private void PostProcess(YamlMappingNode output)
 		{
 			new BuildTimeVariableVisitor().Visit(output);
@@ -192,7 +252,7 @@ namespace DockerGenerator
 			return null;
 		}
 
-		private YamlMappingNode ParseDocument(string fragment)
+		private YamlMappingNode ParseDocument(FragmentName fragment)
 		{
 			var input = new StringReader(File.ReadAllText(GetFragmentLocation(fragment)));
 			YamlStream stream = new YamlStream();
@@ -200,9 +260,9 @@ namespace DockerGenerator
 			return (YamlMappingNode)stream.Documents[0].RootNode;
 		}
 
-		private string GetFragmentLocation(string fragment)
+		private string GetFragmentLocation(FragmentName fragment)
 		{
-			return Path.Combine(FragmentLocation, $"{fragment}.yml");
+			return Path.Combine(FragmentLocation, $"{fragment.Name}.yml");
 		}
 	}
 }

docker-compose-generator/src/FragmentName.cs

@@ -0,0 +1,49 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace DockerGenerator
+{
+	public class FragmentName
+	{
+		public FragmentName(string fragmentName)
+		{
+			if (fragmentName == null)
+				throw new ArgumentNullException(nameof(fragmentName));
+			Name = fragmentName.Trim().ToLowerInvariant();
+			if (Name.EndsWith(".yml", StringComparison.OrdinalIgnoreCase))
+				Name = Name.Substring(0, Name.Length - 4);
+		}
+		public string Name { get; }
+
+		public override bool Equals(object obj)
+		{
+			FragmentName item = obj as FragmentName;
+			if (item == null)
+				return false;
+			return Name.Equals(item.Name);
+		}
+		public static bool operator ==(FragmentName a, FragmentName b)
+		{
+			if (System.Object.ReferenceEquals(a, b))
+				return true;
+			if (((object)a == null) || ((object)b == null))
+				return false;
+			return a.Name == b.Name;
+		}
+
+		public static bool operator !=(FragmentName a, FragmentName b)
+		{
+			return !(a == b);
+		}
+
+		public override int GetHashCode()
+		{
+			return Name.GetHashCode();
+		}
+		public override string ToString()
+		{
+			return Name;
+		}
+	}
+}

docker-compose-generator/src/Program.cs

@@ -22,7 +22,15 @@ namespace DockerGenerator
 
 			var name = Environment.GetEnvironmentVariable("BTCPAYGEN_SUBNAME");
 			name = string.IsNullOrEmpty(name) ? "generated" : name;
-			new Program().Run(composition, name, generatedLocation);
+			try
+			{
+				new Program().Run(composition, name, generatedLocation);
+			}
+			catch (YamlBuildException ex)
+			{
+				ConsoleUtils.WriteLine(ex.Message, ConsoleColor.Red);
+				Environment.ExitCode = 1;
+			}
 		}
 
 		private void Run(DockerComposition composition, string name, string output)
@@ -35,13 +43,10 @@ namespace DockerGenerator
 			switch (composition.SelectedProxy)
 			{
 				case "nginx":
-					fragments.Add("nginx-https");
 					fragments.Add("nginx");
-					fragments.Add("btcpayserver-nginx");
 					break;
 				case "traefik":
 					fragments.Add("traefik");
-					fragments.Add("traefik-labels");
 					break;
 				case "no-reverseproxy":
 				case "none":
@@ -50,9 +55,6 @@ namespace DockerGenerator
 					break;
 			}
 			fragments.Add("btcpayserver");
-			fragments.Add("opt-add-tor");
-			fragments.Add("nbxplorer");
-			fragments.Add("postgres");
 			foreach (var crypto in CryptoDefinition.GetDefinitions())
 			{
 				if (!composition.SelectedCryptos.Contains(crypto.Crypto))
@@ -71,10 +73,12 @@ namespace DockerGenerator
 
 			foreach (var fragment in composition.AdditionalFragments)
 			{
-				fragments.Add(fragment.Trim());
+				fragments.Add(fragment);
 			}
-			fragments = fragments.Where(s => !composition.ExcludeFragments.Contains(s)).ToHashSet();
-			var def = new DockerComposeDefinition(name, fragments);
+			var def = new DockerComposeDefinition(name, fragments.Select(f => new FragmentName(f)).ToHashSet())
+			{
+				ExcludeFragments = composition.ExcludeFragments.Select(f => new FragmentName(f)).ToHashSet()
+			};
 			def.FragmentLocation = fragmentLocation;
 			def.BuildOutputDirectory = output;
 			def.Build();

docker-compose-generator/src/Properties/launchSettings.json

@@ -10,7 +10,9 @@
         "BTCPAYGEN_CRYPTO3": "btg",
         "BTCPAYGEN_CRYPTO2": "ltc",
         "BTCPAYGEN_CRYPTO1": "btc",
-        "BTCPAYGEN_REVERSEPROXY": "nginx"
+        "BTCPAYGEN_REVERSEPROXY": "nginx",
+        "BTCPAYGEN_ADDITIONAL_FRAGMENTS": "opt-save-storage",
+        "BTCPAYGEN_EXCLUDE_FRAGMENTS": "postgres"
       }
     }
   }

docker-compose-generator/src/YamlBuildException.cs

@@ -0,0 +1,14 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace DockerGenerator
+{
+	public class YamlBuildException : Exception
+	{
+		public YamlBuildException(string message): base(message)
+		{
+
+		}
+	}
+}

docker-compose-generator/src/docker-compose-generator.csproj

@@ -1,7 +1,8 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
+    <LangVersion>7.3</LangVersion>
 	<TargetFramework Condition="'$(TargetFrameworkOverride)' == ''">netcoreapp2.1</TargetFramework>
   <TargetFramework Condition="'$(TargetFrameworkOverride)' != ''">$(TargetFrameworkOverride)</TargetFramework>
 	<RootNamespace>DockerGenerator</RootNamespace>