tools: add tests for the doctool

* Test the toHTML function in html.js. Check that given valid markdown
  it produces the expected html. One test case will prevent regressions
  of #5873.
* Check that when given valid markdown toJSON produces valid JSON with
  the expected schema.
* Add doctool to the list of built in tests so it runs in CI.

PR-URL: https://github.com/nodejs/node/pull/6031
Fixes: https://github.com/nodejs/node/issues/5955
Reviewed-By: Roman Reiss <me@silverwind.io>
Reviewed-By: Rich Trott <rtrott@gmail.com>

Makefile

@@ -115,7 +115,7 @@ v8:
 	$(MAKE) -C deps/v8 $(V8_ARCH) $(V8_BUILD_OPTIONS)
 
 test: | cctest  # Depends on 'all'.
-	$(PYTHON) tools/test.py --mode=release message parallel sequential -J
+	$(PYTHON) tools/test.py --mode=release doctool message parallel sequential -J
 	$(MAKE) jslint
 	$(MAKE) cpplint
 
@@ -173,7 +173,7 @@ test-all-valgrind: test-build
 test-ci: | build-addons
 	$(PYTHON) tools/test.py $(PARALLEL_ARGS) -p tap --logfile test.tap \
 		--mode=release --flaky-tests=$(FLAKY_TESTS) \
-		$(TEST_CI_ARGS) addons message parallel sequential
+		$(TEST_CI_ARGS) addons doctool message parallel sequential
 
 test-release: test-build
 	$(PYTHON) tools/test.py --mode=release

test/doctool/test-doctool-html.js

@@ -0,0 +1,48 @@
+'use strict';
+
+const common = require('../common');
+const assert = require('assert');
+const fs = require('fs');
+
+const html = require('../../tools/doc/html.js');
+
+// Test data is a list of objects with two properties.
+// The file property is the file path.
+// The html property is some html which will be generated by the doctool.
+// This html will be stripped of all whitespace because we don't currently
+// have an html parser.
+const testData = [
+  {
+    'file': common.fixturesDir + '/sample_document.md',
+    'html': '<ol><li>fish</li><li><p>fish</p></li><li><p>Redfish</p></li>' +
+      '<li>Bluefish</li></ol>'
+  },
+  {
+    'file': common.fixturesDir + '/order_of_end_tags_5873.md',
+    'html': '<h3>ClassMethod: Buffer.from(array) <span> ' +
+      '<a class="mark" href="#foo_class_method_buffer_from_array" ' +
+      'id="foo_class_method_buffer_from_array">#</a> </span> </h3><div' +
+      'class="signature"><ul><li><code>array</code><a ' +
+      'href="https://developer.mozilla.org/en-US/docs/Web/JavaScript/' +
+      'Reference/Global_Objects/Array" class="type">&lt;Array&gt;</a></li>' +
+      '</ul></div>'
+  },
+];
+
+testData.forEach(function(item) {
+  // Normalize expected data by stripping whitespace
+  const expected = item.html.replace(/\s/g, '');
+
+  fs.readFile(item.file, 'utf8', common.mustCall(function(err, input) {
+    assert.ifError(err);
+    html(input, 'foo', 'doc/template.html',
+      common.mustCall(function(err, output) {
+        assert.ifError(err);
+
+        const actual = output.replace(/\s/g, '');
+        // Assert that the input stripped of all whitespace contains the
+        // expected list
+        assert.notEqual(actual.indexOf(expected), -1);
+      }));
+  }));
+});

test/doctool/test-doctool-json.js

@@ -0,0 +1,78 @@
+'use strict';
+
+const common = require('../common');
+const assert = require('assert');
+const fs = require('fs');
+
+const json = require('../../tools/doc/json.js');
+
+// Outputs valid json with the expected fields when given simple markdown
+// Test data is a list of objects with two properties.
+// The file property is the file path.
+// The json property is some json which will be generated by the doctool.
+var testData = [
+  {
+    'file': common.fixturesDir + '/sample_document.md',
+    'json': {
+      'source': 'foo',
+      'modules': [ { 'textRaw': 'Sample Markdown',
+        'name': 'sample_markdown',
+        'modules': [ { 'textRaw':'Seussian Rhymes',
+          'name': 'seussian_rhymes',
+          'desc': '<ol>\n<li>fish</li>\n<li><p>fish</p>\n</li>\n<li>' +
+                  '<p>Red fish</p>\n</li>\n<li>Blue fish</li>\n</ol>\n',
+          'type': 'module',
+          'displayName': 'Seussian Rhymes'
+         } ],
+        'type': 'module',
+        'displayName': 'Sample Markdown'
+      } ]
+    }
+  },
+  {
+    'file': common.fixturesDir + '/order_of_end_tags_5873.md',
+    'json': {
+      'source':'foo',
+      'modules': [ {
+        'textRaw': 'Title',
+        'name': 'title',
+        'modules': [ {
+          'textRaw': 'Subsection',
+          'name': 'subsection',
+          'classMethods': [ {
+            'textRaw': 'Class Method: Buffer.from(array)',
+            'type':'classMethod',
+            'name':'from',
+            'signatures': [ {
+              'params': [ {
+                'textRaw': '`array` {Array} ',
+                'name': 'array',
+                'type': 'Array'
+              } ]
+            },
+              {
+                'params' : [ {
+                  'name': 'array'
+                } ]
+              }
+            ]
+          } ],
+          'type': 'module',
+          'displayName': 'Subsection'
+        } ],
+        'type': 'module',
+        'displayName': 'Title'
+      } ]
+    }
+  }
+];
+
+testData.forEach(function(item) {
+  fs.readFile(item.file, 'utf8', common.mustCall(function(err, input) {
+    assert.ifError(err);
+    json(input, 'foo', common.mustCall(function(err, output) {
+      assert.ifError(err);
+      assert.deepStrictEqual(output, item.json);
+    }));
+  }));
+});

test/doctool/testcfg.py

@@ -0,0 +1,7 @@
+import os
+import sys
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import testpy
+
+def GetConfiguration(context, root):
+    return testpy.SimpleTestConfiguration(context, root, 'doctool')

test/fixtures/order_of_end_tags_5873.md

@@ -0,0 +1,6 @@
+# Title
+
+## Subsection
+
+### Class Method: Buffer.from(array)
+* `array` {Array}

test/fixtures/sample_document.md

@@ -0,0 +1,8 @@
+# Sample Markdown
+
+## Seussian Rhymes
+1. fish
+2. fish
+
+* Red fish
+* Blue fish

tools/test.py

@@ -1427,6 +1427,7 @@ BUILT_IN_TESTS = [
   'addons',
   'gc',
   'debugger',
+  'doctool',
 ]
 
 

vcbuild.bat

@@ -55,8 +55,8 @@ if /i "%1"=="nosnapshot"    set nosnapshot=1&goto arg-ok
 if /i "%1"=="noetw"         set noetw=1&goto arg-ok
 if /i "%1"=="noperfctr"     set noperfctr=1&goto arg-ok
 if /i "%1"=="licensertf"    set licensertf=1&goto arg-ok
-if /i "%1"=="test"          set test_args=%test_args% addons sequential parallel message -J&set jslint=1&set build_addons=1&goto arg-ok
+if /i "%1"=="test"          set test_args=%test_args% addons doctool sequential parallel message -J&set jslint=1&set build_addons=1&goto arg-ok
-if /i "%1"=="test-ci"       set test_args=%test_args% %test_ci_args% -p tap --logfile test.tap addons message sequential parallel&set build_addons=1&goto arg-ok
+if /i "%1"=="test-ci"       set test_args=%test_args% %test_ci_args% -p tap --logfile test.tap addons doctool message sequential parallel&set build_addons=1&goto arg-ok
 if /i "%1"=="test-addons"   set test_args=%test_args% addons&set build_addons=1&goto arg-ok
 if /i "%1"=="test-simple"   set test_args=%test_args% sequential parallel -J&goto arg-ok
 if /i "%1"=="test-message"  set test_args=%test_args% message&goto arg-ok