[Beta] Cache everything (#5006)

* [Beta] Cache everything

* test cache

* Revert "test cache"

This reverts commit 91ee169ab5c992ac1597bcfc22537bcf74b23b6b.

beta/src/pages/[[...markdownPath]].js

@@ -53,30 +53,20 @@ function reviveNodeOnClient(key, val) {
   }
 }
 
-// Serialize a server React tree node to JSON.
-function stringifyNodeOnServer(key, val) {
-  if (val != null && val.$$typeof === Symbol.for('react.element')) {
-    // Remove fake MDX props.
-    const {mdxType, originalType, parentName, ...cleanProps} = val.props;
-    return [
-      '$r',
-      typeof val.type === 'string' ? val.type : mdxType,
-      val.key,
-      cleanProps,
-    ];
-  } else {
-    return val;
-  }
-}
+// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+// ~~~~ IMPORTANT: BUMP THIS IF YOU CHANGE ANY CODE BELOW ~~~
+const DISK_CACHE_BREAKER = 2;
+// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 // Put MDX output into JSON for client.
 export async function getStaticProps(context) {
   const fs = require('fs');
-  const compileMdx = require('@mdx-js/mdx');
-  const {transform} = require('@babel/core');
-  const fm = require('gray-matter');
-  const {remarkPlugins} = require('../../plugins/markdownToHtml');
+  const {
+    prepareMDX,
+    PREPARE_MDX_CACHE_BREAKER,
+  } = require('../utils/prepareMDX');
   const rootDir = process.cwd() + '/src/content/';
+  const mdxComponentNames = Object.keys(MDXComponents);
 
   // Read MDX from the file. Parse Frontmatter data out of it.
   let path = (context.params.markdownPath || []).join('/') || 'index';
@@ -87,59 +77,59 @@ export async function getStaticProps(context) {
     mdxWithFrontmatter = fs.readFileSync(rootDir + path + '/index.md', 'utf8');
   }
 
-  const DISK_CACHE_BREAKER = 1; // <-- Bump to reset the cache
-  async function compileMdxToJs_FROM_DISK_CACHE(mdxContent, mdxComponentNames) {
-    const {FileStore, stableHash} = require('metro-cache');
-    const store = new FileStore({
-      root: process.cwd() + '/node_modules/.cache/react-docs-mdx/',
-    });
-    const hash = Buffer.from(
-      stableHash({
-        mdxContent,
-        mdxComponentNames,
-        lockfile: fs.readFileSync(process.cwd() + '/yarn.lock', 'utf8'),
-        DISK_CACHE_BREAKER,
-      })
+  // See if we have a cached output first.
+  const {FileStore, stableHash} = require('metro-cache');
+  const store = new FileStore({
+    root: process.cwd() + '/node_modules/.cache/react-docs-mdx/',
+  });
+  const hash = Buffer.from(
+    stableHash({
+      // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+      // ~~~~ IMPORTANT: Everything that the code below may rely on.
+      // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+      mdxWithFrontmatter,
+      mdxComponentNames,
+      DISK_CACHE_BREAKER,
+      PREPARE_MDX_CACHE_BREAKER,
+      lockfile: fs.readFileSync(process.cwd() + '/yarn.lock', 'utf8'),
+    })
+  );
+  const cached = await store.get(hash);
+  if (cached) {
+    console.log(
+      'Reading compiled MDX for /' + path + ' from ./node_modules/.cache/'
     );
-    const cached = await store.get(hash);
-    if (cached) {
-      console.log(
-        'Reading compiled MDX for /' + path + ' from ./node_modules/.cache/'
-      );
-      return cached;
-    }
-    if (process.env.NODE_ENV === 'production') {
-      console.log(
-        'Cache miss for MDX for /' + path + ' from ./node_modules/.cache/'
-      );
-    }
-    const output = await compileMdxToJs(mdxContent, mdxComponentNames);
-    await store.set(hash, output);
-    return output;
+    return cached;
   }
-
-  // IMPORTANT: Keep this a pure function. Its output gets cached on disk.
-  async function compileMdxToJs(mdxContent, mdxComponentNames) {
-    // If we don't add these fake imports, the MDX compiler
-    // will insert a bunch of opaque components we can't introspect.
-    // This will break the prepareMDX() call below.
-    let mdxWithFakeImports = mdxComponentNames
-      .map((key) => 'import ' + key + ' from "' + key + '";\n')
-      .join('\n');
-    mdxWithFakeImports += '\n' + mdxContent;
-    const jsxCode = await compileMdx(mdxWithFakeImports, {
-      remarkPlugins,
-    });
-    return transform(jsxCode, {
-      plugins: ['@babel/plugin-transform-modules-commonjs'],
-      presets: ['@babel/preset-react'],
-    }).code;
+  if (process.env.NODE_ENV === 'production') {
+    console.log(
+      'Cache miss for MDX for /' + path + ' from ./node_modules/.cache/'
+    );
   }
 
+  // Parse Frontmatter headers from MDX.
+  const fm = require('gray-matter');
+  const {content: mdxWithoutFrontmatter, data: meta} = fm(mdxWithFrontmatter);
+
+  // If we don't add these fake imports, the MDX compiler
+  // will insert a bunch of opaque components we can't introspect.
+  // This will break the prepareMDX() call below.
+  let mdxWithFakeImports = mdxComponentNames
+    .map((key) => 'import ' + key + ' from "' + key + '";\n')
+    .join('\n');
+  mdxWithFakeImports += '\n' + mdxWithoutFrontmatter;
+
   // Turn the MDX we just read into some JS we can execute.
-  const {content: mdx, data: meta} = fm(mdxWithFrontmatter);
-  const mdxComponentNames = Object.keys(MDXComponents);
-  const jsCode = await compileMdxToJs_FROM_DISK_CACHE(mdx, mdxComponentNames);
+  const {remarkPlugins} = require('../../plugins/markdownToHtml');
+  const compileMdx = require('@mdx-js/mdx');
+  const jsxCode = await compileMdx(mdxWithFakeImports, {
+    remarkPlugins,
+  });
+  const {transform} = require('@babel/core');
+  const jsCode = await transform(jsxCode, {
+    plugins: ['@babel/plugin-transform-modules-commonjs'],
+    presets: ['@babel/preset-react'],
+  }).code;
 
   // Prepare environment for MDX.
   let fakeExports = {};
@@ -156,18 +146,38 @@ export async function getStaticProps(context) {
   const reactTree = fakeExports.default({});
 
   // Pre-process MDX output and serialize it.
-  const {prepareMDX} = require('../utils/prepareMDX');
   let {toc, children} = prepareMDX(reactTree.props.children);
   if (path === 'index') {
     toc = [];
   }
-  return {
+
+  const output = {
     props: {
       content: JSON.stringify(children, stringifyNodeOnServer),
       toc: JSON.stringify(toc, stringifyNodeOnServer),
       meta,
     },
   };
+
+  // Serialize a server React tree node to JSON.
+  function stringifyNodeOnServer(key, val) {
+    if (val != null && val.$$typeof === Symbol.for('react.element')) {
+      // Remove fake MDX props.
+      const {mdxType, originalType, parentName, ...cleanProps} = val.props;
+      return [
+        '$r',
+        typeof val.type === 'string' ? val.type : mdxType,
+        val.key,
+        cleanProps,
+      ];
+    } else {
+      return val;
+    }
+  }
+
+  // Cache it on the disk.
+  await store.set(hash, output);
+  return output;
 }
 
 // Collect all MDX files for static generation.

beta/src/utils/prepareMDX.js

@@ -3,12 +3,14 @@
  */
 
 import {Children} from 'react';
-import {MDXComponents} from 'components/MDX/MDXComponents';
-
-const {MaxWidth} = MDXComponents;
 
 // TODO: This logic could be in MDX plugins instead.
 
+// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+export const PREPARE_MDX_CACHE_BREAKER = 2;
+// !!! IMPORTANT !!! Bump this if you change any logic.
+// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
 export function prepareMDX(rawChildren) {
   const toc = getTableOfContents(rawChildren);
   const children = wrapChildrenInMaxWidthContainers(rawChildren);