Add customizable docs to build

.gitignore

@@ -274,3 +274,7 @@ local/bin/py/submit_github_status_check.py
 !local/bin/py/build/configuration/pull_config_preview.yaml
 !local/bin/py/build/configuration/pull_config.yaml
 !local/bin/py/build/configuration/integration_merge.yaml
+
+# CDOCS
+layouts/partials/markdoc-assets.html
+static/cdocs

CDOCS_NOTES.md

@@ -0,0 +1,58 @@
+# Cdocs build branch
+
+This file should not be merged to master. If you are reviewing a PR against master and you see this file, make sure it's deleted or moved out of the PR.
+
+This file describes the steps Jen took to build this branch.
+
+## Install the cdocs-hugo-integration package in the documentation repo
+
+The cdocs-hugo-integration package isn't published yet, so you have to build the package tarball locally and install that file.
+
+1. Clone the `corp-node-packages` repo.
+2. Check out the `jen.gilbert/cdocs-build-testing` branch. Changes to this branch will not go live, if you would like to make experimental edits.
+3. From the `cdocs-hugo-integration` folder inside `corp-node-packages/packages`, run `yarn install && yarn test` to verify that everything is healthy in the package.
+4. Run `yarn pack`, and copy the full tarball path provided in the output (e.g., `/Users/jen.gilbert/go/src/github.com/DataDog/corp-node-packages/packages/cdocs-hugo-integration/cdocs-hugo-integration-v1.0.0.tgz`).
+5. In the documentation repo, run `yarn add <ABSOLUTE_TARBALL_PATH>`.
+
+### Warning
+
+If you modify anything about your local copy of cdocs-hugo-integration while working in `documentation` (not recommended):
+
+1. Bump the package version in package.json. You won't commit this change, it's just to ensure that yarn recognizes this as a new build.
+2. Run `yarn pack` to build a new tarball. Copy the absolute tarball path it gives you.
+3. In the `documentation` directory, update the `cdocs-hugo-integration` dependency in `package.json` to use your new tarball.
+4. In the `documentation` directory, run `yarn install`.
+
+Ideally, you won't make updates to the packages while working in the documentation repo. If you find an issue, it's better to create a ticket for it, then update the package separately by using a new test case to verify the changes (rather than using the results you're getting in the documentation repo to confirm fixes).
+
+If you need to debug an unexpected error in the cdocs build, edit the Makefile to use `node inspect` instead of `node` in the `build-cdocs` action. This will trigger the [Node debugger](https://nodejs.org/api/debugger.html). This allows you to inspect variables without adding console.log calls to the packages themselves.
+
+## Add a customization configuration directory
+
+Set up the customization configuration directory at the top level of the documentation repo.
+
+It should look something like the examples provided in [the `cdocs-data` package](https://github.com/DataDog/corp-node-packages/blob/master/packages/cdocs-data/README.md).
+
+## Add a test document
+
+Add a test document at `cdocs_build/test.mdoc.md`.
+
+## Add the cdocs build script
+
+Call the `cdocs-hugo-integration` package inside `assets/scripts/cdocs-build.js` to convert `.mdoc.md` files, generate the `.gitignore` file, etc.
+
+## Add the build script to the Makefile
+
+Add `build-cdocs` to the Makefile, and invoke it in `start-no-pre-build`.
+
+## Include the global cdocs assets partial in the base Hugo layout
+
+Add `{{ partialCached "markdoc-assets.html" . }}` to `layouts/_default/baseof.html`, so the CSS and JS outputted by cdocs-hugo-integration will be included in the Hugo site.
+
+## Update the existing site functionality to build the TOC
+
+The site's [existing TOC logic](./assets/scripts/components/table-of-contents.js) (that builds the righthand nav) has been updated to provide hooks that the Markdoc renderer can run.
+
+## View the test page
+
+Run `make start-no-pre-build` and view the [test page](localhost:1313/cdocs_build/test).

Makefile

@@ -13,6 +13,7 @@ PY3=$(shell if [ `which pyenv` ]; then \
 			else printf "false"; \
 			fi)
 IMAGE_VERSION="latest"
+PROJECT_ROOT := "$(shell git rev-parse --show-toplevel 2>/dev/null || echo $(CURDIR))"
 
 # config
 CONFIG_FILE := Makefile.config
@@ -61,14 +62,22 @@ server:
 	  yarn run prestart && yarn run start; \
 	fi;
 
-# Download all dependencies and run the site
-start: setup-build-scripts ## Build and run docs including external content.
+# compile .mdoc.md files to HTML
+# so Hugo can include them in the site
+build-cdocs: 
+	@echo "Compiling .mdoc files to HTML";
+	@node ./assets/scripts/cdocs-build.js;
+
+start:
+	@make setup-build-scripts ## Build and run docs including external content.
 	@make dependencies
+	@make build-cdocs 
 	@make update_websites_sources_module
 	@make server
 
 # Skip downloading any dependencies and run the site (hugo needs at the least node)
 start-no-pre-build: node_modules  ## Build and run docs excluding external content.
+	@make build-cdocs
 	@make server
 
 # Leave build scripts as is for local testing
@@ -100,7 +109,7 @@ source-dd-source:
 
 # All the requirements for a full build
 dependencies: clean source-dd-source
-	make hugpython all-examples update_pre_build node_modules placeholders
+	make hugpython all-examples update_pre_build node_modules placeholders build-cdocs
 
 integrations_data/extracted/vector:
 	$(call source_repo,vector,https://github.com/vectordotdev/vector.git,master,true,website/)

assets/scripts/cdocs-build.js

@@ -0,0 +1,58 @@
+#!/usr/bin/env node
+
+/**
+ * This script runs as part of the build, before Hugo has processed any files.
+ * It invokes the Markdoc integration to compile all .mdoc files
+ * found in the content directory, and updates the .gitignore file
+ * in that directory to ignore the compiled files.
+ */
+console.time('Markdoc compilation execution time');
+
+const fs = require('fs');
+const path = require('path');
+
+const { CdocsHugoIntegration } = require('cdocs-hugo-integration');
+
+const env = process.env.CI_ENVIRONMENT_NAME || 'development';
+const baseSiteDir = path.resolve(__dirname, '../..');
+
+const ASSETS_PARTIAL_PATH = path.resolve(__dirname, `${baseSiteDir}/layouts/partials/markdoc-assets.html`);
+const CONTENT_DIR = path.resolve(__dirname, `${baseSiteDir}/content`);
+
+// Initialize the Markdoc integration
+const markdocIntegration = new CdocsHugoIntegration({
+    baseSiteDir,
+    env,
+    publishAuthorConsoleInDir: baseSiteDir + '/static/cdocs/console'
+});
+
+// Build the assets partial, and write it to the target file path
+const assetsPartialContents = markdocIntegration.buildAssetsPartial();
+fs.writeFileSync(ASSETS_PARTIAL_PATH, assetsPartialContents);
+
+// Compile all .mdoc files found in the content directory
+const { compiledFilePaths, hasErrors } = markdocIntegration.compileMdocFiles();
+
+console.log(`Markdoc compilation completed. Compiled ${compiledFilePaths.length} files.`);
+
+// Read the contents of the .gitignore file in the content directory into an array of strings
+let gitignoreContents = fs.readFileSync(CONTENT_DIR + '/.gitignore', 'utf8').split('\n');
+
+// Compare gitignoreContents with compiledFilePaths
+// Add any files in compiledFilePaths that are not in gitignoreContents to gitignoreContents
+compiledFilePaths.forEach((file) => {
+    const sanitizedFile = file.replace(CONTENT_DIR, '');
+    if (!gitignoreContents.includes(sanitizedFile)) {
+        gitignoreContents.push(sanitizedFile);
+    }
+});
+
+// Write the updated gitignoreContents to the .gitignore file in the content directory
+fs.writeFileSync(CONTENT_DIR + '/.gitignore', gitignoreContents.join('\n'));
+
+if (hasErrors) {
+    console.error('Markdoc compilation failed with errors:');
+    markdocIntegration.logErrorsToConsole();
+}
+
+console.timeEnd('Markdoc compilation execution time');

assets/scripts/components/table-of-contents.js

@@ -116,7 +116,12 @@ export function buildTOCMap() {
 export function onScroll() {
     const windowTopPosition = scrollTop(window);
     const windowHeight = window.innerHeight;
-    const localOffset = 65;
+    let localOffset = 65;
+
+    const isCustomizableDoc = document.getElementById('cdoc-selector') ? true : false;
+    if (isCustomizableDoc) {
+        localOffset += 65;
+    }
 
     const body = document.body;
     const html = document.documentElement;
@@ -141,6 +146,7 @@ export function onScroll() {
             sideNavItem.navLink.classList.remove('toc_scrolled');
 
             if (
+                windowTopPosition !== 0 &&
                 sideNavItem.header.getBoundingClientRect().top <= 0 + localOffset &&
                 (typeof nextSideNavItem === 'undefined' ||
                     nextSideNavItem.header.getBoundingClientRect().top > 0 + localOffset)
@@ -153,7 +159,7 @@ export function onScroll() {
 
                     if (href) {
                         const id = href.replace('#', '').replace(' ', '-');
-                        const header = document.getElementById(`${(decodeURI(id))}`);
+                        const header = document.getElementById(`${decodeURI(id)}`);
                         if (header && header.nodeName === 'H2') {
                             link.classList.add('toc_open');
                         }
@@ -240,5 +246,24 @@ window.addEventListener('scroll', () => {
     onScroll();
 });
 
-DOMReady(handleAPIPage);
+// Expose the necessary functions to cdocs
+window.cdocsHooks = {};
+
+const phases = ['beforeReveal', 'afterReveal', 'afterRerender'];
+phases.forEach((phase) => {
+    window.cdocsHooks[phase] = [];
+});
 
+// Update the TOC after the page is initially rendered
+// and before it is revealed
+cdocsHooks.beforeReveal.push(buildTOCMap);
+
+// Update the active header in the TOC after the page is revealed
+cdocsHooks.afterReveal.push(buildTOCMap);
+cdocsHooks.afterReveal.push(onScroll);
+
+// Update the TOC any time the page is re-rendered
+cdocsHooks.afterRerender.push(buildTOCMap);
+cdocsHooks.afterRerender.push(onScroll);
+
+DOMReady(handleAPIPage);

assets/styles/components/_table-of-contents.scss

@@ -7,7 +7,7 @@ body > header.scrolled ~ .container .mobile-toc-toggle {
 
 .announcement {
     .mobile-toc-toggle {
-        top: 130px;
+        top: 115px;
     }
 }
 
@@ -88,7 +88,7 @@ body > header.scrolled ~ .container .mobile-toc-toggle {
         @media (min-width: 991px) {
             max-width: 220px !important;
             padding-right: 20px !important;
-        }    
+        }
         @media (min-width: 1440px) {
             margin-right: -40%;
         }

config/_default/config.yaml

@@ -4,6 +4,9 @@ canonifyURLs: true
 pygmentsuseclasses: true
 pygmentsCodeFences: true
 
+# don't publish cdocs source files
+ignoreFiles: ["\\.mdoc\\.md$"]
+
 publishDir: "./public"
 
 # taxonomies

content/.gitignore

@@ -0,0 +1,3 @@
+# GENERATED FILE: DO NOT EDIT
+# Ignore .md files compiled from Markdoc
+/en/real_user_monitoring/guide/proxy-mobile-rum-data.md