feat(contented): ability to customize unified plugins (#629)

#### What this PR does / why we need it:

Make it easier to customize content processor with subset of plugins.

Author: fuxingloh

packages/contented-example/docs/04-markdown.md

@@ -15,10 +15,12 @@ processor
   .use(remarkParse)
   .use(remarkLink)
   .use(remarkDirective)
+  .use(remarkDirectiveRehypeCodeblockHeader)
+  .use(remarkDirectiveRehypeCodeblockGroup)
   .use(remarkDirectiveRehype)
-  .use(collectFields)
-  .use(resolveFields)
-  .use(validateFields)
+  .use(remarkFrontmatterCollect)
+  .use(remarkFrontmatterResolve)
+  .use(remarkFrontmatterValidate)
   .use(options.remarks)
   .use(remarkRehype)
   .use(options.rehypes)
@@ -28,7 +30,7 @@ processor
   .use(rehypeToc)
   .use(rehypeHeading)
   .use(rehypeMermaid)
-  .use(rehypeShiki, { highlighter })
+  .use(rehypeShiki)
   .use(rehypeStringify)
   .use(options.after);
 ```

packages/contented-pipeline-md/src/MarkdownPipeline.ts

@@ -1,7 +1,7 @@
 import console from 'node:console';
 import { join } from 'node:path';
 
-import { ContentedPipeline, FileContent, FileIndex } from '@contentedjs/contented-pipeline';
+import { ContentedPipeline, FileContent, FileIndex, Pipeline } from '@contentedjs/contented-pipeline';
 import { read } from 'to-vfile';
 import { Processor, unified } from 'unified';
 import { VFile } from 'vfile';
@@ -13,7 +13,7 @@ export class MarkdownPipeline extends ContentedPipeline {
   protected readonly processor: Processor = unified();
 
   async init() {
-    await initProcessor(this.processor);
+    initProcessor(this.processor);
   }
 
   protected override async processFileIndex(
@@ -56,6 +56,23 @@ export class MarkdownPipeline extends ContentedPipeline {
       fields: contented.fields,
     };
   }
+
+  /**
+   * Create a new MarkdownPipeline with a custom processor.
+   * This is useful for only using a subset of the plugins that you need.
+   * @param processor is a function that takes a Processor and adds plugins to it.
+   */
+  static withProcessor(
+    processor: (processor: Processor) => void,
+  ): new (rootPath: string, pipeline: Pipeline) => ContentedPipeline {
+    class WithProcessor extends MarkdownPipeline {
+      async init(): Promise<void> {
+        processor(this.processor);
+      }
+    }
+
+    return WithProcessor;
+  }
 }
 
 export class MarkdownVFile extends VFile {

packages/contented-pipeline-md/src/MarkdownPipeline.unit.ts

@@ -1,6 +1,7 @@
 import { join } from 'node:path';
 
 import { MarkdownPipeline } from './MarkdownPipeline';
+import { rehypeStringify, remarkParse, remarkRehype } from './UnifiedProcessor';
 
 const rootPath = join(__dirname, '../fixtures');
 
@@ -95,3 +96,33 @@ describe('Without Config', () => {
     ]);
   });
 });
+
+describe('Custom Pipeline', () => {
+  const pipeline = new (MarkdownPipeline.withProcessor((processor) => {
+    processor.use(remarkParse).use(remarkRehype).use(rehypeStringify);
+  }))(__dirname, {
+    type: 'Markdown',
+    pattern: '/See.Nothing.md',
+    processor: 'md',
+  });
+
+  beforeAll(async () => {
+    await pipeline.init();
+  });
+
+  it('should process See.Nothing.md', async () => {
+    const content = await pipeline.process(rootPath, 'See.Nothing.md');
+    expect(content).toStrictEqual([
+      {
+        type: 'Markdown',
+        fields: {},
+        headings: [],
+        path: '/see-nothing',
+        sections: [],
+        fileId: expect.stringMatching(/[0-f]{64}/),
+        modifiedDate: expect.any(Number),
+        html: '<h1>Nothing To See</h1>\n<p>Markdown for testing <code>MarkdownPipeline.unit.ts</code>.</p>',
+      },
+    ]);
+  });
+});

packages/contented-pipeline-md/src/UnifiedProcessor.ts

@@ -18,7 +18,11 @@ import {
   remarkDirectiveRehypeCodeblockHeader,
 } from './plugins/RemarkCodeblock.js';
 import { remarkDirectiveRehype } from './plugins/RemarkDirectiveRehype.js';
-import { collectFields, resolveFields, validateFields } from './plugins/RemarkFrontmatter.js';
+import {
+  remarkFrontmatterCollect,
+  remarkFrontmatterResolve,
+  remarkFrontmatterValidate,
+} from './plugins/RemarkFrontmatter.js';
 import { remarkLink } from './plugins/RemarkLink.js';
 
 export interface UnifiedOptions {
@@ -28,7 +32,7 @@ export interface UnifiedOptions {
   after?: Plugin[];
 }
 
-export async function initProcessor(processor: Processor, options?: UnifiedOptions) {
+export function initProcessor(processor: Processor, options?: UnifiedOptions): Processor {
   options?.before?.forEach((plugin) => {
     processor.use(plugin);
   });
@@ -43,7 +47,7 @@ export async function initProcessor(processor: Processor, options?: UnifiedOptio
     .use(remarkDirectiveRehypeCodeblockGroup)
     .use(remarkDirectiveRehype);
 
-  processor.use(collectFields).use(resolveFields).use(validateFields);
+  processor.use(remarkFrontmatterCollect).use(remarkFrontmatterResolve).use(remarkFrontmatterValidate);
 
   options?.remarks?.forEach((plugin) => {
     processor.use(plugin);
@@ -69,4 +73,29 @@ export async function initProcessor(processor: Processor, options?: UnifiedOptio
   options?.after?.forEach((plugin) => {
     processor.use(plugin);
   });
+
+  return processor;
 }
+
+export {
+  rehypeAutolinkHeadings,
+  rehypeExternalLinks,
+  rehypeHeading,
+  rehypeMermaid,
+  rehypeShiki,
+  rehypeSlug,
+  rehypeStringify,
+  rehypeToc,
+  remarkDirective,
+  remarkDirectiveRehype,
+  remarkDirectiveRehypeCodeblockGroup,
+  remarkDirectiveRehypeCodeblockHeader,
+  remarkFrontmatter,
+  remarkFrontmatterCollect,
+  remarkFrontmatterResolve,
+  remarkFrontmatterValidate,
+  remarkGfm,
+  remarkLink,
+  remarkParse,
+  remarkRehype,
+};

packages/contented-pipeline-md/src/plugins/RemarkFrontmatter.ts

@@ -9,7 +9,7 @@ import { VFile } from 'vfile';
 
 import { UnifiedContented } from './Plugin.js';
 
-export function collectFields(): Transformer<Parent> {
+export function remarkFrontmatterCollect(): Transformer<Parent> {
   return (tree: Parent, file) => {
     const node = tree.children?.[0];
     if (node?.type === 'yaml') {
@@ -49,7 +49,7 @@ function visitDescription(file: VFile): (node: Paragraph) => void {
   };
 }
 
-export function resolveFields(): Transformer<Parent> {
+export function remarkFrontmatterResolve(): Transformer<Parent> {
   return async (tree: Parent, file) => {
     const contented = file.data.contented as UnifiedContented;
     const entries = Object.entries(contented.pipeline.fields ?? {});
@@ -70,7 +70,7 @@ export function resolveFields(): Transformer<Parent> {
   };
 }
 
-export function validateFields(): Transformer<Parent> {
+export function remarkFrontmatterValidate(): Transformer<Parent> {
   return async (tree: Parent, file) => {
     const contented = file.data.contented as UnifiedContented;
     const entries = Object.entries(contented.pipeline.fields ?? {});