feat(includers/unarchive): unarchive input

introduce new includer that allows to unpack
tar archive before including content

Author: moki

package-lock.json

@@ -53,6 +53,7 @@
     "shelljs": "0.8.5",
     "simple-git": "2.48.0",
     "slugify": "^1.6.5",
+    "tar-stream": "^2.2.0",
     "threads": "1.7.0",
     "threads-plugin": "1.4.0",
     "walk-sync": "2.2.0",
@@ -77,6 +78,7 @@
     "@types/react": "16.14.21",
     "@types/react-dom": "16.9.14",
     "@types/shelljs": "0.8.10",
+    "@types/tar-stream": "^2.2.2",
     "@types/yargs": "15.0.14",
     "@typescript-eslint/eslint-plugin": "2.34.0",
     "@typescript-eslint/parser": "2.34.0",

package.json

@@ -87,7 +87,7 @@ export type YfmTocIncluders = YfmTocIncluder[];
 
 export type YfmTocIncluder = YfmTocIncluderName | YfmTocIncluderObject;
 
-export const includersNames = ['sourcedocs', 'openapi', 'generic'] as const;
+export const includersNames = ['sourcedocs', 'openapi', 'generic', 'unarchive'] as const;
 
 export type YfmTocIncluderName = typeof includersNames[number];
 

src/models.ts

@@ -1,3 +1,4 @@
 export * as generic from './generic';
 export * as sourcedocs from './sourcedocs';
 export * as openapi from './openapi';
+export * as unarchive from './unarchive';

src/services/includers/batteries/index.ts

@@ -0,0 +1,94 @@
+import {mkdirSync, createReadStream, createWriteStream} from 'fs';
+import {join, dirname} from 'path';
+import {extract, Headers} from 'tar-stream';
+
+import type {PassThrough} from 'stream';
+
+import {IncluderFunctionParams} from '../../../models';
+
+const name = 'unarchive';
+
+class UnarchiveIncluderError extends Error {
+    path: string;
+
+    constructor(message: string, path: string) {
+        super(message);
+
+        this.name = 'UnarchiveIncluderError';
+        this.path = path;
+    }
+}
+
+function pipeline(readPath: string, writeBasePath: string): Promise<void> {
+    return new Promise((res, rej) => {
+        const reader = createReadStream(readPath);
+
+        reader.on('error', (err: Error) => {
+            rej(err);
+        });
+
+        const extractor = extract();
+
+        extractor.on('error', (err: Error) => {
+            rej(err);
+        });
+
+        mkdirSync(writeBasePath, {recursive: true});
+
+        extractor.on('entry', (header: Headers, stream: PassThrough, next: Function) => {
+            const {type, name} = header;
+
+            const writePath = join(writeBasePath, name);
+
+            const writeDirPath = type === 'directory' ? writePath : dirname(writePath);
+
+            mkdirSync(writeDirPath, {recursive: true});
+
+            if (type !== 'directory') {
+                const writer = createWriteStream(writePath, {flags: 'w'});
+
+                writer.on('error', (err) => {
+                    rej(err);
+                });
+
+                stream.pipe(writer);
+            }
+
+            stream.on('end', () => {
+                next();
+            });
+
+            stream.resume();
+        });
+
+        reader.pipe(extractor).on('finish', () => {
+            res();
+        });
+    });
+}
+
+async function includerFunction(params: IncluderFunctionParams) {
+    const {readBasePath, writeBasePath, tocPath, passedParams: {input, output}, index} = params;
+
+    if (!input?.length || !output?.length) {
+        throw new UnarchiveIncluderError('provide includer with input parameter', tocPath);
+    }
+
+    const contentPath = index === 0
+        ? join(writeBasePath, input)
+        : join(readBasePath, input);
+
+    const writePath = join(writeBasePath, output);
+
+    try {
+        await pipeline(contentPath, writePath);
+    } catch (err) {
+        throw new UnarchiveIncluderError(err.toString(), tocPath);
+    }
+
+    return {input: output};
+}
+
+export {name, includerFunction};
+
+export default {name, includerFunction};

src/services/includers/batteries/unarchive.ts

@@ -5,7 +5,7 @@ import {isObject} from 'lodash';
 import {ArgvService} from '../index';
 import {IncludeMode} from '../../constants';
 import {logger} from '../../utils/logger';
-import {generic, sourcedocs, openapi} from './batteries';
+import {generic, sourcedocs, openapi, unarchive} from './batteries';
 
 import type {
     Includer,
@@ -51,7 +51,7 @@ class IncludersError extends Error {
 function init(custom: Includer[] = []) {
     if (includersMap) { return; }
 
-    includersMap = {generic, sourcedocs, openapi};
+    includersMap = {generic, sourcedocs, openapi, unarchive};
 
     for (const includer of custom) {
         includersMap[includer.name] = includer;