feat(dgeni): add remove duplicates processor (#2968)

Author: griest024

tools/dgeni/src/transforms/daffodil-api-package/processors/remove-duplicates.ts

@@ -0,0 +1,24 @@
+import {
+  Processor,
+  Document,
+} from 'dgeni';
+
+/**
+ * Removes docs with duplicate IDs.
+ */
+export class RemoveDuplicatesProcessor implements Processor {
+  name = 'removeDuplicates';
+  $runAfter = ['readTypeScriptModules'];
+  $runBefore = ['generateApiList', 'parsing-tags'];
+
+  $process(docs: Array<Document>): Array<Document> {
+    const set = new Set();
+    return docs.filter((doc) => {
+      if (set.has(doc.id)) {
+        return false;
+      }
+      set.add(doc.id);
+      return true;
+    });
+  }
+}