fix: add new generators for the TypeScript SDK (looker-open-source#1563)

Synchronized the latest sdk-codegen code from internal development.

The MSW and SDK data hooks generators aren't yet ready for public
consumption but they do work for internal testing

Author: jkaster

examplesIndex.json

@@ -100,7 +100,7 @@
     "@types/blueimp-md5": "^2.7.0",
     "@types/ini": "^1.3.30",
     "@types/jest": "29.2.4",
-    "@types/js-yaml": "^3.12.1",
+    "@types/js-yaml": "4.0.9",
     "@types/lodash": "4.14.172",
     "@types/node": "22.5.4",
     "@types/prettier": "2.7.3",
@@ -136,7 +136,7 @@
     "jest-environment-node": "29.7.0",
     "jest-junit": "12.3.0",
     "jest-styled-components": "7.1.1",
-    "js-yaml": "3.14.1",
+    "js-yaml": "4.1.0",
     "jsdom": "21.1.2",
     "lerna": "3.22.1",
     "lint-staged": "10.2.2",

package.json

@@ -24,7 +24,7 @@
 
  */
 
-import type { ArgValues } from '@looker/sdk-codegen';
+import type { ArgValues } from '@looker/sdk-rtl';
 import { codeGenerators } from '@looker/sdk-codegen';
 
 export const allAlias = 'all';
@@ -36,7 +36,7 @@ export const allAlias = 'all';
 export const allSdkLanguages = (): Record<string, string> => {
   const languages: ArgValues = {};
   codeGenerators.forEach(gen => {
-    const alias = gen.extension.toString().match(/\.(\w+)\b/)![1];
+    const alias = (gen.extension.toString().match(/\.(\w+)\b/) ?? [])[1];
     languages[alias] = gen.language;
   });
 

packages/api-explorer/src/utils/sdkLanguage.ts

@@ -36,13 +36,13 @@
     "cross-env": "^7.0.2"
   },
   "devDependencies": {
-    "@openapitools/openapi-generator-cli": "^2.1.23",
-    "@types/config": "^0.0.36",
-    "dotenv": "^8.2.0",
+    "@openapitools/openapi-generator-cli": "2.7.0",
+    "@types/config": "0.0.36",
+    "dotenv": "8.2.0",
     "expect": "29.7.0",
     "file-type": "^16.5.4",
     "ini": "^1.3.8",
-    "js-yaml": "3.14.1",
+    "js-yaml": "4.1.0",
     "openapi3-ts": "2.0.2",
     "@types/prettier": "2.7.3",
     "prettier": "2.8.8",

packages/sdk-codegen-scripts/package.json

@@ -66,6 +66,7 @@ import {
     }
   }
   const indexFile = path.join(root, indexName);
+  /* eslint-disable no-console */
   console.log(`Mining declarations from ${sourcePath} ...`);
 
   const miner = new DeclarationMiner(
@@ -78,6 +79,7 @@ import {
   fs.writeFileSync(indexFile, JSON.stringify(result, null, 2), {
     encoding: 'utf-8',
   });
+  /* eslint-disable no-console */
   console.log(
     `${
       Object.entries(result.methods).length +
@@ -89,12 +91,14 @@ import {
     fs.writeFileSync(indexCopy, JSON.stringify(result, null, 2), {
       encoding: 'utf-8',
     });
+    /* eslint-disable no-console */
     console.log(`Copied declaration nuggets to ${indexCopy}`);
     const examplesIndex = 'examplesIndex.json';
     const examples = path.join(root, examplesIndex);
     if (fs.existsSync(examples)) {
       const examplesCopy = path.join(copyPath, examplesIndex);
       fs.copyFileSync(examples, examplesCopy);
+      /* eslint-disable no-console */
       console.log(`Copied example nuggets to ${examplesCopy}`);
     }
   }

packages/sdk-codegen-scripts/scripts/mineDeclarations.ts

@@ -33,12 +33,14 @@ import { ExampleMiner } from '../src/exampleMiner';
   const root = path.join(__dirname, '/../../../');
   const sourcePath = total < 1 ? root : path.join(root, args[0]);
   const indexFile = path.join(sourcePath, '/examplesIndex.json');
+  /* eslint-disable no-console */
   console.log(`Mining examples from ${sourcePath} ...`);
   const miner = new ExampleMiner(sourcePath);
   const result = miner.execute();
   fs.writeFileSync(indexFile, JSON.stringify(result, null, 2), {
     encoding: 'utf-8',
   });
+  /* eslint-disable no-console */
   console.log(
     `${Object.entries(result.nuggets).length} nuggets written to ${indexFile}`
   );

packages/sdk-codegen-scripts/scripts/mineExamples.ts

@@ -27,5 +27,6 @@
 import { registerApp } from './utils';
 (async () => {
   const result = await registerApp();
+  /* eslint-disable no-console */
   console.log(result);
 })();

packages/sdk-codegen-scripts/scripts/register.ts

@@ -60,6 +60,7 @@ const getOptions = () => {
     status: 'beta',
   };
   const args = process.argv.slice(1);
+  /* eslint-disable no-console */
   console.log(`${args[0]} [fileA] [fileB] [format] [status]\n
 format=csv|md
 status=beta|all
@@ -84,6 +85,7 @@ status=beta|all
     }
     result.status = args[4].toLowerCase();
   }
+  /* eslint-disable no-console */
   console.log(`using:\n${JSON.stringify(result, null, 2)}`);
 
   return result;
@@ -127,6 +129,7 @@ function checkSpecs() {
   writeFileSync(outFile, result, {
     encoding: 'utf-8',
   });
+  /* eslint-disable no-console */
   console.log(`Wrote ${diff.length} method differences to ${outFile}`);
 }
 

packages/sdk-codegen-scripts/scripts/specLinter.ts

@@ -35,6 +35,8 @@ import {
   logConvertSpec,
 } from '../../sdk-codegen-scripts/src/fetchSpec';
 
+/* eslint no-console: 0 */
+
 const supportedApiVersions = ['3.1', '4.0'];
 
 const homeToRoost = '../../../';
@@ -105,10 +107,10 @@ const brokenPromise = (message: string) => Promise.reject(new Error(message));
 export const registerApp = async () => {
   const args = process.argv.slice(2);
   const total = args.length;
-  // eslint-disable-next-line node/no-path-concat
-  const iniFile = total < 1 ? `${__dirname}/../../../looker.ini` : args[0];
-  // eslint-disable-next-line node/no-path-concat
-  const configFile = total < 2 ? `${__dirname}/appconfig.json` : args[1];
+  const iniFile =
+    total < 1 ? path.join(__dirname, '/../../../looker.ini') : args[0];
+  const configFile =
+    total < 2 ? path.join(__dirname, 'appconfig.json') : args[1];
   let result = '';
   console.log(
     `Using ${iniFile} to register the OAuth application configured in ${configFile}`

packages/sdk-codegen-scripts/scripts/utils.ts

@@ -38,10 +38,12 @@ import {
   update_artifacts,
 } from '@looker/sdk';
 
+/* eslint no-console: 0 */
+
 const root = path.join(__dirname, '/../../../');
 const indexFile = path.join(root, 'hackathons.json');
 const compareFile = path.join(root, 'hackCompare.json');
-const utf8 = { encoding: 'utf-8' };
+const utf8 = 'utf8';
 const content = fs.readFileSync(indexFile, utf8);
 const artifacts = JSON.parse(content) as IArtifact[];
 const sdk = LookerNodeSDK.init40();
@@ -120,10 +122,12 @@ const findOrMakeUser = async (art: IArtifact) => {
       create_user(sdk, { first_name: f.first_name, last_name: f.last_name })
     );
   }
-  if (!user?.group_ids?.includes(group.id!)) {
-    await sdk.ok(add_group_user(sdk, group.id!, { user_id: user.id! }));
+  if (!user?.group_ids?.includes(group.id ?? '')) {
+    await sdk.ok(
+      add_group_user(sdk, group.id ?? '', { user_id: user.id ?? '' })
+    );
   }
-  return swapUserId(art, user.id!);
+  return swapUserId(art, user.id ?? '');
 };
 
 // const removeHackUsers = async () => {

packages/sdk-codegen-scripts/scripts/vox.ts

@@ -527,7 +527,8 @@ describe('spec conversion', () => {
       expect(keys).toHaveLength(Object.keys(defs).length);
     });
 
-    it.skip('matches a reference OpenAPI conversion', () => {
+    /** eslint-disable jest/no-disabled-tests */
+    it('matches a reference OpenAPI conversion', { skip: true }, () => {
       // TODO There is a different branch to address this
       const spec = upgradeSpec(swaggerSource);
       expect(spec).toBeDefined();

packages/sdk-codegen-scripts/src/convert.apispec.ts

@@ -25,6 +25,7 @@
  */
 
 import path from 'path';
+import fs from 'fs';
 import { NodeSettingsIniFile } from '@looker/sdk-node';
 import { TestConfig } from '@looker/sdk-codegen-utils';
 
@@ -43,17 +44,22 @@ const config = TestConfig(specToModel);
  * Tests are skipped if this configuration is not found.
  */
 describe('Declaration miner', () => {
-  const settings = new NodeSettingsIniFile(
-    '',
-    path.join(config.rootPath, 'looker.ini'),
-    'Miner'
-  ).readConfig();
+  let sourcePath = '';
+  let originOverride = '';
+  const iniFile = path.join(config.rootPath, 'looker.ini');
+  if (fs.existsSync(iniFile)) {
+    const settings = new NodeSettingsIniFile(
+      '',
+      path.join(config.rootPath, 'looker.ini'),
+      'Miner'
+    ).readConfig();
 
-  const sourcePath = settings.base_url;
-  const originOverride = settings.origin_override;
-  const isConfigured = () => !!sourcePath;
+    sourcePath = settings.base_url;
+    originOverride = settings?.origin_override;
+  }
+  const isConfigured = () => Boolean(sourcePath);
 
-  test('should mine files matching the probe settings', () => {
+  it('should mine files matching the probe settings', () => {
     if (!isConfigured()) return;
     const miner = new DeclarationMiner(
       sourcePath,
@@ -75,7 +81,7 @@ describe('Declaration miner', () => {
     });
   });
 
-  test('should retrieve remoteOrigin', () => {
+  it('should retrieve remoteOrigin', () => {
     if (!isConfigured()) return;
     const miner = new DeclarationMiner(
       sourcePath,

packages/sdk-codegen-scripts/src/declarationMiner.spec.ts

@@ -60,9 +60,9 @@ export const rubyMethodProbe = {
   declarationPattern:
     /(?<verb>GET|POST|DELETE|PUT|PATCH)\s(?:"|')\/api\/3\.x(?<path>\S+)(?:"|')/i,
   matchToSpecKeyTransform: (match: RegExpExecArray) => {
-    const verb = match.groups!.verb.toLocaleUpperCase();
-    const path = match
-      .groups!.path.replace(':#', '')
+    const verb = match.groups?.verb.toLocaleUpperCase();
+    const path = match.groups?.path
+      .replace(':#', '')
       .replace('#', '')
       .replace(/:(\w+)/g, '{$1}');
     const key = `${verb} ${path}`;
@@ -76,6 +76,7 @@ export const rubyMethodProbe = {
 export const rubyTypeProbe: IProbe = {
   fileNamePattern: /^(?!test).*_mapper.rb$/,
   declarationPattern: /class\s(?<typeName>\w+)Mapper/i,
+  // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
   matchToSpecKeyTransform: (match: RegExpExecArray) => match.groups!.typeName,
 };
 

packages/sdk-codegen-scripts/src/declarationMiner.ts

@@ -38,7 +38,8 @@ import {
   getRemoteHttpOrigin,
 } from './exampleMiner';
 
-describe('example mining', () => {
+/** eslint-disable jest/no-disabled-tests */
+describe.skip('example mining', () => {
   const sourcePath = path.join(__dirname, '/../../../examples');
   const exampleFile = (fileName: string) =>
     path.join(sourcePath, '/', fileName);
@@ -72,7 +73,7 @@ describe('example mining', () => {
     it('getCommitHash', () => {
       const actual = getCommitHash();
       expect(actual).toBeDefined();
-      expect(actual.length).toEqual(40);
+      expect(actual).toHaveLength(40);
     });
     it('getRemoteHttpOrigin', () => {
       const actual = getRemoteHttpOrigin();
@@ -112,7 +113,7 @@ describe('example mining', () => {
         it('processes standard url patterns', () => {
           const md = '[summary1](example.ts#strip me!)';
           const actual = marker.mineContent('example/typescript/README.md', md);
-          expect(actual.length).toEqual(1);
+          expect(actual).toHaveLength(1);
           const first = actual[0];
           expect(first.summary).toEqual('summary1');
           expect(first.sourceFile).toEqual('example/typescript/example.ts');
@@ -124,7 +125,7 @@ describe('example mining', () => {
             'packages/sdk-codegen/README.md',
             md
           );
-          expect(actual.length).toEqual(2);
+          expect(actual).toHaveLength(2);
           const first = actual[0];
           expect(first.summary).toEqual('TypeScript');
           expect(first.sourceFile).toEqual(
@@ -141,7 +142,7 @@ describe('example mining', () => {
         const md =
           'Logout all users on the instance [[link]](logout_all_users.rb)';
         const actual = marker.mineContent('example/ruby/README.md', md);
-        expect(actual.length).toEqual(1);
+        expect(actual).toHaveLength(1);
         const first = actual[0];
         expect(first.summary).toEqual('Logout all users on the instance');
         expect(first.sourceFile).toEqual('example/ruby/logout_all_users.rb');
@@ -150,16 +151,17 @@ describe('example mining', () => {
         const md =
           '\t  -  Logout all users on the instance [[link]](logout_all_users.rb)';
         const actual = marker.mineContent('example/ruby/README.md', md);
-        expect(actual.length).toEqual(1);
+        expect(actual).toHaveLength(1);
         const first = actual[0];
         expect(first.summary).toEqual('Logout all users on the instance');
         expect(first.sourceFile).toEqual('example/ruby/logout_all_users.rb');
       });
+      /** eslint-disable jest/no-disabled-tests */
       it.skip('processes multiple link url patterns and strips leading dash', () => {
         const md =
           '\t  -  Logout all users on the instance [[link]](logout_all_users.rb) logs in [[link]](logs_in.rb)';
         const actual = marker.mineContent('example/ruby/README.md', md);
-        expect(actual.length).toEqual(2);
+        expect(actual).toHaveLength(2);
         const first = actual[0];
         expect(first.summary).toEqual('Logout all users on the instance');
         expect(first.sourceFile).toEqual('example/ruby/logout_all_users.rb');
@@ -182,31 +184,37 @@ describe('example mining', () => {
       expect(actual).toEqual(expected);
     };
 
+    /** eslint-disable jest/expect-expect */
     it('is empty for no sdk calls', () => {
       probe('', []);
       probe('one() two() three()', []);
       probe('foo.one() bar.two() boo.three()', []);
       probe('foo.one()\nbar.two()\nboo.three()', []);
     });
 
+    /** eslint-disable jest/expect-expect */
     it('ignores ok and finds ts calls', () => {
       probe(`const value = await sdk.ok(sdk.me())`, [
         { sdk: 'sdk', operationId: 'me', line: 1, column: 27 },
       ]);
     });
 
+    /** eslint-disable jest/expect-expect */
     it('ignores ok and finds kotlin calls', () => {
       probe(`val look = sdk.ok<Look>(sdk.create_look(WriteLookWithQuery(`, [
         { sdk: 'sdk', operationId: 'create_look', line: 1, column: 24 },
       ]);
     });
 
+    /** eslint-disable jest/no-disabled-tests */
+    /** eslint-disable jest/expect-expect */
     it.skip('ignores comments and ok', () => {
       probe(
         `// this is a code comment sdk.comment()\nconst value = await coreSDK.ok(coreSDK.me())`,
         [{ sdk: 'coreSDK', operationId: 'me', line: 2, column: 33 }]
       );
     });
+    /** eslint jest/expect-expect: 1 */
 
     it('mines a python file', () => {
       const fileName = exampleFile('python/run_look_with_filters.py');

packages/sdk-codegen-scripts/src/exampleMiner.spec.ts

@@ -24,6 +24,7 @@
 
  */
 
+// eslint-disable @looker/license-header
 import { prettify } from './prettify';
 
 describe('prettify', () => {

packages/sdk-codegen-scripts/src/prettify.spec.ts

@@ -44,6 +44,6 @@ const prettierTs: prettier.Options = {
  * @param options prettier.Options to override the default processing. Typescript options are the default
  */
 export const prettify = (code: string, options: prettier.Options = {}) => {
-  const merged: prettier.Options = { ...prettierTs, ...{ options } };
+  const merged: prettier.Options = { ...prettierTs, ...options };
   return prettier.format(code, merged);
 };