Restructure GraphQL automated content scripts (github#34308)

Author: rachmari

.github/workflows/triage-unallowed-contributions.yml

@@ -14,7 +14,7 @@ on:
       - 'assets/fonts/**'
       - 'data/graphql/**'
       - 'Dockerfile*'
-      - 'lib/graphql/**'
+      - 'src/**'
       - 'lib/redirects/**'
       - 'lib/rest/**'
       - 'lib/webhooks/**'
@@ -58,7 +58,7 @@ jobs:
               - 'assets/fonts/**'
               - 'data/graphql/**'
               - 'Dockerfile*'
-              - 'lib/graphql/**'
+              - 'src/**'
               - 'lib/redirects/**'
               - 'lib/rest/**'
               - 'lib/webhooks/**'
@@ -81,9 +81,9 @@ jobs:
               'assets/fonts/**',
               'data/graphql/**',
               'Dockerfile*',
-              'lib/graphql/**',
+              'src/**',
               'lib/redirects/**',
-              'lib/rest/**',
+              'lib/rest/**',	
               'lib/webhooks/**',
               'package*.json',
               'scripts/**',

.github/workflows/update-graphql-files.yml

@@ -39,7 +39,7 @@ jobs:
           # need to use a token from a user with access to github/github for this step
           GITHUB_TOKEN: ${{ secrets.DOCS_BOT_FR }}
         run: |
-          script/graphql/update-files.js
+          src/graphql/scripts/update-files.js
       - name: Create pull request
         id: create-pull-request
         uses: peter-evans/create-pull-request@2b011faafdcbc9ceb11414d64d0573f37c774b04

Dockerfile

@@ -48,6 +48,7 @@ COPY stylesheets ./stylesheets
 COPY pages ./pages
 COPY components ./components
 COPY lib ./lib
+COPY src ./src
 # Certain content is necessary for being able to build
 COPY content/index.md ./content/index.md
 COPY content/rest ./content/rest
@@ -88,6 +89,7 @@ COPY --chown=node:node package.json ./
 COPY --chown=node:node assets ./assets
 COPY --chown=node:node content ./content
 COPY --chown=node:node lib ./lib
+COPY --chown=node:node src ./src
 COPY --chown=node:node middleware ./middleware
 COPY --chown=node:node data ./data
 COPY --chown=node:node next.config.js ./

content/graphql/README.md

@@ -3,8 +3,6 @@
 The `/content/graphql` directory is where the GitHub GraphQL API docs live!
 
 * The `/content/graphql/guides` and `/content/graphql/overview` directories contain articles that are human-editable.
-* The `/content/graphql/reference` directory contains an article for each GraphQL data type used in the GitHub GraphQL API. Most of the content in this directory is rendered using `include` tags.
+* The `/content/graphql/reference` directory contains an article for each GraphQL data type used in the GitHub GraphQL API. This content is generated from the data in `src/graphql/data` and should not be edited by a human. **As a result, we cannot accept contributions to GraphQL API reference content in this repository.**
 
-  The content rendered by `include` tags is sourced from the `/lib/graphql/static` directory, which is automatically generated from the API source code internally in GitHub, and should not be edited by a human. For more information, see the [`/lib/graphql/README.md`](/lib/graphql/README.md).
-
-  **As a result, we cannot accept contributions to GraphQL API reference content in this repository.**
+For more information, see the [`/src/graphql/README.md`](/src/graphql/README.md).

data/graphql/README.md

@@ -1,7 +1,7 @@
 # GraphQL
 
 The files in this directory are kept in sync with their counterparts on `github/github`
-by [automation](../script/graphql/README.md). These files **should not** be edited by humans.
+by [automation](../src/graphql/README.md). These files **should not** be edited by humans.
 
 Dotcom source files:
 ```

pages/[versionId]/graphql/overview/breaking-changes.tsx

@@ -11,7 +11,7 @@ import {
 import { getAutomatedPageMiniTocItems } from 'lib/get-mini-toc-items.js'
 import { BreakingChanges } from 'components/graphql/BreakingChanges'
 import { BreakingChangesT } from 'components/graphql/types'
-import { getGraphqlBreakingChanges } from 'lib/graphql/index.js'
+import { getGraphqlBreakingChanges } from 'src/graphql/lib/index.js'
 
 type Props = {
   mainContext: MainContextT

pages/[versionId]/graphql/overview/changelog.tsx

@@ -11,7 +11,7 @@ import {
 import { getAutomatedPageMiniTocItems } from 'lib/get-mini-toc-items.js'
 import { Changelog } from 'components/graphql/Changelog'
 import { ChangelogItemT } from 'components/graphql/types'
-import { getGraphqlChangelog } from 'lib/graphql/index.js'
+import { getGraphqlChangelog } from 'src/graphql/lib/index.js'
 
 type Props = {
   mainContext: MainContextT

pages/[versionId]/graphql/overview/schema-previews.tsx

@@ -11,7 +11,7 @@ import {
 import { getAutomatedPageMiniTocItems } from 'lib/get-mini-toc-items.js'
 import { Previews } from 'components/graphql/Previews'
 import { PreviewT } from 'components/graphql/types'
-import { getPreviews } from 'lib/graphql/index.js'
+import { getPreviews } from 'src/graphql/lib/index.js'
 
 type Props = {
   mainContext: MainContextT

pages/[versionId]/graphql/reference/[page].tsx

@@ -2,7 +2,7 @@ import { GetServerSideProps } from 'next'
 import React from 'react'
 
 import { GraphqlPage } from 'components/graphql/GraphqlPage'
-import { getGraphqlSchema, getMiniToc } from 'lib/graphql/index.js'
+import { getGraphqlSchema, getMiniToc } from 'src/graphql/lib/index.js'
 import { MainContextT, MainContext, getMainContext } from 'components/context/MainContext'
 import type { ObjectT, GraphqlT } from 'components/graphql/types'
 import { AutomatedPage } from 'components/article/AutomatedPage'

script/enterprise-server-deprecations/remove-static-files.js

@@ -15,7 +15,7 @@ import { deprecated } from '../../lib/enterprise-server-releases.js'
 
 const graphqlDataDir = path.join(process.cwd(), 'data/graphql')
 const webhooksStaticDir = path.join(process.cwd(), 'lib/webhooks/static')
-const graphqlStaticDir = path.join(process.cwd(), 'lib/graphql/static')
+const graphqlStaticDir = path.join(process.cwd(), 'src/graphql/data')
 const restDecoratedDir = path.join(process.cwd(), 'lib/rest/static/decorated')
 const ghesReleaseNotesDir = 'data/release-notes/enterprise-server'
 

script/enterprise-server-releases/create-graphql-files.js

@@ -12,11 +12,11 @@ import { program } from 'commander'
 import mkdirp from 'mkdirp'
 import { allVersions } from '../../lib/all-versions.js'
 
-const graphqlStaticDir = path.join(process.cwd(), 'lib/graphql/static')
+const graphqlStaticDir = path.join(process.cwd(), 'src/graphql/data')
 const graphqlDataDir = path.join(process.cwd(), 'data/graphql')
 
 program
-  .description('Create GraphQL files in lib/graphql/static based on an existing version.')
+  .description('Create GraphQL files in src/graphql/data based on an existing version.')
   .option(
     '-n, --newVersion <version>',
     'The version to copy the files to. Must be in <plan@release> format.'

lib/graphql/README.md src/graphql/README.md

@@ -2,29 +2,27 @@
 
 ## About this directory
 
-* `lib/graphql/*.json` are human-editable.
-* `lib/graphql/static/*.json` are generated by [scripts](../script/graphql/README.md).
+* `src/graphql/lib` and `src/graphql/scripts` are human-editable.
+* `src/graphql/data/**` are generated by [scripts](../src/graphql/README.md).
 
 ## Editable files
 
-* `lib/graphql/validator.json`
+* `src/graphql/lib/validator.json`
   - JSON schema used in `tests/graphql.js`.
-* `lib/graphql/non-schema-scalars.json`
+* `src/graphql/lib/non-schema-scalars.json`
   - An array of scalar types that live in [`graphql-ruby`](https://github.com/rmosolgo/graphql-ruby/tree/356d9d369e444423bf06cab3dc767ec75fbc6745/lib/graphql/types) only. These are
   not part of the core GraphQL spec.
-* `lib/graphql/redirects.json`
-  - A list of original `developer.github.com/v4` paths and their corresponding new `docs.github.com/graphql` paths. Used in redirect middleware and tests.
-* `lib/graphql/types.json`
+* `src/graphql/lib/types.json`
   - High-level GraphQL types and kinds.
 
-## Static files
+## Data files
 
-Generated by `script/graphql/update-files.js`:
+Generated by `src/graphql/scripts/update-files.js`:
 
-* `lib/graphql/static/schema-VERSION.json` (separate files per version)
-* `lib/graphql/static/previews.json`
-* `lib/graphql/static/upcoming-changes.json`
-* `lib/graphql/static/changelog.json`
+* `src/graphql/data/schema-VERSION.json` (separate files per version)
+* `src/graphql/data/previews.json`
+* `src/graphql/data/upcoming-changes.json`
+* `src/graphql/data/changelog.json`
 
 ## Rendering docs
 

lib/graphql/static/changelog.json src/graphql/data/changelog.json

@@ -1,10 +1,10 @@
 import {
   readCompressedJsonFileFallbackLazily,
   readCompressedJsonFileFallback,
-} from '../../lib/read-json-file.js'
-import { getAutomatedPageMiniTocItems } from '../get-mini-toc-items.js'
-import languages from '../languages.js'
-import { allVersions } from '../all-versions.js'
+} from '../../../lib/read-json-file.js'
+import { getAutomatedPageMiniTocItems } from '../../../lib/get-mini-toc-items.js'
+import languages from '../../../lib/languages.js'
+import { allVersions } from '../../../lib/all-versions.js'
 
 /* ADD LANGUAGE KEY */
 let previews
@@ -22,7 +22,7 @@ export function getGraphqlSchema(version, type) {
   if (!graphqlSchema.has(graphqlVersion)) {
     graphqlSchema.set(
       graphqlVersion,
-      readCompressedJsonFileFallback(`lib/graphql/static/schema-${graphqlVersion}.json`)
+      readCompressedJsonFileFallback(`src/graphql/data/schema-${graphqlVersion}.json`)
     )
   }
   return graphqlSchema.get(graphqlVersion)[type]
@@ -32,7 +32,7 @@ export function getGraphqlChangelog() {
   if (!changelog.has('schema')) {
     changelog.set(
       'schema',
-      readCompressedJsonFileFallbackLazily('./lib/graphql/static/changelog.json')()
+      readCompressedJsonFileFallbackLazily('./src/graphql/data/changelog.json')()
     )
   }
 
@@ -43,7 +43,7 @@ export function getGraphqlBreakingChanges(version) {
   const graphqlVersion = getGraphqlVersion(version)
   if (!upcomingChanges) {
     upcomingChanges = readCompressedJsonFileFallbackLazily(
-      './lib/graphql/static/upcoming-changes.json'
+      './src/graphql/data/upcoming-changes.json'
     )()
   }
   return upcomingChanges[graphqlVersion]
@@ -52,7 +52,7 @@ export function getGraphqlBreakingChanges(version) {
 export function getPreviews(version) {
   const graphqlVersion = getGraphqlVersion(version)
   if (!previews) {
-    previews = readCompressedJsonFileFallbackLazily('./lib/graphql/static/previews.json')()
+    previews = readCompressedJsonFileFallbackLazily('./src/graphql/data/previews.json')()
   }
   return previews[graphqlVersion]
 }

lib/graphql/static/previews.json src/graphql/data/previews.json

@@ -1,5 +1,5 @@
 // the tests in tests/graphql.js use this schema to ensure the integrity
-// of the data in lib/graphql/static/*.json
+// of the data in src/graphql/data/*.json
 
 // PREVIEWS
 export const previewsValidator = {

lib/graphql/static/schema-dotcom.json src/graphql/data/schema-dotcom.json

@@ -3,10 +3,10 @@
 A [scheduled workflow](../.github/workflows/update-graphql-files.yml) runs the following
 scripts on a daily basis:
 ```
-script/graphql/update-files.js
+src/graphql/scripts/update-files.js
 ```
-These scripts update the [static JSON files](../../lib/graphql/static) used to
-render GraphQL docs. See the [`lib/graphql/README`](../../lib/graphql/README.md)
+These scripts update the [JSON data files](src/graphql/data) used to
+render GraphQL docs. See the [`src/graphql/README`](src/graphql/README.md)
 for more info.
 
 **Note**: The changelog script pulls content from the internal-developer repo. It relies on graphql-docs automation running daily to update the changelog files in internal-developer.

lib/graphql/static/schema-ghae.json src/graphql/data/schema-ghae.json

@@ -2,7 +2,7 @@
 import { diff, ChangeType } from '@graphql-inspector/core'
 import { loadSchema } from '@graphql-tools/load'
 import fs from 'fs'
-import renderContent from '../../lib/render-content/index.js'
+import renderContent from '../../../lib/render-content/index.js'
 
 /**
  * Tag `changelogEntry` with `date: YYYY-mm-dd`, then prepend it to the JSON

lib/graphql/static/schema-ghec.json src/graphql/data/schema-ghec.json

@@ -4,17 +4,17 @@ import path from 'path'
 import mkdirp from 'mkdirp'
 import yaml from 'js-yaml'
 import { execSync } from 'child_process'
-import { getContents, listMatchingRefs } from '../helpers/git-utils.js'
-import { allVersions } from '../../lib/all-versions.js'
+import { getContents, listMatchingRefs } from '../../../script/helpers/git-utils.js'
+import { allVersions } from '../../../lib/all-versions.js'
 import processPreviews from './utils/process-previews.js'
 import processUpcomingChanges from './utils/process-upcoming-changes.js'
 import processSchemas from './utils/process-schemas.js'
 import { prependDatedEntry, createChangelogEntry } from './build-changelog.js'
 
 const graphqlDataDir = path.join(process.cwd(), 'data/graphql')
-const graphqlStaticDir = path.join(process.cwd(), 'lib/graphql/static')
+const graphqlStaticDir = path.join(process.cwd(), 'src/graphql/data')
 const dataFilenames = JSON.parse(
-  await fs.readFile(path.join(process.cwd(), './script/graphql/utils/data-filenames.json'))
+  await fs.readFile(path.join(process.cwd(), './src/graphql/scripts/utils/data-filenames.json'))
 )
 
 // check for required PAT
@@ -75,7 +75,7 @@ async function main() {
       if (changelogEntry) {
         prependDatedEntry(
           changelogEntry,
-          path.join(process.cwd(), 'lib/graphql/static/changelog.json')
+          path.join(process.cwd(), 'src/graphql/data/changelog.json')
         )
       }
     }
@@ -104,7 +104,7 @@ async function getRemoteRawContent(filepath, graphqlVersion) {
   return getContents(...Object.values(options))
 }
 
-// find the relevant filepath in script/graphql/utils/data-filenames.json
+// find the relevant filepath in src/graphql/scripts/util/data-filenames.json
 function getDataFilepath(id, graphqlVersion) {
   const versionType = getVersionType(graphqlVersion)
 

lib/graphql/static/schema-ghes-3.4.json src/graphql/data/schema-ghes-3.4.json

@@ -6,7 +6,7 @@ import fs from 'fs/promises'
 import path from 'path'
 
 const externalScalarsJSON = JSON.parse(
-  await fs.readFile(path.join(process.cwd(), './lib/graphql/non-schema-scalars.json'))
+  await fs.readFile(path.join(process.cwd(), './src/graphql/lib/non-schema-scalars.json'))
 )
 const externalScalars = await Promise.all(
   externalScalarsJSON.map(async (scalar) => {
@@ -18,7 +18,7 @@ const externalScalars = await Promise.all(
 )
 
 // select and format all the data from the schema that we need for the docs
-// used in the build step by script/graphql/build-static-files.js
+// used in the build step
 export default async function processSchemas(idl, previewsPerVersion) {
   const schemaAST = parse(idl.toString())
   const schema = buildASTSchema(schemaAST)

lib/graphql/static/schema-ghes-3.5.json src/graphql/data/schema-ghes-3.5.json

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 import yaml from 'js-yaml'
 import { groupBy } from 'lodash-es'
-import renderContent from '../../../lib/render-content/index.js'
+import renderContent from '../../../../lib/render-content/index.js'
 
 export default async function processUpcomingChanges(upcomingChangesYml) {
   const upcomingChanges = yaml.load(upcomingChangesYml).upcoming_changes

lib/graphql/static/schema-ghes-3.6.json src/graphql/data/schema-ghes-3.6.json

@@ -1,11 +1,11 @@
 #!/usr/bin/env node
-import renderContent from '../../../lib/render-content/index.js'
+import renderContent from '../../../../lib/render-content/index.js'
 import fs from 'fs/promises'
 import graphql from 'graphql'
 import path from 'path'
 
 const graphqlTypes = JSON.parse(
-  await fs.readFile(path.join(process.cwd(), './lib/graphql/types.json'))
+  await fs.readFile(path.join(process.cwd(), './src/graphql/lib/types.json'))
 )
 const { isScalarType, isObjectType, isInterfaceType, isUnionType, isEnumType, isInputObjectType } =
   graphql

lib/graphql/static/schema-ghes-3.7.json src/graphql/data/schema-ghes-3.7.json

@@ -3,16 +3,16 @@ import {
   schemaValidator,
   previewsValidator,
   upcomingChangesValidator,
-} from '../../lib/graphql/validator.js'
+} from '../../src/graphql/lib/validator.js'
 import revalidator from 'revalidator'
 import { allVersions } from '../../lib/all-versions.js'
 import { jest } from '@jest/globals'
 
-const previewsJson = readJsonFile('./lib/graphql/static/previews.json')
-const upcomingChangesJson = readJsonFile('./lib/graphql/static/upcoming-changes.json')
+const previewsJson = readJsonFile('./src/graphql/data/previews.json')
+const upcomingChangesJson = readJsonFile('./src/graphql/data/upcoming-changes.json')
 const allVersionValues = Object.values(allVersions)
 const graphqlVersions = allVersionValues.map((v) => v.miscVersionName)
-const graphqlTypes = readJsonFile('./lib/graphql/types.json').map((t) => t.kind)
+const graphqlTypes = readJsonFile('./src/graphql/lib/types.json').map((t) => t.kind)
 
 describe('graphql json files', () => {
   jest.setTimeout(3 * 60 * 1000)
@@ -30,7 +30,7 @@ describe('graphql json files', () => {
     // test up significantly.
     const typeObjsTested = new Set()
     graphqlVersions.forEach((version) => {
-      const schemaJsonPerVersion = readJsonFile(`lib/graphql/static/schema-${version}.json`)
+      const schemaJsonPerVersion = readJsonFile(`src/graphql/data/schema-${version}.json`)
       // all graphql types are arrays except for queries
       graphqlTypes.forEach((type) => {
         schemaJsonPerVersion[type].forEach((typeObj) => {

lib/graphql/static/schema-ghes-3.8.json src/graphql/data/schema-ghes-3.8.json

@@ -4,7 +4,7 @@ import {
   cleanPreviewTitle,
   previewAnchor,
   prependDatedEntry,
-} from '../../script/graphql/build-changelog.js'
+} from '../../src/graphql/scripts/build-changelog.js'
 import fs from 'fs/promises'
 import MockDate from 'mockdate'
 import readJsonFile from '../../lib/read-json-file.js'

lib/graphql/static/upcoming-changes.json src/graphql/data/upcoming-changes.json

@@ -6,10 +6,12 @@ import {
   getGraphqlChangelog,
   getGraphqlBreakingChanges,
   getPreviews,
-} from '../../lib/graphql/index.js'
+} from '../../src/graphql/lib/index.js'
 
 describe('graphql schema', () => {
-  const graphqlTypes = JSON.parse(readFileSync('lib/graphql/types.json')).map((item) => item.kind)
+  const graphqlTypes = JSON.parse(readFileSync('src/graphql/lib/types.json')).map(
+    (item) => item.kind
+  )
   for (const version in allVersions) {
     for (const type of graphqlTypes) {
       test(`getting the GraphQL ${type} schema works for ${version}`, async () => {