sidebars.js

/**
 * Creating a sidebar enables you to:
 - create an ordered group of docs
 - render a sidebar for each doc of that group
 - provide next/previous navigation

 The sidebars can be generated from the filesystem, or explicitly defined here.

 Create as many sidebars as you want.
 */

// @ts-check

/** @type {import('@docusaurus/plugin-content-docs').SidebarsConfig} */
const sidebars = {
  // By default, Docusaurus generates a sidebar from the docs folder structure
  // defaultSidebar: [{ type: "autogenerated", dirName: "." }],

  // But you can create a sidebar manually
  mySidebar: [
    "index",
    {
      type: "html",
      value: '<span class="sidebar-divider" />',
    },
    {
      type: "category",
      label: "Getting started",
      link: { type: "doc", id: "getting-started/getting-started" },
      items: [
        {
          type: "category",
          label: "Quick starts",
          link: {
            type: "doc",
            id: "getting-started/quick-starts/quick-starts",
          },
          items: [
            "getting-started/quick-starts/sql-onboarding",
            "getting-started/quick-starts/spark-onboarding",
          ],
        },
        {
          type: "category",
          label: "Tutorials",
          link: {
            type: "doc",
            id: "getting-started/tutorials/tutorials",
          },
          items: [
            "getting-started/tutorials/spark-with-databricks",
            "getting-started/tutorials/sql-with-snowflake",
            "getting-started/tutorials/sql-with-databricks",
          ],
        },
      ],
    },
    {
      type: "category",
      label: "Concepts",
      link: { type: "doc", id: "concepts/key-concepts" },
      items: [{ type: "autogenerated", dirName: "concepts" }],
    },
    {
      type: "category",
      label: "Data Copilot",
      link: { type: "doc", id: "copilot/copilot" },
      items: [{ type: "autogenerated", dirName: "copilot" }],
    },
    {
      type: "category",
      label: "Release notes",
      link: { type: "doc", id: "release_notes/release_notes" },
      items: [{ type: "autogenerated", dirName: "release_notes" }],
    },
    {
      type: "html",
      value: '<span class="sidebar-divider" />',
    },
    /// PIPELINE DEVELOPMENT
    {
      type: "category",
      label: "Pipeline development",
      link: { type: "doc", id: "Spark/copilot-for-spark-users" },
      items: [
        {
          type: "category",
          label: "Gems",
          link: { type: "doc", id: "Spark/gems/spark-gems" },
          items: [
            {
              type: "category",
              label: "Source and Target",
              link: {
                type: "doc",
                id: "Spark/gems/source-target/source-target",
              },
              items: [
                {
                  type: "category",
                  label: "File",
                  items: [
                    "Spark/gems/source-target/file/avro",
                    "Spark/gems/source-target/file/csv",
                    "Spark/gems/source-target/file/data-generator",
                    "Spark/gems/source-target/file/delta",
                    "Spark/gems/source-target/file/fixed-format",
                    "Spark/gems/source-target/file/json",
                    "Spark/gems/source-target/file/kafka",
                    "Spark/gems/source-target/file/orc",
                    "Spark/gems/source-target/file/parquet",
                    "Spark/gems/source-target/file/seed",
                    "Spark/gems/source-target/file/text",
                    "Spark/gems/source-target/file/upload-file",
                    "Spark/gems/source-target/file/xlsx",
                    "Spark/gems/source-target/file/xml",
                  ],
                },
                {
                  type: "category",
                  label: "Warehouse",
                  items: [
                    "Spark/gems/source-target/warehouse/bigquery",
                    "Spark/gems/source-target/warehouse/cosmos",
                    "Spark/gems/source-target/warehouse/db2",
                    "Spark/gems/source-target/warehouse/jdbc",
                    "Spark/gems/source-target/warehouse/mongodb",
                    "Spark/gems/source-target/warehouse/oracle",
                    "Spark/gems/source-target/warehouse/redshift",
                    "Spark/gems/source-target/warehouse/salesforce",
                    "Spark/gems/source-target/warehouse/snowflake",
                    "Spark/gems/source-target/warehouse/teradata",
                  ],
                },
                {
                  type: "category",
                  label: "Catalog Table",
                  items: [
                    "Spark/gems/source-target/catalog-table/delta",
                    "Spark/gems/source-target/catalog-table/hive",
                    "Spark/gems/source-target/catalog-table/iceberg",
                  ],
                },
                "Spark/gems/source-target/lookup",
              ],
            },
            {
              type: "category",
              label: "Transform",
              link: {
                type: "doc",
                id: "Spark/gems/transform/transform",
              },
              items: [
                "Spark/gems/transform/aggregate",
                "Spark/gems/transform/bulk-column-expressions",
                "Spark/gems/transform/bulk-column-rename",
                "Spark/gems/transform/column-parser",
                "Spark/gems/transform/data-cleansing",
                "Spark/gems/transform/data-quality-check",
                "Spark/gems/transform/deduplicate",
                "Spark/gems/transform/dynamic-replace",
                "Spark/gems/transform/dynamic-select",
                "Spark/gems/transform/filter",
                "Spark/gems/transform/flatten-schema",
                "Spark/gems/transform/fuzzy-match",
                "Spark/gems/transform/limit",
                "Spark/gems/transform/order-by",
                "Spark/gems/transform/reformat",
                "Spark/gems/transform/sample-rows",
                "Spark/gems/transform/schema-transform",
                "Spark/gems/transform/set-operation",
                "Spark/gems/transform/unpivot",
                "Spark/gems/transform/window-function",
              ],
            },
            {
              type: "category",
              label: "Join/Split",
              link: {
                type: "doc",
                id: "Spark/gems/join-split/join-split",
              },
              items: [
                "Spark/gems/join-split/compare-columns",
                "Spark/gems/join-split/join",
                "Spark/gems/join-split/Repartition",
                "Spark/gems/join-split/row-distributor",
              ],
            },
            {
              type: "category",
              label: "Subgraph",
              link: {
                type: "doc",
                id: "Spark/gems/subgraph/subgraph",
              },
              items: [
                "Spark/gems/subgraph/basic-subgraph",
                "Spark/gems/subgraph/table-iterator",
                "Spark/gems/subgraph/while-iterator",
              ],
            },
            {
              type: "category",
              label: "Machine learning",
              link: {
                type: "doc",
                id: "Spark/gems/machine-learning/machine-learning",
              },
              items: [
                "Spark/gems/machine-learning/ml-openai",
                "Spark/gems/machine-learning/ml-pinecone-lookup",
                "Spark/gems/machine-learning/ml-text-processing",
                "Spark/gems/machine-learning/gen-ai-chatbot",
              ],
            },
            {
              type: "category",
              label: "Custom",
              link: {
                type: "doc",
                id: "Spark/gems/custom/custom",
              },
              items: [
                "Spark/gems/custom/script",
                "Spark/gems/custom/sql-statement",
                "Spark/gems/custom/delta-ops",
                "Spark/gems/custom/file-operations",
                "Spark/gems/custom/rest-api-enrich",
                "Spark/gems/custom/directory",
              ],
            },
          ],
        },
        {
          type: "category",
          label: "Execution",
          link: {
            type: "doc",
            id: "Spark/execution/interactive-execution",
          },
          items: [
            "Spark/execution/data-explorer",
            "Spark/execution/conditional-execution",
            "Spark/execution/execution-metrics",
            "Spark/execution/executions_on_databricks_clusters",
            "Spark/execution/executions_on_livy_clusters",
          ],
        },
        {
          type: "category",
          label: "Functions",
          items: [
            {
              type: "category",
              label: "Business rules engine",
              link: {
                type: "doc",
                id: "Spark/functions/business-rules-engine/business-rules-engine",
              },
              items: [],
            },
            "Spark/functions/udfs",
          ],
        },
        "Spark/expression-builder",
        "Spark/configuration",
        "Spark/pipeline-settings",
        {
          type: "category",
          label: "Spark streaming",
          link: { type: "doc", id: "Spark/spark-streaming/streaming" },
          items: [
            {
              type: "category",
              label: "Streaming sources and targets",
              link: {
                type: "doc",
                id: "Spark/spark-streaming/streaming-sources-and-targets/streaming-sources-and-targets",
              },
              items: [
                "Spark/spark-streaming/streaming-sources-and-targets/streaming-file-apps",
                "Spark/spark-streaming/streaming-sources-and-targets/streaming-event-apps",
                "Spark/spark-streaming/streaming-sources-and-targets/streaming-warehouse-apps",
              ],
            },
            "Spark/spark-streaming/transformations-streaming",
          ],
        },
        { "Best practices": ["Spark/best-practices/use-dbx-secrets"] },
      ],
    },
    /// MODEL DEVELOPMENT
    {
      type: "category",
      label: "Model development",
      link: { type: "doc", id: "SQL/copilot-for-sql-users" },
      items: [
        {
          type: "category",
          label: "Gems",
          link: { type: "doc", id: "SQL/gems/sql-gems" },
          items: [
            {
              type: "category",
              label: "Data sources",
              link: { type: "doc", id: "SQL/gems/datasources/datasources-sql" },
              items: ["SQL/gems/datasources/upload-files"],
            },
            {
              type: "category",
              label: "Target models",
              link: { type: "doc", id: "SQL/gems/target-models/target-models" },
              items: [
                "SQL/gems/target-models/location",
                "SQL/gems/target-models/schema",
                "SQL/gems/target-models/sql-query",
                "SQL/gems/target-models/type-and-format",
                "SQL/gems/target-models/write-options",
              ],
            },
            {
              type: "category",
              label: "Transform",
              link: { type: "doc", id: "SQL/gems/transform/transform" },
              items: [
                "SQL/gems/transform/sql-aggregate",
                "SQL/gems/transform/deduplicate",
                "SQL/gems/transform/filter",
                "SQL/gems/transform/flattenschema",
                "SQL/gems/transform/limit",
                "SQL/gems/transform/order-by",
                "SQL/gems/transform/reformat",
                "SQL/gems/transform/set-operation",
                "SQL/gems/transform/unpivot",
                "SQL/gems/transform/window-function",
              ],
            },
            "SQL/gems/data-joins",
            "SQL/gems/subgraph/subgraph",
            {
              type: "category",
              label: "Custom",
              link: { type: "doc", id: "SQL/gems/custom/custom-sql-gems" },
              items: ["SQL/gems/custom/sql-statement"],
            },
          ],
        },
        {
          type: "category",
          label: "Execution",
          link: { type: "doc", id: "SQL/execution/execution" },
          items: ["SQL/execution/data-explorer"],
        },
        {
          type: "category",
          label: "Visual expressions",
          link: {
            type: "doc",
            id: "SQL/visual-expression-builder/visual-expression-builder",
          },
          items: [
            "SQL/visual-expression-builder/visual-expression-builder-reference",
            "SQL/visual-expression-builder/use-the-expression-builder",
            "SQL/visual-expression-builder/variant-schema",
          ],
        },
        "SQL/configuration",
      ],
    },
    /// LINEAGE
    {
      type: "category",
      label: "Lineage",
      link: { type: "doc", id: "lineage/lineage" },
      items: [
        "lineage/lineage-extractor",
        "lineage/lineage-run-and-diagnose",
        "lineage/lineage-view-and-search",
      ],
    },
    /// CI-CD
    {
      type: "category",
      label: "CI/CD",
      link: { type: "doc", id: "ci-cd/reliable-ci-cd" },
      items: [
        "ci-cd/tests",
        {
          type: "category",
          label: "Data tests for SQL",
          link: { type: "doc", id: "ci-cd/data-tests/data-tests" },
          items: [
            "ci-cd/data-tests/use-model-tests",
            "ci-cd/data-tests/use-project-tests",
          ],
        },
        {
          type: "category",
          label: "Release and Deploy",
          link: { type: "doc", id: "ci-cd/deployment/Deployment" },
          items: [
            "ci-cd/deployment/deploy-project",
            "ci-cd/deployment/use-external-release-tags",
          ],
        },
        {
          type: "category",
          label: "Prophecy Build Tool (PBT)",
          link: {
            type: "doc",
            id: "ci-cd/prophecy-build-tool/prophecy-build-tool",
          },
          items: [
            "ci-cd/prophecy-build-tool/prophecy-build-tool-github-actions",
            "ci-cd/prophecy-build-tool/prophecy-build-tool-jenkins",
          ],
        },
      ],
    },
    /// ORCHESTRATION
    {
      type: "category",
      label: "Orchestration",
      link: { type: "doc", id: "Orchestration/Orchestration" },
      items: [
        {
          type: "category",
          label: "Pipeline monitoring",
          link: {
            type: "doc",
            id: "Orchestration/pipeline-monitoring/pipeline-monitoring",
          },
          items: [
            "Orchestration/pipeline-monitoring/enable-pipeline-monitoring",
            "Orchestration/pipeline-monitoring/use-pipeline-monitoring",
          ],
        },
        {
          type: "category",
          label: "Airflow",
          link: { type: "doc", id: "Orchestration/airflow/airflow" },
          items: [
            {
              type: "category",
              label: "Setup",
              link: {
                type: "doc",
                id: "Orchestration/airflow/setup/setup_airflow",
              },
              items: [
                "Orchestration/airflow/setup/composer_fabric",
                "Orchestration/airflow/setup/MWAA_fabric",
              ],
            },
            {
              type: "category",
              label: "Prophecy managed",
              link: {
                type: "doc",
                id: "Orchestration/airflow/prophecy-managed/prophecy_managed_airflow_fabric",
              },
              items: [
                {
                  type: "category",
                  label: "Connections",
                  link: {
                    type: "doc",
                    id: "Orchestration/airflow/prophecy-managed/connections/prophecy_managed_airflow_fabric_connections",
                  },
                  items: [
                    "Orchestration/airflow/prophecy-managed/connections/prophecy_managed_airflow_fabric_aws_connections",
                    "Orchestration/airflow/prophecy-managed/connections/prophecy_managed_airflow_fabric_dbx_spark_connections",
                    "Orchestration/airflow/prophecy-managed/connections/prophecy_managed_airflow_fabric_dbx_sql_connections",
                    "Orchestration/airflow/prophecy-managed/connections/prophecy_managed_airflow_fabric_email_connections",
                    "Orchestration/airflow/prophecy-managed/connections/prophecy_managed_airflow_fabric_snowflake_connections",
                  ],
                },
                "Orchestration/airflow/prophecy-managed/prophecy_managed_airflow_fabric_limits",
              ],
            },
            "Orchestration/airflow/airflow-tutorial",
          ],
        },
        "Orchestration/databricks-jobs",
        "Orchestration/multi-jobs-trigger",
      ],
    },
    /// EXTENSIBILITY
    {
      type: "category",
      label: "Extensibility",
      link: { type: "doc", id: "extensibility/extensibility" },
      items: [
        {
          type: "category",
          label: "Dependencies",
          items: [
            "extensibility/dependencies/spark-dependencies",
            "extensibility/dependencies/sql-dependencies",
            "extensibility/dependencies/prophecy-libraries",
          ],
        },
        {
          type: "category",
          label: "Package Hub",
          link: { type: "doc", id: "extensibility/package-hub/package-hub" },
          items: [
            "extensibility/package-hub/shareable-pipelines",
            "extensibility/package-hub/shareable-datasets",
            "extensibility/package-hub/shareable-subgraphs",
            "extensibility/package-hub/sharable-udfs",
          ],
        },
        {
          type: "category",
          label: "Gem Builder",
          items: [
            "extensibility/gem-builder/sql-gem-builder",
            "extensibility/gem-builder/spark-gem-builder",
            "extensibility/gem-builder/gem-builder-reference",
            "extensibility/gem-builder/optimization-functions",
          ],
        },
      ],
    },
  ],
  adminSidebar: [
    "administration/index",
    {
      type: "html",
      value: '<span class="sidebar-divider" />',
    },
    {
      type: "html",
      className: "sidebar-title",
      value: "Team Admins",
      defaultStyle: true,
    },
    "administration/teamuser",
    {
      type: "category",
      label: "Set up Spark fabrics",
      link: { type: "doc", id: "administration/Spark-fabrics/Fabrics" },
      items: [
        {
          type: "autogenerated",
          dirName: "administration/Spark-fabrics",
        },
      ],
    },
    {
      type: "category",
      label: "Set up SQL fabrics",
      link: { type: "doc", id: "administration/sql-fabrics/Fabrics" },
      items: [
        {
          type: "autogenerated",
          dirName: "administration/sql-fabrics",
        },
      ],
    },
    {
      type: "category",
      label: "Secret management",
      link: {
        type: "doc",
        id: "administration/secret-management/secret-management-spark",
      },
      items: [
        {
          type: "autogenerated",
          dirName: "administration/secret-management",
        },
      ],
    },
    "administration/metadata-connections",
    "administration/settings/settings",
    {
      type: "html",
      value: '<span class="sidebar-divider" />',
    },
    {
      type: "html",
      className: "sidebar-title",
      value: "Prophecy Cluster Admins",
      defaultStyle: true,
    },
    "administration/architecture",
    "administration/prophecy-deployment",
    "administration/security",
    {
      type: "category",
      label: "Authentication",
      link: { type: "doc", id: "administration/authentication/authentication" },
      items: [
        {
          type: "autogenerated",
          dirName: "administration/authentication",
        },
      ],
    },
    {
      type: "category",
      label: "SaaS",
      items: [
        {
          type: "autogenerated",
          dirName: "administration/saas",
        },
      ],
    },
    {
      type: "category",
      label: "Self-hosted",
      link: { type: "doc", id: "administration/self-hosted/self-hosted" },
      items: [
        {
          type: "autogenerated",
          dirName: "administration/self-hosted",
        },
      ],
    },
    {
      type: "category",
      label: "Getting help",
      link: { type: "doc", id: "getting-help/getting-help" },
      items: [{ type: "autogenerated", dirName: "getting-help" }],
    },
  ],
};

module.exports = sidebars;