Merge pull request #107 from UmbrellaDocs/header-support

Feat: Add support for HTTP headers

Author: gaurav-nelson

README.md

@@ -21,7 +21,7 @@ Linkspector is a powerful tool for anyone who creates content using markup langu
 1. **Enhanced Link Checking with Puppeteer**: It uses [Puppeteer](https://pptr.dev/) to check links in Chrome's headless mode, reducing the number of false positives.
 2. **Addresses limitations and adds user-requested features**: It is built to adress the shortcomings in [GitHub Action - Markdown link check](https://github.com/gaurav-nelson/github-action-markdown-link-check) and adds many user requested features.
 3. **Single repository for seamless collaboration**: All the code it needs to run is in a single repository, making it easier for community to collaborate.
-4. **Focused for CI/CD use**: Linkspector is purposefully tailored to run into your CI/CD pipelines. This ensures that link checking becomes an integral part of your development workflow.
+4. **Focused for CI/CD use**: Linkspector ([action-linkspector](https://github.com/UmbrellaDocs/action-linkspector)) is purposefully tailored to run into your CI/CD pipelines. This ensures that link checking becomes an integral part of your development workflow.
 
 ## Installation
 
@@ -112,6 +112,7 @@ Following are the available configuration options:
 | [`aliveStatusCodes`](#alive-status-codes)         | The list of HTTP status codes that are considered as "alive" links.                                   | No                                |
 | [`useGitIgnore`](#use-gitignore)                  | Indicates whether to use the rules defined in the `.gitignore` file to exclude files and directories. | No                                |
 | [`modifiedFilesOnly`](#check-modified-files-only) | Indicates whether to check only the files that have been modified in the last git commit.             | No                                |
+| [`httpHeaders`](#http-headers)                    | The list of URLs and their corresponding HTTP headers to be used during link checking.                | No                                |
 
 ### Files to Check
 
@@ -225,6 +226,31 @@ When enabled, Linkspector will use `git` to find the list of modified files and
 
 Also, if no modified files are found in the list of files to check, Linkspector will skip link checking and exit with a message indicating that no modified files have been edited so it will skip checking.
 
+### HTTP headers
+
+The `httpHeaders` option allows you to specify HTTP headers for specific URLs that require authorization. You can use environment variables for secure values.
+
+1. Create a `.env` file in the root directory of your project and add the environment variables. For example:
+
+   ```env
+   AUTH_TOKEN=abcdef123456
+   ```
+
+1. Add the `httpHeaders` section to the configuration file and specify the URLs and headers. For example:
+
+   ```yaml
+   httpHeaders:
+     - url:
+         - https://example1.com
+       headers:
+         Foo: Bar
+     - url:
+         - https://example2.com
+       headers:
+         Authorization: ${AUTH_TOKEN}
+         Foo: Bar
+   ```
+
 ### Sample configuration
 
 ```yml
@@ -250,6 +276,12 @@ replacementPatterns:
     replacement: '$1/id/$3'
   - pattern: "\\[([^\\]]+)\\]\\((https?://example.com)/file\\)"
     replacement: '<a href="$2/file">$1</a>'
+httpHeaders:
+  - url:
+      - https://example1.com
+    headers:
+      Authorization: Basic Zm9vOmJhcg==
+      Foo: Bar
 aliveStatusCodes:
   - 200
   - 201
@@ -304,20 +336,6 @@ To use Linkspector with Docker, follow these steps:
           bash -c 'linkspector check -c /path/to/custom-config.yml'
    ```
 
-## What's planned
-
-- [x] Spinner for local runs.
-- [x] Create a GitHub action. See [action-linkspector](https://github.com/UmbrellaDocs/action-linkspector)
-- [x] Modified files only check.
-- [!] Asciidoc support. (Limited to hyperlinks only)
-- [ ] ReStructured Text support.
-- [ ] Disable binary files downlaod.
-- [x] JSON output for `failed-only` ~~or `all`~~ links.
-- ~~[ ] CSV output for `all` links.~~ (dropped for now)
-- ~~[ ] Experimaental mode to gather all links and check them in batches to study performance gains.~~ (dropped for now)
-- ~~[ ] Proxy support to connect puppeteer to a remote service.~~ (dropped for now)
-- ~~[ ] Puppeteer config support.~~ (dropped for now)
-
 ## Contributing
 
 If you would like to contribute to Linkspector, please read the [contributing guidelines](/CONTRIBUTING.md).

index.test.js

@@ -34,7 +34,7 @@ test('linkspector should check top-level relative links in Markdown file', async
   }
 
   expect(hasErrorLinks).toBe(false)
-  expect(results.length).toBe(21)
+  expect(results.length).toBe(22)
 })
 
 test('linkspector should track statistics correctly when stats option is enabled', async () => {
@@ -87,7 +87,7 @@ test('linkspector should track statistics correctly when stats option is enabled
 
   // Verify statistics are being tracked correctly
   expect(stats.filesChecked).toBeGreaterThan(0)
-  expect(stats.totalLinks).toBe(21)
+  expect(stats.totalLinks).toBe(22)
   expect(stats.totalLinks).toBe(
     stats.httpLinks +
       stats.fileLinks +

lib/batch-check-links.js

@@ -22,14 +22,22 @@ function createLinkStatus(link, status, statusCode, errorMessage = null) {
   }
 }
 
-async function processLink(link, page, aliveStatusCodes) {
+async function processLink(link, page, aliveStatusCodes, httpHeaders) {
   let status = null
   let statusCode = null
   let errorMessage = null
 
   try {
     if (isUrl(link.url)) {
-      const response = await page.goto(link.url, { waitUntil: 'load' })
+      const headers =
+        httpHeaders.find((header) =>
+          header.url.some((urlPattern) => link.url.includes(urlPattern))
+        )?.headers || {}
+
+      const response = await page.goto(link.url, {
+        waitUntil: 'load',
+        headers,
+      })
       statusCode = response.status()
       if (aliveStatusCodes && aliveStatusCodes.includes(statusCode)) {
         status = 'assumed alive'
@@ -46,7 +54,12 @@ async function processLink(link, page, aliveStatusCodes) {
 }
 
 async function checkHyperlinks(nodes, options = {}, filePath) {
-  const { batchSize = 100, retryCount = 3, aliveStatusCodes } = options
+  const {
+    batchSize = 100,
+    retryCount = 3,
+    aliveStatusCodes,
+    httpHeaders = [],
+  } = options
   const linkStatusList = []
   const tempArray = []
 
@@ -150,7 +163,12 @@ async function checkHyperlinks(nodes, options = {}, filePath) {
 
         while (retryCountLocal < retryCount) {
           try {
-            linkStatus = await processLink(link, page, aliveStatusCodes)
+            linkStatus = await processLink(
+              link,
+              page,
+              aliveStatusCodes,
+              httpHeaders
+            )
             break
           } catch (error) {
             retryCountLocal++

lib/handle-links-modification.js

@@ -9,6 +9,8 @@
  *
  * @returns {Array} The modified nodes.
  */
+import { escapeRegExp } from 'lodash-es'
+
 function doReplacements(nodes, opts = {}) {
   const { ignorePatterns = [], replacementPatterns = [], baseUrl } = opts
 
@@ -17,7 +19,9 @@ function doReplacements(nodes, opts = {}) {
     // Skip link checking if it matches any ignore pattern
     if (
       ignorePatterns.some(({ pattern }) => {
-        const regex = new RegExp(pattern)
+        // Sanitize the pattern before creating the RegExp
+        const sanitizedPattern = escapeRegExp(pattern)
+        const regex = new RegExp(sanitizedPattern)
         return regex.test(url)
       })
     ) {
@@ -31,7 +35,9 @@ function doReplacements(nodes, opts = {}) {
 
     // Replace link URL based on replacement patterns
     replacementPatterns.forEach(({ pattern, replacement }) => {
-      url = url.replace(new RegExp(pattern), replacement)
+      // Sanitize the pattern before creating the RegExp
+      const sanitizedPattern = escapeRegExp(pattern)
+      url = url.replace(new RegExp(sanitizedPattern), replacement)
     })
     node.url = url
 

lib/validate-config.js

@@ -23,15 +23,12 @@ async function validateConfig(config) {
       excludedDirs: Joi.array().items(Joi.string()),
       fileExtensions: Joi.array().items(Joi.string()),
       baseUrl: Joi.string(),
-      httpHeaders: Joi.object({
-        url: Joi.string(),
-        headers: Joi.array().items(
-          Joi.object({
-            name: Joi.string().required(),
-            value: Joi.string().required(),
-          })
-        ),
-      }),
+      httpHeaders: Joi.array().items(
+        Joi.object({
+          url: Joi.array().items(Joi.string().uri()).required(),
+          headers: Joi.object().pattern(Joi.string(), Joi.string()).required(),
+        })
+      ),
       aliveStatusCodes: Joi.array().items(Joi.number()),
       ignorePatterns: Joi.array().items(
         Joi.object({

linkspector.js

@@ -2,6 +2,7 @@ import { execSync } from 'child_process'
 import { readFileSync } from 'fs'
 import path from 'path'
 import yaml from 'js-yaml'
+import dotenv from 'dotenv'
 import { validateConfig } from './lib/validate-config.js'
 import { prepareFilesList } from './lib/prepare-file-list.js'
 import { extractMarkdownHyperlinks } from './lib/extract-markdown-hyperlinks.js'
@@ -10,6 +11,19 @@ import { getUniqueLinks } from './lib/get-unique-links.js'
 import { checkHyperlinks } from './lib/batch-check-links.js'
 import { updateLinkStatusObj } from './lib/update-linkstatus-obj.js'
 
+// Load environment variables from .env file
+dotenv.config()
+
+// Function to replace placeholders with environment variables
+function replaceEnvVariables(config) {
+  const configString = JSON.stringify(config)
+  const replacedConfigString = configString.replace(
+    /\$\{(\w+)\}/g,
+    (_, name) => process.env[name] || ''
+  )
+  return JSON.parse(replacedConfigString)
+}
+
 // Function to check if git is installed
 function isGitInstalled() {
   try {
@@ -44,6 +58,9 @@ export async function* linkspector(configFile, cmd) {
       throw new Error('Failed to parse the YAML content.')
     }
 
+    // Replace environment variables in the configuration
+    config = replaceEnvVariables(config)
+
     try {
       const isValid = await validateConfig(config)
       if (!isValid) {

package-lock.json

@@ -42,12 +42,14 @@
   "homepage": "https://github.com/UmbrellaDocs/linkspector#readme",
   "dependencies": {
     "commander": "^13.1.0",
+    "dotenv": "^16.4.7",
     "github-slugger": "^2.0.0",
     "glob": "^11.0.1",
     "ignore": "^7.0.3",
     "joi": "^17.13.3",
     "js-yaml": "^4.1.0",
     "kleur": "^4.1.5",
+    "lodash-es": "^4.17.21",
     "ora": "^8.2.0",
     "puppeteer": "^24.4.0",
     "remark-gfm": "^4.0.1",