stop using xml sitemap (#517)

* Add sitemap generating machinery

* Update site scanner to use sitemap.json file

* Remove DMS ping

* Fix scan output including sitemap.json

Author: pmac

.github/workflows/mozorg-site-scanning.yaml

@@ -25,14 +25,32 @@ env:
   USER_AGENT: Mozilla/5.0 (Automated; https://github.com/mozmeao/www-site-checker) CheckerBot/1.0
 
 jobs:
+  build_sitemap_json:
+    name: Build sitemap.json file
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Generate sitemap
+        run: sitemap/generate_sitemap_docker.sh
+      - name: Upload sitemap to cache
+        uses: actions/upload-artifact@v4
+        with:
+          name: sitemap_json
+          path: sitemap/sitemap.json
   run_on_cdn:
+    needs: build_sitemap_json
     strategy:
       matrix:
         batch: ["1_8", "2_8", "3_8", "4_8", "5_8", "6_8", "7_8", "8_8"]
     runs-on: ubuntu-latest
     name: Check main CDN - batch ${{ matrix.batch }}
     steps:
       - uses: actions/checkout@v4
+      - name: Download sitemap
+        uses: actions/download-artifact@v4
+        with:
+          name: sitemap_json
+          path: sitemap/
       - uses: actions/setup-python@v5
         with:
           python-version: "3.11"
@@ -41,7 +59,7 @@ jobs:
       - name: Run checker
         run: >
           python bin/scan_site.py
-          --sitemap-url="https://${{ secrets.MOZORG_CDN_HOSTNAME }}/sitemap.xml"
+          --hostname="${{ secrets.MOZORG_CDN_HOSTNAME }}"
           --batch="${{ matrix.batch }}"
           --export-cache
       - name: Upload scan output files
@@ -58,13 +76,19 @@ jobs:
           retention-days: 5
 
   run_on_us_origin:
+    needs: build_sitemap_json
     strategy:
       matrix:
         batch: ["1_8", "2_8", "3_8", "4_8", "5_8", "6_8", "7_8", "8_8"]
     runs-on: ubuntu-latest
     name: Check US origin server - batch ${{ matrix.batch }}
     steps:
       - uses: actions/checkout@v4
+      - name: Download sitemap
+        uses: actions/download-artifact@v4
+        with:
+          name: sitemap_json
+          path: sitemap/
       - uses: actions/setup-python@v5
         with:
           python-version: "3.11"
@@ -73,8 +97,7 @@ jobs:
       - name: Run checker
         run: >
           python bin/scan_site.py
-          --sitemap-url="https://${{ secrets.MOZORG_US_ORIGIN_HOSTNAME }}/sitemap.xml"
-          --maintain-hostname
+          --hostname="${{ secrets.MOZORG_US_ORIGIN_HOSTNAME }}"
           --batch="${{ matrix.batch }}"
       - name: Upload scan output files
         uses: actions/upload-artifact@v4
@@ -84,13 +107,19 @@ jobs:
           retention-days: 15
 
   run_on_eu_origin:
+    needs: build_sitemap_json
     strategy:
       matrix:
         batch: ["1_8", "2_8", "3_8", "4_8", "5_8", "6_8", "7_8", "8_8"]
     runs-on: ubuntu-latest
     name: Check EU origin server - batch ${{ matrix.batch }}
     steps:
       - uses: actions/checkout@v4
+      - name: Download sitemap
+        uses: actions/download-artifact@v4
+        with:
+          name: sitemap_json
+          path: sitemap/
       - uses: actions/setup-python@v5
         with:
           python-version: "3.11"
@@ -99,8 +128,7 @@ jobs:
       - name: Run checker
         run: >
           python bin/scan_site.py
-          --sitemap-url="https://${{ secrets.MOZORG_EU_ORIGIN_HOSTNAME }}/sitemap.xml"
-          --maintain-hostname
+          --hostname="${{ secrets.MOZORG_EU_ORIGIN_HOSTNAME }}"
           --batch="${{ matrix.batch }}"
       - name: Upload scan output files
         uses: actions/upload-artifact@v4

.gitignore

@@ -14,3 +14,4 @@ venv
 TODO.txt
 output/*
 page_cache/*
+sitemap/sitemap.json

README.md

@@ -44,11 +44,14 @@ You can also clone this repo to your local machine and run it there. This is the
 ```bash
 git clone [email protected]:mozmeao/www-site-checker.git
 cd www-site-checker
+# you need docker setup for this step
+# it creates a sitemap.json file in the sitemap dir
+./sitemap/generate_sitemap_docker.sh
 # make a virtualenv, with whatever you prefer, and activate it
 pip install -r requirements.txt
 export ALLOWLIST_FILEPATH=data/allowlist-mozorg.yaml
 export EXTRA_URLS_FILEPATH=data/extra-urls-mozorg.yaml
-python bin/scan_site.py --sitemap-url=https://www.mozilla.org/sitemap.xml
+python bin/scan_site.py --hostname=www.mozilla.org
 ```
 
 The above will start to work through the entire sitemap (and child sitemaps) at that URL
@@ -57,7 +60,7 @@ If you only want to check a smaller batch of URLs (handy in development), add th
 
 ```bash
 # Only inspect batch three of all URLs to check, after slicing the site into 40 batches
-python bin/scan_site.py --sitemap-url=https://www.mozilla.org/sitemap.xml --batch=3:40
+python bin/scan_site.py --hostname=www.mozilla.org --batch=3:40
 ```
 
 And if you only want to check a specific page, you use the `--specific-url` param. The following, for example, checks the homepage and a Fx mobile downbload page
@@ -69,27 +72,27 @@ python bin/scan_site.py --specific-url=https://www.mozilla.org/en-US/firefox/bro
 There is a default allowlist in use (`data/allowlist-mozorg.yaml` - **set via env vars**) but an alernative can be passed via the `--allowlist` param
 
 ```bash
-python bin/scan_site.py --sitemap-url=https://www.mozilla.org/sitemap.xml --allowlist=/path/to/custom/allowlist.yaml
+python bin/scan_site.py --hostname=www.mozilla.org --allowlist=/path/to/custom/allowlist.yaml
 ```
 
-If you want or need to check a site whose sitemap points to a _different_ domain (eg you want to check an origin server whose sitemap is hard-coded to refer to the CDN domain, or a localhost setup) you should ensure the server is listed as an option in the allowlist and also pass the `--maintain-hostname` parameter.
+If you want or need to check a site whose sitemap points to a _different_ domain (eg you want to check an origin server whose sitemap is hard-coded to refer to the CDN domain, or a localhost setup) you should ensure the server is listed as an option in the allowlist.
 
 For example:
 
 ```bash
-python bin/scan_site.py --sitemap-url=http://origin-server.example.com/sitemap.xml --maintain-hostname
+python bin/scan_site.py --hostname=origin-server.example.com
 ```
 
 or, for localhost
 
 ```bash
-python bin/scan_site.py --sitemap-url=http://localhost:8000/sitemap.xml --maintain-hostname
+python bin/scan_site.py --hostname=localhost:8000
 ```
 
 If you want to test the Sentry integration locally, you can pass a Sentry DSN as an environment variable. Here, we're passing a URL to [Kent - a local 'fake Sentry'](https://github.com/willkg/kent)
 
 ```bash
-SENTRY_DSN=http://[email protected]:8011/1 python bin/scan_site.py --sitemap-url=https://www.mozilla.org/sitemap.xml
+SENTRY_DSN=http://[email protected]:8011/1 python bin/scan_site.py --hostname=www.mozilla.org
 ```
 
 ## The output files
@@ -125,4 +128,3 @@ If you come across an alert saying there was an unexpected URL detected and you'
 * Edit the allowlist - eg `data/allowlist-mozorg.yaml`
 * Run the checks locally (see above)
 * Push the branch up and raise a PR.
-

bin/handle_site_scan_output.py

@@ -78,7 +78,7 @@ def _assemble_results(output_path: str) -> set:
 
     # assume we have multiple output files, all scanning different sources of the website data
     for filename in os.listdir(output_path):
-        if filename.endswith(".json"):
+        if filename.startswith("unexpected_urls_for") and filename.endswith(".json"):
             unexpected_url_data.update(
                 _load_results_json(os.path.join(output_path, filename))
             )

bin/scan_site.py

@@ -20,6 +20,7 @@
 import time
 from collections import defaultdict
 from functools import cache
+from pathlib import Path
 from typing import Dict, Iterable, List, Tuple
 from urllib.parse import quote, urlparse
 
@@ -50,6 +51,7 @@
         integrations=[sentry_logging],
     )
 
+DEFAULT_SITEMAP_FILENAME = "sitemap/sitemap.json"
 DEFAULT_BATCH__NOOP = "1_1"  # By default treat all URLs as a single batch
 UNEXPECTED_URLS_FILENAME_FRAGMENT = "unexpected_urls_for"
 URL_RETRY_LIMIT = 3
@@ -63,15 +65,14 @@
 
 @click.command()
 @click.option(
-    "--sitemap-url",
-    default=None,
-    help="URL of an XML sitemap to use as source data",
+    "--hostname",
+    required=True,
+    help="Hostname of the site instance to scan",
 )
 @click.option(
-    "--maintain-hostname",
-    default=False,
-    is_flag=True,
-    help="If the sitemap points to a different domain (eg a CDN domain), override it and replace it with the hostname that served the sitemap",
+    "--sitemap-filename",
+    default=DEFAULT_SITEMAP_FILENAME,
+    help="Filename of a JSON sitemap file",
 )
 @click.option(
     "--specific-url",
@@ -104,8 +105,8 @@
     help="If True, we'll export the cached pages as an artifact to {hostname}-cached-pages/batch{batch-id}, for other checks to use",
 )
 def run_checks(
-    sitemap_url: str,
-    maintain_hostname: bool,
+    sitemap_filename: str,
+    hostname: str,
     specific_url: Iterable,
     batch: str,
     allowlist: str,
@@ -114,24 +115,20 @@ def run_checks(
 ) -> None:
     # Let's tidy up that variables we get from the input option
     specific_urls = specific_url
+    sitemap_path = Path(sitemap_filename)
 
-    if not sitemap_url and not specific_urls:
+    if not sitemap_path.exists() and not specific_urls:
         raise Exception("No sitemap or input URLs specified. Cannot proceed.")
 
-    host_url = (
-        sitemap_url or specific_urls[0]
-    )  # TODO: ensure all specific URLs use the same hostname
-    hostname = urlparse(host_url).netloc
-
     allowlist_config = _get_allowlist_config(
         hostname,
         allowlist_pathname=allowlist,
     )
 
     urls_to_check = _build_urls_to_check(
-        sitemap_url=sitemap_url,
+        sitemap_path=sitemap_path,
+        hostname=hostname,
         specific_urls=specific_url,
-        maintain_hostname=maintain_hostname,
     )
 
     if additional_urls_file:
@@ -437,80 +434,39 @@ def _check_pages_for_outbound_links(urls: List[str], allowlist_config: Dict) ->
 
 
 def _build_urls_to_check(
-    sitemap_url: str,
+    sitemap_path: Path,
+    hostname: str,
     specific_urls: Iterable,
-    maintain_hostname: bool,
 ) -> List[str]:
     """Given a sitemap URL and/or specific URLs to check, put together a list
     of overall URLs whose content wen want to check"""
 
     urls = []
-    if sitemap_url:
-        urls += _get_urls_from_sitemap(sitemap_url, maintain_hostname)
+    if sitemap_path:
+        urls += _get_urls_from_json_file(sitemap_path, hostname)
     if specific_urls:
         # Don't forget any manually specified URLs
         urls += specific_urls
     click.echo(f"Discovered {len(urls)} URLs to check")
     return urls
 
 
-def _get_urls_from_sitemap(
-    sitemap_url: str,
-    maintain_hostname: bool,
+def _get_urls_from_json_file(
+    sitemap_path: Path,
+    hostname: str,
 ) -> List[str]:
-    """Extract URLs to explore from a sitemap, optionally ensuring the hostname in
-    any URLs found is swapped ('maintained') to be the same as that of the source
-    sitemap –- this is needed when checking an origin server whose sitemap returns
-    the CDN's hostname"""
-
     urls = []
+    with sitemap_path.open() as fh:
+        sitemap = json.load(fh)
 
-    _parsed_origin_sitemap_url = urlparse(sitemap_url)
-    origin_hostname_with_scheme = (
-        f"{_parsed_origin_sitemap_url.scheme}://{_parsed_origin_sitemap_url.netloc}"  # noqa E231
-    )
+    for path, locales in sitemap.items():
+        if not locales:
+            urls.append(f"https://{hostname}{path}")
+            continue
+
+        for locale in locales:
+            urls.append(f"https://{hostname}/{locale}{path}")
 
-    resp = _get_url_with_retry(sitemap_url)
-
-    sitemap_xml = resp.text
-    soup = BeautifulSoup(sitemap_xml, "lxml")
-
-    # Look for a <sitemap> node, and get each as a URL for a locale-specific sitemap
-    sitemap_nodes = soup.find_all("sitemap")
-    if len(sitemap_nodes):
-        click.echo(f"Discovered {len(sitemap_nodes)} child sitemaps")
-
-    for sitemap_node in sitemap_nodes:
-        sitemap_url = sitemap_node.loc.text
-
-        if maintain_hostname:
-            sitemap_url = _update_hostname(
-                origin_hostname_with_scheme=origin_hostname_with_scheme,
-                urls=[sitemap_url],
-            )[0]
-
-        click.echo(f"Diving into {sitemap_url}")
-        urls.extend(_get_urls_from_sitemap(sitemap_url, maintain_hostname))
-
-    # look for regular URL nodes, which may or may not co-exist alongside sitemap nodes
-    url_nodes = soup.find_all("url")
-    if url_nodes:
-        click.echo(f"Adding {len(url_nodes)} URLs")
-        for url in url_nodes:
-            try:
-                urls.append(url.loc.text)
-            except AttributeError as ae:
-                sentry_sdk.capture_message(
-                    f"URL node {url} missing '<loc>' - exception to follow"
-                )
-                sentry_sdk.capture_exception(ae)
-
-    # Also remember to update the hostname on the final set of URLs, if required
-    if maintain_hostname:
-        urls = _update_hostname(
-            origin_hostname_with_scheme=origin_hostname_with_scheme,
-            urls=urls,
-        )
     return urls
 
 

sitemap/.bedrock.env

@@ -0,0 +1,9 @@
+DEBUG=False
+DEV=False
+ALLOWED_HOSTS=*
+AWS_DB_S3_BUCKET=bedrock-db-prod
+STATIC_URL=https://www.mozilla.org/media/
+DJANGO_SETTINGS_MODULE=bedrock.settings
+GRANIAN_BACKPRESSURE=64
+GRANIAN_WORKERS=4
+GRANIAN_BLOCKING_THREADS=4

sitemap/Dockerfile

@@ -0,0 +1,8 @@
+FROM mozmeao/bedrock:prod-latest
+USER root
+COPY settings_local.py bedrock/settings/local.py
+COPY run-generator.sh ./
+ARG USER_ID=1000:1000
+ENV USER_ID=${USER_ID}
+RUN chown -R "${USER_ID}" /app
+USER ${USER_ID}

sitemap/generate_sitemap_docker.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -exo pipefail
+
+cd sitemap
+
+docker build --pull \
+             -t sitemap-generator \
+             --build-arg "USER_ID=$(id -u):$(id -g)" \
+             .
+
+docker run --rm \
+           --env-file .bedrock.env \
+           -v "$PWD:/app/sitemap-data" \
+           sitemap-generator ./run-generator.sh

sitemap/run-generator.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -ex
+bin/run-db-download.py
+python manage.py l10n_update
+python manage.py update_sitemaps
+cp /app/root_files/sitemap.json /app/sitemap-data/