test(wpt): include all testing files (nodejs#1954)

* test(wpt): include all testing files

* lint

* fix: mark flaky/failing tests by entire file path

* fix: pass 16/20 tests in request-headers.any.js

* fix: patch referer header never being set

* fix some tests

* fix: node-fetch test

Author: KhafraDev

CONTRIBUTING.md

@@ -77,6 +77,45 @@ cd <your-path-to-undici>
 npm run build:wasm
 ```
 
+<a id="update-wpts"></a>
+### Update `WPTs`
+
+`undici` runs a subset of the [`web-platform-tests`](https://github.com/web-platform-tests/wpt).
+
+Here are the steps to update them.
+
+#### Sparse-clone the [wpt](https://github.com/web-platform-tests/wpt) repo
+
+```bash
+git clone --depth 1 --single-branch --branch epochs/daily --filter=blob:none --sparse https://github.com/web-platform-tests/wpt.git test/wpt/tests
+
+cd test/wpt/tests
+
+```
+
+#### Checkout the tests
+
+Only run the commands for the folder(s) you want to update.
+
+```bash
+git sparse-checkout add /fetch
+git sparse-checkout add /FileAPI
+git sparse-checkout add /xhr
+git sparse-checkout add /websockets
+git sparse-checkout add /resources
+git sparse-checkout add /common
+```
+
+#### Run the tests
+
+Run the tests to ensure that any new failures are marked as such.
+
+You can mark tests as failing in their corresponding [status](./test/wpt/status) file.
+
+```bash
+npm run test:wpt
+```
+
 <a id="lint"></a>
 
 ### Lint

lib/fetch/request.js

@@ -54,7 +54,10 @@ class Request {
     // TODO
     this[kRealm] = {
       settingsObject: {
-        baseUrl: getGlobalOrigin()
+        baseUrl: getGlobalOrigin(),
+        get origin () {
+          return this.baseUrl?.origin
+        }
       }
     }
 

lib/fetch/util.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const { redirectStatus, badPorts, referrerPolicy: referrerPolicyTokens } = require('./constants')
+const { getGlobalOrigin } = require('./global')
 const { performance } = require('perf_hooks')
 const { isBlobLike, toUSVString, ReadableStreamFrom } = require('../core/util')
 const assert = require('assert')
@@ -36,9 +37,11 @@ function responseLocationURL (response, requestFragment) {
   // `Location` and response’s header list.
   let location = response.headersList.get('location')
 
-  // 3. If location is a value, then set location to the result of parsing
-  // location with response’s URL.
-  location = location ? new URL(location, responseURL(response)) : null
+  // 3. If location is a header value, then set location to the result of
+  //    parsing location with response’s URL.
+  if (location !== null && isValidHeaderValue(location)) {
+    location = new URL(location, responseURL(response))
+  }
 
   // 4. If location is a URL whose fragment is null, then set location’s
   // fragment to requestFragment.
@@ -267,7 +270,7 @@ function appendRequestOriginHeader (request) {
   // 2. If request’s response tainting is "cors" or request’s mode is "websocket", then append (`Origin`, serializedOrigin) to request’s header list.
   if (request.responseTainting === 'cors' || request.mode === 'websocket') {
     if (serializedOrigin) {
-      request.headersList.append('Origin', serializedOrigin)
+      request.headersList.append('origin', serializedOrigin)
     }
 
   // 3. Otherwise, if request’s method is neither `GET` nor `HEAD`, then:
@@ -298,7 +301,7 @@ function appendRequestOriginHeader (request) {
 
     if (serializedOrigin) {
       // 2. Append (`Origin`, serializedOrigin) to request’s header list.
-      request.headersList.append('Origin', serializedOrigin)
+      request.headersList.append('origin', serializedOrigin)
     }
   }
 }
@@ -340,106 +343,77 @@ function clonePolicyContainer () {
 // https://w3c.github.io/webappsec-referrer-policy/#determine-requests-referrer
 function determineRequestsReferrer (request) {
   // 1. Let policy be request's referrer policy.
-  const policy = request.referrerPolicy
+  // TODO(@KhafraDev): referrerPolicy is supposed to be non-null & not an empty string.
+  // this is because we don't implement policyContainer.
+  const policy = request.referrerPolicy ?? 'strict-origin-when-cross-origin'
 
-  // Return no-referrer when empty or policy says so
-  if (policy == null || policy === '' || policy === 'no-referrer') {
-    return 'no-referrer'
-  }
+  // 2. Let environment be request’s client.
 
-  // 2. Let environment be the request client
-  const environment = request.client
   let referrerSource = null
 
-  /**
-   * 3, Switch on request’s referrer:
-    "client"
-      If environment’s global object is a Window object, then
-        Let document be the associated Document of environment’s global object.
-        If document’s origin is an opaque origin, return no referrer.
-        While document is an iframe srcdoc document,
-        let document be document’s browsing context’s browsing context container’s node document.
-        Let referrerSource be document’s URL.
-
-      Otherwise, let referrerSource be environment’s creation URL.
-
-    a URL
-    Let referrerSource be request’s referrer.
-   */
+  // 3. Switch on request’s referrer:
   if (request.referrer === 'client') {
-    // Not defined in Node but part of the spec
-    if (request.client?.globalObject?.constructor?.name === 'Window' ) { // eslint-disable-line
-      const origin = environment.globalObject.self?.origin ?? environment.globalObject.location?.origin
-
-      // If document’s origin is an opaque origin, return no referrer.
-      if (origin == null || origin === 'null') return 'no-referrer'
-
-      // Let referrerSource be document’s URL.
-      referrerSource = new URL(environment.globalObject.location.href)
-    } else {
-      // 3(a)(II) If environment's global object is not Window,
-      // Let referrerSource be environments creationURL
-      if (environment?.globalObject?.location == null) {
-        return 'no-referrer'
-      }
+    // Note: node isn't a browser and doesn't implement document/iframes,
+    // so we bypass this step and replace it with our own.
+
+    const globalOrigin = getGlobalOrigin()
 
-      referrerSource = new URL(environment.globalObject.location.href)
+    if (!globalOrigin || globalOrigin.origin === 'null') {
+      return 'no-referrer'
     }
+
+    // note: we need to clone it as it's mutated
+    referrerSource = new URL(globalOrigin)
   } else if (request.referrer instanceof URL) {
-    // 3(b) If requests's referrer is a URL instance, then make
-    // referrerSource be requests's referrer.
+    // Let referrerSource be request’s referrer.
     referrerSource = request.referrer
-  } else {
-    // If referrerSource neither client nor instance of URL
-    // then return "no-referrer".
-    return 'no-referrer'
   }
 
-  const urlProtocol = referrerSource.protocol
+  // 4. Let request’s referrerURL be the result of stripping referrerSource for
+  //    use as a referrer.
+  let referrerURL = stripURLForReferrer(referrerSource)
 
-  // If url's scheme is a local scheme (i.e. one of "about", "data", "javascript", "file")
-  // then return "no-referrer".
-  if (
-    urlProtocol === 'about:' || urlProtocol === 'data:' ||
-    urlProtocol === 'blob:'
-  ) {
-    return 'no-referrer'
+  // 5. Let referrerOrigin be the result of stripping referrerSource for use as
+  //    a referrer, with the origin-only flag set to true.
+  const referrerOrigin = stripURLForReferrer(referrerSource, true)
+
+  // 6. If the result of serializing referrerURL is a string whose length is
+  //    greater than 4096, set referrerURL to referrerOrigin.
+  if (referrerURL.toString().length > 4096) {
+    referrerURL = referrerOrigin
   }
 
-  let temp
-  let referrerOrigin
-  // 4. Let requests's referrerURL be the result of stripping referrer
-  // source for use as referrer (using util function, without origin only)
-  const referrerUrl = (temp = stripURLForReferrer(referrerSource)).length > 4096
-  // 5. Let referrerOrigin be the result of stripping referrer
-  // source for use as referrer (using util function, with originOnly true)
-    ? (referrerOrigin = stripURLForReferrer(referrerSource, true))
-  // 6. If result of seralizing referrerUrl is a string whose length is greater than
-  // 4096, then set referrerURL to referrerOrigin
-    : temp
-  const areSameOrigin = sameOrigin(request, referrerUrl)
-  const isNonPotentiallyTrustWorthy = isURLPotentiallyTrustworthy(referrerUrl) &&
+  const areSameOrigin = sameOrigin(request, referrerURL)
+  const isNonPotentiallyTrustWorthy = isURLPotentiallyTrustworthy(referrerURL) &&
     !isURLPotentiallyTrustworthy(request.url)
 
-  // NOTE: How to treat step 7?
   // 8. Execute the switch statements corresponding to the value of policy:
   switch (policy) {
     case 'origin': return referrerOrigin != null ? referrerOrigin : stripURLForReferrer(referrerSource, true)
-    case 'unsafe-url': return referrerUrl
+    case 'unsafe-url': return referrerURL
     case 'same-origin':
       return areSameOrigin ? referrerOrigin : 'no-referrer'
     case 'origin-when-cross-origin':
-      return areSameOrigin ? referrerUrl : referrerOrigin
-    case 'strict-origin-when-cross-origin':
-      /**
-         * 1. If the origin of referrerURL and the origin of request’s current URL are the same,
-         * then return referrerURL.
-         * 2. If referrerURL is a potentially trustworthy URL and request’s current URL is not a
-         * potentially trustworthy URL, then return no referrer.
-         * 3. Return referrerOrigin
-      */
-      if (areSameOrigin) return referrerOrigin
-      // else return isNonPotentiallyTrustWorthy ? 'no-referrer' : referrerOrigin
+      return areSameOrigin ? referrerURL : referrerOrigin
+    case 'strict-origin-when-cross-origin': {
+      const currentURL = requestCurrentURL(request)
+
+      // 1. If the origin of referrerURL and the origin of request’s current
+      //    URL are the same, then return referrerURL.
+      if (sameOrigin(referrerURL, currentURL)) {
+        return referrerURL
+      }
+
+      // 2. If referrerURL is a potentially trustworthy URL and request’s
+      //    current URL is not a potentially trustworthy URL, then return no
+      //    referrer.
+      if (isURLPotentiallyTrustworthy(referrerURL) && !isURLPotentiallyTrustworthy(currentURL)) {
+        return 'no-referrer'
+      }
+
+      // 3. Return referrerOrigin.
+      return referrerOrigin
+    }
     case 'strict-origin': // eslint-disable-line
       /**
          * 1. If referrerURL is a potentially trustworthy URL and
@@ -458,15 +432,42 @@ function determineRequestsReferrer (request) {
     default: // eslint-disable-line
       return isNonPotentiallyTrustWorthy ? 'no-referrer' : referrerOrigin
   }
+}
 
-  function stripURLForReferrer (url, originOnly = false) {
-    const urlObject = new URL(url.href)
-    urlObject.username = ''
-    urlObject.password = ''
-    urlObject.hash = ''
+/**
+ * @see https://w3c.github.io/webappsec-referrer-policy/#strip-url
+ * @param {URL} url
+ * @param {boolean|undefined} originOnly
+ */
+function stripURLForReferrer (url, originOnly) {
+  // 1. Assert: url is a URL.
+  assert(url instanceof URL)
 
-    return originOnly ? urlObject.origin : urlObject.href
+  // 2. If url’s scheme is a local scheme, then return no referrer.
+  if (url.protocol === 'file:' || url.protocol === 'about:' || url.protocol === 'blank:') {
+    return 'no-referrer'
   }
+
+  // 3. Set url’s username to the empty string.
+  url.username = ''
+
+  // 4. Set url’s password to the empty string.
+  url.password = ''
+
+  // 5. Set url’s fragment to null.
+  url.hash = ''
+
+  // 6. If the origin-only flag is true, then:
+  if (originOnly) {
+    // 1. Set url’s path to « the empty string ».
+    url.pathname = ''
+
+    // 2. Set url’s query to null.
+    url.search = ''
+  }
+
+  // 7. Return url.
+  return url
 }
 
 function isURLPotentiallyTrustworthy (url) {