http,https,tls: switch to WHATWG URL parser

This switches the url parser from `url.parse()` to the WHATWG URL
parser while keeping `url.parse()` as fallback.

Also add tests for invalid url deprecations and correct hostname
checks.

PR-URL: nodejs#20270
Fixes: nodejs#19468
Reviewed-By: Luigi Pinca <[email protected]>
Reviewed-By: James M Snell <[email protected]>
Reviewed-By: Matteo Collina <[email protected]>
Reviewed-By: Tiancheng "Timothy" Gu <[email protected]>
Reviewed-By: Daijiro Wachi <[email protected]>
Reviewed-By: Trivikram Kamat <[email protected]>
Reviewed-By: Ruben Bridgewater <[email protected]>

Author: Hackzzila

doc/api/deprecations.md

@@ -994,6 +994,18 @@ because it also made sense to interpret the value as the number of bytes
 read by the engine, but is inconsistent with other streams in Node.js that
 expose values under these names.
 
+<a id="DEP0109"></a>
+### DEP0109: http, https, and tls support for invalid URLs
+
+Type: Runtime
+
+Some previously supported (but strictly invalid) URLs were accepted through the
+[`http.request()`][], [`http.get()`][], [`https.request()`][],
+[`https.get()`][], and [`tls.checkServerIdentity()`][] APIs because those were
+accepted by the legacy `url.parse()` API. The mentioned APIs now use the WHATWG
+URL parser that requires strictly valid URLs. Passing an invalid URL is
+deprecated and support will be removed in the future.
+
 [`--pending-deprecation`]: cli.html#cli_pending_deprecation
 [`Buffer.allocUnsafeSlow(size)`]: buffer.html#buffer_class_method_buffer_allocunsafeslow_size
 [`Buffer.from(array)`]: buffer.html#buffer_class_method_buffer_from_array
@@ -1035,6 +1047,10 @@ expose values under these names.
 [`fs.read()`]: fs.html#fs_fs_read_fd_buffer_offset_length_position_callback
 [`fs.readSync()`]: fs.html#fs_fs_readsync_fd_buffer_offset_length_position
 [`fs.stat()`]: fs.html#fs_fs_stat_path_callback
+[`http.get()`]: http.html#http_http_get_options_callback
+[`http.request()`]: http.html#http_http_request_options_callback
+[`https.get()`]: https.html#https_https_get_options_callback
+[`https.request()`]: https.html#https_https_request_options_callback
 [`os.networkInterfaces`]: os.html#os_os_networkinterfaces
 [`os.tmpdir()`]: os.html#os_os_tmpdir
 [`process.env`]: process.html#process_process_env
@@ -1046,6 +1062,7 @@ expose values under these names.
 [`tls.SecureContext`]: tls.html#tls_tls_createsecurecontext_options
 [`tls.SecurePair`]: tls.html#tls_class_securepair
 [`tls.TLSSocket`]: tls.html#tls_class_tls_tlssocket
+[`tls.checkServerIdentity()`]: tls.html#tls_tls_checkserveridentity_host_cert
 [`tls.createSecureContext()`]: tls.html#tls_tls_createsecurecontext_options
 [`util._extend()`]: util.html#util_util_extend_target_source
 [`util.debug()`]: util.html#util_util_debug_string

doc/api/errors.md

@@ -1111,11 +1111,6 @@ Invalid characters were detected in headers.
 A cursor on a given stream cannot be moved to a specified row without a
 specified column.
 
-<a id="ERR_INVALID_DOMAIN_NAME"></a>
-### ERR_INVALID_DOMAIN_NAME
-
-`hostname` can not be parsed from a provided URL.
-
 <a id="ERR_INVALID_FD"></a>
 ### ERR_INVALID_FD
 

doc/api/http.md

@@ -1900,7 +1900,7 @@ Node.js maintains several connections per server to make HTTP requests.
 This function allows one to transparently issue requests.
 
 `options` can be an object, a string, or a [`URL`][] object. If `options` is a
-string, it is automatically parsed with [`url.parse()`][]. If it is a [`URL`][]
+string, it is automatically parsed with [`new URL()`][]. If it is a [`URL`][]
 object, it will be automatically converted to an ordinary `options` object.
 
 The optional `callback` parameter will be added as a one-time listener for
@@ -2050,6 +2050,7 @@ not abort the request or do anything besides add a `'timeout'` event.
 [`net.Server`]: net.html#net_class_net_server
 [`net.Socket`]: net.html#net_class_net_socket
 [`net.createConnection()`]: net.html#net_net_createconnection_options_connectlistener
+[`new URL()`]: url.html#url_constructor_new_url_input_base
 [`removeHeader(name)`]: #http_request_removeheader_name
 [`request.end()`]: #http_request_end_data_encoding_callback
 [`request.getHeader()`]: #http_request_getheader_name

doc/api/https.md

@@ -119,7 +119,7 @@ changes:
 Like [`http.get()`][] but for HTTPS.
 
 `options` can be an object, a string, or a [`URL`][] object. If `options` is a
-string, it is automatically parsed with [`url.parse()`][]. If it is a [`URL`][]
+string, it is automatically parsed with [`new URL()`][]. If it is a [`URL`][]
 object, it will be automatically converted to an ordinary `options` object.
 
 Example:
@@ -173,7 +173,7 @@ The following additional `options` from [`tls.connect()`][] are also accepted:
 `secureOptions`, `secureProtocol`, `servername`, `sessionIdContext`
 
 `options` can be an object, a string, or a [`URL`][] object. If `options` is a
-string, it is automatically parsed with [`url.parse()`][]. If it is a [`URL`][]
+string, it is automatically parsed with [`new URL()`][]. If it is a [`URL`][]
 object, it will be automatically converted to an ordinary `options` object.
 
 Example:
@@ -358,8 +358,8 @@ headers: max-age=0; pin-sha256="WoiWRyIOVNa9ihaBciRSC7XHjliYS9VwUGOIud4PB18="; p
 [`https.Agent`]: #https_class_https_agent
 [`https.request()`]: #https_https_request_options_callback
 [`net.Server`]: net.html#net_class_net_server
+[`new URL()`]: url.html#url_constructor_new_url_input_base
 [`server.listen()`]: net.html#net_server_listen
 [`tls.connect()`]: tls.html#tls_tls_connect_options_callback
 [`tls.createSecureContext()`]: tls.html#tls_tls_createsecurecontext_options
 [`tls.createServer()`]: tls.html#tls_tls_createserver_options_secureconnectionlistener
-[`url.parse()`]: url.html#url_url_parse_urlstring_parsequerystring_slashesdenotehost

lib/_http_client.js

@@ -37,12 +37,11 @@ const { OutgoingMessage } = require('_http_outgoing');
 const Agent = require('_http_agent');
 const { Buffer } = require('buffer');
 const { defaultTriggerAsyncIdScope } = require('internal/async_hooks');
-const { urlToOptions, searchParamsSymbol } = require('internal/url');
+const { URL, urlToOptions, searchParamsSymbol } = require('internal/url');
 const { outHeadersKey, ondrain } = require('internal/http');
 const {
   ERR_HTTP_HEADERS_SENT,
   ERR_INVALID_ARG_TYPE,
-  ERR_INVALID_DOMAIN_NAME,
   ERR_INVALID_HTTP_TOKEN,
   ERR_INVALID_PROTOCOL,
   ERR_UNESCAPED_CHARACTERS
@@ -60,13 +59,26 @@ function validateHost(host, name) {
   return host;
 }
 
+let urlWarningEmitted = false;
 function ClientRequest(options, cb) {
   OutgoingMessage.call(this);
 
   if (typeof options === 'string') {
-    options = url.parse(options);
-    if (!options.hostname) {
-      throw new ERR_INVALID_DOMAIN_NAME();
+    const urlStr = options;
+    try {
+      options = urlToOptions(new URL(urlStr));
+    } catch (err) {
+      options = url.parse(urlStr);
+      if (!options.hostname) {
+        throw err;
+      }
+      if (!urlWarningEmitted && !process.noDeprecation) {
+        urlWarningEmitted = true;
+        process.emitWarning(
+          `The provided URL ${urlStr} is not a valid URL, and is supported ` +
+          'in the http module solely for compatibility.',
+          'DeprecationWarning', 'DEP0109');
+      }
     }
   } else if (options && options[searchParamsSymbol] &&
              options[searchParamsSymbol][searchParamsSymbol]) {

lib/https.js

@@ -34,8 +34,7 @@ const {
 const { ClientRequest } = require('_http_client');
 const { inherits } = util;
 const debug = util.debuglog('https');
-const { urlToOptions, searchParamsSymbol } = require('internal/url');
-const { ERR_INVALID_DOMAIN_NAME } = require('internal/errors').codes;
+const { URL, urlToOptions, searchParamsSymbol } = require('internal/url');
 const { IncomingMessage, ServerResponse } = require('http');
 const { kIncomingMessage } = require('_http_common');
 const { kServerResponse } = require('_http_server');
@@ -254,11 +253,24 @@ Agent.prototype._evictSession = function _evictSession(key) {
 
 const globalAgent = new Agent();
 
+let urlWarningEmitted = false;
 function request(options, cb) {
   if (typeof options === 'string') {
-    options = url.parse(options);
-    if (!options.hostname) {
-      throw new ERR_INVALID_DOMAIN_NAME();
+    const urlStr = options;
+    try {
+      options = urlToOptions(new URL(urlStr));
+    } catch (err) {
+      options = url.parse(urlStr);
+      if (!options.hostname) {
+        throw err;
+      }
+      if (!urlWarningEmitted && !process.noDeprecation) {
+        urlWarningEmitted = true;
+        process.emitWarning(
+          `The provided URL ${urlStr} is not a valid URL, and is supported ` +
+          'in the https module solely for compatibility.',
+          'DeprecationWarning', 'DEP0109');
+      }
     }
   } else if (options && options[searchParamsSymbol] &&
              options[searchParamsSymbol][searchParamsSymbol]) {

lib/internal/errors.js

@@ -881,7 +881,6 @@ E('ERR_INVALID_CALLBACK', 'Callback must be a function', TypeError);
 E('ERR_INVALID_CHAR', invalidChar, TypeError);
 E('ERR_INVALID_CURSOR_POS',
   'Cannot set cursor row without setting its column', TypeError);
-E('ERR_INVALID_DOMAIN_NAME', 'Unable to determine the domain name', TypeError);
 E('ERR_INVALID_FD',
   '"fd" must be a positive integer: %s', RangeError);
 E('ERR_INVALID_FD_TYPE', 'Unsupported fd type: %s', TypeError);

lib/tls.js

@@ -32,6 +32,7 @@ const url = require('url');
 const binding = process.binding('crypto');
 const { Buffer } = require('buffer');
 const EventEmitter = require('events');
+const { URL } = require('internal/url');
 const DuplexPair = require('internal/streams/duplexpair');
 const { canonicalizeIP } = process.binding('cares_wrap');
 const _tls_common = require('_tls_common');
@@ -169,6 +170,7 @@ function check(hostParts, pattern, wildcards) {
   return true;
 }
 
+let urlWarningEmitted = false;
 exports.checkServerIdentity = function checkServerIdentity(host, cert) {
   const subject = cert.subject;
   const altNames = cert.subjectaltname;
@@ -183,7 +185,21 @@ exports.checkServerIdentity = function checkServerIdentity(host, cert) {
       if (name.startsWith('DNS:')) {
         dnsNames.push(name.slice(4));
       } else if (name.startsWith('URI:')) {
-        const uri = url.parse(name.slice(4));
+        let uri;
+        try {
+          uri = new URL(name.slice(4));
+        } catch (err) {
+          uri = url.parse(name.slice(4));
+          if (!urlWarningEmitted && !process.noDeprecation) {
+            urlWarningEmitted = true;
+            process.emitWarning(
+              `The URI ${name.slice(4)} found in cert.subjectaltname ` +
+              'is not a valid URI, and is supported in the tls module ' +
+              'solely for compatibility.',
+              'DeprecationWarning', 'DEP0109');
+          }
+        }
+
         uriNames.push(uri.hostname);  // TODO(bnoordhuis) Also use scheme.
       } else if (name.startsWith('IP Address:')) {
         ips.push(canonicalizeIP(name.slice(11)));

test/parallel/test-http-correct-hostname.js

@@ -0,0 +1,28 @@
+/* eslint-disable node-core/crypto-check */
+// Flags: --expose-internals
+'use strict';
+
+const common = require('../common');
+const assert = require('assert');
+
+const { outHeadersKey } = require('internal/http');
+
+const http = require('http');
+const modules = { http };
+
+if (common.hasCrypto) {
+  const https = require('https');
+  modules.https = https;
+}
+
+Object.keys(modules).forEach((module) => {
+  const doNotCall = common.mustNotCall(
+    `${module}.request should not connect to ${module}://example.com%60x.example.com`
+  );
+  const req = modules[module].request(`${module}://example.com%60x.example.com`, doNotCall);
+  assert.deepStrictEqual(req[outHeadersKey].host, [
+    'Host',
+    'example.com`x.example.com',
+  ]);
+  req.abort();
+});

test/parallel/test-http-deprecated-urls.js

@@ -0,0 +1,33 @@
+/* eslint-disable node-core/crypto-check */
+
+'use strict';
+
+const common = require('../common');
+
+const http = require('http');
+const modules = { http };
+
+const deprecations = [
+  ['The provided URL http://[www.nodejs.org] is not a valid URL, and is supported ' +
+  'in the http module solely for compatibility.',
+   'DEP0109'],
+];
+
+if (common.hasCrypto) {
+  const https = require('https');
+  modules.https = https;
+  deprecations.push(
+    ['The provided URL https://[www.nodejs.org] is not a valid URL, and is supported ' +
+    'in the https module solely for compatibility.',
+     'DEP0109'],
+  );
+}
+
+common.expectWarning('DeprecationWarning', deprecations);
+
+Object.keys(modules).forEach((module) => {
+  const doNotCall = common.mustNotCall(
+    `${module}.request should not connect to ${module}://[www.nodejs.org]`
+  );
+  modules[module].request(`${module}://[www.nodejs.org]`, doNotCall).abort();
+});

test/parallel/test-http-invalid-urls.js

@@ -21,7 +21,7 @@ function test(host) {
       const throws = () => { modules[module][fn](host, doNotCall); };
       common.expectsError(throws, {
         type: TypeError,
-        code: 'ERR_INVALID_DOMAIN_NAME'
+        code: 'ERR_INVALID_URL'
       });
     });
   });

test/parallel/test-internal-errors.js

@@ -148,11 +148,6 @@ assert.strictEqual(
   'Cannot render headers after they are sent to the client'
 );
 
-assert.strictEqual(
-  errors.message('ERR_INVALID_DOMAIN_NAME'),
-  'Unable to determine the domain name'
-);
-
 assert.strictEqual(
   errors.message('ERR_INVALID_HTTP_TOKEN', ['Method', 'foo']),
   'Method must be a valid HTTP token ["foo"]'

test/parallel/test-tls-check-server-identity.js

@@ -30,6 +30,13 @@ const util = require('util');
 
 const tls = require('tls');
 
+common.expectWarning('DeprecationWarning', [
+  ['The URI http://[a.b.a.com]/ found in cert.subjectaltname ' +
+  'is not a valid URI, and is supported in the tls module ' +
+  'solely for compatibility.',
+   'DEP0109'],
+]);
+
 const tests = [
   // False-y values.
   {
@@ -218,6 +225,13 @@ const tests = [
     error: 'Host: a.b.a.com. is not in the cert\'s altnames: ' +
            'URI:http://*.b.a.com/'
   },
+  // Invalid URI
+  {
+    host: 'a.b.a.com', cert: {
+      subjectaltname: 'URI:http://[a.b.a.com]/',
+      subject: {}
+    }
+  },
   // IP addresses
   {
     host: 'a.b.a.com', cert: {