Add cookie test to serializer

.github/workflows/ci.yaml

@@ -8,7 +8,10 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [18.x]
+        node-version:
+          - 16.x
+          - 18.x
+          - 20.x
 
     steps:
       - uses: actions/checkout@v3

src/lib/statusandheaders.ts

@@ -123,6 +123,7 @@ export class StatusAndHeadersParser {
     }
 
     const headers = new headersClass();
+    const canAppend = headers instanceof Headers;
 
     const headerBuff = await readToDoubleCRLF(reader);
 
@@ -141,7 +142,11 @@ export class StatusAndHeadersParser {
       } else {
         if (value) {
           try {
-            headers.set(name, value);
+            if (canAppend && name.toLowerCase() === "set-cookie") {
+              headers.append(name, value);
+            } else {
+              headers.set(name, value);
+            }
           } catch (e) {
             // ignore
           }
@@ -171,7 +176,11 @@ export class StatusAndHeadersParser {
 
     if (value) {
       try {
-        headers.set(name, value);
+        if (canAppend && name.toLowerCase() === "set-cookie") {
+          headers.append(name, value);
+        } else {
+          headers.set(name, value);
+        }
       } catch (e) {
         // ignore
       }

src/lib/warcrecord.ts

@@ -40,7 +40,7 @@ export type WARCRecordOpts = {
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   warcHeaders?: any;
   filename?: string;
-  httpHeaders?: Record<string, string>;
+  httpHeaders?: Record<string, any>;
   statusline?: string;
   warcVersion?: typeof WARC_1_0 | typeof WARC_1_1;
   keepHeadersCase?: boolean;
@@ -103,9 +103,7 @@ export class WARCRecord extends BaseAsyncIterReader {
 
     warcHeaders = new StatusAndHeaders({
       statusline: warcVersion,
-      headers: keepHeadersCase
-        ? new Map(Object.entries(warcHeaders))
-        : new Headers(warcHeaders),
+      headers: new Map(Object.entries(warcHeaders))
     });
 
     if (!warcHeaders.headers.get("WARC-Record-ID")) {

test/testSerializer.test.ts

@@ -5,6 +5,22 @@ import { WARCSerializer } from "../src/node/warcserializer";
 const decoder = new TextDecoder("utf-8");
 const encoder = new TextEncoder();
 
+const [ majorVerison, minorVersion, patchVersion ] = process.versions?.node?.split(".").map((v) => Number(v));
+
+// added in 18.14.2
+const nodeHeadersSupportsMultipleCookies = (
+  (majorVerison !== undefined && majorVerison > 18) ||
+  (
+    majorVerison !== undefined && majorVerison === 18 &&
+    minorVersion !== undefined && minorVersion > 14
+  ) ||
+  (
+    majorVerison !== undefined && majorVerison === 18 &&
+    minorVersion !== undefined && minorVersion === 14 &&
+    minorVersion !== undefined && minorVersion >= 2
+  )
+);
+
 async function* iter(data: string) {
   yield encoder.encode(data);
 }
@@ -278,6 +294,152 @@ Accept: */*\r\n\
     );
   });
 
+  test("create request record with cookie array", async () => {
+    const url = "http://example.com/";
+    const date = "2000-01-01T00:00:00Z";
+    const type = "request";
+    const warcHeaders = {
+      "WARC-Record-ID": "<urn:uuid:12345678-feb0-11e6-8f83-68a86d1772ce>",
+    };
+    const httpHeaders = [
+      ["Set-Cookie", "greeting=hello"],
+      ["Set-Cookie", "name=world"],
+    ];
+
+    const statusline = "GET /file HTTP/1.1";
+
+    const record = await WARCRecord.create({
+      url,
+      date,
+      type,
+      warcHeaders,
+      httpHeaders,
+      statusline,
+      keepHeadersCase: false,
+    });
+
+    expect(record.warcType).toBe("request");
+
+    const gzipped = await WARCSerializer.serialize(record, { gzip: true });
+    const res = decoder.decode(pako.inflate(gzipped));
+
+    if (nodeHeadersSupportsMultipleCookies) {
+      expect(res).toBe(
+      "\
+WARC/1.0\r\n\
+WARC-Record-ID: <urn:uuid:12345678-feb0-11e6-8f83-68a86d1772ce>\r\n\
+WARC-Target-URI: http://example.com/\r\n\
+WARC-Date: 2000-01-01T00:00:00Z\r\n\
+WARC-Type: request\r\n\
+Content-Type: application/http; msgtype=request\r\n\
+WARC-Payload-Digest: sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\r\n\
+WARC-Block-Digest: sha256:e37db413e919457008f5f5d9285185eb37d29ad614915c248dd876a81abbd4ae\r\n\
+Content-Length: 74\r\n\
+\r\n\
+GET /file HTTP/1.1\r\n\
+set-cookie: greeting=hello\r\n\
+set-cookie: name=world\r\n\
+\r\n\
+\r\n\
+\r\n\
+"
+      );
+    } else {
+      expect(res).toBe(
+      "\
+WARC/1.0\r\n\
+WARC-Record-ID: <urn:uuid:12345678-feb0-11e6-8f83-68a86d1772ce>\r\n\
+WARC-Target-URI: http://example.com/\r\n\
+WARC-Date: 2000-01-01T00:00:00Z\r\n\
+WARC-Type: request\r\n\
+Content-Type: application/http; msgtype=request\r\n\
+WARC-Payload-Digest: sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\r\n\
+WARC-Block-Digest: sha256:8862c1f0167a2acbeec66b60afa8ffbf855fe666c17cd525238a33bb68c3df02\r\n\
+Content-Length: 62\r\n\
+\r\n\
+GET /file HTTP/1.1\r\n\
+set-cookie: greeting=hello, name=world\r\n\
+\r\n\
+\r\n\
+\r\n\
+"
+      );
+    }
+  });
+
+  test("create request record with cookie header class", async () => {
+    const url = "http://example.com/";
+    const date = "2000-01-01T00:00:00Z";
+    const type = "request";
+    const warcHeaders = {
+      "WARC-Record-ID": "<urn:uuid:12345678-feb0-11e6-8f83-68a86d1772ce>",
+    };
+    const httpHeaders = new Headers();
+    httpHeaders.append('Set-Cookie', 'greeting=hello');
+    httpHeaders.append('Set-Cookie', 'name=world');
+
+    const statusline = "GET /file HTTP/1.1";
+
+    const record = await WARCRecord.create({
+      url,
+      date,
+      type,
+      warcHeaders,
+      httpHeaders,
+      statusline,
+      keepHeadersCase: false,
+    });
+
+    expect(record.warcType).toBe("request");
+
+    const gzipped = await WARCSerializer.serialize(record, { gzip: true });
+    const res = decoder.decode(pako.inflate(gzipped));
+
+    if (nodeHeadersSupportsMultipleCookies) {
+      expect(res).toBe(
+      "\
+WARC/1.0\r\n\
+WARC-Record-ID: <urn:uuid:12345678-feb0-11e6-8f83-68a86d1772ce>\r\n\
+WARC-Target-URI: http://example.com/\r\n\
+WARC-Date: 2000-01-01T00:00:00Z\r\n\
+WARC-Type: request\r\n\
+Content-Type: application/http; msgtype=request\r\n\
+WARC-Payload-Digest: sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\r\n\
+WARC-Block-Digest: sha256:e37db413e919457008f5f5d9285185eb37d29ad614915c248dd876a81abbd4ae\r\n\
+Content-Length: 74\r\n\
+\r\n\
+GET /file HTTP/1.1\r\n\
+set-cookie: greeting=hello\r\n\
+set-cookie: name=world\r\n\
+\r\n\
+\r\n\
+\r\n\
+"
+      );
+    } else {
+      expect(res).toBe(
+      "\
+WARC/1.0\r\n\
+WARC-Record-ID: <urn:uuid:12345678-feb0-11e6-8f83-68a86d1772ce>\r\n\
+WARC-Target-URI: http://example.com/\r\n\
+WARC-Date: 2000-01-01T00:00:00Z\r\n\
+WARC-Type: request\r\n\
+Content-Type: application/http; msgtype=request\r\n\
+WARC-Payload-Digest: sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\r\n\
+WARC-Block-Digest: sha256:8862c1f0167a2acbeec66b60afa8ffbf855fe666c17cd525238a33bb68c3df02\r\n\
+Content-Length: 62\r\n\
+\r\n\
+GET /file HTTP/1.1\r\n\
+set-cookie: greeting=hello, name=world\r\n\
+\r\n\
+\r\n\
+\r\n\
+"
+      );
+    }
+
+  });
+
   test("create warcinfo", async () => {
     const filename = "/my/web/archive.warc";
     const type = "warcinfo";

test/testWARCParser.test.ts

@@ -11,6 +11,22 @@ import { getReader, getReadableStream } from "./utils";
 import fs from "fs";
 import path from "path";
 
+const [ majorVerison, minorVersion, patchVersion ] = process.versions?.node?.split(".").map((v) => Number(v));
+
+// added in 18.14.2
+const nodeHeadersSupportsMultipleCookies = (
+  (majorVerison !== undefined && majorVerison > 18) ||
+  (
+    majorVerison !== undefined && majorVerison === 18 &&
+    minorVersion !== undefined && minorVersion > 14
+  ) ||
+  (
+    majorVerison !== undefined && majorVerison === 18 &&
+    minorVersion !== undefined && minorVersion === 14 &&
+    minorVersion !== undefined && minorVersion >= 2
+  )
+);
+
 const decoder = new TextDecoder("utf-8");
 
 function get_warc_path(filename: string) {
@@ -611,3 +627,102 @@ test("no await catch errors", async () => {
   }
   expect(count).toBe(0);
 });
+
+test("warc1.1 response and request, header checks", async () => {
+  const input =
+    // eslint-disable-next-line quotes -- inner double quote
+    '\
+WARC/1.0\r\n\
+WARC-Type: response\r\n\
+WARC-Record-ID: <urn:uuid:12345678-feb0-11e6-8f83-68a86d1772ce>\r\n\
+WARC-Target-URI: http://example.com/\r\n\
+WARC-Date: 2000-01-01T00:00:00Z\r\n\
+WARC-Payload-Digest: sha1:B6QJ6BNJ3R4B23XXMRKZKHLPGJY2VE4O\r\n\
+WARC-Block-Digest: sha1:OS3OKGCWQIJOAOC3PKXQOQFD52NECQ74\r\n\
+Content-Type: application/http; msgtype=response\r\n\
+Content-Length: 149\r\n\
+\r\n\
+HTTP/1.0 200 OK\r\n\
+Content-Type: text/plain; charset="UTF-8"\r\n\
+Custom-Header: somevalue\r\n\
+Set-Cookie: greeting=hello\r\n\
+Set-Cookie: name=world\r\n\
+\r\n\
+some\n\
+text\r\n\
+\r\n\
+WARC/1.0\r\n\
+WARC-Type: response\r\n\
+WARC-Record-ID: <urn:uuid:12345678-feb0-11e6-8f83-68a86d1772ce>\r\n\
+WARC-Target-URI: http://example.com/\r\n\
+WARC-Date: 2000-01-01T00:00:00Z\r\n\
+WARC-Payload-Digest: sha1:B6QJ6BNJ3R4B23XXMRKZKHLPGJY2VE4O\r\n\
+WARC-Block-Digest: sha1:KMUABC6URWIQ7QXCZDQ5FS6WIBBFRORR\r\n\
+Content-Type: application/http; msgtype=response\r\n\
+Content-Length: 268\r\n\
+\r\n\
+HTTP/1.0 200 OK\r\n\
+Content-Type: text/plain; charset="UTF-8"\r\n\
+Content-Disposition: attachment; filename*=UTF-8\'\'%D0%B8%D1%81%D0%BF%D1%8B%D1%82%D0%B0%D0%BD%D0%B8%D0%B5.txt\r\n\
+Custom-Header: somevalue\r\n\
+Unicode-Header: %F0%9F%93%81%20text%20%F0%9F%97%84%EF%B8%8F\r\n\
+\r\n\
+more\n\
+text\r\n\
+\r\n\
+';
+
+  const reader = new AsyncIterReader(getReader([input]));
+
+  let parser = new WARCParser(reader);
+
+  // eslint-disable-next-line @typescript-eslint/no-non-null-assertion -- checked in expect
+  const record = (await parser.parse())!;
+  expect(record).not.toBeNull();
+  expect(record.warcTargetURI, "http://example.com/");
+
+  let headerEntries = [];
+  if (record.httpHeaders?.headers) {
+    for (const [key, value] of record.httpHeaders?.headers?.entries()) {
+      headerEntries.push([ key, value ]);
+    }
+  }
+
+  if (nodeHeadersSupportsMultipleCookies) {
+    expect(JSON.stringify(headerEntries)).toBe(JSON.stringify([
+      [ 'content-type', 'text/plain; charset="UTF-8"' ],
+      [ 'custom-header', 'somevalue' ],
+      [ 'set-cookie', 'greeting=hello' ],
+      [ 'set-cookie', 'name=world' ]
+    ]));
+  } else {
+    expect(JSON.stringify(headerEntries)).toBe(JSON.stringify([
+      [ 'content-type', 'text/plain; charset="UTF-8"' ],
+      [ 'custom-header', 'somevalue' ],
+      [ 'set-cookie', 'greeting=hello, name=world' ]
+    ]));
+  }
+
+  expect(decoder.decode(await record.readFully())).toBe("some\ntext");
+
+  // eslint-disable-next-line @typescript-eslint/no-non-null-assertion -- checked in expect
+  const record2 = (await parser.parse())!;
+  expect(record2).not.toBeNull();
+
+  let headerEntries2 = [];
+  if (record2.httpHeaders?.headers) {
+    for (const [key, value] of record2.httpHeaders?.headers?.entries()) {
+      headerEntries2.push([ key, value ]);
+    }
+  }
+  expect(JSON.stringify(headerEntries2)).toBe(JSON.stringify([
+    [ 'content-disposition', 'attachment; filename*=UTF-8\'\'%D0%B8%D1%81%D0%BF%D1%8B%D1%82%D0%B0%D0%BD%D0%B8%D0%B5.txt' ],
+    [ 'content-type', 'text/plain; charset="UTF-8"' ],
+    [ 'custom-header', 'somevalue' ],
+    [ 'unicode-header', '%F0%9F%93%81%20text%20%F0%9F%97%84%EF%B8%8F' ]
+  ]));
+
+  expect(decoder.decode(await record2.readFully())).toBe("more\ntext");
+
+  expect(await parser.parse()).toBeNull();
+});