Run test262 tests from a git submodule

Instead of an in-tree checkout use a git submodule of test262 so we can
pin the tests to a particular commit and pull in updates when we are
ready for them.

Requires some changes in various paths.

The tests that were currently marked as expected to fail were not added
to upstream test262 yet, so they are removed from expected-failures.txt.

Author: ptomato

.github/workflows/test.yml

@@ -41,6 +41,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
+      with:
+        submodules: true
     - name: use node.js v16.x
       uses: actions/setup-node@v1
       with:

.gitignore

@@ -3,7 +3,6 @@ polyfill/index.js
 polyfill/index.js.map
 polyfill/script.js
 polyfill/script.js.map
-polyfill/test262/
 polyfill/*.tgz
 polyfill/venv
 polyfill/coverage

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "polyfill/test262"]
+	path = polyfill/test262
+	url = https://github.com/tc39/test262

polyfill/ci_test.sh

@@ -1,27 +1,19 @@
 #!/bin/bash
 
-TESTS=${@:-"./*/**/*.js"}
+TESTS=${@:-"**/*.js"}
 TIMEOUT=${TIMEOUT:-10000}
 PRELUDE=${PRELUDE:-script.js}
 
 export NODE_PATH=$PWD/node_modules
 npm run build262
-if [ ! -d "test262" ]; then
-  git clone --depth 1 https://github.com/tc39/test262.git
-else
-  cd ./test262
-  git fetch origin
-  git merge --ff-only origin/master
-  cd ..
-fi
 
 if [ "x$COVERAGE" = xyes ]; then
   nyc instrument "$PRELUDE" > script-instrumented.js
   PRELUDE=script-instrumented.js
-  TRANSFORMER_ARG="--transformer ./transform.test262.js"
+  TRANSFORMER_ARG="--transformer ../../test/transform.test262.js"
 fi
 
-cd test/
+pushd test262/test/
 
 if [ "$(uname)" = 'Darwin' ]; then
   threads=$(sysctl -n hw.logicalcpu)
@@ -30,22 +22,20 @@ else
 fi
 if [ $threads -gt 8 ]; then threads=8; fi
 
-cp ./helpers/* ../test262/harness/
-
 test262-harness \
   -t $threads \
   -r json \
   --reporter-keys file,rawResult,result,scenario \
-  --test262Dir ../test262 \
-  --prelude "../$PRELUDE" \
+  --test262Dir .. \
+  --prelude "../../$PRELUDE" \
   --timeout "$TIMEOUT" \
-  --preprocessor ./preprocessor.test262.js \
+  --preprocessor ../../test/preprocessor.test262.js \
   $TRANSFORMER_ARG \
-  "$TESTS" \
-  | ./parseResults.js
+  "*/Temporal/$TESTS" \
+  | ../../test/parseResults.js
 RESULT=$?
 
-cd ..
+popd
 
 if [ "x$COVERAGE" = xyes ]; then
   nyc report -t coverage/tmp/transformer --reporter=text-lcov > coverage/test262.lcov

polyfill/test/expected-failures.txt

@@ -1,10 +1,2 @@
-# Bug: https://github.com/tc39/proposal-temporal/issues/1791
-Duration/constructor/compare/relativeto-string-datetime.js
-
-# Blocked on https://github.com/tc39/proposal-temporal/issues/1685
-# (MoveRelativeDate should pass undefined as options)
-Duration/constructor/compare/calendar-dateadd-called-with-options-undefined.js
-Duration/prototype/round/calendar-dateadd-called-with-options-undefined.js
-Duration/prototype/total/calendar-dateadd-called-with-options-undefined.js
-ZonedDateTime/prototype/since/calendar-dateadd-called-with-options-undefined.js
-ZonedDateTime/prototype/until/calendar-dateadd-called-with-options-undefined.js
+# Blocked on https://github.com/tc39/test262/pull/3188
+built-ins/Temporal/Calendar/prototype/fields/long-input.js

polyfill/test/parseResults.js

@@ -12,7 +12,7 @@ let testOutput = '';
 const now = Date.now();
 
 const expectedFailures = new Set();
-const lines = fs.readFileSync('expected-failures.txt', { encoding: 'utf-8' });
+const lines = fs.readFileSync('../../test/expected-failures.txt', { encoding: 'utf-8' });
 for (let line of lines.split('\n')) {
   line = line.trim();
   if (!line) continue;

polyfill/test/transform.test262.js

@@ -2,8 +2,8 @@ module.exports = function addCoverageOutput(code) {
   return `${code}
 const fs = require('fs');
 const {v4: uuid} = require('uuid');
-const filename = '../coverage/tmp/transformer/' + uuid() + '.json';
-fs.mkdirSync('../coverage/tmp/transformer/', { recursive: true });
+const filename = '../../coverage/tmp/transformer/' + uuid() + '.json';
+fs.mkdirSync('../../coverage/tmp/transformer/', { recursive: true });
 fs.writeFileSync(filename, JSON.stringify(globalThis.__coverage__ || {}), 'utf-8');
 `;
 };