Added an execution test pack that uses karma to run test on code generated with ts-loader

.gitignore

@@ -6,5 +6,6 @@ bundle.js
 npm-debug.log
 .test/
 !/test/**/expectedOutput-*/**
+!/test/**/typings/**
 /node_modules
 !build.js

.travis.yml

@@ -6,7 +6,7 @@ install:
   - npm install
   - npm install $TYPESCRIPT
 env:
-#  - [email protected]
-#  - [email protected]
-#  - [email protected]
+  - [email protected]
+  - [email protected]
+  - [email protected]
   - [email protected]

CONTRIBUTING.md

@@ -23,16 +23,34 @@ In short: fork, branch, make your changes, and submit a pull request.
 
 ## Testing
 
-This project makes use of a test suite both to make sure we don't break
-anything and also to make sure new versions of webpack and TypeScript don't
-break anything. The tests are full integration tests, meaning each test is a
-mini project that is run through webpack and then the output is compared to the
-expected output. Not all bugs/features necessarily fit into this framework and
+This project makes use of 2 integration test packs to make sure we don't break
+anything. That's right, count them, 2! There is a comparison test pack which 
+compares compilation outputs and is long established.  There is also an execution 
+test pack which executes the compiled JavaScript. This test pack is young and contains
+few tests so far; but it shows promise.
+
+You can run all the tests (in both test packs) with `npm test`.
+
+To run comparison tests alone use `npm run comparison-tests`.
+To run execution tests alone use `npm run execution-tests`.
+
+Not all bugs/features necessarily fit into either framework and
 that's OK. However, most do and therefore you should make every effort to
 create at least one test which demonstrates the issue or exercises the feature.
+Use your judgement to decide whether you think a comparison test or an
+execution test is most appropriate.
+
+### Comparison Test Pack
+
+This test pack comprises a number of mini-typescript projects which, as part of the test run, are each run through webpack.  
+The outputs (both compiled JavaScript and webpack compilation output) are compared against a set of expected 
+outputs. These are particularly useful for testing failure cases; that is testing scenarios where you expect compilation
+to fail and ensuring the failure is what you expect. For example, ensuring the presence of error messages from the TypeScript 
+compiler in the output etc.
 
+The comparison test pack can be found under `/test/comparison-tests`.
 The test harness uses certain conventions. All tests have their own directory
-under `/test`, eg `/test/someFeature`. Each test should have a
+under `/test/comparison-tests`, eg `/test/comparison-tests/someFeature`. Each test should have a
 `webpack.config.js` file which follows this general convention:
 
 ```javascript
@@ -55,7 +73,8 @@ module.exports = {
 module.exports.resolveLoader = { alias: { 'ts-loader': require('path').join(__dirname, "../../index.js") } }
 ```
 
-You can run all the tests with `npm test`. You can also go into an individual test
+You can run all the tests in the Comparison Test Pack with `npm run comparison-tests`. 
+You can also go into an individual test
 directory and manually build a project using `webpack` or `webpack --watch`.
 This can be useful both when developing the test and also when fixing an issue
 or adding a feature.
@@ -65,17 +84,21 @@ filesystem output (typically `bundle.js` and possibly `bundle.js.map`) and any
 console output. stdout should go in `output.txt` and stderr should go in
 `err.txt`.
 
+To run all the tests use:
+
+`npm run comparison-tests`.
+
 If you would like to run just a single test then:
 
-`npm test -- --single-test nameOfTest`
+`npm run comparison-tests -- --single-test nameOfTest`
 
-### Regenerating test data
+#### Regenerating test data
 
 As a convenience it is possible to regenerate the expected output from the 
 actual output. This is useful when creating new tests and also when making a
 change that affects multiple existing tests. To run use:
 
-`npm test -- --save-output`. 
+`npm run comparison-tests -- --save-output`. 
 
 Note that all tests will automatically pass when
 using this feature. You should double check the generated files to make sure
@@ -84,7 +107,7 @@ the output is indeed correct.
 If you would like to regenerate a single test then combine `--save-output` with 
 `--single-test` like so:
 
-`npm test -- --save-output --single-test nameOfTest`
+`npm run comparison-tests -- --save-output --single-test nameOfTest`
 
 The test harness additionally supports watch mode since that is such an
 integral part of webpack. The initial state is as described above. After the
@@ -99,4 +122,84 @@ Initial state:
 Patch 0
 - test/someFeature/patch0/app.ts - *modified file*
 - test/someFeature/expectedOutput/patch0/bundle.js - *bundle after applying patch*
-- test/someFeature/expectedOutput/patch0/output.txt - *output after applying patch*
+- test/someFeature/expectedOutput/patch0/output.txt - *output after applying patch*
+
+### Execution Test Pack
+
+This test pack is made up of a number of mini-typescript projects which include a test suite. 
+As part of the test run, each project is compiled and the test suite run using Karma. So this 
+test pack is different from the comparison test pack in that it **executes the compiled code**. 
+This test pack is useful for testing expected behaviour.  (It's also reassuring to see your
+code being executed.)
+
+These tests are executed more widely that the comparison tests; we aim to run these against each 
+version of TypeScript defined in our CI build matrices. (Take a look at [`appveyor.yml`](appveyor.yml)
+and [`.travis.yml`](.travis.yml) for details.)
+
+#### Structure
+
+The comparison test pack can be found under `/test/execution-tests`.
+Like the comparison test pack, the execution test pack uses certain conventions.
+All tests have their own directory under `/test/execution-tests`, eg `/test/execution-tests/someFeature`. 
+Each test is expected to have a `karma.conf.js` file and a `webpack.config.js` file.
+
+If a test requires a minimum version of TypeScript then the test directory should be
+prefixed with the minimum TypeScript version.  For example, the `2.0.3_es2016` test requires
+a minimum TypeScript version of 2.0.3; if the installed version is lower than the test
+needs then the test will be skipped.
+
+** IMPORTANT **
+
+In order that the local version of ts-loader is resolved for tests a `webpack.config.js` file will need
+to include this line:
+
+```
+// for test harness purposes only, you would not need this in a normal project
+module.exports.resolveLoader = { alias: { 'ts-loader': path.join(__dirname, "../../../index.js") } }
+```
+
+And likewise the `karma.conf.js` will need to reuse this like so:
+
+```
+    webpack: {
+      devtool: 'inline-source-map',
+      debug: true,
+      module: {
+          loaders: webpackConfig.module.loaders
+      },
+      resolve: webpackConfig.resolve,
+
+      // for test harness purposes only, you would not need this in a normal project
+      resolveLoader: webpackConfig.resolveLoader
+    },
+```
+
+Without this the test won't be able to resolve ts-loader and won't find your TypeScript
+tests.
+
+#### What sort of tests can be included?
+
+It's pretty much your choice what goes in testwise.  
+At present there are only Jasmine tests in place;
+it should be possible to put any test in place that Karma is compatible with. The test pack also expects
+a `typings.json` file and calls `typings install` in each. **Be warned, type definitions are not installed
+until the test framework has been run.**  So if you're wanting to refactor a test you'll need to 
+`typings install` if the requisite typings have not yet been installed. It's possible / probably 
+that this may changed in the future; 
+particularly to cater for situations where types should be acquired via npm etc.
+
+#### Running / debugging the tests
+
+To run all the tests use:
+
+`npm run execution-tests`.
+
+If you would like to run just a single test then:
+
+`npm run execution-tests -- --single-test nameOfTest`
+
+It's pretty handy to be able to debug tests; for that reason you can run a single test in watch mode like this:
+
+`npm run execution-tests -- --single-test nameOfTest --watch`
+
+Then you can fire up http://localhost:9876/ and the world's your oyster.

appveyor.yml

@@ -1,9 +1,9 @@
 environment:
   nodejs_version: "6.0"
   matrix:
-#  - TYPESCRIPT: [email protected]
-#  - TYPESCRIPT: [email protected]
-#  - TYPESCRIPT: [email protected]
+  - TYPESCRIPT: [email protected]
+  - TYPESCRIPT: [email protected]
+  - TYPESCRIPT: [email protected]
   - TYPESCRIPT: [email protected]
 install:
   - ps: Install-Product node $env:nodejs_version

package.json

@@ -5,8 +5,9 @@
   "main": "index.js",
   "scripts": {
     "build": "tsc",
-    "comparison-tests": "npm link ./test/testLib && mocha --reporter spec test/run.js",
-    "test": "npm link ./test/testLib && node test/run-tests-as-child.js ",
+    "comparison-tests": "npm link ./test/comparison-tests/testLib && node test/comparison-tests/run-tests.js",
+    "execution-tests": "node test/execution-tests/run-tests.js",
+    "test": "node test/run-tests.js",
     "prepublish": "npm run build"
   },
   "repository": {
@@ -39,16 +40,27 @@
     "babel": "^6.0.0",
     "babel-core": "^6.0.0",
     "babel-loader": "^6.0.0",
+    "babel-polyfill": "^6.16.0",
     "babel-preset-es2015": "^6.0.0",
+    "babel-preset-es2016": "^6.16.0",
     "babel-preset-react": "^6.0.0",
     "escape-string-regexp": "^1.0.3",
     "fs-extra": "^0.30.0",
     "glob": "^7.1.1",
     "html-webpack-plugin": "^2.17.0",
+    "jasmine-core": "^2.5.2",
+    "karma": "^1.3.0",
+    "karma-jasmine": "^1.0.0",
+    "karma-mocha-reporter": "^2.0.0",
+    "karma-phantomjs-launcher": "^1.0.0",
+    "karma-sourcemap-loader": "^0.3.6",
+    "karma-webpack": "^1.7.0",
     "mkdirp": "^0.5.1",
     "mocha": "^3.1.0",
+    "phantomjs-prebuilt": "^2.1.2",
     "rimraf": "^2.4.2",
     "typescript": "^2.0.3",
+    "typings": "^1.4.0",
     "webpack": "^1.11.0"
   }
 }

test/run.js → test/comparison-tests/create-and-execute-test.js

@@ -16,8 +16,8 @@ require('colors').enabled = true;
 var saveOutputMode = process.argv.indexOf('--save-output') !== -1;
 var excludeVersions = process.argv.indexOf('--exclude-versions') !== -1;
 
-var indexOfSingleTest = process.argv.indexOf('--single-test');
-var singleTestToRun = indexOfSingleTest !== -1 && process.argv[indexOfSingleTest + 1];
+var indexOfTestToRun = process.argv.indexOf('--test-to-run');
+var testToRun = process.argv[indexOfTestToRun + 1];
 
 var savedOutputs = {};
 
@@ -33,28 +33,25 @@ if (saveOutputMode) {
 var typescriptVersion = semver.major(typescript.version) + '.' + semver.minor(typescript.version);
 var FLAKY = '_FLAKY_';
 
-// set up new empty staging area
-var rootPath = path.resolve(__dirname, '..');
+// set up new paths
+var rootPath = path.resolve(__dirname, '../../');
 var rootPathWithIncorrectWindowsSeparator = rootPath.replace(/\\/g, '/');
 var stagingPath = path.resolve(rootPath, '.test');
-rimraf.sync(stagingPath);
 
-// loop through each test directory
-var test = singleTestToRun;
-var testPath = path.join(__dirname, test);
+var testPath = path.join(__dirname, testToRun);
 if (fs.statSync(testPath).isDirectory()) {
 
-    if (test == 'testLib') return;
+    if (testToRun == 'testLib') return;
 
-    if (test == 'issue81' && semver.lt(typescript.version, '1.7.0-0')) return;
+    if (testToRun == 'issue81' && semver.lt(typescript.version, '1.7.0-0')) return;
 
-    describe(test, function () {
-        it('should have the correct output', createTest(test, testPath, {}));
+    describe(testToRun, function () {
+        it('should have the correct output', createTest(testToRun, testPath, {}));
 
-        if (test == 'declarationOutput') { return; }
-        if (test == 'declarationWatch') { return; }
-        if (test == 'issue71') { return; }
-        it('should work with transpile', createTest(test, testPath, { transpile: true }));
+        if (testToRun == 'declarationOutput') { return; }
+        if (testToRun == 'declarationWatch') { return; }
+        if (testToRun == 'issue71') { return; }
+        it('should work with transpile', createTest(testToRun, testPath, { transpile: true }));
     });
 }
 

test/declarationDeps/tsconfig.json → test/comparison-tests/declarationDeps/tsconfig.json

@@ -1,5 +1,5 @@
-{
-    "files": [
-        "./references.d.ts"
-    ]
-}
+{
+    "files": [
+        "./references.d.ts"
+    ]
+}