New api (#15)

* Working test framework

* Switch logging to bunyan, add test config, http test framework

* Switch to using rethinkdbdash

* Complete switch to rethinkdbdash

* Add all deps

* Refactored code layout

* Basic test framework

* Add redis to circle.yml deps

* Add rethinkdb

* Fix test

Author: rosskukulinski

.gitignore

@@ -1,3 +1,4 @@
 .DS_Store
 node_modules
 npm-debug.log
+dump.rdb

circle.yml

@@ -17,9 +17,16 @@ machine:
 
   services:
     - docker
+    - redis
 
 dependencies:
   pre:
+    - source /etc/lsb-release && echo "deb http://download.rethinkdb.com/apt $DISTRIB_CODENAME main" | sudo tee /etc/apt/sources.list.d/rethinkdb.list
+    - wget -qO- http://download.rethinkdb.com/apt/pubkey.gpg | sudo apt-key add -
+    - sudo apt-get update
+    - sudo apt-get install rethinkdb
+    - sudo cp /etc/rethinkdb/default.conf.sample /etc/rethinkdb/instances.d/instance1.conf
+    - sudo /etc/init.d/rethinkdb restart
     - if [[ -z $CIRCLE_TAG ]] ; then npm install; exit $?; fi
     - if [[ -z $CIRCLE_TAG ]] ; then docker login -e [email protected] -u $REGISTRY_USERNAME -p $REGISTRY_PASSWORD quay.io; exit $?; fi
     - if [[ -z $CIRCLE_TAG ]] ; then docker-build -f deploy/common.config; exit $?; fi

index.js

@@ -1,42 +1,29 @@
-var dns = require('dns');
-dnscache = require('dnscache')({
-  enable: true,
-  ttl: 300,
-  cachesize: 1000
-});
-var db = require('./lib/db');
-
-process.title = 'api';
-
-var routes = require('./lib/routes');
-var port = process.env.PORT || 3001;
+var app = require('./lib');
+var bunyan = require('bunyan');
+var log = bunyan(require('./lib/config').bunyan);
 
-var express = require('express');
-var logger = require('morgan');
-var bodyParser = require('body-parser');
-var path = require('path');
-
-var app = express();
-app.use(logger('dev'));
-app.use(bodyParser.json());
-app.use(bodyParser.urlencoded({ extended: false }));
-
-app.use(function (error, request, response, next) {
-    response.status(error.status || 500);
-    response.json({ error: error.message });
-});
-
-app.use(routes);
-
-db.init(function (err) {
+app.start(function onStart (err) {
   if (err) {
-    console.error('Problem setting up rethinkdb, exiting');
-    process.exit(1);
+    log.error(err);
+    process.exit();
   }
-  console.log('Connected to rethinkdb');
-  var server = app.listen(port, function () {
-      var host = server.address().address;
-      var port = server.address().port;
-      console.log('App is listening on http://%s:%s', host, port);
+  process.on('SIGTERM', function onSIGTERM () {
+    log.info('received SIGTERM');
+    app.stop(function () {
+      log.info('Application stopped, exiting');
+      setTimeout(function () {
+        process.exit(0);
+      }, 50);
+    });
   });
-})
+
+  process.on('SIGINT', function onSIGTERM () {
+    log.info('received SIGINT');
+    app.stop(function () {
+      log.info('Application stopped, exiting');
+      setTimeout(function () {
+        process.exit(0);
+      }, 50);
+    });
+  });
+});

lib/config/config.global.js

@@ -0,0 +1,48 @@
+var config = module.exports = {};
+require('pkginfo')(module);
+var os = require('os');
+var bunyan = require('bunyan');
+
+config.modules = [
+  'db',
+  'queue',
+  'snapshots'
+];
+
+config.redis = {
+  host: process.env.REDIS_SERVICE_HOST || '127.0.0.1',
+  port: process.env.REDIS_SERVICE_PORT || 6379
+};
+
+config.bull = {
+  requestQueue: 'screenshot-request',
+  responseQueue: 'screenshot-response'
+};
+
+config.rethinkdb = {
+  host: process.env.RETHINKDB_PROXY_SERVICE_HOST || process.env.RETHINKDB_SERVICE_HOST || 'localhost',
+  port: process.env.RETHINKDB_PROXY_SERVICE_PORT || process.env.RETHINKDB_SERVICE_POST || 28015,
+  db: 'wayback',
+  table: 'snapshots',
+  silent: true
+};
+
+config.info = {
+  version: module.exports.version,
+  name: module.exports.name,
+  instance: os.hostname()
+};
+
+config.port = process.env.PORT || 3001;
+
+config.bunyan = {
+  name: config.info.name,
+  streams: [
+    {
+      level: process.env.LOG_LEVEL || 'info',
+      stream: process.stdout
+    }
+  ],
+  serializers: bunyan.stdSerializers,
+  src: true
+};

lib/config/config.test.js

@@ -0,0 +1,11 @@
+var config = require('./config.global');
+
+config.bunyan.streams = [
+  {
+    level: process.env.LOG_LEVEL || 'error',
+    stream: process.stderr
+  }
+];
+config.bunyan.src = true;
+
+module.exports = config;

lib/config/index.js

@@ -0,0 +1,9 @@
+var env = process.env.NODE_ENV;
+var configFile = '';
+if (env === 'TESTING') {
+  configFile = './config.test';
+} else {
+  configFile = './config.global';
+}
+var config = require(configFile);
+module.exports = config;