Skip to content
This repository was archived by the owner on May 30, 2019. It is now read-only.

Commit aa4d649

Browse files
ryry
committed
tools/dev_website_render.js
1 parent f2d374e commit aa4d649

11 files changed

+228
-79
lines changed

Diff for: ‎tools/build.js

+1-3
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,5 @@ if (process.argv.indexOf("clean") >= 0) {
77

88
(async() => {
99
run.sh("node ./tools/build_binding.js");
10-
run.sh("node ./tools/build_website.js");
11-
await run.parcel("tools/test_dl.ts", "build/website", true);
12-
await run.parcel("tools/test_website.ts", "build/website", true);
10+
run.sh("node ./tools/dev_website_render.js");
1311
})();

Diff for: ‎tools/dev_website.js

+54-32
Original file line numberDiff line numberDiff line change
@@ -1,53 +1,75 @@
11
#!/usr/bin/env node
2-
// This is an alternative to ./tools/build_website which does fast
3-
// incremental builds. The main difference is that the production website
4-
// pre-generates static html for the documentation.
5-
// Eventually we intend to merge the two scripts.
2+
/*
3+
Pre-generates static html.
4+
5+
./tools/dev_website.js prod
6+
7+
prod flag: minimized output
8+
9+
./tools/dev_website.js gendoc
10+
11+
gendoc flag: rebuild docs.json
12+
13+
./tools/dev_website.js build prod
14+
15+
build production website and exit.
16+
*/
617
const run = require("./run");
718
const fs = require("fs");
819
const Bundler = require("parcel-bundler");
920
require("ts-node").register({"typeCheck": true });
10-
const gendoc = require("./gendoc.ts");
1121

12-
(async() => {
22+
const prodFlag = (process.argv.indexOf("prod") >= 0);
23+
exports.prodFlag = prodFlag;
24+
25+
let wdir = "build/dev_website/";
26+
exports.wdir = wdir;
27+
28+
async function bundler(build) {
1329
run.mkdir("build");
14-
run.mkdir("build/website"); // for docs.json
15-
run.mkdir("build/dev_website");
16-
run.mkdir("build/dev_website/docs");
17-
run.mkdir("build/dev_website/notebook");
18-
run.mkdir("build/dev_website/src"); // Needed for npy_test
19-
20-
run.symlink(run.root + "/website/", "build/dev_website/static");
21-
run.symlink(run.root + "/website/img", "build/dev_website/img");
22-
run.symlink(run.root + "/deps/data/", "build/dev_website/data");
30+
run.mkdir(wdir);
31+
run.mkdir(wdir + "src"); // Needed for npy_test
32+
run.symlink(run.root + "/website/", wdir + "static");
33+
run.symlink(run.root + "/website/img", wdir + "img");
34+
run.symlink(run.root + "/deps/data/", wdir + "data");
2335
// Needed for npy_test
24-
run.symlink(run.root + "/src/testdata/", "build/dev_website/src/testdata");
36+
run.symlink(run.root + "/src/testdata/", wdir + "src/testdata");
2537

2638
const opts = {
2739
cache: true,
2840
hmr: false,
2941
logLevel: process.env.CI ? 1 : null,
30-
minify: false,
31-
outDir: "build/dev_website/",
32-
production: false,
42+
minify: prodFlag,
43+
outDir: wdir,
44+
production: prodFlag,
3345
publicUrl: "/",
34-
watch: true
46+
watch: !build,
3547
}
3648

37-
const sandboxBunder = new Bundler("website/sandbox.ts", opts);
38-
await sandboxBunder.bundle();
49+
let b = new Bundler("website/sandbox.ts", opts);
50+
await b.bundle();
51+
52+
b = new Bundler("tools/test_dl.ts", opts);
53+
await b.bundle();
3954

40-
const docs = gendoc.genJSON();
41-
const docsJson = JSON.stringify(docs, null, 2);
42-
fs.writeFileSync("build/dev_website/docs.json", docsJson);
43-
// There's a readFileSync for the docs.json that looks in build/website/
44-
// so we write it there too.
45-
fs.writeFileSync("build/website/docs.json", docsJson);
55+
b = new Bundler("tools/test_website.ts", opts);
56+
await b.bundle();
57+
58+
run.gendoc(wdir + "docs.json");
4659

4760
const indexBunder = new Bundler("website/index.html", opts);
48-
const port = 8080
49-
indexBunder.serve(port);
61+
return indexBunder;
62+
}
63+
64+
const port = 8080
65+
async function devWebsiteServer(build) {
66+
const b = await bundler(build)
67+
return await b.serve(port);
68+
}
69+
exports.devWebsiteServer = devWebsiteServer;
5070

51-
console.log(`Propel dev http://localhost:${port}/`);
52-
})();
5371

72+
if (require.main === module) {
73+
devWebsiteServer(false);
74+
console.log(`Propel http://localhost:${port}/`);
75+
}

Diff for: ‎tools/dev_website_render.js

+92
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
#!/usr/bin/env node
2+
3+
//require("ts-node").register({"typeCheck": true });
4+
5+
let ncp = require("ncp");
6+
let run = require("./run");
7+
const { pages } = require("../website/website");
8+
let fs = require("fs");
9+
let path = require("path");
10+
let { format } = require("util");
11+
let puppeteer = require("puppeteer");
12+
const { devWebsiteServer } = require("./dev_website");
13+
14+
const headless = (process.env.CI != null);
15+
16+
async function puppetRender(browser, url) {
17+
return new Promise(async (resolve, reject) => {
18+
const page = await browser.newPage();
19+
//page.on("load", () => {
20+
// console.log("load");
21+
//});
22+
23+
async function onMessage(msg) {
24+
const values = msg.args.map(
25+
v =>
26+
v._remoteObject.value !== undefined
27+
? v._remoteObject.value
28+
: `[[${v._remoteObject.type}]]`
29+
);
30+
const text = format.apply(null, values);
31+
32+
console.log(text);
33+
34+
if (text.match(/Propel onload/)) {
35+
//console.log("load!!!");
36+
let html = await page.evaluate(() => document.documentElement.innerHTML);
37+
resolve(html);
38+
}
39+
}
40+
41+
page.on("console", onMessage);
42+
//page.on("response", onResponse);
43+
//page.on("pageerror", onError);
44+
page.on("pageerror", (err) => {
45+
console.log("pagerror");
46+
reject(err);
47+
});
48+
page.goto(url, { timeout: 0 });
49+
})
50+
}
51+
52+
function cp(src, dst) {
53+
return new Promise((resolve, reject) => {
54+
ncp(src, dst, (err) => {
55+
if (err) {
56+
reject(err);
57+
} else {
58+
resolve();
59+
}
60+
});
61+
});
62+
}
63+
64+
host = "http://localhost:8080/"
65+
66+
async function writePages(browser, dir) {
67+
for (const page of pages) {
68+
const html = await puppetRender(browser,
69+
host + page.path.replace("index.html", ""));
70+
const fn = path.join(dir, page.path);
71+
const parentDir = path.dirname(fn);
72+
run.mkdir(parentDir);
73+
fs.writeFileSync(fn, html);
74+
console.log("write", fn);
75+
}
76+
}
77+
78+
(async() => {
79+
let dir = "build/website_render";
80+
run.rmrf(dir);
81+
let server = await devWebsiteServer();
82+
const browser = await puppeteer.launch({
83+
args: ["--no-sandbox", "--disable-setuid-sandbox"],
84+
headless,
85+
});
86+
await cp("build/dev_website", dir);
87+
88+
await writePages(browser, dir);
89+
await browser.close();
90+
91+
process.exit(0);
92+
})();

Diff for: ‎tools/run.js

+13
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@ const rimraf = require("rimraf");
66
// Be extra careful to enable ts-node type checking.
77
process.env.TS_NODE_TYPE_CHECK = true;
88

9+
require("ts-node").register({"typeCheck": true });
10+
const gendoc = require("./gendoc.ts");
11+
912
// Always chdir to propel's project root.
1013
const root = resolve(__dirname, "..");
1114
process.chdir(root);
@@ -100,6 +103,16 @@ function version() {
100103
return pkg.version;
101104
}
102105

106+
exports.gendoc = (fn) => {
107+
let gendocFlag = (process.argv.indexOf("gendoc") >= 0);
108+
if (gendocFlag || !fs.existsSync(fn)) {
109+
const docs = gendoc.genJSON();
110+
const docsJson = JSON.stringify(docs, null, 2);
111+
fs.writeFileSync(fn, docsJson);
112+
}
113+
}
114+
115+
103116
exports.mkdir = mkdir;
104117
exports.parcel = parcel;
105118
exports.parcelCli = parcelCli;

Diff for: ‎tools/test.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ process.on("unhandledRejection", e => { throw e; });
1313
run.tsnode("./tools/gendoc_test.ts");
1414

1515
// Web browser tests
16-
run.tsnode("tools/test_browser.ts");
16+
run.tsnode("tools/test_browser.ts use-render");
1717

1818
run.tsnode("tools/jasmine_shim_test.ts");
1919
})();

Diff for: ‎tools/test_browser.ts

+18-28
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,13 @@ import { exitOnFail } from "./tester";
2121
// side-effect of making unhandled rejections crash node.
2222
import { IS_NODE } from "../src/util";
2323

24+
let useRenderFlag = false;
25+
const i = process.argv.indexOf("use-render");
26+
if (i >= 0) {
27+
useRenderFlag = true;
28+
process.argv.splice(i, 1); // delete flag;
29+
}
30+
2431
// Allow people to filter the tests from the command-line.
2532
// Example: ts-node ./tools/test_browser.ts concat
2633
let filterExpr: string = null;
@@ -58,37 +65,14 @@ const propelTests: Test = {
5865
timeout: 2 * 60 * 1000,
5966
};
6067

61-
let TESTS: Test[] = [
68+
const TESTS: Test[] = [
6269
// This page loads and runs all the webpack'ed unit tests.
6370
// The test harness logs "DONE bla bla" to the console when done.
6471
// If this message doesn't appear, or an unhandled error is thrown on the
6572
// page, the test fails.
6673
propelTests,
6774
];
6875

69-
if (!filterExpr) {
70-
TESTS = TESTS.concat([{
71-
path: "index.html",
72-
doneMsg: /Propel onload/,
73-
timeout: 10 * 1000,
74-
},
75-
{
76-
path: "notebook/",
77-
doneMsg: /Propel onload/,
78-
timeout: 10 * 1000,
79-
},
80-
{
81-
path: "notebook/?nbId=default",
82-
doneMsg: /Propel onload/,
83-
timeout: 10 * 1000,
84-
},
85-
{
86-
path: "docs/index.html",
87-
doneMsg: /Propel onload/,
88-
timeout: 10 * 1000,
89-
}]);
90-
}
91-
9276
if (testdl) {
9377
TESTS.unshift({
9478
path: "static/test.html#script=/test_dl.js",
@@ -100,9 +84,14 @@ if (testdl) {
10084
(async() => {
10185
let passed = 0, failed = 0;
10286

103-
const server = createServer({ cors: true, root: "./build/website" });
104-
server.listen();
105-
const port = server.server.address().port;
87+
let server, port;
88+
if (useRenderFlag) {
89+
server = createServer({ cors: true, root: "./build/website_render" });
90+
server.listen();
91+
port = server.server.address().port;
92+
} else {
93+
port = 8080;
94+
}
10695

10796
const browser = await puppeteer.launch({
10897
args: ["--no-sandbox", "--disable-setuid-sandbox"],
@@ -124,11 +113,12 @@ if (testdl) {
124113
}
125114

126115
await browser.close();
127-
server.close();
128116

129117
console.log(`DONE. passed: ${passed}, failed: ${failed}`);
130118
if (failed > 0) {
131119
process.exit(1);
120+
} else {
121+
process.exit(0);
132122
}
133123
})();
134124

Diff for: ‎tools/upload_website.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
#!/usr/bin/env node
22
const run = require("./run");
33
const { execSync } = require("child_process");
4-
run.sh("node ./tools/build_website.js");
4+
run.sh("./tools/dev_website_render.js prod gendoc");
55
// pip install awscli
6-
execSync("aws s3 sync build/website/ s3://propelml.org --follow-symlinks --delete", {
6+
execSync("aws s3 sync build/website_render/ s3://propelml.org --follow-symlinks --delete", {
77
stdio: "inherit"
88
});

Diff for: ‎website/docs.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,7 @@ function startsWithUpperCase(s: string): boolean {
123123
}
124124

125125
export function Docs(props) {
126-
let docs: DocEntry[] = require("../build/website/docs.json");
126+
let docs: DocEntry[] = require("../build/dev_website/docs.json");
127127
docs = docs.sort((a, b) => {
128128
if (!startsWithUpperCase(a.name) && startsWithUpperCase(b.name)) {
129129
return -1;

Diff for: ‎website/main.ts

+12-1
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,27 @@
11
import { h, render, rerender } from "preact";
22
import { assert, IS_WEB } from "../src/util";
33
import { enableFirebase } from "./db";
4-
import { drainExecuteQueue } from "./nb";
4+
import { drainExecuteQueue, registerPrerenderedOutput } from "./nb";
55
import { Router } from "./website";
66

77
assert(IS_WEB);
88

99
enableFirebase();
1010

11+
function cells() {
12+
const outputs = document.querySelectorAll(".output");
13+
for (let i = 0; i < outputs.length; i++) {
14+
registerPrerenderedOutput(outputs[i]);
15+
}
16+
}
17+
1118
window.addEventListener("load", async() => {
19+
cells();
20+
1221
render(h(Router, null), document.body, document.body.children[0]);
22+
1323
await drainExecuteQueue();
24+
1425
// If we're in a testing environment...
1526
if (window.navigator.webdriver) {
1627
// rerender to make sure the dom is up to date and then output a special

0 commit comments

Comments
 (0)