Renaming, plus code to check for renaming of files and to create stubs that redirect (fix #493)

Author: gfwilliams

Code Examples.md

@@ -0,0 +1,21 @@
+# Old File to New file mappings
+# build.sh scans these and makes stubs for the website that redirect
+EspruinoBoard,/Original
+PuckJS,/Puck.js
+Code Examples,/Tutorials
+Notes,/FAQ
+Puck.js Quick Start,/Quick+Start+BLE#puckjs
+Puck.js BLE UART,/BLE UART
+Puck.js Advertising,/BLE Advertising
+Puck.js and Bluetooth Lightbulbs,/BLE Lightbulbs
+Puck.js Eddystone,/Eddystone
+Puck.js iBeacon,/iBeacon
+Puck.js IFTTT,/BLE IFTTT
+Puck.js MIDI,/BLE MIDI
+Puck.js Node-RED,/BLE Node-RED
+Puck.js Printer,/BLE Printers
+Puck.js Security,/BLE Security
+Puck.js Web Bluetooth,/Web Bluetooth
+Puck.js Keyboard,/BLE Keyboard
+Puck.js HTTP Proxy,/BLE HTTP Proxy
+Puck.js Music Control,/BLE Music Control

RENAMED.csv

@@ -15,18 +15,18 @@ Tutorials
 
 * APPEND_KEYWORD: Tutorials
 
-Tasks
------
+Information
+-----------
 
-* APPEND_KEYWORD: tasks
+* APPEND_KEYWORD: info
 
 Peripherals
 ----------
 
 * APPEND_KEYWORD: peripherals
 
 Devices
-------
+--------
 
 * APPEND_KEYWORD: devices,-Module
 
@@ -36,4 +36,3 @@ Examples
 --------
 
 * APPEND_KEYWORD: examples,-Tutorials
-

Tutorials.md

@@ -44,6 +44,13 @@ var ESPRUINO_DIR = path.resolve(BASEDIR, "../Espruino");
 var FUNCTION_KEYWORD_FILE = path.resolve(BASEDIR, "../Espruino/function_keywords.js");
 var KEYWORD_JS_FILE = path.resolve(HTML_DIR, "keywords.js");
 
+var warnings = [];
+function WARNING(s) {
+  console.log("WARNING: "+s);
+  warnings.push(s);
+}
+
+
 var marked = require('marked');
 // Set default options except highlight which has no default
 var markedOptions = {
@@ -81,19 +88,33 @@ if (fs.existsSync(BASEDIR+"/ordering.txt")) {
     fileOrdering[line] = linenumber;
   });
 } else {
-  console.log("WARNING: No 'ordering.txt' - not ordering tutorials");
+  WARNING("No 'ordering.txt' - not ordering tutorials");
   console.log(' - Generate with: git ls-tree -r --name-only HEAD | xargs -I{} git log -1 --format="%at {}" -- {} | sort > ordering.txt');
 }
 
+// Get list of files that have been renamed
+var renamedFiles = {};
+if (fs.existsSync(BASEDIR+"/RENAMED.csv")) {
+  fs.readFileSync(BASEDIR+"/RENAMED.csv").toString().split("\n").map(function(line) {
+    if (line[0]=="#") return;
+    var csv = line.trim().split(",");
+    if (csv=="") return;
+    if (csv.length==2) {
+      renamedFiles[csv[0]] = csv[1];
+    } else WARNING("RENAMED.csv: Badly formatted line "+JSON.stringify(line));
+  });
+} else {
+  console.log("ERROR: No 'RENAMED.csv'");
+  process.exit(1);
+}
+
+
 if (OFFLINE) {
   // if offline, copy the offline.css file in
   fs.createReadStream(path.resolve(BASEDIR, "bin/offline.css")).pipe(
     fs.createWriteStream(path.resolve(HTML_DIR, "offline.css")));
 }
 
-function WARNING(s) {
-  console.log("WARNING: "+s);
-}
 
 function createSafeFilename(filename) {
   filename = filename.replace(/\//g,"_");
@@ -207,8 +228,8 @@ function grabInfo(markdownFiles, preloadedFiles) {
    var contents = preloadedFiles[file] ? preloadedFiles[file] : fs.readFileSync(BASEDIR+"/"+file).toString();
 //   console.log(file,contents.length);
    var contentLines = contents.split("\n");
-   if (contentLines[0].substr(0,15)!="<!--- Copyright") WARNING(file+" doesn't have a copyright line");
-   if (contentLines[1].trim()=="" || contentLines[2].substr(0,3)!="===") WARNING(file+" doesn't have a title on the first line");
+   if (!contentLines[0].match(/<!--+ Copyright/)) WARNING(file+": No copyright line");
+   if (contentLines[1].trim()=="" || contentLines[2].substr(0,3)!="===") WARNING(file+": No title on the first line");
    var fileInfo = {
      path : file,
      title : contentLines[1], // second line
@@ -324,7 +345,7 @@ function handleImages(file, contents) {
           // now rename the image in the tag
           contents = contents.substr(0,tagMid+2)+newPath+contents.substr(tagEnd);
         } else {
-          WARNING(file+": Image '"+imagePath+"' does not exist");
+          WARNING(file+": Image '"+imagePath+"' does not exist ("+tagStart+","+tagMid+","+tagEnd+")");
         }
       }
     }
@@ -358,27 +379,51 @@ exampleFiles.forEach(function(exampleFile) {
     markdownFiles.push(exampleFile);
     preloadedFiles[exampleFile] = newFile;
 
-  } else WARNING(exampleFile+" has no comment block at the start");
+  } else WARNING(exampleFile+": no comment block at the start");
 });
 
 
-
 //console.log(markdownFiles);
 var fileInfo = grabInfo(markdownFiles, preloadedFiles);
 //console.log(fileInfo.keywords);
 grabWebsiteKeywords(fileInfo.keywords);
 
-htmlFiles = {};
-htmlLinks = {};
+htmlFiles = {}; // filename -> file on disk with full path (for copy/etc)
+htmlLinks = {}; // filename -> actual URL link
+knownPageLinks = []; // pages that we know exist on this site
+// Go through all referenced files on the main Espruino site
+// TODO: in offline mode we should detect these links and go direct
+"Order Search Reference Download binaries".split(" ").forEach(function(file) {
+  var htmlLink = file + (OFFLINE?".html":"");
+  knownPageLinks.push((OFFLINE?"":"/")+htmlLink);
+});
+// Add JS files to our known files list
+["devices","modules","boards"].forEach(function(moduledir) {
+  fs.readdirSync(path.resolve(BASEDIR,moduledir)).filter(function(file) {
+    return file.substr(-3) == ".js";
+  }).forEach(function(file) {
+    knownPageLinks.push("/modules/"+file);
+  });
+});
+// Go through all Markdown files
 markdownFiles.forEach(function (file) {
   var htmlFile = file.substring(file.lastIndexOf("/")+1);
   htmlFile = htmlFile.replace(/ /g,"+");
   htmlFile = htmlFile.substring(0,htmlFile.lastIndexOf("."));
   htmlFiles[file] = HTML_DIR+htmlFile+".html";
-  if (OFFLINE)
-    htmlLinks[file] = htmlFile+".html";
-  else
-    htmlLinks[file] = htmlFile;
+  var htmlLink = htmlFile + (OFFLINE?".html":"");
+  htmlLinks[file] = htmlLink;
+  knownPageLinks.push((OFFLINE?"":"/")+htmlLink);
+});
+//console.log(JSON.stringify(knownPageLinks,null,2));process.exit(1);
+
+// For files that have been renamed, create placeholder files that redirect
+Object.keys(renamedFiles).forEach(function(originalFile) {
+  var newFile = renamedFiles[originalFile];
+  var htmlFile = HTML_DIR+originalFile.replace(/ /g,"+")+".html";
+  var html = `<p>This page has moved to <a href="${newFile}">${newFile}</a></p>
+<script>window.location = "${newFile}";</script>`;
+  fs.writeFileSync(htmlFile, html);
 });
 
 // ---------------------------------------------- Inference code
@@ -588,7 +633,7 @@ markdownFiles.forEach(function (file) {
          });
          content = '<div class="thumblinklist thumblinklist-thumbnails">\n  '+contentThumbs.join("\n  ")+'\n</div>';
        } else {
-         WARNING(kwName+" for '"+kw+"' in "+file+" found nothing");
+         WARNING(file+": "+kwName+" for '"+kw+"' found nothing");
          content = ifNone;
        }
        html = html.substr(0,match.index)+content+html.substr(match.index+match[0].length);
@@ -705,9 +750,32 @@ markdownFiles.forEach(function (file) {
    }
 
    // If compiling for offline, quick hack to modify most links to go direct
-   if (OFFLINE)
+   if (OFFLINE) // FIXME - move this into the code below, do it properly with knownPageLinks
      html = html.replace(/(<[aA] .*href=")\/([^"#]*)/g, '$1$2.html');
 
+   // Scan all links in HTML to find possible replaced files
+   {
+     regex = /href="([^"]+)"/g;
+     var m = regex.exec(html);
+     while (m) {
+       var url = m[1];
+       var baseURL = url.replace(/#.*$/,"");
+       if (baseURL!="") {
+         if (baseURL in renamedFiles) {
+           WARNING(`${file}: ${JSON.stringify(baseURL)} was renamed to ${JSON.stringify(renamedFiles[baseURL])}`);
+         } else {
+           if (baseURL.startsWith("http:") || baseURL.startsWith("https:")) {
+             // we're ok...
+           } else { // TODO: check if we know the page link
+             /*if (knownPageLinks.indexOf(baseURL)<0)
+               WARNING(`${file}: ${JSON.stringify(baseURL)} is not found`);*/
+          }
+         }
+       }
+       m = regex.exec(html);
+     }
+   }
+
    // work out of we have any images that might be at the top of the page
    var hasImageContent = html.indexOf("<iframe ")>=0; // do we have a video?
    // do we have an image in the first few lines of markdown?
@@ -750,6 +818,14 @@ fs.writeFileSync(refPath, JSON.stringify(urls,null,1));
 // Write out keywords
 fs.writeFileSync(KEYWORD_JS_FILE, "var keywords = "+JSON.stringify(createKeywordsJS(fileInfo.keywords),null,1)+";");
 
+console.log("====================================================== WARNINGS");
+console.log("====================================================== WARNINGS");
+console.log("====================================================== WARNINGS");
+warnings.forEach(w=>console.log(w));
+console.log("====================================================== WARNINGS");
+console.log("====================================================== WARNINGS");
+console.log("====================================================== WARNINGS");
+
 // -----------------------------------------------------------
 // Newest tutorials
 /*child_process.exec(

bin/build.js

@@ -56,7 +56,7 @@ You can solve this in a few ways:
 * Create a smaller pinhole. Simply get some black tape and put it over the camera, then make a very small hole with a pin. Note that electrical tape won't be suitable as its stretchiness causes the hole you made to close up. You can however use Magic Tape that has been blackened with a marker pen.
 * Use a hemispherical clear stick-on foot as a lens. It is by no means perfect, but is significantly better than nothing:
 
-![Small rubber foot][ADNS5050/optical_lens.jpg]
+![Small rubber foot](ADNS5050/optical_lens.jpg)
 
 **Note:** The sensor comes with a small piece of Kapton Tape over the front of it, which is meant to be removed during production. Without this tape, the silicon of the sensor inside the module appears to be completly unprotected.
 
@@ -74,7 +74,7 @@ sensor.drawImage(sensor.getImage());
 You'll get an ASCII art representation of the sensor's view, a bit like this:
 
 ```
- 
+
                ....
            ........
           .........
@@ -102,7 +102,7 @@ There are also more functions to get movement, or values from just the first lin
 
 * APPEND_JSDOC: ADNS5050.js
 
-Using 
+Using
 -----
 
 * APPEND_USES: ADNS5050

boards/EspruinoBoard.md

@@ -18,11 +18,11 @@ The kit contains the chassis, motors and wheels, but you still need something to
 
 We used:
 
-* An [Espruino Board](EspruinoBoard) 
+* An [Espruino Board](Original) 
 * A [[L293D]] Motor Driver IC soldered on to the board
 * A [[HC-SR04]] Ultrasonic Distance Sensor so that the Robot could sense the distance to obstacles.
 
-Using 
+Using
 -----
 
 * APPEND_USES: Robot

boards/PuckJS.md

@@ -1,5 +1,5 @@
 <!--- Copyright (c) 2016 Gordon Williams, Pur3 Ltd. See the file LICENSE for copying permission. -->
-Puck.js and HTTP Proxies
+Bluetooth LE HTTP Proxies
 =========================
 
 <span style="color:red">:warning: **Please view the correctly rendered version of this page at https://www.espruino.com/Puck.js+HTTP+Proxy. Links, lists, videos, search, and other features will not work correctly when viewed on GitHub** :warning:</span>

devices/ADNS5050.md

@@ -1,5 +1,5 @@
 <!--- Copyright (c) 2016 Gordon Williams, Pur3 Ltd. See the file LICENSE for copying permission. -->
-Puck.js and If This Then That
+Bluetooth LE and If This Then That
 ==============================
 
 <span style="color:red">:warning: **Please view the correctly rendered version of this page at https://www.espruino.com/Puck.js+IFTTT. Links, lists, videos, search, and other features will not work correctly when viewed on GitHub** :warning:</span>

devices/QuectelBG96.png

@@ -1,6 +1,6 @@
 <!--- Copyright (c) 2016 Gordon Williams, Pur3 Ltd. See the file LICENSE for copying permission. -->
-Puck.js and HID Keyboards
-=========================
+Bluetooth LE HID Keyboards
+==========================
 
 <span style="color:red">:warning: **Please view the correctly rendered version of this page at https://www.espruino.com/Puck.js+Keyboard. Links, lists, videos, search, and other features will not work correctly when viewed on GitHub** :warning:</span>
 
@@ -121,3 +121,9 @@ You can find key codes at https://www.usb.org/sites/default/files/documents/hut1
 * `0` is 39
 * Return is 40
 * Space is 44
+
+
+Uses
+----
+
+* APPEND_USES: ble_hid

devices/Robot.md

@@ -1,6 +1,6 @@
 <!--- Copyright (c) 2017 Gordon Williams, Pur3 Ltd. See the file LICENSE for copying permission. -->
-Puck.js MIDI
-============
+Bluetooth LE MIDI
+=================
 
 <span style="color:red">:warning: **Please view the correctly rendered version of this page at https://www.espruino.com/Puck.js+MIDI. Links, lists, videos, search, and other features will not work correctly when viewed on GitHub** :warning:</span>
 

info/Notes.md

@@ -1,11 +1,11 @@
 <!--- Copyright (c) 2017 Gordon Williams, Pur3 Ltd. See the file LICENSE for copying permission. -->
-Puck.js Music Controller
-========================
+Bluetooth Music Controller
+==========================
 
 <span style="color:red">:warning: **Please view the correctly rendered version of this page at https://www.espruino.com/Puck.js+Music+Control. Links, lists, videos, search, and other features will not work correctly when viewed on GitHub** :warning:</span>
 
 * KEYWORDS: Tutorials,Puck.js,BLE,Bluetooth,Car,Hands free,Handsfree,Music,Playback,Play,Pause,Next Track
-* USES: Puck.js,BLE,Only BLE
+* USES: Puck.js,BLE,Only BLE,ble_hid
 
 [[http://youtu.be/3iZ9j_ga6zs]]