Add download script

Author: lutangar

.gitignore

@@ -1,3 +1,4 @@
 node_modules
 npm-debug.log
 cities1000.txt
+cities1000.zip

.prettierignore

@@ -0,0 +1 @@
+cities.json

.prettierrc

@@ -0,0 +1,3 @@
+{
+  "singleQuote": true
+}

README.md

@@ -1,38 +1,48 @@
 # Cities of the World
+
 [![Creative Commons License](https://i.creativecommons.org/l/by/3.0/80x15.png)](https://creativecommons.org/licenses/by/3.0/)
 
-These cities comes from GeoNames Gazetteer:
-http://www.geonames.org
+These cities comes from [GeoNames Gazetteer](http://www.geonames.org).
+
+> See https://www.geonames.org/datasources/ for the list of data sources.
 
 Here is the description of the original dataset:
-> all cities with a population > 1000 or seats of adm div (ca 150.000) [...]
+
+> *all cities with a population > 1000 or seats of adm div (ca 150.000) [...]*
 
 ## Install
+
 ```
 npm install --save cities.json
 ```
 
 ## Usage
+
 Either on **node** or the **browser** (with `webpack`) it get as simple as this:
 
 **ES5**
+
 ```
 const cities = require('cities.json');
 ```
 
 **ES6**
+
 ```
 import cities from 'cities.json';
 ```
 
 > Since webpack >= v2.0.0, importing of JSON files will work by default.
 
 ## Description
+
 This Json version is an array of object of the following shape:
+
 - ISO 3166-1 alpha-2 country code
 - name
 - Latitude
 - Longitude
+
 ```
 [
   {

convert.js

@@ -1,48 +1,53 @@
 var fs = require('fs');
-var jsonfile  = require('jsonfile');
-var readline  = require('readline');
+var jsonfile = require('jsonfile');
+var readline = require('readline');
 
 var file = './cities.json';
-var cities = [], i = 0, city;
+var cities = [],
+  i = 0,
+  city;
 
-readline.createInterface({
-  input: fs.createReadStream('./cities1000.txt'),
-  output: process.stdout,
-  terminal: false
-}).on('line', function(line) {
-  city = line.split("\t");
-  if (i !== 0) {
-    // geonameid         : integer id of record in geonames database
-    // name              : name of geographical point (utf8) varchar(200)
-    // asciiname         : name of geographical point in plain ascii characters, varchar(200)
-    // alternatenames    : alternatenames, comma separated, ascii names automatically transliterated, convenience attribute from alternatename table, varchar(10000)
-    // latitude          : latitude in decimal degrees (wgs84)
-    // longitude         : longitude in decimal degrees (wgs84)
-    // feature class     : see http://www.geonames.org/export/codes.html, char(1)
-    // feature code      : see http://www.geonames.org/export/codes.html, varchar(10)
-    // country code      : ISO-3166 2-letter country code, 2 characters
-    // cc2               : alternate country codes, comma separated, ISO-3166 2-letter country code, 200 characters
-    // admin1 code       : fipscode (subject to change to iso code), see exceptions below, see file admin1Codes.txt for display names of this code; varchar(20)
-    // admin2 code       : code for the second administrative division, a county in the US, see file admin2Codes.txt; varchar(80)
-    // admin3 code       : code for third level administrative division, varchar(20)
-    // admin4 code       : code for fourth level administrative division, varchar(20)
-    // population        : bigint (8 byte int)
-    // elevation         : in meters, integer
-    // dem               : digital elevation model, srtm3 or gtopo30, average elevation of 3''x3'' (ca 90mx90m) or 30''x30'' (ca 900mx900m) area in meters, integer. srtm processed by cgiar/ciat.
-    // timezone          : the iana timezone id (see file timeZone.txt) varchar(40)
-    // modification date : date of last modification in yyyy-MM-dd format
-    cities.push({
-      country: city[8],
-      name: city[1].replace('"', '').replace('"', ''),
-      lat: city[4],
-      lng: city[5]
-    });
-  }
-  i++;
-}).on('close', function() {
-  jsonfile.writeFile(file, cities, {spaces: 2}, function (err) {
-    if (err) {
-      console.error(err)
+readline
+  .createInterface({
+    input: fs.createReadStream('./cities1000.txt'),
+    output: process.stdout,
+    terminal: false,
+  })
+  .on('line', function (line) {
+    city = line.split('\t');
+    if (i !== 0) {
+      // geonameid         : integer id of record in geonames database
+      // name              : name of geographical point (utf8) varchar(200)
+      // asciiname         : name of geographical point in plain ascii characters, varchar(200)
+      // alternatenames    : alternatenames, comma separated, ascii names automatically transliterated, convenience attribute from alternatename table, varchar(10000)
+      // latitude          : latitude in decimal degrees (wgs84)
+      // longitude         : longitude in decimal degrees (wgs84)
+      // feature class     : see http://www.geonames.org/export/codes.html, char(1)
+      // feature code      : see http://www.geonames.org/export/codes.html, varchar(10)
+      // country code      : ISO-3166 2-letter country code, 2 characters
+      // cc2               : alternate country codes, comma separated, ISO-3166 2-letter country code, 200 characters
+      // admin1 code       : fipscode (subject to change to iso code), see exceptions below, see file admin1Codes.txt for display names of this code; varchar(20)
+      // admin2 code       : code for the second administrative division, a county in the US, see file admin2Codes.txt; varchar(80)
+      // admin3 code       : code for third level administrative division, varchar(20)
+      // admin4 code       : code for fourth level administrative division, varchar(20)
+      // population        : bigint (8 byte int)
+      // elevation         : in meters, integer
+      // dem               : digital elevation model, srtm3 or gtopo30, average elevation of 3''x3'' (ca 90mx90m) or 30''x30'' (ca 900mx900m) area in meters, integer. srtm processed by cgiar/ciat.
+      // timezone          : the iana timezone id (see file timeZone.txt) varchar(40)
+      // modification date : date of last modification in yyyy-MM-dd format
+      cities.push({
+        country: city[8],
+        name: city[1].replace('"', '').replace('"', ''),
+        lat: city[4],
+        lng: city[5],
+      });
     }
+    i++;
   })
-});
+  .on('close', function () {
+    jsonfile.writeFile(file, cities, { spaces: 2 }, function (err) {
+      if (err) {
+        console.error(err);
+      }
+    });
+  });

download.js

@@ -0,0 +1,37 @@
+const http = require('https'); // or 'https' for https:// URLs
+const fs = require('fs');
+const yauzl = require('yauzl');
+
+const txtFilename = 'cities1000.txt';
+const zipFilename = 'cities1000.zip';
+const zipFile = fs.createWriteStream(zipFilename);
+const request = http.get(
+  `https://download.geonames.org/export/dump/${zipFilename}`,
+  (response) => {
+    response.pipe(zipFile);
+
+    zipFile.on('finish', () => {
+      zipFile.close();
+      console.log('Download Completed');
+
+      yauzl.open(zipFilename, { lazyEntries: true }, (err, zipfile) => {
+        if (err) throw err;
+        zipfile.readEntry();
+        zipfile.on('entry', (entry) => {
+          if (entry.fileName === txtFilename) {
+            const txtFile = fs.createWriteStream(entry.fileName);
+            zipfile.openReadStream(entry, (err, readStream) => {
+              if (err) {
+                throw err;
+              }
+              readStream.on('end', function () {
+                zipfile.readEntry();
+              });
+              readStream.pipe(txtFile);
+            });
+          }
+        });
+      });
+    });
+  }
+);

package.json

@@ -5,7 +5,10 @@
   "main": "cities.json",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1",
-    "convert": "node convert"
+    "convert": "node convert",
+    "download": "node download",
+    "build": "npm run download && npm run convert",
+    "prettier": "prettier -w -u ."
   },
   "repository": {
     "type": "git",
@@ -23,6 +26,8 @@
   "homepage": "https://github.com/lutangar/cities.json#readme",
   "devDependencies": {
     "jsonfile": "^2.4.0",
-    "readline": "^1.3.0"
+    "prettier": "^2.7.1",
+    "readline": "^1.3.0",
+    "yauzl": "^2.10.0"
   }
 }