more refactoring

Author: alexott

README.md

@@ -105,7 +105,7 @@ If you encounter problems in running some of the examples you can decide to eith
 
 All the scripts work perfectly with the community and the enterprise edition of Neo4j. 
 Moreover, it is possible to use the [Neo4j desktop](https://neo4j.com/download/) to manage the Neo4j instances. 
-The book has been written during the transition from 3.5.x to 4.x so the code has been all updated to run on the version 4.x.
+The book has been written during the transition from 3.5.x to 4.x so the code has been all updated to run on the version 4.x (all code was tested on 4.2.3).
 It introduced some changes, like the variable binding and the multi relationships syntax in Cypher that make the code incompatible with the previous version 3.5.x. 
 
 You can find all the instruction for downloading and installing Neo4j in the way you prefer here:

ch04/imports/movielens/Makefile

@@ -4,9 +4,12 @@ PYTHON=python
 init:
 	$(PIP) install -r requirements.txt
 
-source:
+download:
 	mkdir -p ../../../dataset/movielens
 	curl -f http://files.grouplens.org/datasets/movielens/ml-latest-small.zip -o ../../../dataset/movielens/ml-latest-small.zip
 	unzip ../../../dataset/movielens/ml-latest-small.zip -d ../../../dataset/movielens/
+
+
+
 test:
 	nosetests tests

ch04/imports/movielens/README.md

@@ -8,12 +8,12 @@ To install what is necessary run:
 make
 ```
 
-## Download the data source
+## Download the source data
 The Makefile contains also the command to download the necessary data sources.
 Run:
 
 ```sh
-make source
+make download
 ```
 
 You can also download it manually from [project's site](http://files.grouplens.org/datasets/movielens/ml-latest-small.zip)
@@ -28,7 +28,7 @@ The default location is in the home of this code repository in the directory dat
 python import_movielens.py -u <neo4j username> -p <password>  -b <bolt uri> -s <source directory>
 ```
 
-If you used the makefile for downloading the directory you don't need to specify the datasource. 
+If you used the makefile for downloading the directory you don't need to specify the datasource.  If you specified username, password & bolt URI in the config file, then you don't need to specify these parameters.
 The simple version takes a while to be completed. I recommend to run the parallel version as follows:
 
 ```sh
@@ -42,4 +42,4 @@ it can happen that it will start rejecting the requests. It is perfectly normal.
 
 After the chapter has been released the full version of the IMDB has been released [here](https://www.imdb.com/interfaces/).
 
-In the future I'll make some changes in order to load from files instead.
+In the future I'll make some changes in order to load from files instead.

ch05/recommendation/Makefile renamed to ch05/recommendation/collaborative_filtering/Makefile

@@ -16,8 +16,8 @@ pip install -r requirements.txt
 
 ## Import the data
 
-Before starting to work with the code, you need to import the data as described [here](../imports/retail_rocket/README.md).
+Before starting to work with the code, you need to import the data as described [here](../../imports/retail_rocket/README.md).
 
 ## Working with the code
 
-The code for chapter 5 is located in the [`collaborative_filtering`](collaborative_filtering/) directory.  Just change into it
+Code is in the file `recommender.py`, just execute it with `python recommender.py`.

ch05/recommendation/README.md renamed to ch05/recommendation/collaborative_filtering/README.md

@@ -122,7 +122,7 @@ class UserRecommender(BaseRecommender):
         WITH otherUser, count(otherUser) as size
         MATCH (otherUser)-[r:PURCHASES]->(target:Target)
         WHERE target.itemId = $itemId
-        return (1.0f/size)*count(r) as score
+        return (+1.0/size)*count(r) as score
     """
 
 

ch05/recommendation/collaborative_filtering/__init__.py

@@ -5,4 +5,5 @@ test:
 	nosetests tests
 
 download:
-	curl -L -o ratings.txt https://raw.githubusercontent.com/JDonini/depaulmovie-recommender-system/master/dataset/Movie_DePaulMovie/ratings.txt
+	mkdir -p ../../../dataset/deeppaulmovie && \
+	curl -L -o ../../../dataset/deeppaulmovie/ratings.txt https://raw.githubusercontent.com/JDonini/depaulmovie-recommender-system/master/dataset/Movie_DePaulMovie/ratings.txt

ch05/recommendation/collaborative_filtering/recommender.py

@@ -207,7 +207,7 @@ def write_movie_on_db(self):
     importing = DePaulMovieImporter(sys.argv[1:])
     base_path = importing.source_dataset_path
     if not base_path:
-        base_path = "/Users/ale/neo4j-servers/gpml/dataset/Movie_DePaulMovie"
+        base_path = "../../../dataset/deeppaulmovie"
     importing.import_event_data(file=os.path.join(base_path, "ratings.txt"))
     importing.import_movie_details()
     end = time.time() - start

ch05/recommendation/requirements.txt renamed to ch05/recommendation/collaborative_filtering/requirements.txt

@@ -2,24 +2,25 @@
 import time
 import threading
 from queue import Queue
-from neo4j import GraphDatabase
 import math
 import sys
 
-class IEEEImporter(object):
+from util.graphdb_base import GraphDBBase
 
-    def __init__(self, uri, user, password):
-        self._driver = GraphDatabase.driver(uri, auth=(user, password), encrypted=0)
+class IEEEImporter(GraphDBBase):
+
+    def __init__(self, argv):
+        super().__init__(command=__file__, argv=argv)
         self._transactions = Queue()
         self._dictionaries = {}
         self._print_lock = threading.Lock()
         with self._driver.session() as session:
-            self.executeNoException(session, "CREATE CONSTRAINT ON (s:Transaction) ASSERT s.transactionId IS UNIQUE")
-            self.executeNoException(session, "CREATE INDEX ON :Transaction(isFraud)")
-            self.executeNoException(session, "CREATE INDEX ON :Transaction(isTrain)")
+            self.execute_without_exception("CREATE CONSTRAINT ON (s:Transaction) ASSERT s.transactionId IS UNIQUE")
+            self.execute_without_exception("CREATE INDEX ON :Transaction(isFraud)")
+            self.execute_without_exception("CREATE INDEX ON :Transaction(isTrain)")
 
     def close(self):
-        self._driver.close()
+        self.close()
 
     def import_transaction(self, directory):
         j = 0
@@ -113,26 +114,16 @@ def write_transaction(self):
                     print(e, row)
             self._transactions.task_done()
 
-    def executeNoException(self, session, query):
-        try:
-            session.run(query)
-        except Exception as e:
-            pass
-
-
-def strip(string): return ''.join([c if 0 < ord(c) < 128 else ' ' for c in string])
-
 
 if __name__ == '__main__':
-    uri = "bolt://localhost:7687"
-    importer = IEEEImporter(uri=uri, user="neo4j", password="q1")
+    importer = IEEEImporter(sys.argv[1:])
 
     start = time.time()
-    base_path = "/Users/ale/neo4j-servers/gpml/dataset/ieee/"
-    if len(sys.argv) > 1:
-        base_path = sys.argv[1]
+    base_path = importer.source_dataset_path
+    if not base_path:
+        base_path = "../../../dataset/ieee"
     importer.import_transaction(directory=base_path)
-    print("Time to complete paysim ingestion:", time.time() - start)
+    print("Time to complete IEEE ingestion:", time.time() - start)
 
     # intermediate = time.time()
     # importer.post_processing(sess_clicks=sessions)

ch07/imports/depaulmovie/Makefile

@@ -1,16 +1,18 @@
 import pandas as pd
 import numpy as np
 import time
-from neo4j import GraphDatabase
 import sys
+import os
 
-class PaySimImporter(object):
+from util.graphdb_base import GraphDBBase
 
-    def __init__(self, uri, user, password):
-        self._driver = GraphDatabase.driver(uri, auth=(user, password))
+class PaySimImporter(GraphDBBase):
+
+    def __init__(self, argv):
+        super().__init__(command=__file__, argv=argv)
 
     def close(self):
-        self._driver.close()
+        self.close()
 
     def import_paysim(self, file):
         dtype = {
@@ -144,19 +146,16 @@ def post_processing(self, sess_clicks):
         return sess_clicks
 
 
-def strip(string): return ''.join([c if 0 < ord(c) < 128 else ' ' for c in string])
-
-
 if __name__ == '__main__':
-    uri = "bolt://localhost:7687"
-    importer = PaySimImporter(uri=uri, user="neo4j", password="q1")
+    importer = PaySimImporter(sys.argv[1:])
 
     start = time.time()
-    file_path = "/Users/ale/neo4j-servers/gpml/dataset/paysim/PS_20174392719_1491204439457_log.csv"
-    if len(sys.argv) > 1:
-        file_path = sys.argv[1]
-    importer.import_paysim(file=file_path)
-    print("Time to complete paysim ingestion:", time.time() - start)
+    base_path = importer.source_dataset_path
+    if not base_path:
+        base_path = "../../../dataset/paysim"
+
+    importer.import_paysim(file=os.path.join(base_path, "PS_20174392719_1491204439457_log.csv"))
+    print("Time to complete PaySim ingestion:", time.time() - start)
 
     # intermediate = time.time()
     # importer.post_processing(sess_clicks=sessions)