Merge pull request #2 from Acollie/feat/relational-db

Acollie · web-flow · commit 9a72090e6e19 · 2024-04-01T19:25:45.000+01:00
Feat/relational db
diff --git a/awsx/const.go b/awsx/const.go
@@ -0,0 +1,5 @@
+package awsx
+
+const (
+	Region = "eu-west-1"
+)
diff --git a/config.yml b/config.yml
@@ -1,6 +1,9 @@
+relational: true
 websites:
   - "https://www.example.com"
   - "https://www.github.com"
+  - "https://docs.github.com"
+  - "https://partner.github.com"
   - "https://www.google.com"
   - "https://www.facebook.com"
   - "https://www.youtube.com"
diff --git a/config/fetch.go b/config/fetch.go
@@ -1,4 +1,4 @@
-package ignore_list
+package config
 
 import (
 	"gopkg.in/yaml.v3"
@@ -8,14 +8,16 @@ import (
 )
 
 type IgnoreList struct {
-	Website map[string]bool
+	Website    map[string]bool
+	Relational bool
 }
 type rawIgnoreList struct {
-	Websites []string `yaml:"websites"`
+	Websites   []string `yaml:"websites"`
+	Relational bool     `yaml:"relational"`
 }
 
 func Fetch() *IgnoreList {
-	yamlFile, err := ioutil.ReadFile("ignore_list.yml")
+	yamlFile, err := ioutil.ReadFile("config.yml")
 	if err != nil {
 		log.Printf("yamlFile.Get err #%v ", err)
 	}
@@ -26,7 +28,8 @@ func Fetch() *IgnoreList {
 	}
 
 	conf := &IgnoreList{
-		Website: make(map[string]bool),
+		Website:    make(map[string]bool),
+		Relational: raw.Relational,
 	}
 
 	for _, website := range raw.Websites {
diff --git a/config/fetch_test.go b/config/fetch_test.go
@@ -1,4 +1,4 @@
-package ignore_list
+package config
 
 import (
 	"github.com/stretchr/testify/require"
@@ -7,7 +7,7 @@ import (
 
 func TestIgnoreList(t *testing.T) {
 
-	t.Run("Fetch ignore list ", func(t *testing.T) {
+	t.Run("FetchDDB ignore list ", func(t *testing.T) {
 		conf := IgnoreList{
 			Website: map[string]bool{
 				"github.com": true,
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -21,4 +21,13 @@ services:
       - DATA_DIR=/tmp/localstack-data
     volumes:
       - "./localstack-data:/tmp/localstack-data"
-      - "/var/run/docker.sock:/var/run/docker.sock"
+      - "/var/run/docker.sock:/var/run/docker.sock"
+  postgres:
+    container_name: webcrawler-postgres
+    image: postgres
+    ports:
+      - "5431:5432"
+    environment:
+      POSTGRES_USER: root
+      POSTGRES_PASSWORD: root
+      POSTGRES_DB: webcrawler
diff --git a/dynamoDBx/add.go b/dynamoDBx/add.go
@@ -1,4 +1,4 @@
-package dynamo_db_x
+package dynamoDBx
 
 import (
 	"context"
diff --git a/dynamoDBx/fetch.go b/dynamoDBx/fetch.go
@@ -1,4 +1,4 @@
-package dynamo_db_x
+package dynamoDBx
 
 import (
 	"context"
diff --git a/dynamoDBx/remove.go b/dynamoDBx/remove.go
@@ -1,4 +1,4 @@
-package dynamo_db_x
+package dynamoDBx
 
 import (
 	"context"
diff --git a/dynamoDBx/type.go b/dynamoDBx/type.go
@@ -1,9 +1,10 @@
-package dynamo_db_x
+package dynamoDBx
 
 import (
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/service/dynamodb"
 	"os"
+	"webcrawler/awsx"
 )
 
 type DB struct {
@@ -13,15 +14,15 @@ type DB struct {
 }
 
 func New(pageName string, websiteName string, cfg aws.Config) *DB {
-	cfg.Region = "us-west-2"
+	cfg.Region = awsx.Region
 	sessionClient := dynamodb.NewFromConfig(cfg)
 
 	if os.Getenv("ENVIRONMENT") == "local" {
 		cfg.EndpointResolver = aws.EndpointResolverFunc(func(service, region string) (aws.Endpoint, error) {
 			return aws.Endpoint{
 				PartitionID:   "aws",
 				URL:           "http://localhost:8000",
-				SigningRegion: "us-west-2",
+				SigningRegion: awsx.Region,
 			}, nil
 		})
 		sessionClient = dynamodb.NewFromConfig(cfg)
diff --git a/dynamoDBx/update.go b/dynamoDBx/update.go
@@ -1,4 +1,4 @@
-package dynamo_db_x
+package dynamoDBx
 
 import (
 	"context"
diff --git a/go.mod b/go.mod
@@ -10,8 +10,10 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/dynamodb v1.27.1
 	github.com/aws/aws-sdk-go-v2/service/sqs v1.29.6
 	github.com/joho/godotenv v1.5.1
+	github.com/lib/pq v1.10.9
 	github.com/stretchr/testify v1.6.1
 	github.com/temoto/robotstxt v1.1.2
+	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c
 )
 
 require (
@@ -33,5 +35,4 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
 )
diff --git a/go.sum b/go.sum
@@ -46,6 +46,8 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
 github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
+github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
+github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
diff --git a/handler/flow.go b/handler/flow.go
@@ -28,6 +28,7 @@ func (h *Server) Scan(ctx context.Context) {
 				defer wg.Done()
 				if h.Config.Ignore(link.Url) {
 					log.Printf("skipping domain")
+					h.Queue.Remove(ctx, *link.Handler)
 					return
 				}
 
diff --git a/handler/flow_test.go b/handler/flow_test.go
@@ -6,7 +6,7 @@ import (
 	"os"
 	"testing"
 	"webcrawler/awsx"
-	"webcrawler/dynamo_db_x"
+	"webcrawler/dynamoDBx"
 
 	"webcrawler/queue"
 )
@@ -19,7 +19,7 @@ func setup(t *testing.T, ctx context.Context) Server {
 		os.Getenv("LINKS_QUEUE"),
 		cfg,
 	)
-	dbClient := dynamo_db_x.New(
+	dbClient := dynamoDBx.New(
 		os.Getenv("DB_TABLE_PAGE"),
 		os.Getenv("DB_TABLE_WEBSITE"),
 		cfg,
diff --git a/handler/type.go b/handler/type.go
@@ -1,15 +1,17 @@
 package handler
 
 import (
-	"webcrawler/ignore_list"
+	"webcrawler/config"
 	"webcrawler/queue"
 	"webcrawler/site"
+	"webcrawler/sqlRelational"
 )
 
 type Server struct {
 	Queue  queue.HandlerI
 	Db     DBi
-	Config *ignore_list.IgnoreList
+	Config *config.IgnoreList
+	DB     *sqlRelational.SqlDB
 }
 
 type DBi interface {
@@ -22,10 +24,11 @@ type DBi interface {
 	UpdateWebsite(page site.Page, website site.Website) error
 }
 
-func New(db DBi, queue *queue.Handler, conf *ignore_list.IgnoreList) Server {
+func New(db DBi, queue *queue.Handler, sqlDB *sqlRelational.SqlDB, conf *config.IgnoreList) Server {
 	return Server{
 		Db:     db,
 		Queue:  queue,
+		DB:     sqlDB,
 		Config: conf,
 	}
 
diff --git a/main.go b/main.go
@@ -2,20 +2,24 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"github.com/joho/godotenv"
 	"log"
 	"os"
 	"webcrawler/awsx"
-	"webcrawler/dynamo_db_x"
+	"webcrawler/config"
+	"webcrawler/dynamoDBx"
 	"webcrawler/handler"
-	"webcrawler/ignore_list"
 	"webcrawler/queue"
+	"webcrawler/sqlRelational"
 )
 
 func main() {
-	err := godotenv.Load()
+
 	ctx := context.Background()
-	conf := ignore_list.Fetch()
+	err := godotenv.Load()
+	//ctx := context.Background()
+	conf := config.Fetch()
 	if err != nil {
 		log.Fatalf("Failed to load .env file with error: %v", err)
 	}
@@ -32,15 +36,22 @@ func main() {
 		os.Getenv("LINKS_QUEUE"),
 		cfg,
 	)
-	dbClient := dynamo_db_x.New(
+	dbClient := dynamoDBx.New(
 		os.Getenv("DB_TABLE_PAGE"),
 		os.Getenv("DB_TABLE_WEBSITE"),
 		cfg,
 	)
-	initalLink := queue.NewMessage("https://alexcollie.com")
+	sqlClient := sqlRelational.New(
+		os.Getenv("SQL_TABLE"),
+	)
+	initialLink := queue.NewMessage("https://bbc.co.uk")
 
-	server := handler.New(dbClient, sqsClient, conf)
-	server.Queue.Add(ctx, initalLink)
+	server := handler.New(dbClient, sqsClient, sqlClient, conf)
+	err = server.Queue.Add(ctx, initialLink)
+	if err != nil {
+		err = fmt.Errorf("add %s", err)
+		panic(err)
+	}
 	server.Scan(ctx)
 
 }
diff --git a/main.tf b/main.tf
@@ -2,10 +2,8 @@ provider "aws" {
   region = "eu-west-1"
 }
 resource "aws_sqs_queue" "LinksQueue" {
-  name                        = "LinksQueue.fifo"
-  message_retention_seconds   = 3600
-  fifo_queue                  = true
-  content_based_deduplication = true
+  name                      = "LinksQueue"
+  message_retention_seconds = 3600
 }
 
 
diff --git a/site/types.go b/site/types.go
@@ -1,22 +1,14 @@
 package site
 
-import (
-	"github.com/anaskhan96/soup"
-)
-
-type pageI interface {
-	Fetch(url string) (*soup.Root, error)
-	Save(website Page) error
-}
 type Page struct {
-	Url     string `dynamodbav:"PageURL"`
-	Title   string `dynamodbav:"title"`
-	Body    string `dynamodbav:"body"`
-	BaseURL string `dynamodbav:"BaseURL"`
+	Url     string `dynamodbav:"PageURL" sql:"url"`
+	Title   string `dynamodbav:"title" sql:"title"`
+	Body    string `dynamodbav:"body" sql:"body"`
+	BaseURL string `dynamodbav:"BaseURL" sql:"baseUrl"`
 }
 
 type Website struct {
-	Url             string   `dynamodbav:"BaseURL"`
-	Links           []string `dynamodbav:"links"`
-	ProminenceValue float64  `dynamodbav:"promanceValue"`
+	Url             string   `dynamodbav:"BaseURL" sql:"baseurl"`
+	Links           []string `dynamodbav:"links" sql:"links"`
+	ProminenceValue float64  `dynamodbav:"promanceValue" sql:"promanceValue"`
 }
diff --git a/sqlRelational/add.go b/sqlRelational/add.go
@@ -0,0 +1,20 @@
+package sqlRelational
+
+import (
+	"fmt"
+	"webcrawler/site"
+)
+
+func (c *SqlDB) AddWebsite(website site.Website) error {
+	queryString := fmt.Sprintf("INSERT INTO website (baseurl, promancevalue) VALUES ('%s', %f)", website.Url, website.ProminenceValue)
+	_, err := c.Client.Exec(queryString)
+
+	return err
+}
+
+func (c *SqlDB) AddPage(page site.Page) error {
+	queryString := fmt.Sprintf("INSERT INTO page (pageurl, title, body,baseurl) VALUES ('%s', '%s', '%s', '%s')", page.Url, page.Title, page.Body, page.BaseURL)
+	_, err := c.Client.Exec(queryString)
+
+	return err
+}
diff --git a/sqlRelational/conn.go b/sqlRelational/conn.go
@@ -1 +1,36 @@
 package sqlRelational
+
+import (
+	"database/sql"
+	"fmt"
+	_ "github.com/lib/pq"
+	"os"
+)
+
+type SqlDB struct {
+	Client *sql.DB
+	DBName string
+}
+
+func connect(dbName string) *sql.DB {
+	host := os.Getenv("POSTGRES_HOST")
+	password := os.Getenv("POSTGRES_PASSWORD")
+	user := os.Getenv("POSTGRES_USER")
+
+	psqlInfo := fmt.Sprintf("host=%s port=%d user=%s "+
+		"password=%s dbname=%s sslmode=disable",
+		host, port, user, password, dbName)
+
+	client, err := sql.Open("postgres", psqlInfo)
+	if err != nil {
+		panic(err)
+	}
+	return client
+}
+
+func New(dbName string) *SqlDB {
+	return &SqlDB{
+		Client: connect(dbName),
+		DBName: dbName,
+	}
+}
diff --git a/sqlRelational/consts.go b/sqlRelational/consts.go
diff --git a/sqlRelational/db_test.go b/sqlRelational/db_test.go
diff --git a/sqlRelational/fetch.go b/sqlRelational/fetch.go
diff --git a/sqlRelational/update.go b/sqlRelational/update.go

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-package ignore_list`
	`1`	`+package config`
`2`	`2`
`3`	`3`	`import (`
`4`	`4`	`"gopkg.in/yaml.v3"`
`@@ -8,14 +8,16 @@ import (`
`8`	`8`	`)`
`9`	`9`
`10`	`10`	`type IgnoreList struct {`
`11`		`- Website map[string]bool`
	`11`	`+ Website map[string]bool`
	`12`	`+ Relational bool`
`12`	`13`	`}`
`13`	`14`	`type rawIgnoreList struct {`
`14`		- Websites []string `yaml:"websites"`
	`15`	+ Websites []string `yaml:"websites"`
	`16`	+ Relational bool `yaml:"relational"`
`15`	`17`	`}`
`16`	`18`
`17`	`19`	`func Fetch() *IgnoreList {`
`18`		`- yamlFile, err := ioutil.ReadFile("ignore_list.yml")`
	`20`	`+ yamlFile, err := ioutil.ReadFile("config.yml")`
`19`	`21`	`if err != nil {`
`20`	`22`	`log.Printf("yamlFile.Get err #%v ", err)`
`21`	`23`	`}`
`@@ -26,7 +28,8 @@ func Fetch() *IgnoreList {`
`26`	`28`	`}`
`27`	`29`
`28`	`30`	`conf := &IgnoreList{`
`29`		`- Website: make(map[string]bool),`
	`31`	`+ Website: make(map[string]bool),`
	`32`	`+ Relational: raw.Relational,`
`30`	`33`	`}`
`31`	`34`
`32`	`35`	`for _, website := range raw.Websites {`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-package dynamo_db_x`
	`1`	`+package dynamoDBx`
`2`	`2`
`3`	`3`	`import (`
`4`	`4`	`"context"`
Original file line number	Diff line number	Diff line change
`@@ -28,6 +28,7 @@ func (h *Server) Scan(ctx context.Context) {`
`28`	`28`	`defer wg.Done()`
`29`	`29`	`if h.Config.Ignore(link.Url) {`
`30`	`30`	`log.Printf("skipping domain")`
	`31`	`+ h.Queue.Remove(ctx, *link.Handler)`
`31`	`32`	`return`
`32`	`33`	`}`
`33`	`34`
Original file line number	Diff line number	Diff line change
`@@ -2,10 +2,8 @@ provider "aws" {`
`2`	`2`	`region = "eu-west-1"`
`3`	`3`	`}`
`4`	`4`	`resource "aws_sqs_queue" "LinksQueue" {`
`5`		`- name = "LinksQueue.fifo"`
`6`		`- message_retention_seconds = 3600`
`7`		`- fifo_queue = true`
`8`		`- content_based_deduplication = true`
	`5`	`+ name = "LinksQueue"`
	`6`	`+ message_retention_seconds = 3600`
`9`	`7`	`}`
`10`	`8`
`11`	`9`