Merge pull request #1 from Acollie/feat/adding-db-tests

Feat adding tests

Author: Acollie

.github/workflows/terraform.yml

@@ -0,0 +1,26 @@
+name: 'Terraform'
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  terraform:
+    name: 'Terraform'
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install Terraform
+        uses: hashicorp/setup-terraform@v1
+        with:
+          terraform_version: 1.0.5
+
+      - name: Terraform Init
+        run: terraform init
+
+      - name: Terraform Validate
+        run: terraform validate

dynamoDBx/add.go dynamo_db_x/add.go

@@ -1,4 +1,4 @@
-package dynamoDBx
+package dynamo_db_x
 
 import (
 	"context"

dynamoDBx/fetch.go dynamo_db_x/fetch.go

@@ -1,4 +1,4 @@
-package dynamoDBx
+package dynamo_db_x
 
 import (
 	"context"

dynamoDBx/remove.go dynamo_db_x/remove.go

@@ -1,4 +1,4 @@
-package dynamoDBx
+package dynamo_db_x
 
 import (
 	"context"

dynamoDBx/type.go dynamo_db_x/type.go

@@ -1,4 +1,4 @@
-package dynamoDBx
+package dynamo_db_x
 
 import (
 	"github.com/aws/aws-sdk-go-v2/aws"

dynamoDBx/update.go dynamo_db_x/update.go

@@ -1,4 +1,4 @@
-package dynamoDBx
+package dynamo_db_x
 
 import (
 	"context"

handler/flow.go

@@ -25,8 +25,12 @@ func (h *Server) Scan(ctx context.Context) {
 
 			wg.Add(1)
 			go func() {
-
 				defer wg.Done()
+				if h.Config.Ignore(link.Url) {
+					log.Printf("skipping domain")
+					return
+				}
+
 				valid, err := site.FetchRobots(link.Url)
 				if err != nil {
 					log.Printf("fetching robots %v", err)

handler/flow_test.go

@@ -6,7 +6,7 @@ import (
 	"os"
 	"testing"
 	"webcrawler/awsx"
-	"webcrawler/dynamoDBx"
+	"webcrawler/dynamo_db_x"
 
 	"webcrawler/queue"
 )
@@ -19,7 +19,7 @@ func setup(t *testing.T, ctx context.Context) Server {
 		os.Getenv("LINKS_QUEUE"),
 		cfg,
 	)
-	dbClient := dynamoDBx.New(
+	dbClient := dynamo_db_x.New(
 		os.Getenv("DB_TABLE_PAGE"),
 		os.Getenv("DB_TABLE_WEBSITE"),
 		cfg,

handler/type.go

@@ -1,13 +1,15 @@
 package handler
 
 import (
+	"webcrawler/ignore_list"
 	"webcrawler/queue"
 	"webcrawler/site"
 )
 
 type Server struct {
-	Queue *queue.Handler
-	Db    DBi
+	Queue  queue.HandlerI
+	Db     DBi
+	Config *ignore_list.IgnoreList
 }
 
 type DBi interface {
@@ -20,10 +22,11 @@ type DBi interface {
 	UpdateWebsite(page site.Page, website site.Website) error
 }
 
-func New(db DBi, queue *queue.Handler) Server {
+func New(db DBi, queue *queue.Handler, conf *ignore_list.IgnoreList) Server {
 	return Server{
-		Db:    db,
-		Queue: queue,
+		Db:     db,
+		Queue:  queue,
+		Config: conf,
 	}
 
 }

ignore_list.yml

@@ -0,0 +1,14 @@
+websites:
+  - "https://www.example.com"
+  - "https://www.github.com"
+  - "https://www.google.com"
+  - "https://www.facebook.com"
+  - "https://www.youtube.com"
+  - "https://www.instagram.com"
+  - "https://www.twitter.com"
+  - "https://www.linkedin.com"
+  - "https://www.pinterest.com"
+  - "https://www.tumblr.com"
+  - "https://www.reddit.com"
+  - "https://www.wordpress.com"
+  - "https://www.medium.com"

ignore_list/fetch.go

@@ -0,0 +1,54 @@
+package ignore_list
+
+import (
+	"gopkg.in/yaml.v3"
+	"io/ioutil"
+	"log"
+	"net/url"
+)
+
+type IgnoreList struct {
+	Website map[string]bool
+}
+type rawIgnoreList struct {
+	Websites []string `yaml:"websites"`
+}
+
+func Fetch() *IgnoreList {
+	yamlFile, err := ioutil.ReadFile("ignore_list.yml")
+	if err != nil {
+		log.Printf("yamlFile.Get err #%v ", err)
+	}
+	raw := &rawIgnoreList{}
+	err = yaml.Unmarshal(yamlFile, raw)
+	if err != nil {
+		log.Fatalf("Unmarshal: %v", err)
+	}
+
+	conf := &IgnoreList{
+		Website: make(map[string]bool),
+	}
+
+	for _, website := range raw.Websites {
+		// Extract host from the url
+		parsedUrl, err := url.Parse(website)
+		if err != nil {
+			log.Printf("Failed to parse URL from yaml file: %v, err: %v", website, err)
+			continue
+		}
+		conf.Website[parsedUrl.Host] = true
+	}
+	return conf
+}
+
+func (i *IgnoreList) Ignore(urlStr string) bool {
+
+	parsedURL, err := url.Parse(urlStr)
+	if err != nil {
+		log.Printf("Url parse error %v ", err)
+		return false
+	}
+
+	_, isIgnored := i.Website[parsedURL.Host]
+	return isIgnored
+}

ignore_list/fetch_test.go

@@ -0,0 +1,20 @@
+package ignore_list
+
+import (
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+func TestIgnoreList(t *testing.T) {
+
+	t.Run("Fetch ignore list ", func(t *testing.T) {
+		conf := IgnoreList{
+			Website: map[string]bool{
+				"github.com": true,
+			},
+		}
+		url := "https://github.com"
+		require.Equal(t, conf.Ignore(url), true)
+		require.Equal(t, conf.Ignore("https://alexcollie.com"), false)
+	})
+}

main.go

@@ -6,15 +6,16 @@ import (
 	"log"
 	"os"
 	"webcrawler/awsx"
-	"webcrawler/dynamoDBx"
+	"webcrawler/dynamo_db_x"
 	"webcrawler/handler"
+	"webcrawler/ignore_list"
 	"webcrawler/queue"
 )
 
 func main() {
-	// Load .env file
 	err := godotenv.Load()
 	ctx := context.Background()
+	conf := ignore_list.Fetch()
 	if err != nil {
 		log.Fatalf("Failed to load .env file with error: %v", err)
 	}
@@ -31,14 +32,14 @@ func main() {
 		os.Getenv("LINKS_QUEUE"),
 		cfg,
 	)
-	dbClient := dynamoDBx.New(
+	dbClient := dynamo_db_x.New(
 		os.Getenv("DB_TABLE_PAGE"),
 		os.Getenv("DB_TABLE_WEBSITE"),
 		cfg,
 	)
 	initalLink := queue.NewMessage("https://alexcollie.com")
 
-	server := handler.New(dbClient, sqsClient)
+	server := handler.New(dbClient, sqsClient, conf)
 	server.Queue.Add(ctx, initalLink)
 	server.Scan(ctx)
 

main.tf

@@ -2,8 +2,10 @@ provider "aws" {
   region = "eu-west-1"
 }
 resource "aws_sqs_queue" "LinksQueue" {
-  name                      = "LinksQueue"
-  message_retention_seconds = 3600
+  name                        = "LinksQueue.fifo"
+  message_retention_seconds   = 3600
+  fifo_queue                  = true
+  content_based_deduplication = true
 }
 
 

sqlRelational/conn.go

@@ -0,0 +1 @@
+package sqlRelational

sql_relational/consts.go sqlRelational/consts.go

@@ -1,4 +1,4 @@
-package sql_relational
+package sqlRelational
 
 const (
 	addWebsite      = `insert () values into websites`