feat:

- Adding local testing options

Author: Acollie

.github/workflows.yml

@@ -0,0 +1,22 @@
+name: Go Test
+
+on: [ push, pull_request ]
+
+jobs:
+  test:
+    name: Run Tests
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v2
+        with:
+          go-version: ^1.21  # SemVer range, or 1.x to match the latest stable Go version
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Test
+        run: |
+          cd mydirectory
+          go test ./...

dynamoDBx/update.go

@@ -15,13 +15,18 @@ func (db *DB) UpdateWebsite(page site.Page, website site.Website) error {
 	if err != nil {
 		return err
 	}
-	if reflect.DeepEqual(websiteDB, site.Website{}) {
-		return db.AddWebsite(website)
+	if reflect.DeepEqual(websiteDB, &site.Website{}) && err == nil {
+		println("Website not found")
+		websiteDB = &website
+	} else {
+		website.Links = append(websiteDB.Links, page.Url)
 	}
 
-	websiteDB.Links = append(websiteDB.Links, page.Url)
 	websiteDB.ProminenceValue += 1
-	av, err := attributevalue.MarshalMap(websiteDB)
+	websiteDB.Links = append(websiteDB.Links, page.Url)
+	website = *websiteDB
+
+	av, err := attributevalue.MarshalMap(website)
 	if err != nil {
 		return err
 	}

formating/text.go

@@ -47,9 +47,9 @@ func resolveURL(baseURL, relURL string) string {
 func removeDuplicates(links []string) []string {
 	encountered := map[string]bool{}
 	var result []string
+
 	for v := range links {
-		if encountered[links[v]] == true {
-		} else {
+		if !encountered[links[v]] {
 			encountered[links[v]] = true
 			result = append(result, links[v])
 		}

handler/flow.go

@@ -3,70 +3,74 @@ package handler
 import (
 	"context"
 	"log"
+	"sync"
 	"webcrawler/formating"
 	"webcrawler/site"
 )
 
 func (h *Server) Scan(ctx context.Context) {
 
-	//Wait group
-	for i := 0; i < 5; i++ {
+	for i := 0; i < 20; i++ {
 
 		links, err := h.Queue.Fetch(ctx)
 
 		log.Printf("Length of links %d", len(links))
+
 		if err != nil {
 			log.Printf("fetching %v", err)
 		}
 
+		wg := sync.WaitGroup{}
 		for _, link := range links {
-			//item, err := h.Db.FetchWebsite(link.Url)
-			//if err == nil && reflect.DeepEqual(item, site.Website{}) {
-			//	h.Queue.Remove(ctx, *link.Handler)
-			//	log.Printf("Skipping")
-			//	continue
-			//
-			//}
-			log.Printf("Scanning %s", link.Url)
-			valid, err := site.FetchRobots(link.Url)
-			if err != nil {
-				log.Printf("fetching robots %v", err)
-			}
-			if !valid {
-				log.Printf("Robots disallowed")
-				h.Queue.Remove(ctx, *link.Handler)
-				continue
-			}
-
-			page, resp, err := site.NewPage(link.Url)
-			if err != nil {
-				log.Printf("creating page %v", err)
-			}
-			err = h.Db.AddPage(page)
-
-			if err != nil {
-				log.Printf("adding page %v", err)
-			}
-			err = h.Queue.Remove(ctx, *link.Handler)
-			if err != nil {
-				log.Printf("failed to remove item from queue for url %s with error %s", link.Url, err)
-			}
-
-			linksNew, err := formating.GetLinks(link.Url, resp)
-			if err != nil {
-				log.Printf("getting links %v", err)
-			}
-			website := site.NewWebsite(link.Url, linksNew)
-
-			err = h.Queue.BatchAdd(ctx, linksNew)
-			if err != nil {
-				log.Printf("adding links to queue %v", err)
-			}
-
-			err = h.Db.UpdateWebsite(page, website)
-			if err != nil {
-				log.Printf("updating website %v", err)
-			}
+
+			wg.Add(1)
+			go func() {
+
+				defer wg.Done()
+				valid, err := site.FetchRobots(link.Url)
+				if err != nil {
+					log.Printf("fetching robots %v", err)
+				}
+				if !valid {
+					log.Printf("Robots disallowed")
+					h.Queue.Remove(ctx, *link.Handler)
+					return
+				}
+
+				page, resp, err := site.NewPage(link.Url)
+				if err != nil {
+					return
+				}
+				if err := h.Db.AddPage(page); err != nil {
+					log.Printf("adding page %v", err)
+					return
+				}
+
+				linksNew, err := formating.GetLinks(link.Url, resp)
+				if err != nil {
+					log.Printf("getting links %v", err)
+					return
+				}
+				website := site.NewWebsite(link.Url, linksNew)
+
+				err = h.Queue.BatchAdd(ctx, linksNew)
+				if err != nil {
+					log.Printf("adding links to queue %v", err)
+					return
+				}
+
+				err = h.Db.UpdateWebsite(page, website)
+				if err != nil {
+					log.Printf("updating website %v", err)
+				}
+
+				if err := h.Queue.Remove(ctx, *link.Handler); err != nil {
+					log.Printf("removing link from queue %v", err)
+				}
+
+			}()
+
+			wg.Wait()
 		}
 
 	}