Limit works (#354)

Author: gauntface

Diff for: cmds/htmlassets/htmlassets.go

@@ -54,7 +54,9 @@ import (
 	"github.com/gauntface/go-html-asset-manager/v4/utils/html/htmlencoding"
 	"github.com/gauntface/go-html-asset-manager/v4/utils/vimeoapi"
 	"github.com/mitchellh/go-homedir"
+	"github.com/schollz/progressbar/v3"
 	"golang.org/x/net/html"
+	"golang.org/x/sync/semaphore"
 )
 
 var (
@@ -216,15 +218,25 @@ func (c *client) manipulations(manager assetmanagerManager, manipulators []manip
 }
 
 func (c *client) manipulateHTMLFiles(assets []assetmanagerLocalAsset, manager assetmanagerManager, manipulators []manipulations.Manipulator) []error {
-	var wg sync.WaitGroup
-	wg.Add(len(assets))
+	ctx := context.Background()
 
 	errs := []error{}
 	var errMu sync.Mutex
 
+	maxWorkers := int64(24)
+	sem := semaphore.NewWeighted(maxWorkers)
+	bar := progressbar.Default(int64(len(assets)), "HTML files processed")
+
 	for _, htmlAsset := range assets {
+		if err := sem.Acquire(ctx, 1); err != nil {
+			errMu.Lock()
+			defer errMu.Unlock()
+			errs = append(errs, fmt.Errorf("%w %q: %v", errManipulate, htmlAsset.Path(), err))
+		}
+
 		go func(htmlAsset assetmanagerLocalAsset) {
-			defer wg.Done()
+			defer sem.Release(1)
+			defer bar.Add(1)
 
 			err := c.manipulateHTMLFile(htmlAsset, manager, manipulators)
 			if err != nil {
@@ -235,7 +247,7 @@ func (c *client) manipulateHTMLFiles(assets []assetmanagerLocalAsset, manager as
 		}(htmlAsset)
 	}
 
-	wg.Wait()
+	sem.Acquire(ctx, maxWorkers)
 
 	return errs
 }

Diff for: go.mod

@@ -13,10 +13,9 @@ require (
 	github.com/otiai10/copy v1.9.0
 	github.com/schollz/progressbar/v3 v3.12.2
 	golang.org/x/net v0.4.0
+	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
 )
 
-require github.com/tdewolff/parse/v2 v2.6.1 // indirect
-
 require (
 	github.com/aws/aws-sdk-go-v2 v1.17.2 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
@@ -34,14 +33,11 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.9 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.17.6 // indirect
 	github.com/aws/smithy-go v1.13.5 // indirect
-	github.com/gauntface/go-html-asset-manager/v3 v3.1.4 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/mattn/go-runewidth v0.0.14 // indirect
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
 	github.com/rivo/uniseg v0.4.3 // indirect
 	github.com/stretchr/testify v1.7.0 // indirect
-	github.com/tdewolff/minify/v2 v2.12.0 // indirect
-	golang.org/x/crypto v0.0.0-20220824171710-5757bc0c5503 // indirect
 	golang.org/x/image v0.2.0 // indirect
 	golang.org/x/sys v0.3.0 // indirect
 	golang.org/x/term v0.3.0 // indirect