fix(providers): Sanitize nuts properties from configuration (#221)

* fix(providers): Sanitize nuts properties from configuration

* Support cache_keys JSON configuration

* keep stale on purge if strategy is not hard

* Fix stale surrogate invalidation

* Update E2E tests

* Group configuration object

* Surrogate_keys enhancement

* Fix surrogate with commas

* Support the etcd provider

* Fix nutsdb regex based cache purge. (#222)

* Handle delete many by regexp on etcd provider

* Increase wait for Chi & Webgo run to 30s

* Remove Yoda conditions

* Remove useless print

Co-authored-by: Kiss Károly Pál <[email protected]>

Author: darkweak

.github/workflows/plugins-master.yml

@@ -24,12 +24,7 @@ jobs:
         uses: actions/checkout@v2
       -
         name: Install xcaddy
-        run: |
-          sudo apt install -y debian-keyring debian-archive-keyring apt-transport-https
-          curl -1sLf 'https://dl.cloudsmith.io/public/caddy/xcaddy/gpg.key' | sudo apt-key add -
-          curl -1sLf 'https://dl.cloudsmith.io/public/caddy/xcaddy/debian.deb.txt' | sudo tee /etc/apt/sources.list.d/caddy-xcaddy.list
-          sudo apt update
-          sudo apt install xcaddy
+        run: go install github.com/caddyserver/xcaddy/cmd/xcaddy@latest
       -
         name: Build current Souin as caddy module with referenced Souin core version when merge on master
         run: cd plugins/caddy && xcaddy build --with github.com/${{ github.repository }}/plugins/caddy@$(git rev-parse --short "$GITHUB_SHA")

.github/workflows/plugins.yml

@@ -144,7 +144,7 @@ jobs:
         name: Wait for Souin is really loaded inside Chi as middleware
         uses: jakejarvis/wait-action@master
         with:
-          time: 20s
+          time: 30s
       -
         name: Set Chi logs configuration result as environment variable
         run: cd plugins/chi/examples && echo "CHI_MIDDLEWARE_RESULT=$(docker-compose logs chi | grep Souin)" >> $GITHUB_ENV
@@ -531,7 +531,7 @@ jobs:
         name: Wait for Souin is really loaded inside Webgo as middleware
         uses: jakejarvis/wait-action@master
         with:
-          time: 20s
+          time: 30s
       -
         name: Set Webgo logs configuration result as environment variable
         run: cd plugins/webgo/examples && echo "WEBGO_MIDDLEWARE_RESULT=$(docker-compose logs webgo | grep Souin)" >> $GITHUB_ENV

README.md

@@ -90,14 +90,20 @@ default_cache:
     - GET
     - POST
     - HEAD
-  distributed: true # Use Olric distributed storage
+  distributed: true # Use Olric or Etcd distributed storage
   headers: # Default headers concatenated in stored keys
     - Authorization
   key:
     disable_body: true
     disable_host: true
     disable_method: true
-  olric: # If distributed is set to true, you'll have to define the olric section
+  etcd: # If distributed is set to true, you'll have to define either the etcd or olric section
+    configuration: # Configure directly the Etcd client
+      endpoints: # Define multiple endpoints
+        - etcd-1:2379 # First node
+        - etcd-2:2379 # Second node
+        - etcd-3:2379 # Third node
+  olric: # If distributed is set to true, you'll have to define either the etcd or olric section
     url: 'olric:3320' # Olric server
   regex:
     exclude: 'ARegexHere' # Regex to exclude from cache
@@ -171,6 +177,8 @@ surrogate_keys:
 | `default_cache.key.disable_body`                  | Disable the body part in the key (GraphQL context)                                                                                          | `true`<br/><br/>`(default: false)`                                                                                        |
 | `default_cache.key.disable_host`                  | Disable the host part in the key                                                                                                            | `true`<br/><br/>`(default: false)`                                                                                        |
 | `default_cache.key.disable_method`                | Disable the method part in the key                                                                                                          | `true`<br/><br/>`(default: false)`                                                                                        |
+| `default_cache.etcd`                              | Configure the Etcd cache storage                                                                                                            |                                                                                                                           |
+| `default_cache.etcd.configuration`                | Configure Etcd directly in the Caddyfile or your JSON caddy configuration                                                                   | [See the Etcd configuration for the options](https://pkg.go.dev/go.etcd.io/etcd/clientv3#Config)                          |
 | `default_cache.olric`                             | Configure the Olric cache storage                                                                                                           |                                                                                                                           |
 | `default_cache.olric.path`                        | Configure Olric with a file                                                                                                                 | `/anywhere/olric_configuration.json`                                                                                      |
 | `default_cache.olric.configuration`               | Configure Olric directly in the Caddyfile or your JSON caddy configuration                                                                  | [See the Olric configuration for the options](https://github.com/buraksezer/olric/blob/master/cmd/olricd/olricd.yaml/)    |
@@ -243,9 +251,10 @@ See the sequence diagram for the minimal version below
 Supported providers
  - [Badger](https://github.com/dgraph-io/badger)
  - [NutsDB](https://github.com/nutsdb/nutsdb)
+ - [Etcd](https://github.com/etcd-io/etcd)
  - [Olric](https://github.com/buraksezer/olric)
 
-The cache system sits on top of three providers at the moment. It provides two in-memory storage solutions (badger and nuts), and a distributed storage called Olric because setting, getting, updating and deleting keys in these providers is as easy as it gets.  
+The cache system sits on top of three providers at the moment. It provides two in-memory storage solutions (badger and nuts), and two distributed storages Olric and Etcd because setting, getting, updating and deleting keys in these providers is as easy as it gets.  
 **The Badger provider (default one)**: you can tune its configuration using the badger configuration inside your Souin configuration. In order to do that, you have to declare the `badger` block. See the following json example.
 ```json
 "badger": {
@@ -284,8 +293,19 @@ The cache system sits on top of three providers at the moment. It provides two i
 }
 ```
 In order to do that, the Olric provider need to be either on the same network as the Souin instance when using docker-compose or over the internet, then it will use by default in-memory to avoid network latency as much as possible. 
+
+**The Etcd provider**: you can tune its configuration using the etcd configuration inside your Souin configuration and declare Souin has to use the distributed provider. In order to do that, you have to declare the `etcd` block and the `distributed` directive. See the following json example.
+```json
+"distributed": true,
+"etcd": {
+  "configuration": {
+    # Etcd configuration here...
+  }
+}
+```
+In order to do that, the Etcd provider need to be either on the same network as the Souin instance when using docker-compose or over the internet, then it will use by default in-memory to avoid network latency as much as possible. 
 Souin will return at first the response from the choosen provider when it gives a non-empty response, or fallback to the reverse proxy otherwise.
-Since v1.4.2, Souin supports [Olric](https://github.com/buraksezer/olric) to handle distributed cache.
+Since v1.4.2, Souin supports [Olric](https://github.com/buraksezer/olric) and since v1.6.10 it supports [Etcd](https://github.com/etcd-io/etcd) to handle distributed cache.
 
 ## GraphQL
 This feature is currently in beta.  
@@ -435,11 +455,16 @@ There is the fully configuration below
             disable_method
         }
         log_level debug
+        etcd {
+            configuration {
+                # Your Etcd configuration here
+            }
+        }
         olric {
             url url_to_your_cluster:3320
             path the_path_to_a_file.yaml
             configuration {
-                # Your badger configuration here
+                # Your Olric configuration here
             }
         }
         regex {
@@ -918,3 +943,4 @@ Thanks to these users for contributing or helping this project in any way
 * [Menci](https://github.com/menci)
 * [Duy Nguyen](https://github.com/duy-nguyen-devops)
 * [Kiss Karoly](https://github.com/kresike)
+* [Matthias von Bargen](https://github.com/mattvb91)

cache/providers/abstractProvider.go

@@ -16,10 +16,14 @@ const stalePrefix = "STALE_"
 func InitializeProvider(configuration configurationtypes.AbstractConfigurationInterface) types.AbstractProviderInterface {
 	var r types.AbstractProviderInterface
 	if configuration.GetDefaultCache().GetDistributed() {
-		if configuration.GetDefaultCache().GetOlric().URL != "" {
-			r, _ = OlricConnectionFactory(configuration)
+		if configuration.GetDefaultCache().GetEtcd().Configuration != nil {
+			r, _ = EtcdConnectionFactory(configuration)
 		} else {
-			r, _ = EmbeddedOlricConnectionFactory(configuration)
+			if configuration.GetDefaultCache().GetOlric().URL != "" {
+				r, _ = OlricConnectionFactory(configuration)
+			} else {
+				r, _ = EmbeddedOlricConnectionFactory(configuration)
+			}
 		}
 	} else if configuration.GetDefaultCache().GetNuts().Configuration != nil || configuration.GetDefaultCache().GetNuts().Path != "" {
 		r, _ = NutsConnectionFactory(configuration)
@@ -46,10 +50,12 @@ func varyVoter(baseKey string, req *http.Request, currentKey string) bool {
 
 		for _, item := range strings.Split(list, ";") {
 			index := strings.LastIndex(item, ":")
-			if len(item) >= index+1 && strings.Contains(req.Header.Get(item[:index]), item[index+1:]) {
-				return true
+			if !(len(item) >= index+1 && req.Header.Get(item[:index]) == item[index+1:]) {
+				return false
 			}
 		}
+
+		return true
 	}
 
 	return false

cache/providers/abstractProvider_test.go

@@ -1,12 +1,33 @@
 package providers
 
 import (
+	"fmt"
 	"testing"
+	"time"
 
+	"github.com/darkweak/souin/cache/types"
+	"github.com/darkweak/souin/configurationtypes"
 	"github.com/darkweak/souin/errors"
 	"github.com/darkweak/souin/tests"
 )
 
+const BYTEKEY = "MyByteKey"
+const NONEXISTENTKEY = "NonexistentKey"
+
+func verifyNewValueAfterSet(client types.AbstractProviderInterface, key string, value []byte, t *testing.T) {
+	newValue := client.Get(key)
+
+	if len(newValue) != len(value) {
+		errors.GenerateError(t, fmt.Sprintf("Key %s should be equals to %s, %s provided", key, value, newValue))
+	}
+}
+
+func setValueThenVerify(client types.AbstractProviderInterface, key string, value []byte, matchedURL configurationtypes.URL, ttl time.Duration, t *testing.T) {
+	client.Set(key, value, matchedURL, ttl)
+	time.Sleep(1 * time.Second)
+	verifyNewValueAfterSet(client, key, value, t)
+}
+
 func TestInitializeProvider(t *testing.T) {
 	c := tests.MockConfiguration(tests.BaseConfiguration)
 	p := InitializeProvider(c)

cache/providers/badgerProvider.go

@@ -112,6 +112,7 @@ func (provider *Badger) Prefix(key string, req *http.Request) []byte {
 		defer it.Close()
 		for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() {
 			if varyVoter(key, req, string(it.Item().Key())) {
+				fmt.Println(string(it.Item().Key()))
 				_ = it.Item().Value(func(val []byte) error {
 					result = val
 					return nil

cache/providers/badgerProvider_test.go

@@ -14,8 +14,6 @@ import (
 )
 
 const BADGERVALUE = "My first data"
-const BYTEKEY = "MyByteKey"
-const NONEXISTENTKEY = "NonexistentKey"
 
 func getBadgerClientAndMatchedURL(key string) (types.AbstractProviderInterface, configurationtypes.URL) {
 	return tests.GetCacheProviderClientAndMatchedURL(
@@ -89,7 +87,7 @@ func TestBadger_GetSetRequestInCache_OneByte(t *testing.T) {
 	time.Sleep(1 * time.Second)
 
 	res := client.Get(BYTEKEY)
-	if 0 == len(res) {
+	if len(res) == 0 {
 		errors.GenerateError(t, fmt.Sprintf("Key %s should exist", BYTEKEY))
 	}
 
@@ -98,20 +96,6 @@ func TestBadger_GetSetRequestInCache_OneByte(t *testing.T) {
 	}
 }
 
-func verifyNewValueAfterSet(client types.AbstractProviderInterface, key string, value []byte, t *testing.T) {
-	newValue := client.Get(key)
-
-	if len(newValue) != len(value) {
-		errors.GenerateError(t, fmt.Sprintf("Key %s should be equals to %s, %s provided", key, value, newValue))
-	}
-}
-
-func setValueThenVerify(client types.AbstractProviderInterface, key string, value []byte, matchedURL configurationtypes.URL, ttl time.Duration, t *testing.T) {
-	client.Set(key, value, matchedURL, ttl)
-	time.Sleep(1 * time.Second)
-	verifyNewValueAfterSet(client, key, value, t)
-}
-
 func TestBadger_SetRequestInCache_TTL(t *testing.T) {
 	key := "MyEmptyKey"
 	client, matchedURL := getBadgerClientAndMatchedURL(key)

cache/providers/etcdProvider.go

@@ -0,0 +1,140 @@
+package providers
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"regexp"
+	"time"
+
+	t "github.com/darkweak/souin/configurationtypes"
+	clientv3 "go.etcd.io/etcd/client/v3"
+)
+
+// Etcd provider type
+type Etcd struct {
+	*clientv3.Client
+	stale time.Duration
+	ctx   context.Context
+}
+
+// EtcdConnectionFactory function create new Nuts instance
+func EtcdConnectionFactory(c t.AbstractConfigurationInterface) (*Etcd, error) {
+	dc := c.GetDefaultCache()
+	bc, _ := json.Marshal(dc.GetEtcd().Configuration)
+	etcdConfiguration := clientv3.Config{
+		DialTimeout:      5 * time.Second,
+		AutoSyncInterval: 1 * time.Second,
+		Logger:           c.GetLogger(),
+	}
+	_ = json.Unmarshal(bc, &etcdConfiguration)
+
+	cli, err := clientv3.New(etcdConfiguration)
+
+	if err != nil {
+		fmt.Println("Impossible to initialize the Etcd DB.", err)
+		return nil, err
+	}
+
+	return &Etcd{
+		Client: cli,
+		ctx:    context.Background(),
+		stale:  dc.GetStale(),
+	}, nil
+}
+
+// ListKeys method returns the list of existing keys
+func (provider *Etcd) ListKeys() []string {
+	keys := []string{}
+
+	r, e := provider.Client.Get(provider.ctx, "\x00", clientv3.WithFromKey())
+
+	if e != nil {
+		return []string{}
+	}
+	for _, k := range r.Kvs {
+		keys = append(keys, string(k.Key))
+	}
+
+	return keys
+}
+
+// Get method returns the populated response if exists, empty response then
+func (provider *Etcd) Get(key string) (item []byte) {
+	r, e := provider.Client.Get(provider.ctx, key)
+
+	if e == nil && r != nil && len(r.Kvs) > 0 {
+		item = r.Kvs[0].Value
+	}
+
+	return
+}
+
+// Prefix method returns the populated response if exists, empty response then
+func (provider *Etcd) Prefix(key string, req *http.Request) []byte {
+	r, e := provider.Client.Get(provider.ctx, key, clientv3.WithPrefix())
+
+	if e == nil && r != nil {
+		for _, v := range r.Kvs {
+			if varyVoter(key, req, string(v.Key)) {
+				return v.Value
+			}
+		}
+	}
+
+	return []byte{}
+}
+
+// Set method will store the response in Etcd provider
+func (provider *Etcd) Set(key string, value []byte, url t.URL, duration time.Duration) {
+	if duration == 0 {
+		duration = url.TTL.Duration
+	}
+
+	rs, _ := provider.Client.Grant(context.TODO(), int64(duration.Seconds()))
+	_, err := provider.Client.Put(provider.ctx, key, string(value), clientv3.WithLease(rs.ID))
+
+	if err != nil {
+		panic(fmt.Sprintf("Impossible to set value into Etcd, %s", err))
+	}
+
+	_, err = provider.Client.Put(provider.ctx, stalePrefix+key, string(value), clientv3.WithLease(rs.ID))
+
+	if err != nil {
+		panic(fmt.Sprintf("Impossible to set value into Etcd, %s", err))
+	}
+}
+
+// Delete method will delete the response in Etcd provider if exists corresponding to key param
+func (provider *Etcd) Delete(key string) {
+	_, _ = provider.Client.Delete(provider.ctx, key)
+}
+
+// DeleteMany method will delete the responses in Nuts provider if exists corresponding to the regex key param
+func (provider *Etcd) DeleteMany(key string) {
+	re, e := regexp.Compile(key)
+
+	if e != nil {
+		return
+	}
+
+	if r, e := provider.Client.Get(provider.ctx, "\x00", clientv3.WithFromKey()); e == nil {
+		for _, k := range r.Kvs {
+			key := string(k.Key)
+			if re.MatchString(key) {
+				provider.Delete(key)
+			}
+		}
+	}
+}
+
+// Init method will
+func (provider *Etcd) Init() error {
+	return nil
+}
+
+// Reset method will reset or close provider
+func (provider *Etcd) Reset() error {
+	return provider.Client.Close()
+}