Distributed cache support (#38)

* feat: Support distributed cache with olric

* feat: Support external distributed cache

* feat: Fix lint

* feat: Fix docker-compose to be able to build and tests on CI

* feat: Fix docker-compose and create directory for olric configuration

* feat: Fix docker-compose

* feat: Fix

* Update the doc

* feat: Update plantuml

Author: darkweak

.traefik.yml

@@ -5,14 +5,14 @@ DC_BUILD=$(DC) build
 DC_EXEC=$(DC) exec
 
 build-app: env-prod ## Build containers with prod env vars
-	$(DC_BUILD) souin
+	$(DC_BUILD) olric souin
 	$(MAKE) up
 
 build-and-run-caddy: ## Build caddy binary
 	cd plugins/caddy && xcaddy build --with github.com/darkweak/souin/plugins/caddy=./ --with github.com/darkweak/souin@latest=../.. && ./caddy run
 
 build-dev: env-dev ## Build containers with dev env vars
-	$(DC_BUILD) souin
+	$(DC_BUILD) olric souin
 	$(MAKE) up
 
 coverage: ## Show code coverage

Makefile

@@ -70,8 +70,10 @@ default_cache:
     - Authorization
   cache_providers:
     - all # Enable all providers by default
-  redis: # Redis configuration
+  redis: # Redis provider configuration
     url: 'redis:6379'
+  olric: # Olric provider configuration
+    url: 'olric:3320'
   regex:
     exclude: 'ARegexHere' # Regex to exclude from cache
 ssl_providers: # The {providers}.json to use
@@ -100,6 +102,7 @@ urls:
 | `default_cache.headers`              | List of headers to include to the cache                                    | `- Authorization`<br/><br/>`- Content-Type`<br/><br/>`- X-Additional-Header` |
 | `default_cache.cache_providers`      | Your providers list to cache your data, by default it will use all systems | `- all`<br/><br/>`- ristretto`<br/><br/>`- redis`                            |
 | `default_cache.redis.url`            | The redis url, used if you enabled it in the provider section              | `redis:6379` (container way) and `http://yourdomain.com:6379` (network way)  |
+| `default_cache.olric.url`            | The olric url, used if you enabled it in the provider section              | `olric:3320` (container way) and `http://yourdomain.com:3320` (network way)  |
 | `default_cache.regex.exclude`        | The regex used to prevent paths being cached                               | `^[A-z]+.*$`                                                                 |
 | `ssl_providers`                      | List of your providers handling certificates                               | `- traefik`<br/><br/>`- nginx`<br/><br/>`- apache`                           |
 | `urls.{your url or regex}`           | List of your custom configuration depending each URL or regex              | 'https:\/\/yourdomain.com'                                                   |
@@ -136,13 +139,13 @@ See the sequence for the minimal version below
 
 ## Cache systems
 Supported providers
- - [Redis](https://github.com/go-redis/redis)
- - [Olric](https://github.com/buraksezer/olric)
+- [Redis](https://github.com/go-redis/redis)
+- [Olric](https://github.com/buraksezer/olric)
 
- The cache system sits on top of three providers at the moment. It provides an in-memory, redis and Olric cache systems because setting, getting, updating and deleting keys in these providers is as easy as it gets.  
- In order to do that, Redis and Olric providers need to be either on the same network as the Souin instance when using docker-compose or over the internet, then it will use by default in-memory to avoid network latency as much as possible. 
- Souin will return at first the in-memory response when it gives a non-empty response, then the olric followed by the redis one with same condition, or fallback to the reverse proxy otherwise.
- Since 1.4.2, Souin supports [Olric](https://github.com/buraksezer/olric) to handle distributed cache.
+The cache system sits on top of three providers at the moment. It provides an in-memory, redis and Olric cache systems because setting, getting, updating and deleting keys in these providers is as easy as it gets.  
+In order to do that, Redis and Olric providers need to be either on the same network as the Souin instance when using docker-compose or over the internet, then it will use by default in-memory to avoid network latency as much as possible. 
+Souin will return at first the in-memory response when it gives a non-empty response, then the olric followed by the redis one with same condition, or fallback to the reverse proxy otherwise.
+Since 1.4.2, Souin supports [Olric](https://github.com/buraksezer/olric) to handle distributed cache.
 
 ### Cache invalidation
 The cache invalidation is build for CRUD requests, if you're doing a GET HTTP request, it will serve the cached response when it exists, otherwise the reverse-proxy response will be served.  
@@ -194,6 +197,7 @@ services:
       - 80:80
       - 443:443
     depends_on:
+      - olric
       - redis
     environment:
       GOPATH: /app
@@ -202,6 +206,14 @@ services:
       - /anywhere/configuration.yml:/configuration/configuration.yml
     <<: *networks
 
+  olric:
+    build:
+      context: ./olric
+      dockerfile: Dockerfile-olric
+      target: olric
+    restart: on-failure
+    <<: *networks
+
   redis:
     image: redis:alpine
     <<: *networks

README.md

@@ -20,4 +20,5 @@ func TestInitialize(t *testing.T) {
 	if !endpoints[0].IsEnabled() {
 		errors.GenerateError(t, fmt.Sprintf("Endpoint should be enabled"))
 	}
+	prs["olric"].Reset()
 }

api/main_test.go

@@ -24,6 +24,7 @@ func mockSouinAPI() *SouinAPI {
 
 func TestSouinAPI_BulkDelete(t *testing.T) {
 	souinMock := mockSouinAPI()
+	defer souinMock.providers["olric"].Reset()
 	for _, provider := range souinMock.providers {
 		provider.Set("key", []byte("value"), tests.GetMatchedURL("key"), 20 * time.Second)
 		provider.Set("key2", []byte("value"), tests.GetMatchedURL("key"), 20 * time.Second)
@@ -45,6 +46,7 @@ func TestSouinAPI_BulkDelete(t *testing.T) {
 
 func TestSouinAPI_Delete(t *testing.T) {
 	souinMock := mockSouinAPI()
+	defer souinMock.providers["olric"].Reset()
 	for _, provider := range souinMock.providers {
 		provider.Set("key", []byte("value"), tests.GetMatchedURL("key"), 20 * time.Second)
 	}
@@ -65,6 +67,7 @@ func TestSouinAPI_Delete(t *testing.T) {
 
 func TestSouinAPI_GetAll(t *testing.T) {
 	souinMock := mockSouinAPI()
+	defer souinMock.providers["olric"].Reset()
 	for _, v := range souinMock.GetAll() {
 		if len(v) > 0 {
 			errors.GenerateError(t, "Souin API shouldn't have a record")
@@ -81,6 +84,7 @@ func TestSouinAPI_GetAll(t *testing.T) {
 		}
 	}
 	souinMock.providers["redis"].Delete("key")
+	souinMock.providers["olric"].Delete("key")
 	time.Sleep(10 * time.Second)
 	for _, v := range souinMock.GetAll() {
 		if len(v) == 1 {
@@ -91,13 +95,15 @@ func TestSouinAPI_GetAll(t *testing.T) {
 
 func TestSouinAPI_GetBasePath(t *testing.T) {
 	souinMock := mockSouinAPI()
+	defer souinMock.providers["olric"].Reset()
 	if souinMock.GetBasePath() != "/souinbasepath" {
 		errors.GenerateError(t, "Souin API should be enabled")
 	}
 }
 
 func TestSouinAPI_IsEnabled(t *testing.T) {
 	souinMock := mockSouinAPI()
+	defer souinMock.providers["olric"].Reset()
 	if !souinMock.IsEnabled() {
 		errors.GenerateError(t, "Souin API should be enabled")
 	}

api/souin_test.go

@@ -10,10 +10,12 @@ import (
 	"testing"
 )
 
-func commonInitializer() (*httptest.ResponseRecorder, *http.Request, *types.RetrieverResponseProperties) {
+func TestServeResponse(t *testing.T) {
 	c := tests.MockConfiguration()
-	prs := providers.InitializeProvider(c)
 	regexpUrls := helpers.InitializeRegexp(c)
+	prs := providers.InitializeProvider(c)
+	defer prs["olric"].Reset()
+	rc := Initialize()
 	retriever := &types.RetrieverResponseProperties{
 		Configuration: c,
 		Providers:      prs,
@@ -23,12 +25,6 @@ func commonInitializer() (*httptest.ResponseRecorder, *http.Request, *types.Retr
 	r := httptest.NewRequest("GET", "http://"+tests.DOMAIN+tests.PATH, nil)
 	w := httptest.NewRecorder()
 
-	return w, r, retriever
-}
-
-func TestServeResponse(t *testing.T) {
-	rc := Initialize()
-	w, r, retriever := commonInitializer()
 	ServeResponse(
 		w,
 		r,

cache/coalescing/requestCoalescing_test.go

@@ -20,17 +20,27 @@ func InitializeProvider(configuration configurationtypes.AbstractConfigurationIn
 	if len(configuration.GetDefaultCache().Providers) == 0 || contains(configuration.GetDefaultCache().Providers, "all") {
 		redis, _ := RedisConnectionFactory(configuration)
 		providers["redis"] = redis
+		olric, _ := OlricConnectionFactory(configuration)
+		providers["olric"] = olric
 		ristretto, _ := RistrettoConnectionFactory(configuration)
 		providers["ristretto"] = ristretto
 	} else {
 		if contains(configuration.GetDefaultCache().Providers, "redis") {
 			redis, _ := RedisConnectionFactory(configuration)
 			providers["redis"] = redis
 		}
+		if contains(configuration.GetDefaultCache().Providers, "olric") {
+			olric, _ := OlricConnectionFactory(configuration)
+			providers["olric"] = olric
+		}
 		if contains(configuration.GetDefaultCache().Providers, "ristretto") {
 			ristretto, _ := RistrettoConnectionFactory(configuration)
 			providers["ristretto"] = ristretto
 		}
 	}
+
+	for _, p := range providers {
+		_ = p.Init()
+	}
 	return providers
 }

cache/providers/abstractProvider.go

@@ -1,69 +1,22 @@
 package providers
 
 import (
-	"github.com/darkweak/souin/configuration"
-	"github.com/darkweak/souin/configurationtypes"
 	"github.com/darkweak/souin/errors"
 	"github.com/darkweak/souin/helpers"
 	"github.com/darkweak/souin/tests"
-	"log"
-	"regexp"
 	"testing"
 )
 
-func MockConfiguration() configurationtypes.AbstractConfigurationInterface {
-	var config configuration.Configuration
-	e := config.Parse([]byte(`
-default_cache:
-  headers:
-    - Authorization
-  port:
-    web: 80
-    tls: 443
-  redis:
-    url: 'redis:6379'
-  regex:
-    exclude: 'ARegexHere'
-  ttl: 1000
-reverse_proxy_url: 'http://traefik'
-ssl_providers:
-  - traefik
-urls:
-  'domain.com/':
-    ttl: 1000
-    headers:
-      - Authorization
-  'mysubdomain.domain.com':
-    ttl: 50
-    headers:
-      - Authorization
-      - 'Content-Type'
-`))
-	if e != nil {
-		log.Fatal(e)
-	}
-	return &config
-}
-
-func MockInitializeRegexp(configurationInstance configurationtypes.AbstractConfigurationInterface) regexp.Regexp {
-	u := ""
-	for k := range configurationInstance.GetUrls() {
-		if "" != u {
-			u += "|"
-		}
-		u += "(" + k + ")"
-	}
-
-	return *regexp.MustCompile(u)
-}
-
 func TestInitializeProvider(t *testing.T) {
 	c := tests.MockConfiguration()
 	ps := InitializeProvider(c)
-	for _, p := range ps {
-		err := p.Init()
-		if nil != err {
-			errors.GenerateError(t, "Init shouldn't crash")
+	defer ps["olric"].Reset()
+	for k, p := range ps {
+		if k != "olric" {
+			err := p.Init()
+			if nil != err {
+				errors.GenerateError(t, "Init shouldn't crash")
+			}
 		}
 	}
 }

cache/providers/abstractProvider_test.go

@@ -0,0 +1,110 @@
+package providers
+
+import (
+	"github.com/buraksezer/olric/client"
+	"github.com/buraksezer/olric/config"
+	"github.com/darkweak/souin/cache/keysaver"
+	t "github.com/darkweak/souin/configurationtypes"
+	"strconv"
+	"time"
+)
+
+// Olric provider type
+type Olric struct {
+	*client.Client
+	dm *client.DMap
+	keySaver *keysaver.ClearKey
+}
+
+// OlricConnectionFactory function create new Olric instance
+func OlricConnectionFactory(configuration t.AbstractConfigurationInterface) (*Olric, error) {
+	var keySaver *keysaver.ClearKey
+	if configuration.GetAPI().Souin.Enable {
+		keySaver = keysaver.NewClearKey()
+	}
+
+	c, err := client.New(&client.Config{
+		Servers: []string{configuration.GetDefaultCache().Olric.URL},
+		Client: &config.Client{
+			DialTimeout: time.Second,
+			KeepAlive:   time.Second,
+			MaxConn:     10,
+		},
+	})
+	if err != nil {
+		panic(err)
+	}
+
+	return &Olric{
+		c,
+		nil,
+		keySaver,
+	}, nil
+}
+
+// ListKeys method returns the list of existing keys
+func (provider *Olric) ListKeys() []string {
+	if nil != provider.keySaver {
+		return provider.keySaver.ListKeys()
+	}
+	return []string{}
+}
+
+// Get method returns the populated response if exists, empty response then
+func (provider *Olric) Get(key string) []byte {
+	val2, err := provider.dm.Get(key)
+
+	if err != nil {
+		return []byte{}
+	}
+
+	return val2.([]byte)
+}
+
+// Set method will store the response in Redis provider
+func (provider *Olric) Set(key string, value []byte, url t.URL, duration time.Duration) {
+	if duration == 0 {
+		ttl, _ := strconv.Atoi(url.TTL)
+		duration = time.Duration(ttl)*time.Second
+	}
+
+	err := provider.dm.PutEx(key, value, duration)
+	if err != nil {
+		panic(err)
+	} else {
+		go func() {
+			if nil != provider.keySaver {
+				provider.keySaver.AddKey(key)
+			}
+		}()
+	}
+}
+
+// Delete method will delete the response in Redis provider if exists corresponding to key param
+func (provider *Olric) Delete(key string) {
+	go func() {
+		err := provider.dm.Delete(key)
+		if err != nil {
+			panic(err)
+		} else {
+			go func() {
+				if nil != provider.keySaver {
+					provider.keySaver.DelKey(key, 0)
+				}
+			}()
+		}
+	}()
+}
+
+// Init method will initialize Olric provider if needed
+func (provider *Olric) Init() error {
+	dm := provider.Client.NewDMap("souin-map")
+
+	provider.dm = dm
+	return nil
+}
+
+// Reset method will reset or close provider
+func (provider *Olric) Reset() {
+	provider.Client.Close()
+}