Skip to content

Commit 816afa8

Browse files
darkweakdarkweak
authored
fix(providers): Search by prefix and regexp in NutsDB (#224)
* fix(providers): Search by prefix and regexp in NutsDB * Increase nutsdb limit * Fix tests * Replace fmt to logger * Automatically generate workflow * Add Surrogate-Key purge/store debug messages * Support Træfik v2.7+ * Bump version
1 parent d1b8dd7 commit 816afa8

File tree

137 files changed

+3975
-2462
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

137 files changed

+3975
-2462
lines changed

‎.github/workflows/plugins.yml

+130-97
Large diffs are not rendered by default.

‎.github/workflows/workflow_plugins_generator.sh

+99
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
#!/bin/bash
2+
3+
plugins=("beego" "chi" "dotweb" "echo" "fiber" "gin" "go-zero" "goyave" "skipper" "souin" "traefik" "tyk" "webgo")
4+
durations=("35" "30" "30" "30" "45" "30" "50" "35" "50" "40" "30" "30" "30")
5+
versions=("16" "16" "16" "16" "16" "16" "16" "16" "18" "16" "16" "16" "16")
6+
7+
IFS= read -r -d '' tpl <<EOF
8+
name: Build and validate Souin as plugins
9+
10+
on:
11+
- pull_request
12+
13+
jobs:
14+
build-caddy-validator:
15+
name: Check that Souin build as caddy module
16+
runs-on: ubuntu-latest
17+
steps:
18+
-
19+
name: Add domain.com host to /etc/hosts
20+
run: |
21+
sudo echo "127.0.0.1 domain.com" | sudo tee -a /etc/hosts
22+
-
23+
name: Install Go
24+
uses: actions/setup-go@v2
25+
with:
26+
go-version: 1.17
27+
-
28+
name: Checkout code
29+
uses: actions/checkout@v2
30+
-
31+
name: Install xcaddy
32+
run: go install github.com/caddyserver/xcaddy/cmd/xcaddy@latest
33+
-
34+
name: Build Souin as caddy module
35+
run: cd plugins/caddy && xcaddy build --with github.com/darkweak/souin/plugins/caddy=./ --with github.com/darkweak/souin@latest=../..
36+
-
37+
name: Run detached caddy
38+
run: cd plugins/caddy && ./caddy run &
39+
-
40+
name: Run Caddy E2E tests
41+
uses: anthonyvscode/newman-action@v1
42+
with:
43+
collection: "docs/e2e/Souin E2E.postman_collection.json"
44+
folder: Caddy
45+
reporters: cli
46+
delayRequest: 5000
47+
EOF
48+
workflow+="$tpl"
49+
50+
for i in ${!plugins[@]}; do
51+
lower="${plugins[$i]}"
52+
capitalized="$(tr '[:lower:]' '[:upper:]' <<< ${lower:0:1})${lower:1}"
53+
IFS= read -d '' tpl <<EOF
54+
build-$lower-validator:
55+
name: Check that Souin build as $capitalized middleware
56+
runs-on: ubuntu-latest
57+
steps:
58+
-
59+
name: Add domain.com host to /etc/hosts
60+
run: |
61+
sudo echo "127.0.0.1 domain.com" | sudo tee -a /etc/hosts
62+
-
63+
name: Install Go
64+
uses: actions/setup-go@v2
65+
with:
66+
go-version: 1.${versions[$i]}
67+
-
68+
name: Checkout code
69+
uses: actions/checkout@v2
70+
-
71+
name: Build Souin as $capitalized plugin
72+
run: make build-and-run-$lower
73+
-
74+
name: Wait for Souin is really loaded inside $capitalized as middleware
75+
uses: jakejarvis/wait-action@master
76+
with:
77+
time: ${durations[$i]}s
78+
-
79+
name: Set $capitalized logs configuration result as environment variable
80+
run: cd plugins/$lower && echo "\$(make load-checker)" >> \$GITHUB_ENV
81+
-
82+
name: Check if the configuration is loaded to define if Souin is loaded too
83+
uses: nick-invision/assert-action@v1
84+
with:
85+
expected: '"Souin configuration is now loaded."'
86+
actual: \${{ env.MIDDLEWARE_RESULT }}
87+
comparison: contains
88+
-
89+
name: Run $capitalized E2E tests
90+
uses: anthonyvscode/newman-action@v1
91+
with:
92+
collection: "docs/e2e/Souin E2E.postman_collection.json"
93+
folder: $capitalized
94+
reporters: cli
95+
delayRequest: 5000
96+
EOF
97+
workflow+="$tpl"
98+
done
99+
echo "${workflow%$'\n'}" > "$( dirname -- "$0"; )/plugins.yml"

‎Makefile

+4-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
DC=docker-compose
77
DC_BUILD=$(DC) build
88
DC_EXEC=$(DC) exec
9-
PLUGINS_LIST=beego caddy chi dotweb echo fiber skipper gin go-zero goyave traefik tyk webgo
9+
PLUGINS_LIST=beego caddy chi dotweb echo fiber skipper gin go-zero goyave traefik tyk webgo souin
1010

1111
base-build-and-run-%:
1212
cd plugins/$* && $(MAKE) prepare
@@ -89,6 +89,9 @@ gatling: ## Launch gatling scenarios
8989
generate-plantUML: ## Generate plantUML diagrams
9090
cd ./docs/plantUML && sh generate.sh && cd ../..
9191

92+
generate-workflow: ## Generate plugin workflow
93+
bash .github/workflows/workflow_plugins_generator.sh
94+
9295
golangci-lint: ## Run golangci-lint to ensure the code quality
9396
docker run --rm -v $(pwd):/app -w /app golangci/golangci-lint:v1.46.2 golangci-lint run -v
9497

‎README.md

+9-6
Original file line numberDiff line numberDiff line change
@@ -226,13 +226,16 @@ The base path for the souin API is `/souin`.
226226
The Souin API supports the invalidation by surrogate keys such as Fastly which will replace the Varnish system. You can read the doc [about this system](https://github.com/darkweak/souin/blob/master/cache/surrogate/README.md).
227227
This system is able to invalidate by tags your cloud provider cache. Actually it supports Akamai and Fastly but in a near future some other providers would be implemented like Cloudflare or Varnish.
228228

229-
| Method | Endpoint | Description |
230-
|:--------|:------------------|:-----------------------------------------------------------------------------------------|
231-
| `GET` | `/` | List stored keys cache |
232-
| `PURGE` | `/{id or regexp}` | Purge selected item(s) depending. The parameter can be either a specific key or a regexp |
233-
| `PURGE` | `/?ykey={key}` | Purge selected item(s) corresponding to the target ykey such as Varnish (deprecated) |
229+
| Method | Endpoint | Headers | Description |
230+
|:--------|:------------------|:-----------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
231+
| `GET` | `/` | - | List stored keys cache |
232+
| `GET` | `/surrogate_keys` | - | List stored keys cache |
233+
| `PURGE` | `/{id or regexp}` | - | Purge selected item(s) depending. The parameter can be either a specific key or a regexp |
234+
| `PURGE` | `/?ykey={key}` | - | Purge selected item(s) corresponding to the target ykey such as Varnish (deprecated) |
235+
| `PURGE` | `/` | `Surrogate-Key: Surrogate-Key-First, Surrogate-Key-Second` | Purge selected item(s) belong to the target key in the header `Surrogate-Key` (see [Surrogate-Key system](https://github.com/darkweak/souin/blob/master/cache/surrogate/README.md)) |
234236

235237
### Security API
238+
**DEPRECATED**
236239
Security API allows users to protect other APIs with JWT authentication.
237240
The base path for the security API is `/authentication`.
238241

@@ -760,7 +763,7 @@ experimental:
760763
plugins:
761764
souin:
762765
moduleName: github.com/darkweak/souin
763-
version: v1.6.9
766+
version:
764767
```
765768
After that you can declare either the whole configuration at once in the middleware block or by service. See the examples below.
766769
```yaml

‎cache/providers/badgerProvider.go

+9-8
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,14 @@ import (
1010
t "github.com/darkweak/souin/configurationtypes"
1111
badger "github.com/dgraph-io/badger/v3"
1212
"github.com/imdario/mergo"
13+
"go.uber.org/zap"
1314
)
1415

1516
// Badger provider type
1617
type Badger struct {
1718
*badger.DB
18-
stale time.Duration
19+
stale time.Duration
20+
logger *zap.Logger
1921
}
2022

2123
var enabledBadgerInstances = make(map[string]*Badger)
@@ -29,15 +31,15 @@ func BadgerConnectionFactory(c t.AbstractConfigurationInterface) (*Badger, error
2931
var parsedBadger badger.Options
3032
if b, e := json.Marshal(badgerConfiguration.Configuration); e == nil {
3133
if e = json.Unmarshal(b, &parsedBadger); e != nil {
32-
fmt.Println("Impossible to parse the configuration for the default provider (Badger)", e)
34+
c.GetLogger().Sugar().Error("Impossible to parse the configuration for the default provider (Badger)", e)
3335
}
3436
}
3537

3638
if err := mergo.Merge(&badgerOptions, parsedBadger, mergo.WithOverride); err != nil {
37-
fmt.Println("An error occurred during the badgerOptions merge from the default options with your configuration.")
39+
c.GetLogger().Sugar().Error("An error occurred during the badgerOptions merge from the default options with your configuration.")
3840
}
39-
} else {
40-
badgerOptions = badgerOptions.WithInMemory(true).WithNumMemtables(1).WithNumLevelZeroTables(1)
41+
} else if badgerConfiguration.Path == "" {
42+
badgerOptions = badgerOptions.WithInMemory(true)
4143
}
4244

4345
uid := badgerOptions.Dir + badgerOptions.ValueDir
@@ -48,10 +50,10 @@ func BadgerConnectionFactory(c t.AbstractConfigurationInterface) (*Badger, error
4850
db, e := badger.Open(badgerOptions)
4951

5052
if e != nil {
51-
fmt.Println("Impossible to open the Badger DB.", e)
53+
c.GetLogger().Sugar().Error("Impossible to open the Badger DB.", e)
5254
}
5355

54-
i := &Badger{DB: db, stale: dc.GetStale()}
56+
i := &Badger{DB: db, logger: c.GetLogger(), stale: dc.GetStale()}
5557
enabledBadgerInstances[uid] = i
5658

5759
return i, nil
@@ -112,7 +114,6 @@ func (provider *Badger) Prefix(key string, req *http.Request) []byte {
112114
defer it.Close()
113115
for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() {
114116
if varyVoter(key, req, string(it.Item().Key())) {
115-
fmt.Println(string(it.Item().Key()))
116117
_ = it.Item().Value(func(val []byte) error {
117118
result = val
118119
return nil

‎cache/providers/embeddedOlricProvider.go

+16-15
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package providers
22

33
import (
44
"context"
5-
"fmt"
65
"io/ioutil"
76
"net/http"
87
"os"
@@ -19,10 +18,11 @@ import (
1918

2019
// EmbeddedOlric provider type
2120
type EmbeddedOlric struct {
22-
dm *olric.DMap
23-
db *olric.Olric
24-
stale time.Duration
25-
ct context.Context
21+
dm *olric.DMap
22+
db *olric.Olric
23+
stale time.Duration
24+
logger *zap.Logger
25+
ct context.Context
2626
}
2727

2828
func tryToLoadConfiguration(olricInstance *config.Config, olricConfiguration t.CacheProvider, logger *zap.Logger) (*config.Config, bool) {
@@ -70,7 +70,7 @@ func EmbeddedOlricConnectionFactory(configuration t.AbstractConfigurationInterfa
7070

7171
started, cancel := context.WithCancel(context.Background())
7272
olricInstance.Started = func() {
73-
fmt.Println("Embedded Olric is ready")
73+
configuration.GetLogger().Sugar().Error("Embedded Olric is ready")
7474
defer cancel()
7575
}
7676

@@ -96,13 +96,14 @@ func EmbeddedOlricConnectionFactory(configuration t.AbstractConfigurationInterfa
9696
}
9797
dm, e := db.NewDMap("souin-map")
9898

99-
fmt.Println("Embedded Olric is ready for this node.")
99+
configuration.GetLogger().Sugar().Info("Embedded Olric is ready for this node.")
100100

101101
return &EmbeddedOlric{
102-
dm: dm,
103-
db: db,
104-
stale: configuration.GetDefaultCache().GetStale(),
105-
ct: context.Background(),
102+
dm: dm,
103+
db: db,
104+
stale: configuration.GetDefaultCache().GetStale(),
105+
logger: configuration.GetLogger(),
106+
ct: context.Background(),
106107
}, e
107108
}
108109

@@ -122,7 +123,7 @@ func (provider *EmbeddedOlric) ListKeys() []string {
122123
defer c.Close()
123124
}
124125
if err != nil {
125-
fmt.Printf("An error occurred while trying to list keys in Olric: %s\n", err)
126+
provider.logger.Sugar().Errorf("An error occurred while trying to list keys in Olric: %s\n", err)
126127
return []string{}
127128
}
128129

@@ -139,14 +140,14 @@ func (provider *EmbeddedOlric) ListKeys() []string {
139140
func (provider *EmbeddedOlric) Prefix(key string, req *http.Request) []byte {
140141
c, err := provider.dm.Query(query.M{
141142
"$onKey": query.M{
142-
"$regexMatch": "^" + key,
143+
"$regexMatch": "^" + key + "({|$)",
143144
},
144145
})
145146
if c != nil {
146147
defer c.Close()
147148
}
148149
if err != nil {
149-
fmt.Printf("An error occurred while trying to retrieve data in Olric: %s\n", err)
150+
provider.logger.Sugar().Errorf("An error occurred while trying to retrieve data in Olric: %s\n", err)
150151
return []byte{}
151152
}
152153

@@ -240,7 +241,7 @@ func (provider *EmbeddedOlric) Reset() error {
240241

241242
// Destruct method will reset or close provider
242243
func (provider *EmbeddedOlric) Destruct() error {
243-
fmt.Println("Destruct current embedded olric...")
244+
provider.logger.Sugar().Debug("Destruct current embedded olric...")
244245
return provider.Reset()
245246
}
246247

‎cache/providers/etcdProvider.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ func EtcdConnectionFactory(c t.AbstractConfigurationInterface) (*Etcd, error) {
3333
cli, err := clientv3.New(etcdConfiguration)
3434

3535
if err != nil {
36-
fmt.Println("Impossible to initialize the Etcd DB.", err)
36+
c.GetLogger().Sugar().Error("Impossible to initialize the Etcd DB.", err)
3737
return nil, err
3838
}
3939

‎cache/providers/nutsProvider.go

+18-13
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,8 @@ type Nuts struct {
1919
}
2020

2121
const (
22-
bucket = "souin-bucket"
22+
bucket = "souin-bucket"
23+
nutsLimit = 1 << 16
2324
)
2425

2526
func sanitizeProperties(m map[string]interface{}) map[string]interface{} {
@@ -35,6 +36,12 @@ func sanitizeProperties(m map[string]interface{}) map[string]interface{} {
3536
}
3637
}
3738

39+
for _, i := range []string{"SegmentSize", "NodeNum", "MaxFdNumsInCache"} {
40+
if v := m[i]; v != nil {
41+
m[i], _ = v.(int64)
42+
}
43+
}
44+
3845
if v := m["EntryIdxMode"]; v != nil {
3946
m["EntryIdxMode"] = nutsdb.HintKeyValAndRAMIdxMode
4047
switch v {
@@ -46,11 +53,11 @@ func sanitizeProperties(m map[string]interface{}) map[string]interface{} {
4653
}
4754

4855
if v := m["SyncEnable"]; v != nil {
49-
b, ok := v.(bool)
50-
if ok {
56+
m["SyncEnable"] = true
57+
if b, ok := v.(bool); ok {
5158
m["SyncEnable"] = b
52-
} else {
53-
m["SyncEnable"], _ = strconv.ParseBool(v.(string))
59+
} else if s, ok := v.(string); ok {
60+
m["SyncEnable"], _ = strconv.ParseBool(s)
5461
}
5562
}
5663

@@ -68,12 +75,12 @@ func NutsConnectionFactory(c t.AbstractConfigurationInterface) (*Nuts, error) {
6875
nutsConfiguration.Configuration = sanitizeProperties(nutsConfiguration.Configuration.(map[string]interface{}))
6976
if b, e := json.Marshal(nutsConfiguration.Configuration); e == nil {
7077
if e = json.Unmarshal(b, &parsedNuts); e != nil {
71-
fmt.Println("Impossible to parse the configuration for the Nuts provider", e)
78+
c.GetLogger().Sugar().Error("Impossible to parse the configuration for the Nuts provider", e)
7279
}
7380
}
7481

7582
if err := mergo.Merge(&nutsOptions, parsedNuts, mergo.WithOverride); err != nil {
76-
fmt.Println("An error occurred during the nutsOptions merge from the default options with your configuration.")
83+
c.GetLogger().Sugar().Error("An error occurred during the nutsOptions merge from the default options with your configuration.")
7784
}
7885
} else {
7986
nutsOptions.RWMode = nutsdb.MMap
@@ -85,12 +92,10 @@ func NutsConnectionFactory(c t.AbstractConfigurationInterface) (*Nuts, error) {
8592
db, e := nutsdb.Open(nutsOptions)
8693

8794
if e != nil {
88-
fmt.Println("Impossible to open the Nuts DB.", e)
95+
c.GetLogger().Sugar().Error("Impossible to open the Nuts DB.", e)
8996
}
9097

91-
i := &Nuts{DB: db, stale: dc.GetStale()}
92-
93-
return i, nil
98+
return &Nuts{DB: db, stale: dc.GetStale()}, nil
9499
}
95100

96101
// ListKeys method returns the list of existing keys
@@ -132,7 +137,7 @@ func (provider *Nuts) Prefix(key string, req *http.Request) []byte {
132137
_ = provider.DB.View(func(tx *nutsdb.Tx) error {
133138
prefix := []byte(key)
134139

135-
if entries, _, err := tx.PrefixScan(bucket, prefix, 0, 50); err != nil {
140+
if entries, _, err := tx.PrefixSearchScan(bucket, prefix, "^({|$)", 0, 50); err != nil {
136141
return err
137142
} else {
138143
for _, entry := range entries {
@@ -181,7 +186,7 @@ func (provider *Nuts) Delete(key string) {
181186
// DeleteMany method will delete the responses in Nuts provider if exists corresponding to the regex key param
182187
func (provider *Nuts) DeleteMany(key string) {
183188
_ = provider.DB.Update(func(tx *nutsdb.Tx) error {
184-
if entries, _, err := tx.PrefixSearchScan(bucket, []byte(""), key, 0, 100); err != nil {
189+
if entries, _, err := tx.PrefixSearchScan(bucket, []byte(""), key, 0, nutsLimit); err != nil {
185190
return err
186191
} else {
187192
for _, entry := range entries {

0 commit comments

Comments
 (0)