Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Initial support for OSS Cluster API #170

Merged
merged 44 commits into from
Sep 13, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
44 commits
Select commit Hold shift + click to select a range
af34b9c
delete mock
shacharPash Aug 10, 2023
ff101b5
coverage
shacharPash Aug 10, 2023
c2b5fe6
fix test
shacharPash Aug 10, 2023
f8f2aec
indent
shacharPash Aug 13, 2023
dbe0b74
Merge branch 'master' into DeleteMoq
shacharPash Aug 14, 2023
27a69c0
change to var - check
shacharPash Aug 14, 2023
dc0d3c0
cluster test
shacharPash Aug 14, 2023
eeb2edf
add opthin to connect cluster with dotnet test
shacharPash Aug 15, 2023
0d100d7
use key in topk tests
shacharPash Aug 16, 2023
53c0163
Merge branch 'master' into ClusterSupport
shacharPash Aug 16, 2023
02d1837
get env vars inside RedisFixture
shacharPash Aug 22, 2023
5ace5b9
Merge branch 'master' into ClusterSupport
shacharPash Aug 22, 2023
65a49b6
skip if redis
shacharPash Aug 22, 2023
c7b6cd9
add skip where needed
shacharPash Aug 24, 2023
6e82630
Execute broadcast
shacharPash Aug 24, 2023
b5cdf87
delete cluster tests
shacharPash Aug 24, 2023
376ccdf
RedisFixture fix
shacharPash Aug 24, 2023
495dce1
add to contributing
shacharPash Aug 24, 2023
47e2d7f
run cluster on CI
shacharPash Aug 28, 2023
d87cec0
wip
shacharPash Aug 28, 2023
b933427
fix /
shacharPash Aug 28, 2023
ccb7625
-d
shacharPash Aug 28, 2023
908699d
delete restore
shacharPash Aug 28, 2023
b5318bf
return restore
shacharPash Aug 28, 2023
31bef69
add -RC3
shacharPash Aug 30, 2023
e544fe0
add RC3 to docker-compose
shacharPash Aug 30, 2023
5f67f42
try define both .net 6 and 7
shacharPash Aug 30, 2023
9724f9b
Skip if cluster where needed
shacharPash Aug 31, 2023
d998125
add names
shacharPash Aug 31, 2023
2842a29
skip configOnTimeout if cluster
shacharPash Aug 31, 2023
d07f225
try to fix win tests
shacharPash Aug 31, 2023
661cb1c
tests names +fix win version
shacharPash Aug 31, 2023
b4a5ff2
fix versions
shacharPash Aug 31, 2023
6c98a16
versions
shacharPash Aug 31, 2023
b1feed4
win verer
shacharPash Aug 31, 2023
3b79773
wording
shacharPash Aug 31, 2023
ad44131
Merge branch 'master' into ClusterSupport
chayim Sep 3, 2023
341161a
Merge branch 'master' into ClusterSupport
chayim Sep 10, 2023
36d32dd
Merge branch 'master' into ClusterSupport
shacharPash Sep 10, 2023
f5cd46d
change to OSSCluster
shacharPash Sep 13, 2023
64e9da1
isOSSCluster
shacharPash Sep 13, 2023
3476abb
update skip reason to OSS cluster
shacharPash Sep 13, 2023
74b0c13
general dispose
shacharPash Sep 13, 2023
611f0e8
general dispose for the rest of the disposes
shacharPash Sep 13, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/cluster.env
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
REDIS_CLUSTER=127.0.0.1:16379
NUM_REDIS_CLUSTER_NODES=6
30 changes: 30 additions & 0 deletions .github/docker-compose.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
---
version: "3.8"
services:

redis-stack-7.2.0-RC3:
image: redis/redis-stack-server:7.2.0-RC3
ports: ["6379:6379"]

redis-stack-6.2.6:
image: redis/redis-stack-server:6.2.6-v9
ports: ["6379:6379"]

redis-stack-edge:
image: redis/redis-stack-server:edge
ports: ["6379:6379"]

redis-stack-cluster:
container_name: redis-cluster
build:
context: .
dockerfile: dockers/Dockerfile.cluster
ports:
- 16379:16379
- 16380:16380
- 16381:16381
- 16382:16382
- 16383:16383
- 16384:16384
volumes:
- "./dockers/cluster.redis.conf:/redis.conf:ro"
7 changes: 7 additions & 0 deletions .github/dockers/Dockerfile.cluster
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
FROM redis/redis-stack-server:edge as rss

COPY dockers/create_cluster.sh /create_cluster.sh
RUN ls -R /opt/redis-stack
RUN chmod a+x /create_cluster.sh

ENTRYPOINT [ "/create_cluster.sh"]
8 changes: 8 additions & 0 deletions .github/dockers/cluster.redis.conf
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
protected-mode no
enable-debug-command yes
loadmodule /opt/redis-stack/lib/redisearch.so
loadmodule /opt/redis-stack/lib/redisgraph.so
loadmodule /opt/redis-stack/lib/redistimeseries.so
loadmodule /opt/redis-stack/lib/rejson.so
loadmodule /opt/redis-stack/lib/redisbloom.so
loadmodule /opt/redis-stack/lib/redisgears.so v8-plugin-path /opt/redis-stack/lib/libredisgears_v8_plugin.so
47 changes: 47 additions & 0 deletions .github/dockers/create_cluster.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
#! /bin/bash

mkdir -p /nodes
touch /nodes/nodemap
if [ -z ${START_PORT} ]; then
START_PORT=16379
fi
if [ -z ${END_PORT} ]; then
END_PORT=16384
fi
if [ ! -z "$3" ]; then
START_PORT=$2
START_PORT=$3
fi
echo "STARTING: ${START_PORT}"
echo "ENDING: ${END_PORT}"

for PORT in `seq ${START_PORT} ${END_PORT}`; do
mkdir -p /nodes/$PORT
if [[ -e /redis.conf ]]; then
cp /redis.conf /nodes/$PORT/redis.conf
else
touch /nodes/$PORT/redis.conf
fi
cat << EOF >> /nodes/$PORT/redis.conf
port ${PORT}
cluster-enabled yes
daemonize yes
logfile /redis.log
dir /nodes/$PORT
EOF

set -x
/opt/redis-stack/bin/redis-server /nodes/$PORT/redis.conf
sleep 1
if [ $? -ne 0 ]; then
echo "Redis failed to start, exiting."
continue
fi
echo 127.0.0.1:$PORT >> /nodes/nodemap
done
if [ -z "${REDIS_PASSWORD}" ]; then
echo yes | /opt/redis-stack/bin/redis-cli --cluster create `seq -f 127.0.0.1:%g ${START_PORT} ${END_PORT}` --cluster-replicas 1
else
echo yes | opt/redis-stack/bin/redis-cli -a ${REDIS_PASSWORD} --cluster create `seq -f 127.0.0.1:%g ${START_PORT} ${END_PORT}` --cluster-replicas 1
fi
tail -f /redis.log
1 change: 1 addition & 0 deletions .github/standalone.env
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
REDIS=localhost:6379
98 changes: 51 additions & 47 deletions .github/workflows/integration.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,61 +12,65 @@ on:
- cron: "0 1 * * *"

jobs:
build_and_Test:
name: Test [redis-stack ${{matrix.redis-stack-version}}]
runs-on: ubuntu-latest
dotnet_6_cluster:
name: .NET 6 on [redis-stack cluster]
uses: ./.github/workflows/reusable.yml
with:
redis_stack_type: cluster
dotnet_version: 6.0.x
clr_version: net6.0
dotenv_file: .github/cluster.env
secrets: inherit

dotnet_6:
name: .NET 6 on [redis-stack ${{matrix.redis-stack-type}}]
uses: ./.github/workflows/reusable.yml
strategy:
fail-fast: false
max-parallel: 5
matrix:
redis-stack-version: ["6.2.6-v9", "7.2.0-RC3", "edge"]
env:
USER_NAME: ${{ secrets.USER_NAME }}
PASSWORD: ${{ secrets.PASSWORD }}
ENDPOINT: ${{ secrets.ENDPOINT }}
REDIS_VERSION: ${{ matrix.redis-stack-version }}
steps:
- uses: actions/checkout@v3
- name: .NET Core 6
uses: actions/setup-dotnet@v2
with:
dotnet-version: '6.0.x'
- name: .NET Core 7
uses: actions/setup-dotnet@v2
with:
dotnet-version: '7.0.x'
- name: run redis-stack-server docker
run: docker run -p 6379:6379 -d redis/redis-stack-server:${{matrix.redis-stack-version}}
- name: Restore dependencies
run: dotnet restore
- name: Build
run: dotnet build --no-restore /p:ContinuousIntegrationBuild=true
- name: Test
run: |
echo "${{secrets.REDIS_CA_PEM}}" > tests/NRedisStack.Tests/bin/Debug/net6.0/redis_ca.pem
echo "${{secrets.REDIS_USER_CRT}}" > tests/NRedisStack.Tests/bin/Debug/net6.0/redis_user.crt
echo "${{secrets.REDIS_USER_PRIVATE_KEY}}" > tests/NRedisStack.Tests/bin/Debug/net6.0/redis_user_private.key
ls -R
dotnet test -f net6.0 --no-build --verbosity normal /p:CollectCoverage=true /p:CoverletOutputFormat=opencover
- name: Test
run: |
echo "${{secrets.REDIS_CA_PEM}}" > tests/NRedisStack.Tests/bin/Debug/net7.0/redis_ca.pem
echo "${{secrets.REDIS_USER_CRT}}" > tests/NRedisStack.Tests/bin/Debug/net7.0/redis_user.crt
echo "${{secrets.REDIS_USER_PRIVATE_KEY}}" > tests/NRedisStack.Tests/bin/Debug/net7.0/redis_user_private.key
ls -R
dotnet test -f net7.0 --no-build --verbosity normal /p:CollectCoverage=true /p:CoverletOutputFormat=opencover
- name: Codecov
uses: codecov/codecov-action@v3
with:
token: ${{secrets.CODECOV_TOKEN}}
verbose: true
- name: Build
run: dotnet pack -c Release
redis-stack-type: ['edge', '6.2.6', '7.2.0-RC3']
with:
redis_stack_type: ${{matrix.redis-stack-type}}
dotnet_version: 6.0.x
clr_version: net6.0
dotenv_file: .github/standalone.env
secrets: inherit

dotnet_7_cluster:
name: .NET 7 on [redis-stack cluster]
uses: ./.github/workflows/reusable.yml
with:
redis_stack_type: cluster
dotnet_version: 7.0.x
clr_version: net7.0
dotenv_file: .github/cluster.env
secrets: inherit

dotnet_7:
name: .NET 7 on [redis-stack ${{matrix.redis-stack-type}}]
uses: ./.github/workflows/reusable.yml
strategy:
fail-fast: false
max-parallel: 5
matrix:
redis-stack-type: ['edge', '6.2.6', '7.2.0-RC3']
with:
redis_stack_type: ${{matrix.redis-stack-type}}
dotnet_version: 7.0.x
clr_version: net7.0
dotenv_file: .github/standalone.env
secrets: inherit

build_and_test_windows:
name: Windows Test ${{matrix.redis-stack-version}}
runs-on: windows-latest
strategy:
fail-fast: false
matrix:
redis-stack-version: ['6.2.6-v9', '7.2.0-RC3']
env:
redis_stack_version: 6.2.6-v7
redis_stack_version: ${{matrix.redis-stack-version}}
USER_NAME: ${{ secrets.USER_NAME }}
PASSWORD: ${{ secrets.PASSWORD }}
ENDPOINT: ${{ secrets.ENDPOINT }}
Expand Down
72 changes: 72 additions & 0 deletions .github/workflows/reusable.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
name: Build and Test
on:
workflow_call:
inputs:

redis_stack_type:
required: true
type: string

dotnet_version:
required: true
type: string

clr_version:
required: true
type: string

dotenv_file:
required: true
type: string
jobs:

build_and_test:
name: Test
runs-on: ubuntu-latest

env:
USER_NAME: ${{ secrets.USER_NAME }}
PASSWORD: ${{ secrets.PASSWORD }}
ENDPOINT: ${{ secrets.ENDPOINT }}
steps:

- uses: actions/checkout@v3

- name: .NET Core 6
uses: actions/setup-dotnet@v2
with:
dotnet-version: '6.0.x'

- name: .NET Core 7
uses: actions/setup-dotnet@v2
with:
dotnet-version: '7.0.x'

- name: run redis-stack-server docker
working-directory: .github
run: docker-compose up -d redis-stack-${{inputs.redis_stack_type}}

- name: set variables in dotenv
uses: c-py/action-dotenv-to-setenv@v2
with:
env-file: ${{inputs.dotenv_file}}

- name: Restore dependencies
run: dotnet restore
- name: Build
run: dotnet build --no-restore /p:ContinuousIntegrationBuild=true
- name: Test
run: |
echo "${{secrets.REDIS_CA_PEM}}" > tests/NRedisStack.Tests/bin/Debug/net6.0/redis_ca.pem
echo "${{secrets.REDIS_USER_CRT}}" > tests/NRedisStack.Tests/bin/Debug/net6.0/redis_user.crt
echo "${{secrets.REDIS_USER_PRIVATE_KEY}}" > tests/NRedisStack.Tests/bin/Debug/net6.0/redis_user_private.key
ls -R
dotnet test -f ${{inputs.clr_version}} --no-build --verbosity normal /p:CollectCoverage=true /p:CoverletOutputFormat=opencover
- name: Codecov
uses: codecov/codecov-action@v3
with:
token: ${{secrets.CODECOV_TOKEN}}
verbose: true
- name: Build
run: dotnet pack -c Release

9 changes: 9 additions & 0 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,15 @@ e.g:
```bash
dotnet test --environment="REDIS=172.17.0.1:6379"
```

To run your tests against an oss cluster:
```bash
dotnet test --environment "REDIS_CLUSTER=<redisServer:port>" --environment "NUM_REDIS_CLUSTER_NODES=<number of nodes in the cluster>"
```
e.g. :
```bash
dotnet test --environment "REDIS_CLUSTER=127.0.0.1:16379" --environment "NUM_REDIS_CLUSTER_NODES=6"
```
## How to Report a Bug

### Security Vulnerabilities
Expand Down
48 changes: 48 additions & 0 deletions src/NRedisStack/Auxiliary.cs
Original file line number Diff line number Diff line change
Expand Up @@ -35,5 +35,53 @@ public async static Task<RedisResult> ExecuteAsync(this IDatabaseAsync db, Seria
{
return await db.ExecuteAsync(command.Command, command.Args);
}

public static List<RedisResult> ExecuteBroadcast(this IDatabaseAsync db, string command)
=> db.ExecuteBroadcast(new SerializedCommand(command));

public static List<RedisResult> ExecuteBroadcast(this IDatabaseAsync db, SerializedCommand command)
{
var redis = db.Multiplexer;
var endpoints = redis.GetEndPoints();
var results = new List<RedisResult>(endpoints.Length);

foreach (var endPoint in endpoints)
{
var server = redis.GetServer(endPoint);

if (server.IsReplica)
{
continue; // Skip replica nodes
}
// Send your command to the master node

results.Add(server.Multiplexer.GetDatabase().Execute(command));
}
return results;
}

public async static Task<List<RedisResult>> ExecuteBroadcastAsync(this IDatabaseAsync db, string command)
=> await db.ExecuteBroadcastAsync(new SerializedCommand(command));

public async static Task<List<RedisResult>> ExecuteBroadcastAsync(this IDatabaseAsync db, SerializedCommand command)
{
var redis = db.Multiplexer;
var endpoints = redis.GetEndPoints();
var results = new List<RedisResult>(endpoints.Length);

foreach (var endPoint in endpoints)
{
var server = redis.GetServer(endPoint);

if (server.IsReplica)
{
continue; // Skip replica nodes
}
// Send your command to the master node

results.Add(await server.Multiplexer.GetDatabase().ExecuteAsync(command));
}
return results;
}
}
}
1 change: 1 addition & 0 deletions tests/Doc/Doc.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -20,5 +20,6 @@
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\src\NRedisStack\NRedisStack.csproj" />
<ProjectReference Include="..\..\tests\NRedisStack.Tests\NRedisStack.Tests.csproj" />
</ItemGroup>
</Project>
3 changes: 2 additions & 1 deletion tests/Doc/HashExample.cs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
// EXAMPLE: hash_tutorial
// HIDE_START
using NRedisStack.Tests;
using StackExchange.Redis;

//REMOVE_START
Expand All @@ -8,7 +9,7 @@ namespace NRedisStack.Doc;
//REMOVE_END
public class HashExample
{
[Fact]
[SkipIfRedis(Is.OSSCluster)]
public void run()
{
var muxer = ConnectionMultiplexer.Connect("localhost:6379");
Expand Down
Loading