Skip to content

Commit dac072f

Browse files
feat: HTTP REST API: Filter support v2 (#1890)
Filter v2 rest api support implemented Filter rest api documentation updated with v1 and v2 interface support. Separated legacy filter rest interface Fix code and tests of v2 Filter rest api Filter v2 message push test added Applied autoshard to Filter V2 Redesigned FilterPushHandling, code style, catch up apps and tests with filter v2 interface changes Rename of FilterV1SubscriptionsRequest to FilterLegacySubscribeRequest, fix broken chat2 app, fix tests Changed Filter v2 push handler subscription to simple register Separate node's filterUnsubscribe and filterUnsubscribeAll
1 parent 9085b1b commit dac072f

32 files changed

+1895
-366
lines changed

apps/chat2/chat2.nim

+12-6
Original file line numberDiff line numberDiff line change
@@ -262,10 +262,12 @@ proc writeAndPrint(c: Chat) {.async.} =
262262
echo "You are now known as " & c.nick
263263

264264
elif line.startsWith("/exit"):
265-
if not c.node.wakuFilter.isNil():
265+
if not c.node.wakuFilterLegacy.isNil():
266266
echo "unsubscribing from content filters..."
267267

268-
await c.node.unsubscribe(pubsubTopic=some(DefaultPubsubTopic), contentTopics=c.contentTopic)
268+
let peerOpt = c.node.peerManager.selectPeer(WakuLegacyFilterCodec)
269+
if peerOpt.isSome():
270+
await c.node.legacyFilterUnsubscribe(pubsubTopic=some(DefaultPubsubTopic), contentTopics=c.contentTopic, peer=peerOpt.get())
269271

270272
echo "quitting..."
271273

@@ -464,14 +466,18 @@ proc processInput(rfd: AsyncFD, rng: ref HmacDrbgContext) {.async.} =
464466
if peerInfo.isOk():
465467
await node.mountFilter()
466468
await node.mountFilterClient()
467-
node.peerManager.addServicePeer(peerInfo.value, WakuFilterCodec)
469+
node.peerManager.addServicePeer(peerInfo.value, WakuLegacyFilterCodec)
468470

469471
proc filterHandler(pubsubTopic: PubsubTopic, msg: WakuMessage) {.async, gcsafe, closure.} =
470472
trace "Hit filter handler", contentTopic=msg.contentTopic
471473
chat.printReceivedMessage(msg)
472474

473-
await node.subscribe(pubsubTopic=some(DefaultPubsubTopic), contentTopics=chat.contentTopic, filterHandler)
474-
475+
await node.legacyFilterSubscribe(pubsubTopic=some(DefaultPubsubTopic),
476+
contentTopics=chat.contentTopic,
477+
filterHandler,
478+
peerInfo.value)
479+
# TODO: Here to support FilterV2 relevant subscription, but still
480+
# Legacy Filter is concurrent to V2 untill legacy filter will be removed
475481
else:
476482
error "Filter not mounted. Couldn't parse conf.filternode",
477483
error = peerInfo.error
@@ -485,7 +491,7 @@ proc processInput(rfd: AsyncFD, rng: ref HmacDrbgContext) {.async.} =
485491
chat.printReceivedMessage(msg)
486492

487493
let topic = DefaultPubsubTopic
488-
await node.subscribe(some(topic), @[ContentTopic("")], handler)
494+
node.subscribe(topic, handler)
489495

490496
if conf.rlnRelay:
491497
info "WakuRLNRelay is enabled"

apps/chat2bridge/chat2bridge.nim

+3-1
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import
1818
../../../waku/waku_node,
1919
../../../waku/node/peer_manager,
2020
../../waku/waku_filter,
21+
../../waku/waku_filter_v2,
2122
../../waku/waku_store,
2223
# Chat 2 imports
2324
../chat2/chat2,
@@ -297,7 +298,8 @@ when isMainModule:
297298
if conf.filternode != "":
298299
let filterPeer = parsePeerInfo(conf.filternode)
299300
if filterPeer.isOk():
300-
bridge.nodev2.peerManager.addServicePeer(filterPeer.value, WakuFilterCodec)
301+
bridge.nodev2.peerManager.addServicePeer(filterPeer.value, WakuLegacyFilterCodec)
302+
bridge.nodev2.peerManager.addServicePeer(filterPeer.value, WakuFilterSubscribeCodec)
301303
else:
302304
error "Error parsing conf.filternode", error = filterPeer.error
303305

apps/wakunode2/app.nim

+11-4
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,8 @@ import
3737
../../waku/waku_store,
3838
../../waku/waku_lightpush,
3939
../../waku/waku_filter,
40+
../../waku/waku_filter_v2,
41+
../../waku/waku_filter_v2/client as waku_filter_client,
4042
./wakunode2_validator_signed,
4143
./internal_config,
4244
./external_config
@@ -46,6 +48,7 @@ import
4648
../../waku/node/rest/debug/handlers as rest_debug_api,
4749
../../waku/node/rest/relay/handlers as rest_relay_api,
4850
../../waku/node/rest/relay/topic_cache,
51+
../../waku/node/rest/filter/legacy_handlers as rest_legacy_filter_api,
4952
../../waku/node/rest/filter/handlers as rest_filter_api,
5053
../../waku/node/rest/store/handlers as rest_store_api,
5154
../../waku/node/rest/health/handlers as rest_health_api,
@@ -470,8 +473,9 @@ proc setupProtocols(node: WakuNode,
470473
if conf.filternode != "":
471474
let filterNode = parsePeerInfo(conf.filternode)
472475
if filterNode.isOk():
473-
await mountFilterClient(node)
474-
node.peerManager.addServicePeer(filterNode.value, WakuFilterCodec)
476+
await node.mountFilterClient()
477+
node.peerManager.addServicePeer(filterNode.value, WakuLegacyFilterCodec)
478+
node.peerManager.addServicePeer(filterNode.value, WakuFilterSubscribeCodec)
475479
else:
476480
return err("failed to set node waku filter peer: " & filterNode.error)
477481

@@ -577,8 +581,11 @@ proc startRestServer(app: App, address: ValidIpAddress, port: Port, conf: WakuNo
577581

578582
## Filter REST API
579583
if conf.filter:
580-
let filterCache = rest_filter_api.MessageCache.init(capacity=rest_filter_api.filterMessageCacheDefaultCapacity)
581-
installFilterApiHandlers(server.router, app.node, filterCache)
584+
let legacyFilterCache = rest_legacy_filter_api.MessageCache.init()
585+
rest_legacy_filter_api.installLegacyFilterRestApiHandlers(server.router, app.node, legacyFilterCache)
586+
587+
let filterCache = rest_filter_api.MessageCache.init()
588+
rest_filter_api.installFilterRestApiHandlers(server.router, app.node, filterCache)
582589

583590
## Store REST API
584591
installStoreApiHandlers(server.router, app.node)

examples/filter_subscriber.nim

+6-2
Original file line numberDiff line numberDiff line change
@@ -28,13 +28,15 @@ proc unsubscribe(wfc: WakuFilterClient,
2828
else:
2929
notice "unsubscribe request successful"
3030

31-
proc messagePushHandler(pubsubTopic: PubsubTopic, message: WakuMessage) =
31+
proc messagePushHandler(pubsubTopic: PubsubTopic, message: WakuMessage)
32+
{.async, gcsafe.} =
3233
let payloadStr = string.fromBytes(message.payload)
3334
notice "message received", payload=payloadStr,
3435
pubsubTopic=pubsubTopic,
3536
contentTopic=message.contentTopic,
3637
timestamp=message.timestamp
3738

39+
3840
proc maintainSubscription(wfc: WakuFilterClient,
3941
filterPeer: RemotePeerInfo,
4042
filterPubsubTopic: PubsubTopic,
@@ -68,11 +70,13 @@ proc setupAndSubscribe(rng: ref HmacDrbgContext) =
6870
var
6971
switch = newStandardSwitch()
7072
pm = PeerManager.new(switch)
71-
wfc = WakuFilterClient.new(rng, messagePushHandler, pm)
73+
wfc = WakuFilterClient.new(pm, rng)
7274

7375
# Mount filter client protocol
7476
switch.mount(wfc)
7577

78+
wfc.registerPushHandler(messagePushHandler)
79+
7680
# Start maintaining subscription
7781
asyncSpawn maintainSubscription(wfc, filterPeer, FilterPubsubTopic, FilterContentTopic)
7882

tests/all_tests_waku.nim

+5-3
Original file line numberDiff line numberDiff line change
@@ -58,8 +58,8 @@ import
5858
./test_waku_lightpush,
5959
./test_wakunode_lightpush,
6060
# Waku Filter
61-
./test_waku_filter,
62-
./test_wakunode_filter,
61+
./test_waku_filter_legacy,
62+
./test_wakunode_filter_legacy,
6363
./test_waku_peer_exchange,
6464
./test_peer_store_extended,
6565
./test_message_cache,
@@ -95,7 +95,9 @@ import
9595
./wakunode_rest/test_rest_relay,
9696
./wakunode_rest/test_rest_relay_serdes,
9797
./wakunode_rest/test_rest_serdes,
98-
./wakunode_rest/test_rest_store
98+
./wakunode_rest/test_rest_store,
99+
./wakunode_rest/test_rest_filter,
100+
./wakunode_rest/test_rest_legacy_filter
99101

100102
import
101103
./waku_rln_relay/test_waku_rln_relay,

tests/test_peer_manager.nim

+28-28
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ procSuite "Peer Manager":
5252
await allFutures(nodes.mapIt(it.mountFilter()))
5353

5454
# Dial node2 from node1
55-
let conn = await nodes[0].peerManager.dialPeer(nodes[1].peerInfo.toRemotePeerInfo(), WakuFilterCodec)
55+
let conn = await nodes[0].peerManager.dialPeer(nodes[1].peerInfo.toRemotePeerInfo(), WakuLegacyFilterCodec)
5656
# Check connection
5757
check:
5858
conn.isSome()
@@ -81,12 +81,12 @@ procSuite "Peer Manager":
8181
let nonExistentPeer = nonExistentPeerRes.value
8282

8383
# Dial non-existent peer from node1
84-
let conn1 = await nodes[0].peerManager.dialPeer(nonExistentPeer, WakuFilterCodec)
84+
let conn1 = await nodes[0].peerManager.dialPeer(nonExistentPeer, WakuLegacyFilterCodec)
8585
check:
8686
conn1.isNone()
8787

8888
# Dial peer not supporting given protocol
89-
let conn2 = await nodes[0].peerManager.dialPeer(nodes[1].peerInfo.toRemotePeerInfo(), WakuFilterCodec)
89+
let conn2 = await nodes[0].peerManager.dialPeer(nodes[1].peerInfo.toRemotePeerInfo(), WakuLegacyFilterCodec)
9090
check:
9191
conn2.isNone()
9292

@@ -109,14 +109,14 @@ procSuite "Peer Manager":
109109
node.mountStoreClient()
110110

111111
node.peerManager.addServicePeer(storePeer.toRemotePeerInfo(), WakuStoreCodec)
112-
node.peerManager.addServicePeer(filterPeer.toRemotePeerInfo(), WakuFilterCodec)
112+
node.peerManager.addServicePeer(filterPeer.toRemotePeerInfo(), WakuLegacyFilterCodec)
113113

114114
# Check peers were successfully added to peer manager
115115
check:
116116
node.peerManager.peerStore.peers().len == 2
117-
node.peerManager.peerStore.peers(WakuFilterCodec).allIt(it.peerId == filterPeer.peerId and
117+
node.peerManager.peerStore.peers(WakuLegacyFilterCodec).allIt(it.peerId == filterPeer.peerId and
118118
it.addrs.contains(filterLoc) and
119-
it.protocols.contains(WakuFilterCodec))
119+
it.protocols.contains(WakuLegacyFilterCodec))
120120
node.peerManager.peerStore.peers(WakuStoreCodec).allIt(it.peerId == storePeer.peerId and
121121
it.addrs.contains(storeLoc) and
122122
it.protocols.contains(WakuStoreCodec))
@@ -429,7 +429,7 @@ procSuite "Peer Manager":
429429

430430
# service peers
431431
node.peerManager.addServicePeer(peers[0], WakuStoreCodec)
432-
node.peerManager.addServicePeer(peers[1], WakuFilterCodec)
432+
node.peerManager.addServicePeer(peers[1], WakuLegacyFilterCodec)
433433
node.peerManager.addServicePeer(peers[2], WakuLightPushCodec)
434434
node.peerManager.addServicePeer(peers[3], WakuPeerExchangeCodec)
435435

@@ -449,7 +449,7 @@ procSuite "Peer Manager":
449449
# all service peers are added to its service slot
450450
check:
451451
node.peerManager.serviceSlots[WakuStoreCodec].peerId == peers[0].peerId
452-
node.peerManager.serviceSlots[WakuFilterCodec].peerId == peers[1].peerId
452+
node.peerManager.serviceSlots[WakuLegacyFilterCodec].peerId == peers[1].peerId
453453
node.peerManager.serviceSlots[WakuLightPushCodec].peerId == peers[2].peerId
454454
node.peerManager.serviceSlots[WakuPeerExchangeCodec].peerId == peers[3].peerId
455455

@@ -474,38 +474,38 @@ procSuite "Peer Manager":
474474
(await nodes[0].peerManager.connectRelay(pInfos[2])) == true
475475
(await nodes[1].peerManager.connectRelay(pInfos[2])) == true
476476

477-
(await nodes[0].peerManager.dialPeer(pInfos[1], WakuFilterCodec)).isSome() == true
478-
(await nodes[0].peerManager.dialPeer(pInfos[2], WakuFilterCodec)).isSome() == true
477+
(await nodes[0].peerManager.dialPeer(pInfos[1], WakuLegacyFilterCodec)).isSome() == true
478+
(await nodes[0].peerManager.dialPeer(pInfos[2], WakuLegacyFilterCodec)).isSome() == true
479479

480480
# isolated dial creates a relay conn under the hood (libp2p behaviour)
481-
(await nodes[2].peerManager.dialPeer(pInfos[3], WakuFilterCodec)).isSome() == true
481+
(await nodes[2].peerManager.dialPeer(pInfos[3], WakuLegacyFilterCodec)).isSome() == true
482482

483483

484484
# assert physical connections
485485
check:
486486
nodes[0].peerManager.connectedPeers(WakuRelayCodec)[0].len == 0
487487
nodes[0].peerManager.connectedPeers(WakuRelayCodec)[1].len == 2
488488

489-
nodes[0].peerManager.connectedPeers(WakuFilterCodec)[0].len == 0
490-
nodes[0].peerManager.connectedPeers(WakuFilterCodec)[1].len == 2
489+
nodes[0].peerManager.connectedPeers(WakuLegacyFilterCodec)[0].len == 0
490+
nodes[0].peerManager.connectedPeers(WakuLegacyFilterCodec)[1].len == 2
491491

492492
nodes[1].peerManager.connectedPeers(WakuRelayCodec)[0].len == 1
493493
nodes[1].peerManager.connectedPeers(WakuRelayCodec)[1].len == 1
494494

495-
nodes[1].peerManager.connectedPeers(WakuFilterCodec)[0].len == 1
496-
nodes[1].peerManager.connectedPeers(WakuFilterCodec)[1].len == 0
495+
nodes[1].peerManager.connectedPeers(WakuLegacyFilterCodec)[0].len == 1
496+
nodes[1].peerManager.connectedPeers(WakuLegacyFilterCodec)[1].len == 0
497497

498498
nodes[2].peerManager.connectedPeers(WakuRelayCodec)[0].len == 2
499499
nodes[2].peerManager.connectedPeers(WakuRelayCodec)[1].len == 1
500500

501-
nodes[2].peerManager.connectedPeers(WakuFilterCodec)[0].len == 1
502-
nodes[2].peerManager.connectedPeers(WakuFilterCodec)[1].len == 1
501+
nodes[2].peerManager.connectedPeers(WakuLegacyFilterCodec)[0].len == 1
502+
nodes[2].peerManager.connectedPeers(WakuLegacyFilterCodec)[1].len == 1
503503

504504
nodes[3].peerManager.connectedPeers(WakuRelayCodec)[0].len == 1
505505
nodes[3].peerManager.connectedPeers(WakuRelayCodec)[1].len == 0
506506

507-
nodes[3].peerManager.connectedPeers(WakuFilterCodec)[0].len == 1
508-
nodes[3].peerManager.connectedPeers(WakuFilterCodec)[1].len == 0
507+
nodes[3].peerManager.connectedPeers(WakuLegacyFilterCodec)[0].len == 1
508+
nodes[3].peerManager.connectedPeers(WakuLegacyFilterCodec)[1].len == 0
509509

510510
asyncTest "getNumStreams() returns expected number of connections per protocol":
511511
# Create 2 nodes
@@ -521,17 +521,17 @@ procSuite "Peer Manager":
521521
require:
522522
# multiple streams are multiplexed over a single connection.
523523
# note that a relay connection is created under the hood when dialing a peer (libp2p behaviour)
524-
(await nodes[0].peerManager.dialPeer(pInfos[1], WakuFilterCodec)).isSome() == true
525-
(await nodes[0].peerManager.dialPeer(pInfos[1], WakuFilterCodec)).isSome() == true
526-
(await nodes[0].peerManager.dialPeer(pInfos[1], WakuFilterCodec)).isSome() == true
527-
(await nodes[0].peerManager.dialPeer(pInfos[1], WakuFilterCodec)).isSome() == true
524+
(await nodes[0].peerManager.dialPeer(pInfos[1], WakuLegacyFilterCodec)).isSome() == true
525+
(await nodes[0].peerManager.dialPeer(pInfos[1], WakuLegacyFilterCodec)).isSome() == true
526+
(await nodes[0].peerManager.dialPeer(pInfos[1], WakuLegacyFilterCodec)).isSome() == true
527+
(await nodes[0].peerManager.dialPeer(pInfos[1], WakuLegacyFilterCodec)).isSome() == true
528528

529529
check:
530530
nodes[0].peerManager.getNumStreams(WakuRelayCodec) == (1, 1)
531-
nodes[0].peerManager.getNumStreams(WakuFilterCodec) == (0, 4)
531+
nodes[0].peerManager.getNumStreams(WakuLegacyFilterCodec) == (0, 4)
532532

533533
nodes[1].peerManager.getNumStreams(WakuRelayCodec) == (1, 1)
534-
nodes[1].peerManager.getNumStreams(WakuFilterCodec) == (4, 0)
534+
nodes[1].peerManager.getNumStreams(WakuLegacyFilterCodec) == (4, 0)
535535

536536
test "selectPeer() returns the correct peer":
537537
# Valid peer id missing the last digit
@@ -552,7 +552,7 @@ procSuite "Peer Manager":
552552

553553
# Add a peer[0] to the peerstore
554554
pm.peerStore[AddressBook][peers[0].peerId] = peers[0].addrs
555-
pm.peerStore[ProtoBook][peers[0].peerId] = @[WakuRelayCodec, WakuStoreCodec, WakuFilterCodec]
555+
pm.peerStore[ProtoBook][peers[0].peerId] = @[WakuRelayCodec, WakuStoreCodec, WakuLegacyFilterCodec]
556556

557557
# When no service peers, we get one from the peerstore
558558
let selectedPeer1 = pm.selectPeer(WakuStoreCodec)
@@ -561,7 +561,7 @@ procSuite "Peer Manager":
561561
selectedPeer1.get().peerId == peers[0].peerId
562562

563563
# Same for other protocol
564-
let selectedPeer2 = pm.selectPeer(WakuFilterCodec)
564+
let selectedPeer2 = pm.selectPeer(WakuLegacyFilterCodec)
565565
check:
566566
selectedPeer2.isSome() == true
567567
selectedPeer2.get().peerId == peers[0].peerId
@@ -757,7 +757,7 @@ procSuite "Peer Manager":
757757
discard await nodes[0].peerManager.connectRelay(pInfos[3])
758758
discard await nodes[0].peerManager.connectRelay(pInfos[4])
759759

760-
# they are also prunned 
760+
# they are also prunned
761761
check nodes[0].peerManager.switch.connManager.getConnections().len == 1
762762

763763
# we should have 4 peers (2in/2out) but due to collocation limit
File renamed without changes.

tests/test_wakunode_filter.nim tests/test_wakunode_filter_legacy.nim

+1-1
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ suite "WakuNode - Filter":
4444
filterPushHandlerFut.complete((pubsubTopic, msg))
4545

4646
## When
47-
await client.filterSubscribe(some(pubsubTopic), contentTopic, filterPushHandler, peer=serverPeerInfo)
47+
await client.legacyFilterSubscribe(some(pubsubTopic), contentTopic, filterPushHandler, peer=serverPeerInfo)
4848

4949
# Wait for subscription to take effect
5050
waitFor sleepAsync(100.millis)

tests/waku_filter_v2/client_utils.nim

+2-3
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
import
22
std/[options,tables],
3-
testutils/unittests,
43
chronos,
54
chronicles
65

@@ -22,10 +21,10 @@ proc newTestWakuFilter*(switch: Switch): Future[WakuFilter] {.async.} =
2221

2322
return proto
2423

25-
proc newTestWakuFilterClient*(switch: Switch, messagePushHandler: MessagePushHandler): Future[WakuFilterClient] {.async.} =
24+
proc newTestWakuFilterClient*(switch: Switch): Future[WakuFilterClient] {.async.} =
2625
let
2726
peerManager = PeerManager.new(switch)
28-
proto = WakuFilterClient.new(rng, messagePushHandler, peerManager)
27+
proto = WakuFilterClient.new(peerManager, rng)
2928

3029
await proto.start()
3130
switch.mount(proto)

0 commit comments

Comments
 (0)