Skip to content

Commit 98abcd5

Browse files
authored
Firestore: Optimize local cache sync when resuming a query that had docs deleted (#7229)
1 parent 41f06be commit 98abcd5

File tree

60 files changed

+2110
-95
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

60 files changed

+2110
-95
lines changed

.changeset/swift-eels-change.md

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
---
2+
'@firebase/webchannel-wrapper': minor
3+
'@firebase/firestore': minor
4+
'firebase': minor
5+
---
6+
7+
Implemented an optimization in the local cache synchronization logic that reduces the number of billed document reads when documents were deleted on the server while the client was not actively listening to the query (e.g. while the client was offline).

packages/firestore/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
"dev": "rollup -c -w",
2121
"lint": "eslint -c .eslintrc.js '**/*.ts' --ignore-path '../../.gitignore'",
2222
"lint:fix": "eslint --fix -c .eslintrc.js '**/*.ts' --ignore-path '../../.gitignore'",
23-
"prettier": "prettier --write '*.js' '@(lite|src|test)/**/*.ts'",
23+
"prettier": "prettier --write '*.js' '@(lite|src|test)/**/*.ts' 'test/unit/remote/bloom_filter_golden_test_data/*.json'",
2424
"test:lite": "ts-node ./scripts/run-tests.ts --emulator --platform node_lite --main=lite/index.ts 'test/lite/**/*.test.ts'",
2525
"test:lite:prod": "ts-node ./scripts/run-tests.ts --platform node_lite --main=lite/index.ts 'test/lite/**/*.test.ts'",
2626
"test:lite:browser": "karma start --single-run --lite",

packages/firestore/src/core/sync_engine_impl.ts

+7-5
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,6 @@ import { primitiveComparator } from '../util/misc';
7171
import { ObjectMap } from '../util/obj_map';
7272
import { Deferred } from '../util/promise';
7373
import { SortedMap } from '../util/sorted_map';
74-
import { SortedSet } from '../util/sorted_set';
7574
import { BATCHID_UNKNOWN } from '../util/types';
7675

7776
import {
@@ -316,9 +315,6 @@ export async function syncEngineListen(
316315
syncEngineImpl.localStore,
317316
queryToTarget(query)
318317
);
319-
if (syncEngineImpl.isPrimaryClient) {
320-
remoteStoreListen(syncEngineImpl.remoteStore, targetData);
321-
}
322318

323319
const status = syncEngineImpl.sharedClientState.addLocalQueryTarget(
324320
targetData.targetId
@@ -331,6 +327,10 @@ export async function syncEngineListen(
331327
status === 'current',
332328
targetData.resumeToken
333329
);
330+
331+
if (syncEngineImpl.isPrimaryClient) {
332+
remoteStoreListen(syncEngineImpl.remoteStore, targetData);
333+
}
334334
}
335335

336336
return viewSnapshot;
@@ -638,7 +638,9 @@ export async function syncEngineRejectListen(
638638
const event = new RemoteEvent(
639639
SnapshotVersion.min(),
640640
/* targetChanges= */ new Map<TargetId, TargetChange>(),
641-
/* targetMismatches= */ new SortedSet<TargetId>(primitiveComparator),
641+
/* targetMismatches= */ new SortedMap<TargetId, TargetPurpose>(
642+
primitiveComparator
643+
),
642644
documentUpdates,
643645
resolvedLimboDocuments
644646
);

packages/firestore/src/local/local_store_impl.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -601,7 +601,7 @@ export function localStoreApplyRemoteEventToLocalCache(
601601
let newTargetData = oldTargetData.withSequenceNumber(
602602
txn.currentSequenceNumber
603603
);
604-
if (remoteEvent.targetMismatches.has(targetId)) {
604+
if (remoteEvent.targetMismatches.get(targetId) !== null) {
605605
newTargetData = newTargetData
606606
.withResumeToken(
607607
ByteString.EMPTY_BYTE_STRING,

packages/firestore/src/local/target_data.ts

+37-5
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,17 @@ export const enum TargetPurpose {
2626
Listen,
2727

2828
/**
29-
* The query target was used to refill a query after an existence filter mismatch.
29+
* The query target was used to refill a query after an existence filter
30+
* mismatch.
3031
*/
3132
ExistenceFilterMismatch,
3233

34+
/**
35+
* The query target was used if the query is the result of a false positive in
36+
* the bloom filter.
37+
*/
38+
ExistenceFilterMismatchBloom,
39+
3340
/** The query target was used to resolve a limbo document. */
3441
LimboResolution
3542
}
@@ -66,7 +73,13 @@ export class TargetData {
6673
* matches the target. The resume token essentially identifies a point in
6774
* time from which the server should resume sending results.
6875
*/
69-
readonly resumeToken: ByteString = ByteString.EMPTY_BYTE_STRING
76+
readonly resumeToken: ByteString = ByteString.EMPTY_BYTE_STRING,
77+
/**
78+
* The number of documents that last matched the query at the resume token or
79+
* read time. Documents are counted only when making a listen request with
80+
* resume token or read time, otherwise, keep it null.
81+
*/
82+
readonly expectedCount: number | null = null
7083
) {}
7184

7285
/** Creates a new target data instance with an updated sequence number. */
@@ -78,7 +91,8 @@ export class TargetData {
7891
sequenceNumber,
7992
this.snapshotVersion,
8093
this.lastLimboFreeSnapshotVersion,
81-
this.resumeToken
94+
this.resumeToken,
95+
this.expectedCount
8296
);
8397
}
8498

@@ -97,7 +111,24 @@ export class TargetData {
97111
this.sequenceNumber,
98112
snapshotVersion,
99113
this.lastLimboFreeSnapshotVersion,
100-
resumeToken
114+
resumeToken,
115+
/* expectedCount= */ null
116+
);
117+
}
118+
119+
/**
120+
* Creates a new target data instance with an updated expected count.
121+
*/
122+
withExpectedCount(expectedCount: number): TargetData {
123+
return new TargetData(
124+
this.target,
125+
this.targetId,
126+
this.purpose,
127+
this.sequenceNumber,
128+
this.snapshotVersion,
129+
this.lastLimboFreeSnapshotVersion,
130+
this.resumeToken,
131+
expectedCount
101132
);
102133
}
103134

@@ -115,7 +146,8 @@ export class TargetData {
115146
this.sequenceNumber,
116147
this.snapshotVersion,
117148
lastLimboFreeSnapshotVersion,
118-
this.resumeToken
149+
this.resumeToken,
150+
this.expectedCount
119151
);
120152
}
121153
}

packages/firestore/src/protos/firestore_proto_api.ts

+11
Original file line numberDiff line numberDiff line change
@@ -224,6 +224,15 @@ export declare namespace firestoreV1ApiClientInterfaces {
224224
interface ExistenceFilter {
225225
targetId?: number;
226226
count?: number;
227+
unchangedNames?: BloomFilter;
228+
}
229+
interface BloomFilter {
230+
bits?: BitSequence;
231+
hashCount?: number;
232+
}
233+
interface BitSequence {
234+
bitmap?: string | Uint8Array;
235+
padding?: number;
227236
}
228237
interface FieldFilter {
229238
field?: FieldReference;
@@ -390,6 +399,7 @@ export declare namespace firestoreV1ApiClientInterfaces {
390399
readTime?: Timestamp;
391400
targetId?: number;
392401
once?: boolean;
402+
expectedCount?: number | { value: number };
393403
}
394404
interface TargetChange {
395405
targetChangeType?: TargetChangeTargetChangeType;
@@ -454,6 +464,7 @@ export declare type BeginTransactionRequest =
454464
firestoreV1ApiClientInterfaces.BeginTransactionRequest;
455465
export declare type BeginTransactionResponse =
456466
firestoreV1ApiClientInterfaces.BeginTransactionResponse;
467+
export declare type BloomFilter = firestoreV1ApiClientInterfaces.BloomFilter;
457468
export declare type CollectionSelector =
458469
firestoreV1ApiClientInterfaces.CollectionSelector;
459470
export declare type CommitRequest =
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
// Copyright 2022 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
syntax = "proto3";
16+
17+
package google.firestore.v1;
18+
19+
option csharp_namespace = "Google.Cloud.Firestore.V1";
20+
option go_package = "google.golang.org/genproto/googleapis/firestore/v1;firestore";
21+
option java_multiple_files = true;
22+
option java_outer_classname = "BloomFilterProto";
23+
option java_package = "com.google.firestore.v1";
24+
option objc_class_prefix = "GCFS";
25+
option php_namespace = "Google\\Cloud\\Firestore\\V1";
26+
option ruby_package = "Google::Cloud::Firestore::V1";
27+
28+
// A sequence of bits, encoded in a byte array.
29+
//
30+
// Each byte in the `bitmap` byte array stores 8 bits of the sequence. The only
31+
// exception is the last byte, which may store 8 _or fewer_ bits. The `padding`
32+
// defines the number of bits of the last byte to be ignored as "padding". The
33+
// values of these "padding" bits are unspecified and must be ignored.
34+
//
35+
// To retrieve the first bit, bit 0, calculate: (bitmap[0] & 0x01) != 0.
36+
// To retrieve the second bit, bit 1, calculate: (bitmap[0] & 0x02) != 0.
37+
// To retrieve the third bit, bit 2, calculate: (bitmap[0] & 0x04) != 0.
38+
// To retrieve the fourth bit, bit 3, calculate: (bitmap[0] & 0x08) != 0.
39+
// To retrieve bit n, calculate: (bitmap[n / 8] & (0x01 << (n % 8))) != 0.
40+
//
41+
// The "size" of a `BitSequence` (the number of bits it contains) is calculated
42+
// by this formula: (bitmap.length * 8) - padding.
43+
message BitSequence {
44+
// The bytes that encode the bit sequence.
45+
// May have a length of zero.
46+
bytes bitmap = 1;
47+
48+
// The number of bits of the last byte in `bitmap` to ignore as "padding".
49+
// If the length of `bitmap` is zero, then this value must be 0.
50+
// Otherwise, this value must be between 0 and 7, inclusive.
51+
int32 padding = 2;
52+
}
53+
54+
// A bloom filter (https://en.wikipedia.org/wiki/Bloom_filter).
55+
//
56+
// The bloom filter hashes the entries with MD5 and treats the resulting 128-bit
57+
// hash as 2 distinct 64-bit hash values, interpreted as unsigned integers
58+
// using 2's complement encoding.
59+
//
60+
// These two hash values, named h1 and h2, are then used to compute the
61+
// `hash_count` hash values using the formula, starting at i=0:
62+
//
63+
// h(i) = h1 + (i * h2)
64+
//
65+
// These resulting values are then taken modulo the number of bits in the bloom
66+
// filter to get the bits of the bloom filter to test for the given entry.
67+
message BloomFilter {
68+
// The bloom filter data.
69+
BitSequence bits = 1;
70+
71+
// The number of hashes used by the algorithm.
72+
int32 hash_count = 2;
73+
}

packages/firestore/src/protos/google/firestore/v1/firestore.proto

+10
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import "google/firestore/v1/query.proto";
2626
import "google/firestore/v1/write.proto";
2727
import "google/protobuf/empty.proto";
2828
import "google/protobuf/timestamp.proto";
29+
import "google/protobuf/wrappers.proto";
2930
import "google/rpc/status.proto";
3031

3132
option csharp_namespace = "Google.Cloud.Firestore.V1";
@@ -857,6 +858,15 @@ message Target {
857858

858859
// If the target should be removed once it is current and consistent.
859860
bool once = 6;
861+
862+
// The number of documents that last matched the query at the resume token or
863+
// read time.
864+
//
865+
// This value is only relevant when a `resume_type` is provided. This value
866+
// being present and greater than zero signals that the client wants
867+
// `ExistenceFilter.unchanged_names` to be included in the response.
868+
//
869+
google.protobuf.Int32Value expected_count = 12;
860870
}
861871

862872
// Targets being watched have changed.

packages/firestore/src/protos/google/firestore/v1/write.proto

+15
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ syntax = "proto3";
1616

1717
package google.firestore.v1;
1818

19+
import "google/firestore/v1/bloom_filter.proto";
1920
import "google/firestore/v1/common.proto";
2021
import "google/firestore/v1/document.proto";
2122
import "google/protobuf/timestamp.proto";
@@ -261,4 +262,18 @@ message ExistenceFilter {
261262
// If different from the count of documents in the client that match, the
262263
// client must manually determine which documents no longer match the target.
263264
int32 count = 2;
265+
266+
// A bloom filter that contains the UTF-8 byte encodings of the resource names
267+
// of the documents that match [target_id][google.firestore.v1.ExistenceFilter.target_id], in the
268+
// form `projects/{project_id}/databases/{database_id}/documents/{document_path}`
269+
// that have NOT changed since the query results indicated by the resume token
270+
// or timestamp given in `Target.resume_type`.
271+
//
272+
// This bloom filter may be omitted at the server's discretion, such as if it
273+
// is deemed that the client will not make use of it or if it is too
274+
// computationally expensive to calculate or transmit. Clients must gracefully
275+
// handle this field being absent by falling back to the logic used before
276+
// this field existed; that is, re-add the target without a resume token to
277+
// figure out which documents in the client's cache are out of sync.
278+
BloomFilter unchanged_names = 3;
264279
}

packages/firestore/src/protos/protos.json

+32
Original file line numberDiff line numberDiff line change
@@ -928,6 +928,30 @@
928928
}
929929
}
930930
},
931+
"BitSequence": {
932+
"fields": {
933+
"bitmap": {
934+
"type": "bytes",
935+
"id": 1
936+
},
937+
"padding": {
938+
"type": "int32",
939+
"id": 2
940+
}
941+
}
942+
},
943+
"BloomFilter": {
944+
"fields": {
945+
"bits": {
946+
"type": "BitSequence",
947+
"id": 1
948+
},
949+
"hashCount": {
950+
"type": "int32",
951+
"id": 2
952+
}
953+
}
954+
},
931955
"DocumentMask": {
932956
"fields": {
933957
"fieldPaths": {
@@ -2052,6 +2076,10 @@
20522076
"once": {
20532077
"type": "bool",
20542078
"id": 6
2079+
},
2080+
"expectedCount": {
2081+
"type": "google.protobuf.Int32Value",
2082+
"id": 12
20552083
}
20562084
},
20572085
"nested": {
@@ -2660,6 +2688,10 @@
26602688
"count": {
26612689
"type": "int32",
26622690
"id": 2
2691+
},
2692+
"unchangedNames": {
2693+
"type": "BloomFilter",
2694+
"id": 3
26632695
}
26642696
}
26652697
}

0 commit comments

Comments
 (0)