Skip to content

Commit 69fa5c6

Browse files
gaobinlonggaobinlong
committed
merge main
Signed-off-by: Gao Binlong <[email protected]>
2 parents a981af4 + 6c17119 commit 69fa5c6

File tree

31 files changed

+2781
-436
lines changed

31 files changed

+2781
-436
lines changed

Diff for: ‎.github/benchmark-configs.json

+18
Original file line numberDiff line numberDiff line change
@@ -221,5 +221,23 @@
221221
"data_instance_config": "4vCPU, 32G Mem, 16G Heap"
222222
},
223223
"baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
224+
},
225+
"id_14": {
226+
"description": "Search only test-procedure for big5, uses snapshot to restore the data for OS-3.0.0. Enables range query approximation.",
227+
"supported_major_versions": ["3"],
228+
"cluster-benchmark-configs": {
229+
"SINGLE_NODE_CLUSTER": "true",
230+
"MIN_DISTRIBUTION": "true",
231+
"TEST_WORKLOAD": "big5",
232+
"ADDITIONAL_CONFIG": "opensearch.experimental.feature.approximate_point_range_query.enabled:true",
233+
"WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard_ordered\"}",
234+
"CAPTURE_NODE_STAT": "true",
235+
"TEST_PROCEDURE": "restore-from-snapshot"
236+
},
237+
"cluster_configuration": {
238+
"size": "Single-Node",
239+
"data_instance_config": "4vCPU, 32G Mem, 16G Heap"
240+
},
241+
"baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
224242
}
225243
}

Diff for: ‎.github/workflows/links.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ jobs:
1313
- uses: actions/checkout@v4
1414
- name: lychee Link Checker
1515
id: lychee
16-
uses: lycheeverse/lychee-action@v1.10.0
16+
uses: lycheeverse/lychee-action@v2.0.2
1717
with:
1818
args: --accept=200,403,429 --exclude-mail **/*.html **/*.md **/*.txt **/*.json --exclude-file .lychee.excludes
1919
fail: true

Diff for: ‎CHANGELOG.md

+11-4
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
2424
- New `phone` & `phone-search` analyzer + tokenizer ([#15915](https://github.com/opensearch-project/OpenSearch/pull/15915))
2525
- Add _list/shards API as paginated alternate to _cat/shards ([#14641](https://github.com/opensearch-project/OpenSearch/pull/14641))
2626
- Latency and Memory allocation improvements to Multi Term Aggregation queries ([#14993](https://github.com/opensearch-project/OpenSearch/pull/14993))
27+
- Flat object field use IndexOrDocValuesQuery to optimize query ([#14383](https://github.com/opensearch-project/OpenSearch/issues/14383))
2728

2829
### Dependencies
2930
- Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578))
@@ -32,7 +33,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
3233
- Bump `peter-evans/create-pull-request` from 6 to 7 ([#15863](https://github.com/opensearch-project/OpenSearch/pull/15863))
3334
- Bump `com.nimbusds:oauth2-oidc-sdk` from 11.9.1 to 11.19.1 ([#15862](https://github.com/opensearch-project/OpenSearch/pull/15862))
3435
- Bump `com.microsoft.azure:msal4j` from 1.17.0 to 1.17.1 ([#15945](https://github.com/opensearch-project/OpenSearch/pull/15945))
35-
- Bump `ch.qos.logback:logback-core` from 1.5.6 to 1.5.8 ([#15946](https://github.com/opensearch-project/OpenSearch/pull/15946))
36+
- Bump `ch.qos.logback:logback-core` from 1.5.6 to 1.5.10 ([#15946](https://github.com/opensearch-project/OpenSearch/pull/15946), [#16307](https://github.com/opensearch-project/OpenSearch/pull/16307))
3637
- Update protobuf from 3.25.4 to 3.25.5 ([#16011](https://github.com/opensearch-project/OpenSearch/pull/16011))
3738
- Bump `org.roaringbitmap:RoaringBitmap` from 1.2.1 to 1.3.0 ([#16040](https://github.com/opensearch-project/OpenSearch/pull/16040))
3839
- Bump `com.nimbusds:nimbus-jose-jwt` from 9.40 to 9.41.1 ([#16038](https://github.com/opensearch-project/OpenSearch/pull/16038))
@@ -41,21 +42,26 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
4142
- Bump `com.maxmind.geoip2:geoip2` from 4.2.0 to 4.2.1 ([#16042](https://github.com/opensearch-project/OpenSearch/pull/16042))
4243
- Bump `com.maxmind.db:maxmind-db` from 3.1.0 to 3.1.1 ([#16137](https://github.com/opensearch-project/OpenSearch/pull/16137))
4344
- Bump Apache lucene from 9.11.1 to 9.12.0 ([#15333](https://github.com/opensearch-project/OpenSearch/pull/15333))
44-
- Bump `com.azure:azure-core-http-netty` from 1.15.3 to 1.15.4 ([#16133](https://github.com/opensearch-project/OpenSearch/pull/16133))
45+
- Bump `com.azure:azure-core-http-netty` from 1.15.3 to 1.15.5 ([#16133](https://github.com/opensearch-project/OpenSearch/pull/16133), [#16311](https://github.com/opensearch-project/OpenSearch/pull/16311))
4546
- Bump `org.jline:jline` from 3.26.3 to 3.27.0 ([#16135](https://github.com/opensearch-project/OpenSearch/pull/16135))
4647
- Bump `netty` from 4.1.112.Final to 4.1.114.Final ([#16182](https://github.com/opensearch-project/OpenSearch/pull/16182))
4748
- Bump `com.google.api-client:google-api-client` from 2.2.0 to 2.7.0 ([#16216](https://github.com/opensearch-project/OpenSearch/pull/16216))
4849
- Bump `com.azure:azure-json` from 1.1.0 to 1.3.0 ([#16217](https://github.com/opensearch-project/OpenSearch/pull/16217))
4950
- Bump `io.grpc:grpc-api` from 1.57.2 to 1.68.0 ([#16213](https://github.com/opensearch-project/OpenSearch/pull/16213))
5051
- Bump `com.squareup.okio:okio` from 3.9.0 to 3.9.1 ([#16212](https://github.com/opensearch-project/OpenSearch/pull/16212))
52+
- Bump `me.champeau.gradle.japicmp` from 0.4.3 to 0.4.4 ([#16309](https://github.com/opensearch-project/OpenSearch/pull/16309))
53+
- Bump `com.google.oauth-client:google-oauth-client` from 1.35.0 to 1.36.0 ([#16306](https://github.com/opensearch-project/OpenSearch/pull/16306))
54+
- Bump `lycheeverse/lychee-action` from 1.10.0 to 2.0.2 ([#16310](https://github.com/opensearch-project/OpenSearch/pull/16310))
55+
- Bump `com.google.code.gson:gson` from 2.10.1 to 2.11.0 ([#16308](https://github.com/opensearch-project/OpenSearch/pull/16308))
5156

5257
### Changed
5358
- Add support for docker compose v2 in TestFixturesPlugin ([#16049](https://github.com/opensearch-project/OpenSearch/pull/16049))
5459
- Remove identity-related feature flagged code from the RestController ([#15430](https://github.com/opensearch-project/OpenSearch/pull/15430))
5560
- Remove Identity FeatureFlag ([#16024](https://github.com/opensearch-project/OpenSearch/pull/16024))
5661
- Ensure RestHandler.Wrapper delegates all implementations to the wrapped handler ([#16154](https://github.com/opensearch-project/OpenSearch/pull/16154))
62+
- Enable coordinator search.request_stats_enabled by default ([#16290](https://github.com/opensearch-project/OpenSearch/pull/16290))
5763
- Code cleanup: Remove ApproximateIndexOrDocValuesQuery ([#16273](https://github.com/opensearch-project/OpenSearch/pull/16273))
58-
64+
- Optimise clone operation for incremental full cluster snapshots ([#16296](https://github.com/opensearch-project/OpenSearch/pull/16296))
5965

6066
### Deprecated
6167

@@ -73,7 +79,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
7379
- Streaming bulk request hangs ([#16158](https://github.com/opensearch-project/OpenSearch/pull/16158))
7480
- Fix warnings from SLF4J on startup when repository-s3 is installed ([#16194](https://github.com/opensearch-project/OpenSearch/pull/16194))
7581
- Fix protobuf-java leak through client library dependencies ([#16254](https://github.com/opensearch-project/OpenSearch/pull/16254))
76-
- Fix get index settings API doesn't show `number_of_routing_shards` setting when it was explicitly set
82+
- Fix get index settings API doesn't show `number_of_routing_shards` setting when it was explicitly set ([#16294](https://github.com/opensearch-project/OpenSearch/pull/16294))
83+
- Fix multi-search with template doesn't return status code ([#16265](https://github.com/opensearch-project/OpenSearch/pull/16265))
7784

7885
### Security
7986

Diff for: ‎modules/lang-mustache/src/main/java/org/opensearch/script/mustache/MultiSearchTemplateResponse.java

+3
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232

3333
package org.opensearch.script.mustache;
3434

35+
import org.opensearch.ExceptionsHelper;
3536
import org.opensearch.OpenSearchException;
3637
import org.opensearch.action.search.MultiSearchResponse;
3738
import org.opensearch.common.Nullable;
@@ -167,6 +168,7 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par
167168
if (item.isFailure()) {
168169
builder.startObject();
169170
OpenSearchException.generateFailureXContent(builder, params, item.getFailure(), true);
171+
builder.field(Fields.STATUS, ExceptionsHelper.status(item.getFailure()).getStatus());
170172
builder.endObject();
171173
} else {
172174
item.getResponse().toXContent(builder, params);
@@ -179,6 +181,7 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par
179181

180182
static final class Fields {
181183
static final String RESPONSES = "responses";
184+
static final String STATUS = "status";
182185
}
183186

184187
public static MultiSearchTemplateResponse fromXContext(XContentParser parser) {

Diff for: ‎modules/lang-mustache/src/main/java/org/opensearch/script/mustache/SearchTemplateResponse.java

+4-1
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,10 @@ public static SearchTemplateResponse fromXContent(XContentParser parser) throws
120120
@Override
121121
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
122122
if (hasResponse()) {
123-
response.toXContent(builder, params);
123+
builder.startObject();
124+
response.innerToXContent(builder, params);
125+
builder.field(MultiSearchTemplateResponse.Fields.STATUS, response.status().getStatus());
126+
builder.endObject();
124127
} else {
125128
builder.startObject();
126129
// we can assume the template is always json as we convert it before compiling it

Diff for: ‎modules/lang-mustache/src/test/java/org/opensearch/script/mustache/MultiSearchTemplateResponseTests.java

+79
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,12 @@
3434
import org.opensearch.OpenSearchException;
3535
import org.opensearch.action.search.SearchResponse;
3636
import org.opensearch.action.search.ShardSearchFailure;
37+
import org.opensearch.common.xcontent.XContentType;
3738
import org.opensearch.core.common.Strings;
39+
import org.opensearch.core.common.bytes.BytesReference;
40+
import org.opensearch.core.xcontent.MediaTypeRegistry;
3841
import org.opensearch.core.xcontent.ToXContent;
42+
import org.opensearch.core.xcontent.XContentBuilder;
3943
import org.opensearch.core.xcontent.XContentParser;
4044
import org.opensearch.search.internal.InternalSearchResponse;
4145
import org.opensearch.test.AbstractXContentTestCase;
@@ -44,6 +48,7 @@
4448
import java.util.function.Predicate;
4549
import java.util.function.Supplier;
4650

51+
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertToXContentEquivalent;
4752
import static org.hamcrest.Matchers.containsString;
4853
import static org.hamcrest.Matchers.equalTo;
4954
import static org.hamcrest.Matchers.nullValue;
@@ -177,4 +182,78 @@ public void testFromXContentWithFailures() throws IOException {
177182
ToXContent.EMPTY_PARAMS
178183
);
179184
}
185+
186+
public void testToXContentWithFailures() throws Exception {
187+
long overallTookInMillis = randomNonNegativeLong();
188+
MultiSearchTemplateResponse.Item[] items = new MultiSearchTemplateResponse.Item[2];
189+
190+
long tookInMillis = 1L;
191+
int totalShards = 2;
192+
int successfulShards = 2;
193+
int skippedShards = 0;
194+
InternalSearchResponse internalSearchResponse = InternalSearchResponse.empty();
195+
SearchTemplateResponse searchTemplateResponse = new SearchTemplateResponse();
196+
SearchResponse searchResponse = new SearchResponse(
197+
internalSearchResponse,
198+
null,
199+
totalShards,
200+
successfulShards,
201+
skippedShards,
202+
tookInMillis,
203+
ShardSearchFailure.EMPTY_ARRAY,
204+
SearchResponse.Clusters.EMPTY
205+
);
206+
searchTemplateResponse.setResponse(searchResponse);
207+
items[0] = new MultiSearchTemplateResponse.Item(searchTemplateResponse, null);
208+
209+
items[1] = new MultiSearchTemplateResponse.Item(null, new IllegalArgumentException("Invalid argument"));
210+
211+
MultiSearchTemplateResponse response = new MultiSearchTemplateResponse(items, overallTookInMillis);
212+
213+
XContentType contentType = randomFrom(XContentType.values());
214+
XContentBuilder expectedResponse = MediaTypeRegistry.contentBuilder(contentType)
215+
.startObject()
216+
.field("took", overallTookInMillis)
217+
.startArray("responses")
218+
.startObject()
219+
.field("took", 1)
220+
.field("timed_out", false)
221+
.startObject("_shards")
222+
.field("total", 2)
223+
.field("successful", 2)
224+
.field("skipped", 0)
225+
.field("failed", 0)
226+
.endObject()
227+
.startObject("hits")
228+
.startObject("total")
229+
.field("value", 0)
230+
.field("relation", "eq")
231+
.endObject()
232+
.field("max_score", 0.0F)
233+
.startArray("hits")
234+
.endArray()
235+
.endObject()
236+
.field("status", 200)
237+
.endObject()
238+
.startObject()
239+
.startObject("error")
240+
.field("type", "illegal_argument_exception")
241+
.field("reason", "Invalid argument")
242+
.startArray("root_cause")
243+
.startObject()
244+
.field("type", "illegal_argument_exception")
245+
.field("reason", "Invalid argument")
246+
.endObject()
247+
.endArray()
248+
.endObject()
249+
.field("status", 400)
250+
.endObject()
251+
.endArray()
252+
.endObject();
253+
254+
XContentBuilder actualResponse = MediaTypeRegistry.contentBuilder(contentType);
255+
response.toXContent(actualResponse, ToXContent.EMPTY_PARAMS);
256+
257+
assertToXContentEquivalent(BytesReference.bytes(expectedResponse), BytesReference.bytes(actualResponse), contentType);
258+
}
180259
}

Diff for: ‎modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateResponseTests.java

+1
Original file line numberDiff line numberDiff line change
@@ -234,6 +234,7 @@ public void testSearchResponseToXContent() throws IOException {
234234
.endObject()
235235
.endArray()
236236
.endObject()
237+
.field("status", 200)
237238
.endObject();
238239

239240
XContentBuilder actualResponse = MediaTypeRegistry.contentBuilder(contentType);

Diff for: ‎modules/lang-mustache/src/yamlRestTest/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml

+48
Original file line numberDiff line numberDiff line change
@@ -205,3 +205,51 @@ setup:
205205
- match: { responses.1.hits.total.relation: eq }
206206
- match: { responses.2.hits.total.value: 1 }
207207
- match: { responses.1.hits.total.relation: eq }
208+
209+
---
210+
"Test multi-search template returns status code":
211+
- skip:
212+
version: " - 2.17.99"
213+
reason: Fixed in 2.18.0.
214+
- do:
215+
msearch_template:
216+
rest_total_hits_as_int: true
217+
body:
218+
# Search 0 is OK
219+
- index: index_*
220+
- source: '{"query": {"match": {"foo": "{{value}}"} } }'
221+
params:
222+
value: "foo"
223+
# Search 1 has an unclosed JSON template
224+
- index: index_*
225+
- source: '{"query": {"match": {'
226+
params:
227+
field: "foo"
228+
value: "bar"
229+
# Search 2 is OK
230+
- index: _all
231+
- source:
232+
query:
233+
match: {foo: "{{text}}"}
234+
params:
235+
text: "baz"
236+
# Search 3 has an unknown query type
237+
- index: index_*
238+
- source: '{"query": {"{{query_type}}": {} }' # Unknown query type
239+
params:
240+
query_type: "unknown"
241+
# Search 4 has an unsupported track_total_hits
242+
- index: index_*
243+
- source: '{"query": {"match_all": {} }, "track_total_hits": "{{trackTotalHits}}" }'
244+
params:
245+
trackTotalHits: 1
246+
# Search 5 has an unknown index
247+
- index: unknown_index
248+
- source: '{"query": {"match_all": {} } }'
249+
250+
- match: { responses.0.status: 200 }
251+
- match: { responses.1.status: 400 }
252+
- match: { responses.2.status: 200 }
253+
- match: { responses.3.status: 400 }
254+
- match: { responses.4.status: 400 }
255+
- match: { responses.5.status: 404 }

Diff for: ‎plugins/discovery-gce/build.gradle

+1-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ opensearchplugin {
2020
dependencies {
2121
api "com.google.apis:google-api-services-compute:v1-rev20240407-2.0.0"
2222
api "com.google.api-client:google-api-client:1.35.2"
23-
api "com.google.oauth-client:google-oauth-client:1.35.0"
23+
api "com.google.oauth-client:google-oauth-client:1.36.0"
2424
api "com.google.http-client:google-http-client:${versions.google_http_client}"
2525
api "com.google.http-client:google-http-client-gson:${versions.google_http_client}"
2626
api "com.google.http-client:google-http-client-jackson2:${versions.google_http_client}"

Diff for: ‎plugins/discovery-gce/licenses/google-oauth-client-1.35.0.jar.sha1

-1
This file was deleted.

Diff for: ‎plugins/discovery-gce/licenses/google-oauth-client-1.36.0.jar.sha1

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
dc3f07bc8f49dd52fe8fcc15958f3cfeb003e20f

Diff for: ‎plugins/repository-azure/build.gradle

+1-1
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ dependencies {
4848
api 'com.azure:azure-json:1.3.0'
4949
api 'com.azure:azure-xml:1.1.0'
5050
api 'com.azure:azure-storage-common:12.25.1'
51-
api 'com.azure:azure-core-http-netty:1.15.4'
51+
api 'com.azure:azure-core-http-netty:1.15.5'
5252
api "io.netty:netty-codec-dns:${versions.netty}"
5353
api "io.netty:netty-codec-socks:${versions.netty}"
5454
api "io.netty:netty-codec-http2:${versions.netty}"

Diff for: ‎plugins/repository-azure/licenses/azure-core-http-netty-1.15.4.jar.sha1

-1
This file was deleted.

Diff for: ‎plugins/repository-azure/licenses/azure-core-http-netty-1.15.5.jar.sha1

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
44d99705d3759e2ad7ee8110f811d4ed304a6a7c

Diff for: ‎plugins/repository-gcs/build.gradle

+1-1
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ dependencies {
7070
api 'com.google.cloud:google-cloud-core-http:2.23.0'
7171
api 'com.google.cloud:google-cloud-storage:1.113.1'
7272

73-
api 'com.google.code.gson:gson:2.10.1'
73+
api 'com.google.code.gson:gson:2.11.0'
7474

7575
runtimeOnly "com.google.guava:guava:${versions.guava}"
7676
api 'com.google.guava:failureaccess:1.0.1'

Diff for: ‎plugins/repository-gcs/licenses/gson-2.10.1.jar.sha1

-1
This file was deleted.

Diff for: ‎plugins/repository-gcs/licenses/gson-2.11.0.jar.sha1

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
527175ca6d81050b53bdd4c457a6d6e017626b0e

Diff for: ‎rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml

-32
Original file line numberDiff line numberDiff line change
@@ -671,38 +671,6 @@ teardown:
671671
- match: { error.root_cause.0.reason: "Mapping definition for [data] has unsupported parameters: [analyzer : standard]"}
672672
- match: { status: 400 }
673673

674-
# Wildcard Query with dot path.
675-
- do:
676-
catch: bad_request
677-
search:
678-
body: {
679-
_source: true,
680-
query: {
681-
"wildcard": {
682-
"catalog.title": "Mock*"
683-
}
684-
}
685-
}
686-
- match: { error.root_cause.0.type: "query_shard_exception" }
687-
- match: { error.root_cause.0.reason: "Can only use wildcard queries on keyword and text fields - not on [catalog.title] which is of type [flat_object]"}
688-
- match: { status: 400 }
689-
690-
# Wildcard Query without dot path.
691-
- do:
692-
catch: bad_request
693-
search:
694-
body: {
695-
_source: true,
696-
query: {
697-
"wildcard": {
698-
"catalog": "Mock*"
699-
}
700-
}
701-
}
702-
- match: { error.root_cause.0.type: "query_shard_exception" }
703-
- match: { error.root_cause.0.reason: "Can only use wildcard queries on keyword and text fields - not on [catalog] which is of type [flat_object]" }
704-
- match: { status: 400 }
705-
706674
# Aggregation and Match Query with dot path.
707675
- do:
708676
catch: bad_request

0 commit comments

Comments
 (0)