Fix to #21006 - Support a default value for non-nullable properties

Only for scalar properties when projecting Json-mapped entity.
Only need to change code for Cosmos - relational already works in the desired way after the change to streaming (properties that are not encountered maintain their default value)
We still throw exception if JSON contains explicit null where non-nullable scalar is expected.

Fixes #21006

Author: maumar

Diff for: src/EFCore.Cosmos/Query/Internal/CosmosShapedQueryCompilingExpressionVisitor.CosmosProjectionBindingRemovingExpressionVisitorBase.cs

@@ -691,7 +691,11 @@ private Expression CreateGetValueExpression(
                 && !property.IsShadowProperty())
             {
                 var readExpression = CreateGetValueExpression(
-                    jTokenExpression, storeName, type.MakeNullable(), property.GetTypeMapping());
+                    jTokenExpression,
+                    storeName,
+                    type.MakeNullable(),
+                    property.GetTypeMapping(),
+                    isNonNullableScalar: false);
 
                 var nonNullReadExpression = readExpression;
                 if (nonNullReadExpression.Type != type)
@@ -712,15 +716,23 @@ private Expression CreateGetValueExpression(
             }
 
             return Convert(
-                CreateGetValueExpression(jTokenExpression, storeName, type.MakeNullable(), property.GetTypeMapping()),
+                CreateGetValueExpression(
+                    jTokenExpression,
+                    storeName,
+                    type.MakeNullable(),
+                    property.GetTypeMapping(),
+                    // special case keys - we check them for null to see if the entity needs to be materialized, so we want to keep the null, rather than non-nullable default
+                    // returning defaults is supposed to help with evolving the schema - so this doesn't concern keys anyway (they shouldn't evolve)
+                    isNonNullableScalar: !property.IsNullable && !property.IsKey()),
                 type);
         }
 
         private Expression CreateGetValueExpression(
             Expression jTokenExpression,
             string storeName,
             Type type,
-            CoreTypeMapping typeMapping = null)
+            CoreTypeMapping typeMapping = null,
+            bool isNonNullableScalar = false)
         {
             Check.DebugAssert(type.IsNullableType(), "Must read nullable type from JObject.");
 
@@ -763,6 +775,7 @@ var body
                             Constant(CosmosClientWrapper.Serializer)),
                         converter.ConvertFromProviderExpression.Body);
 
+                var originalBodyType = body.Type;
                 if (body.Type != type)
                 {
                     body = Convert(body, type);
@@ -783,7 +796,11 @@ var body
                 }
                 else
                 {
-                    replaceExpression = Default(type);
+                    replaceExpression = isNonNullableScalar
+                        ? Expression.Convert(
+                            Default(originalBodyType),
+                            type)
+                        : Default(type);
                 }
 
                 body = Condition(
@@ -799,7 +816,11 @@ var body
             }
             else
             {
-                valueExpression = ConvertJTokenToType(jTokenExpression, typeMapping?.ClrType.MakeNullable() ?? type);
+                valueExpression = ConvertJTokenToType(
+                    jTokenExpression,
+                    (isNonNullableScalar
+                        ? typeMapping?.ClrType
+                        : typeMapping?.ClrType.MakeNullable()) ?? type);
 
                 if (valueExpression.Type != type)
                 {

Diff for: test/EFCore.Cosmos.FunctionalTests/Query/AdHocCosmosTestHelpers.cs

@@ -0,0 +1,43 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Net;
+using Microsoft.Azure.Cosmos;
+using Microsoft.EntityFrameworkCore.Cosmos.Storage.Internal;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+
+namespace Microsoft.EntityFrameworkCore.Query;
+
+public class AdHocCosmosTestHelpers
+{
+    public static async Task CreateCustomEntityHelperAsync(
+        Container container,
+        string json,
+        CancellationToken cancellationToken)
+    {
+        var document = JObject.Parse(json);
+
+        var stream = new MemoryStream();
+        await using var __ = stream.ConfigureAwait(false);
+        var writer = new StreamWriter(stream, new UTF8Encoding(), bufferSize: 1024, leaveOpen: false);
+        await using var ___ = writer.ConfigureAwait(false);
+        using var jsonWriter = new JsonTextWriter(writer);
+
+        CosmosClientWrapper.Serializer.Serialize(jsonWriter, document);
+        await jsonWriter.FlushAsync(cancellationToken).ConfigureAwait(false);
+
+        var response = await container.CreateItemStreamAsync(
+                stream,
+                PartitionKey.None,
+                requestOptions: null,
+                cancellationToken)
+            .ConfigureAwait(false);
+
+
+        if (response.StatusCode != HttpStatusCode.Created)
+        {
+            throw new InvalidOperationException($"Failed to create entitty (status code: {response.StatusCode}) for json: {json}");
+        }
+    }
+}

Diff for: test/EFCore.Cosmos.FunctionalTests/Query/AdHocJsonQueryCosmosTest.cs

@@ -0,0 +1,269 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using Microsoft.EntityFrameworkCore.Cosmos.Storage.Internal;
+
+namespace Microsoft.EntityFrameworkCore.Query;
+
+public class AdHocJsonQueryCosmosTest : AdHocJsonQueryTestBase
+{
+    public override async Task Project_root_with_missing_scalars(bool async)
+    {
+        if (async)
+        {
+            await base.Project_root_with_missing_scalars(async);
+
+            AssertSql(
+                """
+SELECT VALUE c
+FROM root c
+""");
+        }
+    }
+
+    [ConditionalTheory(Skip = "issue #35702")]
+    public override async Task Project_top_level_json_entity_with_missing_scalars(bool async)
+    {
+        if (async)
+        {
+            await base.Project_top_level_json_entity_with_missing_scalars(async);
+
+            AssertSql();
+        }
+    }
+
+    public override async Task Project_nested_json_entity_with_missing_scalars(bool async)
+    {
+        if (async)
+        {
+            await AssertTranslationFailed(
+                () => base.Project_nested_json_entity_with_missing_scalars(async));
+
+            AssertSql();
+        }
+    }
+
+    protected override void OnModelCreating21006(ModelBuilder modelBuilder)
+    {
+        base.OnModelCreating21006(modelBuilder);
+
+        modelBuilder.Entity<Context21006.Entity>().ToContainer("Entities");
+    }
+
+    protected override async Task Seed21006(Context21006 context)
+    {
+        await base.Seed21006(context);
+
+        var wrapper = (CosmosClientWrapper)context.GetService<ICosmosClientWrapper>();
+        var singletonWrapper = context.GetService<ISingletonCosmosClientWrapper>();
+        var entitiesContainer = singletonWrapper.Client.GetContainer(StoreName, containerId: "Entities");
+
+        var missingTopLevel =
+$$"""
+{
+  "Id": 2,
+  "$type": "Entity",
+  "Name": "e2",
+  "id": "2",
+  "Collection": [
+    {
+      "Text": "e2 c1",
+      "NestedCollection": [
+        {
+          "DoB": "2000-01-01T00:00:00",
+          "Text": "e2 c1 c1"
+        },
+        {
+          "DoB": "2000-01-01T00:00:00",
+          "Text": "e2 c1 c2"
+        }
+      ],
+      "NestedOptionalReference": {
+        "DoB": "2000-01-01T00:00:00",
+        "Text": "e2 c1 nor"
+      },
+      "NestedRequiredReference": {
+        "DoB": "2000-01-01T00:00:00",
+        "Text": "e2 c1 nrr"
+      }
+    },
+    {
+      "Text": "e2 c2",
+      "NestedCollection": [
+        {
+          "DoB": "2000-01-01T00:00:00",
+          "Text": "e2 c2 c1"
+        },
+        {
+          "DoB": "2000-01-01T00:00:00",
+          "Text": "e2 c2 c2"
+        }
+      ],
+      "NestedOptionalReference": {
+        "DoB": "2000-01-01T00:00:00",
+        "Text": "e2 c2 nor"
+      },
+      "NestedRequiredReference": {
+        "DoB": "2000-01-01T00:00:00",
+        "Text": "e2 c2 nrr"
+      }
+    }
+  ],
+  "OptionalReference": {
+    "Text": "e2 or",
+    "NestedCollection": [
+      {
+        "DoB": "2000-01-01T00:00:00",
+        "Text": "e2 or c1"
+      },
+      {
+        "DoB": "2000-01-01T00:00:00",
+        "Text": "e2 or c2"
+      }
+    ],
+    "NestedOptionalReference": {
+      "DoB": "2000-01-01T00:00:00",
+      "Text": "e2 or nor"
+    },
+    "NestedRequiredReference": {
+      "DoB": "2000-01-01T00:00:00",
+      "Text": "e2 or nrr"
+    }
+  },
+  "RequiredReference": {
+    "Text": "e2 rr",
+    "NestedCollection": [
+      {
+        "DoB": "2000-01-01T00:00:00",
+        "Text": "e2 rr c1"
+      },
+      {
+        "DoB": "2000-01-01T00:00:00",
+        "Text": "e2 rr c2"
+      }
+    ],
+    "NestedOptionalReference": {
+      "DoB": "2000-01-01T00:00:00",
+      "Text": "e2 rr nor"
+    },
+    "NestedRequiredReference": {
+      "DoB": "2000-01-01T00:00:00",
+      "Text": "e2 rr nrr"
+    }
+  }
+}
+""";
+
+        await AdHocCosmosTestHelpers.CreateCustomEntityHelperAsync(
+            entitiesContainer,
+            missingTopLevel,
+            CancellationToken.None);
+
+        var missingNested =
+$$"""
+{
+  "Id": 3,
+  "$type": "Entity",
+  "Name": "e3",
+  "id": "3",
+  "Collection": [
+    {
+      "Number": 7.0,
+      "Text": "e3 c1",
+      "NestedCollection": [
+        {
+          "Text": "e3 c1 c1"
+        },
+        {
+          "Text": "e3 c1 c2"
+        }
+      ],
+      "NestedOptionalReference": {
+        "Text": "e3 c1 nor"
+      },
+      "NestedRequiredReference": {
+        "Text": "e3 c1 nrr"
+      }
+    },
+    {
+      "Number": 7.0,
+      "Text": "e3 c2",
+      "NestedCollection": [
+        {
+          "Text": "e3 c2 c1"
+        },
+        {
+          "Text": "e3 c2 c2"
+        }
+      ],
+      "NestedOptionalReference": {
+        "Text": "e3 c2 nor"
+      },
+      "NestedRequiredReference": {
+        "Text": "e3 c2 nrr"
+      }
+    }
+  ],
+  "OptionalReference": {
+    "Number": 7.0,
+    "Text": "e3 or",
+    "NestedCollection": [
+      {
+        "Text": "e3 or c1"
+      },
+      {
+        "Text": "e3 or c2"
+      }
+    ],
+    "NestedOptionalReference": {
+      "Text": "e3 or nor"
+    },
+    "NestedRequiredReference": {
+      "Text": "e3 or nrr"
+    }
+  },
+  "RequiredReference": {
+    "Number": 7.0,
+    "Text": "e3 rr",
+    "NestedCollection": [
+      {
+        "Text": "e3 rr c1"
+      },
+      {
+        "Text": "e3 rr c2"
+      }
+    ],
+    "NestedOptionalReference": {
+      "Text": "e3 rr nor"
+    },
+    "NestedRequiredReference": {
+      "Text": "e3 rr nrr"
+    }
+  }
+}
+""";
+
+        await AdHocCosmosTestHelpers.CreateCustomEntityHelperAsync(
+            entitiesContainer,
+            missingNested,
+            CancellationToken.None);
+    }
+
+    protected TestSqlLoggerFactory TestSqlLoggerFactory
+        => (TestSqlLoggerFactory)ListLoggerFactory;
+
+    private void AssertSql(params string[] expected)
+        => TestSqlLoggerFactory.AssertBaseline(expected);
+
+    protected static async Task AssertTranslationFailed(Func<Task> query)
+        => Assert.Contains(
+            CoreStrings.TranslationFailed("")[48..],
+            (await Assert.ThrowsAsync<InvalidOperationException>(query))
+            .Message);
+
+    protected override DbContextOptionsBuilder AddOptions(DbContextOptionsBuilder builder)
+        => builder.ConfigureWarnings(b => b.Ignore(CosmosEventId.NoPartitionKeyDefined));
+
+    protected override ITestStoreFactory TestStoreFactory
+        => CosmosTestStoreFactory.Instance;
+}