Fix to #21006 - Support a default value for non-nullable properties

Description
Problem was that we don't have a good story for schema evolution in Cosmos (e.g. adding a new non-nullable property). We start generating shaper code expecting the new schema, but the old documents are not being updated. Effectively, querying for those modified entities stops working because the document no longer matches the expected shape. This is a major pain point and adoption blocker for EF Core Cosmos users. While designing a compelling schema evolution story is a large task and out of scope for now, we believe we can solve majority of the problems with a targeted change - when generating shaper for Cosmos entities, if required value is not present, we generate the CLR default rather than throw.

Customer impact
EF Core customers using Cosmos experience exceptions when querying for an entity which had it's schema modified, e.g. by adding a non-nullable property. There is no good workaround, apart from manually adding missing data to the document or using nullable properties instead.

How found
Reported by multiple customers on 9.

Regression
No.

Testing
Added multiple tests for scalar and reference scenarios, all supported types, converters vs no. missing value vs explicit nulls.

Risk
Low - Targeted fix, when constructing non-nullable property of an entity type we produce default of that type, rather than null. This does not affect keys, just regular scalar properties.
Potential negative side-effect is that we lose the validation for scenarios where legitimate required property is introduced, but the value hasn't been provided for it in the document by accident.
Added a quirk as an escape hatch to revert to previous behavior.

Fixes #21006

Author: maumar

Diff for: src/EFCore.Cosmos/Query/Internal/CosmosShapedQueryCompilingExpressionVisitor.CosmosProjectionBindingRemovingExpressionVisitorBase.cs

@@ -20,6 +20,9 @@ private abstract class CosmosProjectionBindingRemovingExpressionVisitorBase(
         bool trackQueryResults)
         : ExpressionVisitor
     {
+        private static readonly bool UseOldBehavior21006 =
+              AppContext.TryGetSwitch("Microsoft.EntityFrameworkCore.Issue21006", out var enabled21006) && enabled21006;
+
         private static readonly MethodInfo GetItemMethodInfo
             = typeof(JObject).GetRuntimeProperties()
                 .Single(pi => pi.Name == "Item" && pi.GetIndexParameters()[0].ParameterType == typeof(string))
@@ -691,7 +694,11 @@ private Expression CreateGetValueExpression(
                 && !property.IsShadowProperty())
             {
                 var readExpression = CreateGetValueExpression(
-                    jTokenExpression, storeName, type.MakeNullable(), property.GetTypeMapping());
+                    jTokenExpression,
+                    storeName,
+                    type.MakeNullable(),
+                    property.GetTypeMapping(),
+                    isNonNullableScalar: false);
 
                 var nonNullReadExpression = readExpression;
                 if (nonNullReadExpression.Type != type)
@@ -712,15 +719,23 @@ private Expression CreateGetValueExpression(
             }
 
             return Convert(
-                CreateGetValueExpression(jTokenExpression, storeName, type.MakeNullable(), property.GetTypeMapping()),
+                CreateGetValueExpression(
+                    jTokenExpression,
+                    storeName,
+                    type.MakeNullable(),
+                    property.GetTypeMapping(),
+                    // special case keys - we check them for null to see if the entity needs to be materialized, so we want to keep the null, rather than non-nullable default
+                    // returning defaults is supposed to help with evolving the schema - so this doesn't concern keys anyway (they shouldn't evolve)
+                    isNonNullableScalar: !property.IsNullable && !property.IsKey()),
                 type);
         }
 
         private Expression CreateGetValueExpression(
             Expression jTokenExpression,
             string storeName,
             Type type,
-            CoreTypeMapping typeMapping = null)
+            CoreTypeMapping typeMapping = null,
+            bool isNonNullableScalar = false)
         {
             Check.DebugAssert(type.IsNullableType(), "Must read nullable type from JObject.");
 
@@ -763,6 +778,7 @@ var body
                             Constant(CosmosClientWrapper.Serializer)),
                         converter.ConvertFromProviderExpression.Body);
 
+                var originalBodyType = body.Type;
                 if (body.Type != type)
                 {
                     body = Convert(body, type);
@@ -783,7 +799,11 @@ var body
                 }
                 else
                 {
-                    replaceExpression = Default(type);
+                    replaceExpression = isNonNullableScalar && !UseOldBehavior21006
+                        ? Expression.Convert(
+                            Default(originalBodyType),
+                            type)
+                        : Default(type);
                 }
 
                 body = Condition(
@@ -799,7 +819,11 @@ var body
             }
             else
             {
-                valueExpression = ConvertJTokenToType(jTokenExpression, typeMapping?.ClrType.MakeNullable() ?? type);
+                valueExpression = ConvertJTokenToType(
+                    jTokenExpression,
+                    (isNonNullableScalar && !UseOldBehavior21006
+                        ? typeMapping?.ClrType
+                        : typeMapping?.ClrType.MakeNullable()) ?? type);
 
                 if (valueExpression.Type != type)
                 {

Diff for: test/EFCore.Cosmos.FunctionalTests/Query/AdHocCosmosTestHelpers.cs

@@ -0,0 +1,43 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Net;
+using Microsoft.Azure.Cosmos;
+using Microsoft.EntityFrameworkCore.Cosmos.Storage.Internal;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+
+namespace Microsoft.EntityFrameworkCore.Query;
+
+public class AdHocCosmosTestHelpers
+{
+    public static async Task CreateCustomEntityHelperAsync(
+        Container container,
+        string json,
+        CancellationToken cancellationToken)
+    {
+        var document = JObject.Parse(json);
+
+        var stream = new MemoryStream();
+        await using var __ = stream.ConfigureAwait(false);
+        var writer = new StreamWriter(stream, new UTF8Encoding(), bufferSize: 1024, leaveOpen: false);
+        await using var ___ = writer.ConfigureAwait(false);
+        using var jsonWriter = new JsonTextWriter(writer);
+
+        CosmosClientWrapper.Serializer.Serialize(jsonWriter, document);
+        await jsonWriter.FlushAsync(cancellationToken).ConfigureAwait(false);
+
+        var response = await container.CreateItemStreamAsync(
+                stream,
+                PartitionKey.None,
+                requestOptions: null,
+                cancellationToken)
+            .ConfigureAwait(false);
+
+
+        if (response.StatusCode != HttpStatusCode.Created)
+        {
+            throw new InvalidOperationException($"Failed to create entity (status code: {response.StatusCode}) for json: {json}");
+        }
+    }
+}