Skip to content

Commit 5ff0244

Browse files
maumarmaumar
authored
Fix to #30604 - Implement JSON serialization/deserialization via Utf8JsonReader/Utf8JsonWriter (#31160)
Using Utf8JsonReader to read JSON data rather than caching it using DOM. This should reduce allocations significantly. Tricky part is that entity materializers are build in a way that assumes we have random access to all the data we need. This is not the case here. We read JSON data sequentially and can only do it once, and we don't know the order in which we get the data. This is somewhat problematic in case where entity takes argument in the constructor. Those could be at the very end of the JSON string, so we must read all the data before we can instantiate the object, and populate it's properties and do navigation fixup. This requires us reading all the JSON data, store them in local variables, and only when we are done reading we instantiate the entity and populate all the properties with data stored in those variables. This adds some allocations (specifically navigations). We also have to disable de-duplication logic - we can't always safely re-read the JSON string, and definitely can't start reading it from arbitrary position, so now we have to add JSON string for every aggregate projected, even if we already project it's parent. Serialization implementation (i.e. Utf8JsonWriter) is pretty straighforward. Also fix to #30993 - Query/Json: data corruption for tracking queries with nested json entities, then updating nested entities outside EF and re-querying Fix is to recognize and modify shaper in case of tracking query, so that nav expansions are not skipped when parent entity is found in Change Tracker. This is necessary to fix alongside streaming, because now we throw exception from reader (unexpected token) if we don't process the entire stream correctly. Before it would be silently ignored apart from the edge case described in the bug. Fixes #30604 Fixes #30993
1 parent 1bb18b9 commit 5ff0244

22 files changed

+1966
-654
lines changed

Diff for: ‎src/EFCore.Relational/Properties/RelationalStrings.Designer.cs

+8
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Diff for: ‎src/EFCore.Relational/Properties/RelationalStrings.resx

+3
Original file line numberDiff line numberDiff line change
@@ -547,6 +547,9 @@
547547
<data name="JsonPropertyNameShouldBeConfiguredOnNestedNavigation" xml:space="preserve">
548548
<value>The JSON property name should only be configured on nested owned navigations.</value>
549549
</data>
550+
<data name="JsonReaderInvalidTokenType" xml:space="preserve">
551+
<value>Invalid token type: '{tokenType}'.</value>
552+
</data>
550553
<data name="JsonRequiredEntityWithNullJson" xml:space="preserve">
551554
<value>Entity {entity} is required but the JSON element containing it is null.</value>
552555
</data>

Diff for: ‎src/EFCore.Relational/Query/Internal/JsonProjectionInfo.cs

+1-15
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,10 @@ public readonly struct JsonProjectionInfo
1919
/// </summary>
2020
public JsonProjectionInfo(
2121
int jsonColumnIndex,
22-
List<(IProperty?, int?, int?)> keyAccessInfo,
23-
(string?, int?, int?)[] additionalPath)
22+
List<(IProperty?, int?, int?)> keyAccessInfo)
2423
{
2524
JsonColumnIndex = jsonColumnIndex;
2625
KeyAccessInfo = keyAccessInfo;
27-
AdditionalPath = additionalPath;
2826
}
2927

3028
/// <summary>
@@ -55,16 +53,4 @@ public JsonProjectionInfo(
5553
/// doing so can result in application failures when updating to a new Entity Framework Core release.
5654
/// </remarks>
5755
public List<(IProperty? KeyProperty, int? ConstantKeyValue, int? KeyProjectionIndex)> KeyAccessInfo { get; }
58-
59-
/// <summary>
60-
/// List of additional path elements, only one of the values in the tuple is non-null
61-
/// this information is used to access the correct sub-element of a JsonElement that we materialized
62-
/// </summary>
63-
/// <remarks>
64-
/// This is an internal API that supports the Entity Framework Core infrastructure and not subject to
65-
/// the same compatibility standards as public APIs. It may be changed or removed without notice in
66-
/// any release. You should only use it directly in your code with extreme caution and knowing that
67-
/// doing so can result in application failures when updating to a new Entity Framework Core release.
68-
/// </remarks>
69-
public (string? JsonPropertyName, int? ConstantArrayIndex, int? NonConstantArrayIndex)[] AdditionalPath { get; }
7056
}

Diff for: ‎src/EFCore.Relational/Query/Internal/RelationalProjectionBindingExpressionVisitor.cs

+2-8
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@ private static readonly MethodInfo GetParameterValueMethodInfo
2525

2626
private bool _indexBasedBinding;
2727
private Dictionary<EntityProjectionExpression, ProjectionBindingExpression>? _entityProjectionCache;
28-
private Dictionary<JsonQueryExpression, ProjectionBindingExpression>? _jsonQueryCache;
2928
private List<Expression>? _clientProjections;
3029

3130
private readonly Dictionary<ProjectionMember, Expression> _projectionMapping = new();
@@ -66,7 +65,6 @@ public virtual Expression Translate(SelectExpression selectExpression, Expressio
6665
{
6766
_indexBasedBinding = true;
6867
_entityProjectionCache = new Dictionary<EntityProjectionExpression, ProjectionBindingExpression>();
69-
_jsonQueryCache = new Dictionary<JsonQueryExpression, ProjectionBindingExpression>();
7068
_projectionMapping.Clear();
7169
_clientProjections = new List<Expression>();
7270

@@ -307,11 +305,8 @@ protected override Expression VisitExtension(Expression extensionExpression)
307305
{
308306
if (_indexBasedBinding)
309307
{
310-
if (!_jsonQueryCache!.TryGetValue(jsonQueryExpression, out var jsonProjectionBinding))
311-
{
312-
jsonProjectionBinding = AddClientProjection(jsonQueryExpression, typeof(ValueBuffer));
313-
_jsonQueryCache[jsonQueryExpression] = jsonProjectionBinding;
314-
}
308+
_clientProjections!.Add(jsonQueryExpression);
309+
var jsonProjectionBinding = new ProjectionBindingExpression(_selectExpression, _clientProjections.Count - 1, typeof(ValueBuffer));
315310

316311
return entityShaperExpression.Update(jsonProjectionBinding);
317312
}
@@ -654,7 +649,6 @@ private ProjectionBindingExpression AddClientProjection(Expression expression, T
654649
return new ProjectionBindingExpression(_selectExpression, existingIndex, type);
655650
}
656651

657-
#pragma warning disable IDE0052 // Remove unread private members
658652
private static T GetParameterValue<T>(QueryContext queryContext, string parameterName)
659653
#pragma warning restore IDE0052 // Remove unread private members
660654
=> (T)queryContext.ParameterValues[parameterName]!;

Diff for: ‎src/EFCore.Relational/Query/RelationalShapedQueryCompilingExpressionVisitor.ShaperProcessingExpressionVisitor.ClientMethods.cs

+145-67
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
using System.Text.Json;
66
using Microsoft.EntityFrameworkCore.Internal;
77
using Microsoft.EntityFrameworkCore.Query.Internal;
8+
using Microsoft.EntityFrameworkCore.Storage.Json;
89

910
namespace Microsoft.EntityFrameworkCore.Query;
1011

@@ -67,8 +68,8 @@ private static readonly MethodInfo MaterializeJsonEntityMethodInfo
6768
private static readonly MethodInfo MaterializeJsonEntityCollectionMethodInfo
6869
= typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(MaterializeJsonEntityCollection))!;
6970

70-
private static readonly MethodInfo ExtractJsonPropertyMethodInfo
71-
= typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(ExtractJsonProperty))!;
71+
private static readonly MethodInfo InverseCollectionFixupMethod
72+
= typeof(ShaperProcessingExpressionVisitor).GetTypeInfo().GetDeclaredMethod(nameof(InverseCollectionFixup))!;
7273

7374
[MethodImpl(MethodImplOptions.AggressiveInlining)]
7475
private static TValue ThrowReadValueException<TValue>(
@@ -123,13 +124,6 @@ private static TValue ThrowExtractJsonPropertyException<TValue>(
123124
exception);
124125
}
125126

126-
private static T? ExtractJsonProperty<T>(JsonElement element, string propertyName, bool nullable)
127-
=> nullable
128-
? element.TryGetProperty(propertyName, out var jsonValue)
129-
? jsonValue.Deserialize<T>()
130-
: default
131-
: element.GetProperty(propertyName).Deserialize<T>();
132-
133127
private static void IncludeReference<TEntity, TIncludingEntity, TIncludedEntity>(
134128
QueryContext queryContext,
135129
TEntity entity,
@@ -869,105 +863,189 @@ static async Task<RelationalDataReader> InitializeReaderAsync(
869863
dataReaderContext.HasNext = false;
870864
}
871865

872-
private static void IncludeJsonEntityReference<TIncludingEntity, TIncludedEntity>(
866+
private static TEntity? MaterializeJsonEntity<TEntity>(
873867
QueryContext queryContext,
874-
JsonElement? jsonElement,
875868
object[] keyPropertyValues,
876-
TIncludingEntity entity,
877-
Func<QueryContext, object[], JsonElement, TIncludedEntity> innerShaper,
878-
Action<TIncludingEntity, TIncludedEntity> fixup)
879-
where TIncludingEntity : class
880-
where TIncludedEntity : class
869+
JsonReaderData? jsonReaderData,
870+
bool nullable,
871+
Func<QueryContext, object[], JsonReaderData, TEntity> shaper)
872+
where TEntity : class
881873
{
882-
if (jsonElement.HasValue && jsonElement.Value.ValueKind != JsonValueKind.Null)
874+
if (jsonReaderData == null)
883875
{
884-
var included = innerShaper(queryContext, keyPropertyValues, jsonElement.Value);
885-
fixup(entity, included);
876+
return nullable
877+
? null
878+
: throw new InvalidOperationException(
879+
RelationalStrings.JsonRequiredEntityWithNullJson(typeof(TEntity).Name));
880+
}
881+
882+
var manager = new Utf8JsonReaderManager(jsonReaderData);
883+
var tokenType = manager.CurrentReader.TokenType;
884+
885+
if (tokenType == JsonTokenType.Null)
886+
{
887+
return nullable
888+
? null
889+
: throw new InvalidOperationException(
890+
RelationalStrings.JsonRequiredEntityWithNullJson(typeof(TEntity).Name));
891+
}
892+
893+
if (tokenType != JsonTokenType.StartObject)
894+
{
895+
throw new InvalidOperationException(
896+
RelationalStrings.JsonReaderInvalidTokenType(tokenType.ToString()));
886897
}
898+
899+
manager.CaptureState();
900+
var result = shaper(queryContext, keyPropertyValues, jsonReaderData);
901+
902+
return result;
887903
}
888904

889-
private static void IncludeJsonEntityCollection<TIncludingEntity, TIncludedCollectionElement>(
905+
private static TResult? MaterializeJsonEntityCollection<TEntity, TResult>(
890906
QueryContext queryContext,
891-
JsonElement? jsonElement,
892907
object[] keyPropertyValues,
893-
TIncludingEntity entity,
894-
Func<QueryContext, object[], JsonElement, TIncludedCollectionElement> innerShaper,
895-
Action<TIncludingEntity, TIncludedCollectionElement> fixup)
896-
where TIncludingEntity : class
897-
where TIncludedCollectionElement : class
908+
JsonReaderData? jsonReaderData,
909+
INavigationBase navigation,
910+
Func<QueryContext, object[], JsonReaderData, TEntity> innerShaper)
911+
where TEntity : class
912+
where TResult : ICollection<TEntity>
898913
{
899-
if (jsonElement.HasValue && jsonElement.Value.ValueKind != JsonValueKind.Null)
914+
if (jsonReaderData == null)
900915
{
901-
var newKeyPropertyValues = new object[keyPropertyValues.Length + 1];
902-
Array.Copy(keyPropertyValues, newKeyPropertyValues, keyPropertyValues.Length);
916+
return default;
917+
}
918+
919+
var manager = new Utf8JsonReaderManager(jsonReaderData);
920+
var tokenType = manager.CurrentReader.TokenType;
921+
922+
if (tokenType == JsonTokenType.Null)
923+
{
924+
return default;
925+
}
903926

904-
var i = 0;
905-
foreach (var jsonArrayElement in jsonElement.Value.EnumerateArray())
927+
if (tokenType != JsonTokenType.StartArray)
928+
{
929+
throw new InvalidOperationException(
930+
RelationalStrings.JsonReaderInvalidTokenType(tokenType.ToString()));
931+
}
932+
933+
var collectionAccessor = navigation.GetCollectionAccessor();
934+
var result = (TResult)collectionAccessor!.Create();
935+
936+
var newKeyPropertyValues = new object[keyPropertyValues.Length + 1];
937+
Array.Copy(keyPropertyValues, newKeyPropertyValues, keyPropertyValues.Length);
938+
939+
tokenType = manager.MoveNext();
940+
941+
var i = 0;
942+
while (tokenType != JsonTokenType.EndArray)
943+
{
944+
newKeyPropertyValues[^1] = ++i;
945+
946+
if (tokenType == JsonTokenType.StartObject)
906947
{
907-
newKeyPropertyValues[^1] = ++i;
948+
manager.CaptureState();
949+
var entity = innerShaper(queryContext, newKeyPropertyValues, jsonReaderData);
950+
result.Add(entity);
951+
manager = new Utf8JsonReaderManager(manager.Data);
908952

909-
var resultElement = innerShaper(queryContext, newKeyPropertyValues, jsonArrayElement);
953+
if (manager.CurrentReader.TokenType != JsonTokenType.EndObject)
954+
{
955+
throw new InvalidOperationException(
956+
RelationalStrings.JsonReaderInvalidTokenType(tokenType.ToString()));
957+
}
910958

911-
fixup(entity, resultElement);
959+
tokenType = manager.MoveNext();
912960
}
913961
}
962+
963+
manager.CaptureState();
964+
965+
return result;
914966
}
915967

916-
private static TEntity? MaterializeJsonEntity<TEntity>(
968+
private static void IncludeJsonEntityReference<TIncludingEntity, TIncludedEntity>(
917969
QueryContext queryContext,
918-
JsonElement? jsonElement,
919970
object[] keyPropertyValues,
920-
bool nullable,
921-
Func<QueryContext, object[], JsonElement, TEntity> shaper)
922-
where TEntity : class
971+
JsonReaderData? jsonReaderData,
972+
TIncludingEntity entity,
973+
Func<QueryContext, object[], JsonReaderData, TIncludedEntity> innerShaper,
974+
Action<TIncludingEntity, TIncludedEntity> fixup,
975+
bool trackingQuery)
976+
where TIncludingEntity : class
977+
where TIncludedEntity : class
923978
{
924-
if (jsonElement.HasValue && jsonElement.Value.ValueKind != JsonValueKind.Null)
979+
if (jsonReaderData == null)
925980
{
926-
var result = shaper(queryContext, keyPropertyValues, jsonElement.Value);
927-
928-
return result;
981+
return;
929982
}
930983

931-
if (nullable)
984+
var included = innerShaper(queryContext, keyPropertyValues, jsonReaderData);
985+
986+
if (!trackingQuery)
932987
{
933-
return default;
988+
fixup(entity, included);
934989
}
935-
936-
throw new InvalidOperationException(
937-
RelationalStrings.JsonRequiredEntityWithNullJson(typeof(TEntity).Name));
938990
}
939991

940-
private static TResult? MaterializeJsonEntityCollection<TEntity, TResult>(
992+
private static void IncludeJsonEntityCollection<TIncludingEntity, TIncludedCollectionElement>(
941993
QueryContext queryContext,
942-
JsonElement? jsonElement,
943994
object[] keyPropertyValues,
944-
INavigationBase navigation,
945-
Func<QueryContext, object[], JsonElement, TEntity> innerShaper)
946-
where TEntity : class
947-
where TResult : ICollection<TEntity>
995+
JsonReaderData? jsonReaderData,
996+
TIncludingEntity entity,
997+
Func<QueryContext, object[], JsonReaderData, TIncludedCollectionElement> innerShaper,
998+
Action<TIncludingEntity, TIncludedCollectionElement> fixup,
999+
bool trackingQuery)
1000+
where TIncludingEntity : class
1001+
where TIncludedCollectionElement : class
9481002
{
949-
if (jsonElement.HasValue && jsonElement.Value.ValueKind != JsonValueKind.Null)
1003+
if (jsonReaderData == null)
9501004
{
951-
var collectionAccessor = navigation.GetCollectionAccessor();
952-
var result = (TResult)collectionAccessor!.Create();
1005+
return;
1006+
}
1007+
1008+
var manager = new Utf8JsonReaderManager(jsonReaderData);
1009+
var tokenType = manager.CurrentReader.TokenType;
1010+
1011+
if (tokenType != JsonTokenType.StartArray)
1012+
{
1013+
throw new InvalidOperationException(
1014+
RelationalStrings.JsonReaderInvalidTokenType(tokenType.ToString()));
1015+
}
9531016

954-
var newKeyPropertyValues = new object[keyPropertyValues.Length + 1];
955-
Array.Copy(keyPropertyValues, newKeyPropertyValues, keyPropertyValues.Length);
1017+
var newKeyPropertyValues = new object[keyPropertyValues.Length + 1];
1018+
Array.Copy(keyPropertyValues, newKeyPropertyValues, keyPropertyValues.Length);
9561019

957-
var i = 0;
958-
foreach (var jsonArrayElement in jsonElement.Value.EnumerateArray())
1020+
tokenType = manager.MoveNext();
1021+
1022+
var i = 0;
1023+
while (tokenType != JsonTokenType.EndArray)
1024+
{
1025+
newKeyPropertyValues[^1] = ++i;
1026+
1027+
if (tokenType == JsonTokenType.StartObject)
9591028
{
960-
newKeyPropertyValues[^1] = ++i;
1029+
manager.CaptureState();
1030+
var resultElement = innerShaper(queryContext, newKeyPropertyValues, jsonReaderData);
1031+
1032+
if (!trackingQuery)
1033+
{
1034+
fixup(entity, resultElement);
1035+
}
9611036

962-
var resultElement = innerShaper(queryContext, newKeyPropertyValues, jsonArrayElement);
1037+
manager = new Utf8JsonReaderManager(manager.Data);
1038+
if (manager.CurrentReader.TokenType != JsonTokenType.EndObject)
1039+
{
1040+
throw new InvalidOperationException(
1041+
RelationalStrings.JsonReaderInvalidTokenType(tokenType.ToString()));
1042+
}
9631043

964-
result.Add(resultElement);
1044+
tokenType = manager.MoveNext();
9651045
}
966-
967-
return result;
9681046
}
9691047

970-
return default;
1048+
manager.CaptureState();
9711049
}
9721050

9731051
private static async Task TaskAwaiter(Func<Task>[] taskFactories)

0 commit comments

Comments
 (0)