Skip to content

Commit 62c7ec1

Browse files
jeanbernjeanbern
committed
Add unit tests.
Remove Fody and LocalsInit. Upgrade to .net 5
1 parent 8172fb8 commit 62c7ec1

17 files changed

+501332
-63
lines changed

‎.gitmodules

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
[submodule "ChacoSharp"]
2+
path = ChacoSharp
3+
url = https://github.com/jeanbern/ChacoSharp

‎AllProjects.sln

+29-1
Original file line numberDiff line numberDiff line change
@@ -28,30 +28,58 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Portent.Test", "Portent.Tes
2828
EndProject
2929
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Portent.Benchmark", "Portent.Benchmark\Portent.Benchmark.csproj", "{BD1A249D-0576-49EE-BC96-63B9545674BE}"
3030
EndProject
31-
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MinLA", "MinLA\MinLA.csproj", "{382F1CE8-1B54-4CFC-801A-05883A89295C}"
31+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MinLA", "MinLA\MinLA.csproj", "{382F1CE8-1B54-4CFC-801A-05883A89295C}"
32+
EndProject
33+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ChacoSharp", "ChacoSharp\ChacoSharp.csproj", "{379A0679-7276-45D3-89CE-02B753809E99}"
3234
EndProject
3335
Global
3436
GlobalSection(SolutionConfigurationPlatforms) = preSolution
3537
Debug|Any CPU = Debug|Any CPU
38+
Debug|x64 = Debug|x64
3639
Release|Any CPU = Release|Any CPU
40+
Release|x64 = Release|x64
3741
EndGlobalSection
3842
GlobalSection(ProjectConfigurationPlatforms) = postSolution
3943
{B51F0229-53A3-4773-8F45-5CFC3636C65E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
4044
{B51F0229-53A3-4773-8F45-5CFC3636C65E}.Debug|Any CPU.Build.0 = Debug|Any CPU
45+
{B51F0229-53A3-4773-8F45-5CFC3636C65E}.Debug|x64.ActiveCfg = Debug|Any CPU
46+
{B51F0229-53A3-4773-8F45-5CFC3636C65E}.Debug|x64.Build.0 = Debug|Any CPU
4147
{B51F0229-53A3-4773-8F45-5CFC3636C65E}.Release|Any CPU.ActiveCfg = Release|Any CPU
4248
{B51F0229-53A3-4773-8F45-5CFC3636C65E}.Release|Any CPU.Build.0 = Release|Any CPU
49+
{B51F0229-53A3-4773-8F45-5CFC3636C65E}.Release|x64.ActiveCfg = Release|Any CPU
50+
{B51F0229-53A3-4773-8F45-5CFC3636C65E}.Release|x64.Build.0 = Release|Any CPU
4351
{2745B9B8-FCA5-4648-994F-69A47C6AFA86}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
4452
{2745B9B8-FCA5-4648-994F-69A47C6AFA86}.Debug|Any CPU.Build.0 = Debug|Any CPU
53+
{2745B9B8-FCA5-4648-994F-69A47C6AFA86}.Debug|x64.ActiveCfg = Debug|Any CPU
54+
{2745B9B8-FCA5-4648-994F-69A47C6AFA86}.Debug|x64.Build.0 = Debug|Any CPU
4555
{2745B9B8-FCA5-4648-994F-69A47C6AFA86}.Release|Any CPU.ActiveCfg = Release|Any CPU
4656
{2745B9B8-FCA5-4648-994F-69A47C6AFA86}.Release|Any CPU.Build.0 = Release|Any CPU
57+
{2745B9B8-FCA5-4648-994F-69A47C6AFA86}.Release|x64.ActiveCfg = Release|Any CPU
58+
{2745B9B8-FCA5-4648-994F-69A47C6AFA86}.Release|x64.Build.0 = Release|Any CPU
4759
{BD1A249D-0576-49EE-BC96-63B9545674BE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
4860
{BD1A249D-0576-49EE-BC96-63B9545674BE}.Debug|Any CPU.Build.0 = Debug|Any CPU
61+
{BD1A249D-0576-49EE-BC96-63B9545674BE}.Debug|x64.ActiveCfg = Debug|Any CPU
62+
{BD1A249D-0576-49EE-BC96-63B9545674BE}.Debug|x64.Build.0 = Debug|Any CPU
4963
{BD1A249D-0576-49EE-BC96-63B9545674BE}.Release|Any CPU.ActiveCfg = Release|Any CPU
5064
{BD1A249D-0576-49EE-BC96-63B9545674BE}.Release|Any CPU.Build.0 = Release|Any CPU
65+
{BD1A249D-0576-49EE-BC96-63B9545674BE}.Release|x64.ActiveCfg = Release|Any CPU
66+
{BD1A249D-0576-49EE-BC96-63B9545674BE}.Release|x64.Build.0 = Release|Any CPU
5167
{382F1CE8-1B54-4CFC-801A-05883A89295C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
5268
{382F1CE8-1B54-4CFC-801A-05883A89295C}.Debug|Any CPU.Build.0 = Debug|Any CPU
69+
{382F1CE8-1B54-4CFC-801A-05883A89295C}.Debug|x64.ActiveCfg = Debug|Any CPU
70+
{382F1CE8-1B54-4CFC-801A-05883A89295C}.Debug|x64.Build.0 = Debug|Any CPU
5371
{382F1CE8-1B54-4CFC-801A-05883A89295C}.Release|Any CPU.ActiveCfg = Release|Any CPU
5472
{382F1CE8-1B54-4CFC-801A-05883A89295C}.Release|Any CPU.Build.0 = Release|Any CPU
73+
{382F1CE8-1B54-4CFC-801A-05883A89295C}.Release|x64.ActiveCfg = Release|Any CPU
74+
{382F1CE8-1B54-4CFC-801A-05883A89295C}.Release|x64.Build.0 = Release|Any CPU
75+
{379A0679-7276-45D3-89CE-02B753809E99}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
76+
{379A0679-7276-45D3-89CE-02B753809E99}.Debug|Any CPU.Build.0 = Debug|Any CPU
77+
{379A0679-7276-45D3-89CE-02B753809E99}.Debug|x64.ActiveCfg = Debug|Any CPU
78+
{379A0679-7276-45D3-89CE-02B753809E99}.Debug|x64.Build.0 = Debug|Any CPU
79+
{379A0679-7276-45D3-89CE-02B753809E99}.Release|Any CPU.ActiveCfg = Release|Any CPU
80+
{379A0679-7276-45D3-89CE-02B753809E99}.Release|Any CPU.Build.0 = Release|Any CPU
81+
{379A0679-7276-45D3-89CE-02B753809E99}.Release|x64.ActiveCfg = Release|Any CPU
82+
{379A0679-7276-45D3-89CE-02B753809E99}.Release|x64.Build.0 = Release|Any CPU
5583
EndGlobalSection
5684
GlobalSection(SolutionProperties) = preSolution
5785
HideSolutionNode = FALSE

‎AllProjects.sln.DotSettings

+1
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
<wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
2+
<s:Boolean x:Key="/Default/UserDictionary/Words/=Dawg/@EntryIndexedValue">True</s:Boolean>
23
<s:Boolean x:Key="/Default/UserDictionary/Words/=nint/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>

‎ChacoSharp

Submodule ChacoSharp added at f1c3c73

‎MinLA/MinLA.csproj

+2-7
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
<PropertyGroup>
44
<OutputType>Exe</OutputType>
5-
<TargetFramework>netcoreapp3.1</TargetFramework>
5+
<TargetFramework>net5.0</TargetFramework>
66
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
77
<Nullable>enable</Nullable>
88
</PropertyGroup>
@@ -15,13 +15,8 @@
1515
</PropertyGroup>
1616

1717
<ItemGroup>
18+
<ProjectReference Include="..\ChacoSharp\ChacoSharp.csproj" />
1819
<ProjectReference Include="..\Portent\Portent.csproj" />
1920
</ItemGroup>
2021

21-
<ItemGroup>
22-
<Reference Include="ChacoSharp">
23-
<HintPath>..\..\..\Users\JPelleri\source\repos\ChacoSharp\bin\Release\netcoreapp3.1\ChacoSharp.dll</HintPath>
24-
</Reference>
25-
</ItemGroup>
26-
2722
</Project>

‎Portent.Benchmark/Portent.Benchmark.csproj

+20-10
Original file line numberDiff line numberDiff line change
@@ -2,19 +2,33 @@
22

33
<PropertyGroup>
44
<OutputType>Exe</OutputType>
5-
<TargetFramework>netcoreapp3.1</TargetFramework>
5+
<TargetFramework>net5.0</TargetFramework>
66
<LangVersion>8.0</LangVersion>
7+
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
78
<PlatformTarget>x64</PlatformTarget>
9+
810
<NullableContextOptions>enable</NullableContextOptions>
911
<Nullable>enable</Nullable>
10-
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
1112
<GenerateSerializationAssemblies>Off</GenerateSerializationAssemblies>
1213
</PropertyGroup>
1314

1415
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|AnyCPU'">
15-
<DebugType>none</DebugType>
16-
<DebugSymbols>false</DebugSymbols>
16+
<DebugType>pdbonly</DebugType>
17+
<DebugSymbols>true</DebugSymbols>
18+
<Optimize>true</Optimize>
1719
<WarningLevel>2</WarningLevel>
20+
21+
<!-- Use together -->
22+
<PublishTrimmed>true</PublishTrimmed>
23+
<PublishReadyToRun>true</PublishReadyToRun>
24+
25+
<!-- Use together -->
26+
<TieredCompilation>true</TieredCompilation>
27+
<TieredCompilationQuickJit>true</TieredCompilationQuickJit>
28+
29+
<ServerGarbageCollection>true</ServerGarbageCollection>
30+
<GCLargePages>true</GCLargePages>
31+
1832
</PropertyGroup>
1933

2034
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
@@ -24,8 +38,8 @@
2438
</PropertyGroup>
2539

2640
<ItemGroup>
27-
<PackageReference Include="BenchmarkDotNet" Version="0.12.0" />
28-
<PackageReference Include="BenchmarkDotNet.Diagnostics.Windows" Version="0.12.0" />
41+
<PackageReference Include="BenchmarkDotNet" Version="0.12.1" />
42+
<PackageReference Include="BenchmarkDotNet.Diagnostics.Windows" Version="0.12.1" />
2943
</ItemGroup>
3044

3145
<ItemGroup>
@@ -41,8 +55,4 @@
4155
</None>
4256
</ItemGroup>
4357

44-
<Target Name="PostBuild" AfterTargets="PostBuildEvent">
45-
<Exec Command="&quot;C:\Program Files\PowerShell\6\pwsh.exe&quot; -ExecutionPolicy ByPass C:\doctemp\signingScript.ps1 -filepath $(TargetPath)" />
46-
</Target>
47-
4858
</Project>

‎Portent.Test/Portent.Test.csproj

+13-4
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
<Project Sdk="Microsoft.NET.Sdk">
22

33
<PropertyGroup>
4-
<TargetFramework>netcoreapp3.1</TargetFramework>
4+
<TargetFramework>net5.0</TargetFramework>
55
<LangVersion>8.0</LangVersion>
66
<IsPackable>false</IsPackable>
77
<PlatformTarget>x64</PlatformTarget>
@@ -14,13 +14,13 @@
1414
</PropertyGroup>
1515

1616
<ItemGroup>
17-
<PackageReference Include="coverlet.collector" Version="1.1.0">
17+
<PackageReference Include="coverlet.collector" Version="1.3.0">
1818
<PrivateAssets>all</PrivateAssets>
1919
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
2020
</PackageReference>
21-
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.3.0-preview-20190828-03" />
21+
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.7.0-preview-20200519-01" />
2222
<PackageReference Include="xunit" Version="2.4.1" />
23-
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.1">
23+
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.2">
2424
<PrivateAssets>all</PrivateAssets>
2525
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
2626
</PackageReference>
@@ -30,4 +30,13 @@
3030
<ProjectReference Include="..\Portent\Portent.csproj" />
3131
</ItemGroup>
3232

33+
<ItemGroup>
34+
<None Update="TestData\frequency_dictionary_en_500_000.txt">
35+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
36+
</None>
37+
<None Update="TestData\noisy_query_en_1000.txt">
38+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
39+
</None>
40+
</ItemGroup>
41+
3342
</Project>

‎Portent.Test/UnitTest1.cs

-14
This file was deleted.

‎Portent/Allocation/LargePageMemoryChunk.cs

+12-8
Original file line numberDiff line numberDiff line change
@@ -65,18 +65,22 @@ public LargePageMemoryChunk(ulong length)
6565
const MemoryAllocationType flags = MemoryAllocationType.MemReserve | MemoryAllocationType.MemCommit;
6666
const string lockMemory = "SeLockMemoryPrivilege";
6767
_bytesReserved = MemoryAlignmentHelper.LargePageMultiple(length);
68-
69-
using var privs = PrivilegeHolder.EnablePrivilege(lockMemory);
70-
if (privs != null)
68+
try
7169
{
72-
const MemoryAllocationType largeFlag = flags | MemoryAllocationType.MemLargePages;
73-
_ptr = NativeMethods.VirtualAlloc(IntPtr.Zero, (IntPtr)_bytesReserved, largeFlag, MemoryProtectionConstants.PageReadwrite);
74-
GC.KeepAlive(privs);
70+
using var privs = PrivilegeHolder.EnablePrivilege(lockMemory);
71+
if (privs != null)
72+
{
73+
const MemoryAllocationType largeFlag = flags | MemoryAllocationType.MemLargePages;
74+
_ptr = NativeMethods.VirtualAlloc(IntPtr.Zero, (IntPtr) _bytesReserved, largeFlag, MemoryProtectionConstants.PageReadwrite);
75+
GC.KeepAlive(privs);
76+
}
7577
}
76-
else
78+
catch (ObjectDisposedException)
7779
{
78-
_ptr = NativeMethods.VirtualAlloc(IntPtr.Zero, (IntPtr)_bytesReserved, flags, MemoryProtectionConstants.PageReadwrite);
80+
// TODO: fix the underlying cause
7981
}
82+
83+
_ptr = NativeMethods.VirtualAlloc(IntPtr.Zero, (IntPtr)_bytesReserved, flags, MemoryProtectionConstants.PageReadwrite);
8084
}
8185

8286
public static MemoryChunkBuilder Builder()

‎Portent/Dawg.cs

-3
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111
using System.Runtime.Intrinsics.X86;
1212
#endif
1313
using System.Threading.Tasks;
14-
using LocalsInit;
1514

1615
// ReSharper disable BuiltInTypeReferenceStyle
1716

@@ -25,7 +24,6 @@ namespace Portent
2524
{
2625
public sealed unsafe class Dawg : IDisposable
2726
{
28-
[LocalsInit(false)]
2927
[MethodImpl(MethodImplOptions.AggressiveOptimization)]
3028
public IEnumerable<SuggestItem> Lookup(in string word, uint maxEdits)
3129
{
@@ -127,7 +125,6 @@ public IEnumerable<SuggestItem> Lookup(in string word, uint maxEdits)
127125
return _compoundResultCollection;
128126
}
129127

130-
[LocalsInit(false)]
131128
[MethodImpl(MethodImplOptions.AggressiveOptimization)]
132129
[SuppressMessage("Major Code Smell", "S907:\"goto\" statement should not be used")]
133130
private static void Search5(Job me)

‎Portent/Portent.csproj

+4-14
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
<Project Sdk="Microsoft.NET.Sdk">
22

33
<PropertyGroup>
4-
<TargetFramework>netcoreapp3.1</TargetFramework>
4+
<TargetFramework>net5.0</TargetFramework>
55
<LangVersion>preview</LangVersion>
66
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
77
<PlatformTarget>x64</PlatformTarget>
@@ -56,20 +56,10 @@
5656
</ItemGroup>
5757

5858
<ItemGroup>
59-
<PackageReference Include="Fody" Version="6.0.0">
60-
<PrivateAssets>all</PrivateAssets>
61-
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
62-
</PackageReference>
63-
<PackageReference Include="JetBrains.Annotations" Version="2019.1.3" />
64-
<PackageReference Include="LocalsInit.Fody" Version="1.1.0" />
59+
<PackageReference Include="JetBrains.Annotations" Version="2020.2.0-eap" />
6560
<PackageReference Include="System.Numerics.Vectors" Version="4.6.0-preview5.19224.8" />
6661
<PackageReference Include="System.Runtime" Version="4.3.1" />
67-
<PackageReference Include="System.Runtime.CompilerServices.Unsafe" Version="4.6.0-preview9.19421.4" />
62+
<PackageReference Include="System.Runtime.CompilerServices.Unsafe" Version="5.0.0-preview.7.20364.11" />
6863
<PackageReference Include="System.Runtime.Numerics" Version="4.3.0" />
6964
</ItemGroup>
70-
71-
<Target Name="PostBuild" AfterTargets="PostBuildEvent">
72-
<Exec Command="&quot;C:\Program Files\PowerShell\6\pwsh.exe&quot; -ExecutionPolicy ByPass C:\doctemp\signingScript.ps1 -filepath $(TargetPath)" />
73-
</Target>
74-
75-
</Project>
65+
</Project>

‎Portent/Serialization/StreamExtensions.cs

-2
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
using System.IO;
44
using System.Runtime.CompilerServices;
55
using System.Text;
6-
using LocalsInit;
76

87
// ReSharper disable ForCanBeConvertedToForeach
98

@@ -278,7 +277,6 @@ public static void WriteSequentialCompressedToUshort(this Stream stream, Span<ui
278277
#endregion Unsigned
279278

280279
[MethodImpl(MethodImplOptions.AggressiveInlining)]
281-
[LocalsInit(false)]
282280
public static void WriteUtf8(this Stream stream, char[] value)
283281
{
284282
var encoding = Utf8NoBom;

‎portent.Test/DawgTests/DawgHelper.cs

+83
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
#nullable enable
2+
using System;
3+
using System.IO;
4+
using System.Linq;
5+
6+
namespace Portent.Test.DawgTests
7+
{
8+
internal static class DawgHelper
9+
{
10+
private const string TempAugPath = "writefile.aug";
11+
12+
public static Dawg Create(params string[] words)
13+
{
14+
var builder = new PartitionedGraphBuilder();
15+
foreach (var word in words.OrderBy(x => x))
16+
{
17+
builder.Insert(word, 0);
18+
}
19+
20+
using var compressed = builder.AsCompressedSparseRows();
21+
compressed.Save(TempAugPath);
22+
23+
using var read = File.OpenRead(TempAugPath);
24+
return new Dawg(read);
25+
}
26+
27+
public static Dawg CreateFromCorpus(string corpusLocation)
28+
{
29+
var builder = new PartitionedGraphBuilder();
30+
using var stream = File.OpenRead(corpusLocation);
31+
using var reader = new StreamReader(stream);
32+
string? line;
33+
while ((line = reader.ReadLine()) != null)
34+
{
35+
var lineTokens = line.Split(' ');
36+
if (lineTokens.Length != 2)
37+
{
38+
continue;
39+
}
40+
41+
if (!ulong.TryParse(lineTokens[1], out var count))
42+
{
43+
continue;
44+
}
45+
46+
builder.Insert(lineTokens[0], count);
47+
}
48+
49+
using var compressedGraph = builder.AsCompressedSparseRows();
50+
compressedGraph.Save(TempAugPath);
51+
52+
using var dawgStream = File.OpenRead(TempAugPath);
53+
return new Dawg(dawgStream);
54+
}
55+
56+
public static string[] BuildQuery1K(string queryLocation)
57+
{
58+
using var stream = File.OpenRead(queryLocation);
59+
var testList = new string[1000];
60+
var i = 0;
61+
62+
using var reader = new StreamReader(stream);
63+
64+
string? line;
65+
while ((line = reader.ReadLine()) != null)
66+
{
67+
var lineParts = line.Split(default(char[]), StringSplitOptions.None);
68+
if (lineParts.Length == 3)
69+
{
70+
testList[i++] = lineParts[0];
71+
}
72+
}
73+
74+
if (i != 1000)
75+
{
76+
// ReSharper disable once RedundantToStringCallForValueType - would box value type?
77+
throw new InvalidOperationException("Unexpected number of query inputs: " + i.ToString());
78+
}
79+
80+
return testList;
81+
}
82+
}
83+
}

0 commit comments

Comments
 (0)