Merge pull request #74 from Nexus-Mods/better-caching

Better caching

NexusMods.MnemonicDB.sln

@@ -35,8 +35,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.MnemonicDB.Tests"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.MnemonicDB.Benchmarks", "benchmarks\NexusMods.MnemonicDB.Benchmarks\NexusMods.MnemonicDB.Benchmarks.csproj", "{930B3AB7-56EA-48D6-B603-24D79C7DD00A}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.MnemonicDB.Storage", "src\NexusMods.MnemonicDB.Storage\NexusMods.MnemonicDB.Storage.csproj", "{73E074F9-250F-4D8A-8038-5B12DB761E98}"
-EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.MnemonicDB.Storage.Tests", "tests\NexusMods.MnemonicDB.Storage.Tests\NexusMods.MnemonicDB.Storage.Tests.csproj", "{33A3DA79-D3FD-46DC-8D14-82E23D5B608D}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OneBillionDatomsTest", "benchmarks\OneBillionDatomsTest\OneBillionDatomsTest.csproj", "{EA397BAE-9726-486F-BC9B-87BD86DF157F}"
@@ -54,7 +52,6 @@ Global
 		{EC1570A4-18B9-4A76-84FF-275BAA76A357} = {6ED01F9D-5E12-4EB2-9601-64A2ADC719DE}
 		{07E2C578-8644-474D-8F07-B25CFEB28408} = {6ED01F9D-5E12-4EB2-9601-64A2ADC719DE}
 		{930B3AB7-56EA-48D6-B603-24D79C7DD00A} = {72AFE85F-8C12-436A-894E-638ED2C92A76}
-		{73E074F9-250F-4D8A-8038-5B12DB761E98} = {0377EBE6-F147-4233-86AD-32C821B9567E}
 		{33A3DA79-D3FD-46DC-8D14-82E23D5B608D} = {6ED01F9D-5E12-4EB2-9601-64A2ADC719DE}
 		{EA397BAE-9726-486F-BC9B-87BD86DF157F} = {72AFE85F-8C12-436A-894E-638ED2C92A76}
 		{0EE4BFCE-9E72-4BCC-B179-416E16136A1E} = {0377EBE6-F147-4233-86AD-32C821B9567E}
@@ -80,10 +77,6 @@ Global
 		{930B3AB7-56EA-48D6-B603-24D79C7DD00A}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{930B3AB7-56EA-48D6-B603-24D79C7DD00A}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{930B3AB7-56EA-48D6-B603-24D79C7DD00A}.Release|Any CPU.Build.0 = Release|Any CPU
-		{73E074F9-250F-4D8A-8038-5B12DB761E98}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{73E074F9-250F-4D8A-8038-5B12DB761E98}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{73E074F9-250F-4D8A-8038-5B12DB761E98}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{73E074F9-250F-4D8A-8038-5B12DB761E98}.Release|Any CPU.Build.0 = Release|Any CPU
 		{33A3DA79-D3FD-46DC-8D14-82E23D5B608D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{33A3DA79-D3FD-46DC-8D14-82E23D5B608D}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{33A3DA79-D3FD-46DC-8D14-82E23D5B608D}.Release|Any CPU.ActiveCfg = Release|Any CPU

benchmarks/NexusMods.MnemonicDB.Benchmarks/Benchmarks/ABenchmark.cs

@@ -22,7 +22,7 @@ public async Task InitializeAsync()
         var builder = Host.CreateDefaultBuilder()
             .ConfigureServices(services =>
             {
-                services.AddMnemonicDBStorage()
+                services
                     .AddRocksDbBackend()
                     .AddMnemonicDB()
                     .AddTestModel()

benchmarks/NexusMods.MnemonicDB.Benchmarks/Benchmarks/ReadThenWriteBenchmarks.cs

@@ -0,0 +1,81 @@
+using System.Linq;
+using System.Threading.Tasks;
+using BenchmarkDotNet.Attributes;
+using NexusMods.Hashing.xxHash64;
+using NexusMods.MnemonicDB.Abstractions;
+using NexusMods.MnemonicDB.TestModel;
+using NexusMods.Paths;
+
+namespace NexusMods.MnemonicDB.Benchmarks.Benchmarks;
+
+[MemoryDiagnoser]
+[MaxIterationCount(20)]
+public class ReadThenWriteBenchmarks : ABenchmark
+{
+
+    private EntityId _modId;
+
+    [GlobalSetup]
+    public async Task Setup()
+    {
+        await InitializeAsync();
+
+        using var tx = Connection.BeginTransaction();
+
+        var loadout = new Loadout.New(tx)
+        {
+            Name = "My Loadout"
+        };
+
+        for (int i = 0; i < 90; i++)
+        {
+            var mod = new Mod.New(tx)
+            {
+                Name = $"Mod {i}",
+                Source = new System.Uri($"http://mod{i}.com"),
+                LoadoutId = loadout,
+                OptionalHash = Hash.FromLong(0)
+            };
+
+            _modId = mod.Id;
+
+            for (int j = 0; j < 1000; j++)
+            {
+                var file = new File.New(tx)
+                {
+                    Path = $"File {j}",
+                    ModId = mod,
+                    Size = Size.FromLong(j),
+                    Hash = Hash.FromLong(j)
+                };
+            }
+        }
+
+        var result = await tx.Commit();
+        _modId = result[_modId];
+    }
+
+    [Benchmark]
+    public async Task<ulong> ReadThenWrite()
+    {
+        using var tx = Connection.BeginTransaction();
+        var mod = Mod.Load(Connection.Db, _modId);
+        var oldHash = mod.OptionalHash;
+        tx.Add(_modId, Mod.OptionalHash, Hash.From(oldHash.Value + 1));
+        var nextdb = await tx.Commit();
+
+        var loadout = Loadout.Load(Connection.Db, mod.LoadoutId);
+
+
+        ulong totalSize = 0;
+        foreach (var mod2 in loadout.Mods)
+        {
+            foreach (var file in mod2.Files)
+            {
+                totalSize += file.Size.Value;
+            }
+        }
+        return totalSize;
+    }
+
+}

benchmarks/NexusMods.MnemonicDB.Benchmarks/Program.cs

@@ -11,28 +11,25 @@
 
 #if DEBUG
 
-var benchmark = new ReadTests
-{
-    Count = 128
-};
+var benchmark = new ReadThenWriteBenchmarks();
 
 var sw = Stopwatch.StartNew();
 await benchmark.Setup();
 
 ulong result = 0;
 
-MeasureProfiler.StartCollectingData();
+//MeasureProfiler.StartCollectingData();
 //MemoryProfiler.CollectAllocations(true);
-for (var i = 0; i < 10000; i++)
-    result = benchmark.ReadAllFromMod();
+for (var i = 0; i < 1; i++)
+    result = await benchmark.ReadThenWrite();
 //MemoryProfiler.CollectAllocations(false);
 
-MeasureProfiler.SaveData();
+//MeasureProfiler.SaveData();
 Console.WriteLine("Elapsed: " + sw.Elapsed + " Result: " + result);
 
 #else
 
-BenchmarkRunner.Run<IndexSegmentEBenchmarks>(config: DefaultConfig.Instance.WithOption(ConfigOptions.DisableOptimizationsValidator, true));
+BenchmarkRunner.Run<ReadThenWriteBenchmarks>(config: DefaultConfig.Instance.WithOption(ConfigOptions.DisableOptimizationsValidator, true));
 #endif
 
 

benchmarks/OneBillionDatomsTest/OneBillionDatomsTest.csproj

@@ -12,7 +12,6 @@
     </ItemGroup>
 
     <ItemGroup>
-        <ProjectReference Include="..\..\src\NexusMods.MnemonicDB.Storage\NexusMods.MnemonicDB.Storage.csproj"/>
         <ProjectReference Include="..\..\src\NexusMods.MnemonicDB\NexusMods.MnemonicDB.csproj"/>
         <ProjectReference Include="..\..\tests\NexusMods.MnemonicDB.Storage.Tests\NexusMods.MnemonicDB.Storage.Tests.csproj"/>
         <ProjectReference Include="..\..\tests\NexusMods.MnemonicDB.TestModel\NexusMods.MnemonicDB.TestModel.csproj"/>

src/NexusMods.MnemonicDB.Abstractions/IConnection.cs

@@ -43,7 +43,7 @@ public interface IConnection
     /// <summary>
     ///     A sequential stream of database revisions.
     /// </summary>
-    public IObservable<Revision> Revisions { get; }
+    public IObservable<IDb> Revisions { get; }
 
     /// <summary>
     /// A service provider that entities can use to resolve their values

src/NexusMods.MnemonicDB.Abstractions/IDatomStore.cs

@@ -17,7 +17,7 @@ public interface IDatomStore : IDisposable
     ///     An observable of the transaction log, for getting the latest changes to the store. This observable
     /// will always start with the most recent value, so there is no reason to use `StartWith` or `Replay` on it.
     /// </summary>
-    public IObservable<(TxId TxId, ISnapshot Snapshot)> TxLog { get; }
+    public IObservable<IDb> TxLog { get; }
 
     /// <summary>
     ///     Gets the latest transaction id found in the log.
@@ -32,20 +32,13 @@ public interface IDatomStore : IDisposable
     /// <summary>
     ///     Transacts (adds) the given datoms into the store.
     /// </summary>
-    public Task<StoreResult> TransactAsync(IndexSegment datoms, HashSet<ITxFunction>? txFunctions = null, Func<ISnapshot, IDb>? databaseFactory = null);
+    public Task<(StoreResult, IDb)> TransactAsync(IndexSegment datoms, HashSet<ITxFunction>? txFunctions = null);
 
 
     /// <summary>
     ///     Transacts (adds) the given datoms into the store.
     /// </summary>
-    public StoreResult Transact(IndexSegment datoms, HashSet<ITxFunction>? txFunctions = null, Func<ISnapshot, IDb>? databaseFactory = null);
-
-    /// <summary>
-    /// Executes an empty transaction. Returns a StoreResult valid asof the latest
-    /// transaction
-    /// </summary>
-    /// <returns></returns>
-    public Task<StoreResult> Sync();
+    public (StoreResult, IDb) Transact(IndexSegment datoms, HashSet<ITxFunction>? txFunctions = null);
 
     /// <summary>
     ///     Registers new attributes with the store.

src/NexusMods.MnemonicDB.Abstractions/IDb.cs

@@ -24,6 +24,11 @@ public interface IDb : IEquatable<IDb>
     ///     The connection that this database is using for its state.
     /// </summary>
     IConnection Connection { get; }
+
+    /// <summary>
+    ///     The datoms that were added in the most recent transaction (indicated by the basis TxId).
+    /// </summary>
+    IndexSegment RecentlyAdded { get; }
 
     /// <summary>
     /// The snapshot that this database is based on.
@@ -40,13 +45,6 @@ public interface IDb : IEquatable<IDb>
     /// </summary>
     public IndexSegment Get(EntityId entityId);
 
-    /// <summary>
-    ///     Gets a read model for every enitity that references the given entity id
-    ///     with the given attribute.
-    /// </summary>
-    public Entities<EntityIds, TModel> GetReverse<TModel>(EntityId id, Attribute<EntityId, ulong> attribute)
-        where TModel : IReadOnlyModel<TModel>;
-
     /// <summary>
     /// Get all the datoms for the given entity id.
     /// </summary>

src/NexusMods.MnemonicDB.Abstractions/IndexType.cs

@@ -5,7 +5,7 @@
 
 namespace NexusMods.MnemonicDB.Abstractions;
 
-public enum IndexType
+public enum IndexType : byte
 {
     // Transaction log, the final source of truth, used
     // for replaying the database

src/NexusMods.MnemonicDB.Abstractions/Query/ObservableDatoms.cs

@@ -44,19 +44,19 @@ public static IObservable<IChangeSet<Datom>> ObserveDatoms(this IConnection conn
         var lastTxId = TxId.From(0);
 
         return conn.Revisions
-            .Where(rev => rev.AddedDatoms.Valid)
+            .Where(rev => rev.RecentlyAdded.Count > 0)
             .Select((rev, idx) =>
         {
             lock (set)
             {
-                if (rev.Database.BasisTxId <= lastTxId)
+                if (rev.BasisTxId <= lastTxId)
                     return ChangeSet<Datom>.Empty;
 
-                lastTxId = rev.Database.BasisTxId;
+                lastTxId = rev.BasisTxId;
 
                 if (idx == 0)
-                    return Setup(set, rev.Database, descriptor);
-                return Diff(conn.Registry, set, rev.AddedDatoms, descriptor, equality);
+                    return Setup(set, rev, descriptor);
+                return Diff(conn.Registry, set, rev.RecentlyAdded, descriptor, equality);
             }
         });
     }