diff --git a/NexusMods.MnemonicDB.sln b/NexusMods.MnemonicDB.sln index 959fba1d..c19949ec 100644 --- a/NexusMods.MnemonicDB.sln +++ b/NexusMods.MnemonicDB.sln @@ -35,8 +35,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.MnemonicDB.Tests" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.MnemonicDB.Benchmarks", "benchmarks\NexusMods.MnemonicDB.Benchmarks\NexusMods.MnemonicDB.Benchmarks.csproj", "{930B3AB7-56EA-48D6-B603-24D79C7DD00A}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.MnemonicDB.Storage", "src\NexusMods.MnemonicDB.Storage\NexusMods.MnemonicDB.Storage.csproj", "{73E074F9-250F-4D8A-8038-5B12DB761E98}" -EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.MnemonicDB.Storage.Tests", "tests\NexusMods.MnemonicDB.Storage.Tests\NexusMods.MnemonicDB.Storage.Tests.csproj", "{33A3DA79-D3FD-46DC-8D14-82E23D5B608D}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OneBillionDatomsTest", "benchmarks\OneBillionDatomsTest\OneBillionDatomsTest.csproj", "{EA397BAE-9726-486F-BC9B-87BD86DF157F}" @@ -54,7 +52,6 @@ Global {EC1570A4-18B9-4A76-84FF-275BAA76A357} = {6ED01F9D-5E12-4EB2-9601-64A2ADC719DE} {07E2C578-8644-474D-8F07-B25CFEB28408} = {6ED01F9D-5E12-4EB2-9601-64A2ADC719DE} {930B3AB7-56EA-48D6-B603-24D79C7DD00A} = {72AFE85F-8C12-436A-894E-638ED2C92A76} - {73E074F9-250F-4D8A-8038-5B12DB761E98} = {0377EBE6-F147-4233-86AD-32C821B9567E} {33A3DA79-D3FD-46DC-8D14-82E23D5B608D} = {6ED01F9D-5E12-4EB2-9601-64A2ADC719DE} {EA397BAE-9726-486F-BC9B-87BD86DF157F} = {72AFE85F-8C12-436A-894E-638ED2C92A76} {0EE4BFCE-9E72-4BCC-B179-416E16136A1E} = {0377EBE6-F147-4233-86AD-32C821B9567E} @@ -80,10 +77,6 @@ Global {930B3AB7-56EA-48D6-B603-24D79C7DD00A}.Debug|Any CPU.Build.0 = Debug|Any CPU {930B3AB7-56EA-48D6-B603-24D79C7DD00A}.Release|Any CPU.ActiveCfg = Release|Any CPU {930B3AB7-56EA-48D6-B603-24D79C7DD00A}.Release|Any CPU.Build.0 = Release|Any CPU - {73E074F9-250F-4D8A-8038-5B12DB761E98}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {73E074F9-250F-4D8A-8038-5B12DB761E98}.Debug|Any CPU.Build.0 = Debug|Any CPU - {73E074F9-250F-4D8A-8038-5B12DB761E98}.Release|Any CPU.ActiveCfg = Release|Any CPU - {73E074F9-250F-4D8A-8038-5B12DB761E98}.Release|Any CPU.Build.0 = Release|Any CPU {33A3DA79-D3FD-46DC-8D14-82E23D5B608D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {33A3DA79-D3FD-46DC-8D14-82E23D5B608D}.Debug|Any CPU.Build.0 = Debug|Any CPU {33A3DA79-D3FD-46DC-8D14-82E23D5B608D}.Release|Any CPU.ActiveCfg = Release|Any CPU diff --git a/benchmarks/NexusMods.MnemonicDB.Benchmarks/Benchmarks/ABenchmark.cs b/benchmarks/NexusMods.MnemonicDB.Benchmarks/Benchmarks/ABenchmark.cs index f0e89b02..00344bd3 100644 --- a/benchmarks/NexusMods.MnemonicDB.Benchmarks/Benchmarks/ABenchmark.cs +++ b/benchmarks/NexusMods.MnemonicDB.Benchmarks/Benchmarks/ABenchmark.cs @@ -22,7 +22,7 @@ public async Task InitializeAsync() var builder = Host.CreateDefaultBuilder() .ConfigureServices(services => { - services.AddMnemonicDBStorage() + services .AddRocksDbBackend() .AddMnemonicDB() .AddTestModel() diff --git a/benchmarks/NexusMods.MnemonicDB.Benchmarks/Benchmarks/ReadThenWriteBenchmarks.cs b/benchmarks/NexusMods.MnemonicDB.Benchmarks/Benchmarks/ReadThenWriteBenchmarks.cs new file mode 100644 index 00000000..5729f525 --- /dev/null +++ b/benchmarks/NexusMods.MnemonicDB.Benchmarks/Benchmarks/ReadThenWriteBenchmarks.cs @@ -0,0 +1,81 @@ +using System.Linq; +using System.Threading.Tasks; +using BenchmarkDotNet.Attributes; +using NexusMods.Hashing.xxHash64; +using NexusMods.MnemonicDB.Abstractions; +using NexusMods.MnemonicDB.TestModel; +using NexusMods.Paths; + +namespace NexusMods.MnemonicDB.Benchmarks.Benchmarks; + +[MemoryDiagnoser] +[MaxIterationCount(20)] +public class ReadThenWriteBenchmarks : ABenchmark +{ + + private EntityId _modId; + + [GlobalSetup] + public async Task Setup() + { + await InitializeAsync(); + + using var tx = Connection.BeginTransaction(); + + var loadout = new Loadout.New(tx) + { + Name = "My Loadout" + }; + + for (int i = 0; i < 90; i++) + { + var mod = new Mod.New(tx) + { + Name = $"Mod {i}", + Source = new System.Uri($"http://mod{i}.com"), + LoadoutId = loadout, + OptionalHash = Hash.FromLong(0) + }; + + _modId = mod.Id; + + for (int j = 0; j < 1000; j++) + { + var file = new File.New(tx) + { + Path = $"File {j}", + ModId = mod, + Size = Size.FromLong(j), + Hash = Hash.FromLong(j) + }; + } + } + + var result = await tx.Commit(); + _modId = result[_modId]; + } + + [Benchmark] + public async Task ReadThenWrite() + { + using var tx = Connection.BeginTransaction(); + var mod = Mod.Load(Connection.Db, _modId); + var oldHash = mod.OptionalHash; + tx.Add(_modId, Mod.OptionalHash, Hash.From(oldHash.Value + 1)); + var nextdb = await tx.Commit(); + + var loadout = Loadout.Load(Connection.Db, mod.LoadoutId); + + + ulong totalSize = 0; + foreach (var mod2 in loadout.Mods) + { + foreach (var file in mod2.Files) + { + totalSize += file.Size.Value; + } + } + return totalSize; + } + +} diff --git a/benchmarks/NexusMods.MnemonicDB.Benchmarks/Program.cs b/benchmarks/NexusMods.MnemonicDB.Benchmarks/Program.cs index 8c20ac7e..bd313190 100644 --- a/benchmarks/NexusMods.MnemonicDB.Benchmarks/Program.cs +++ b/benchmarks/NexusMods.MnemonicDB.Benchmarks/Program.cs @@ -11,28 +11,25 @@ #if DEBUG -var benchmark = new ReadTests -{ - Count = 128 -}; +var benchmark = new ReadThenWriteBenchmarks(); var sw = Stopwatch.StartNew(); await benchmark.Setup(); ulong result = 0; -MeasureProfiler.StartCollectingData(); +//MeasureProfiler.StartCollectingData(); //MemoryProfiler.CollectAllocations(true); -for (var i = 0; i < 10000; i++) - result = benchmark.ReadAllFromMod(); +for (var i = 0; i < 1; i++) + result = await benchmark.ReadThenWrite(); //MemoryProfiler.CollectAllocations(false); -MeasureProfiler.SaveData(); +//MeasureProfiler.SaveData(); Console.WriteLine("Elapsed: " + sw.Elapsed + " Result: " + result); #else -BenchmarkRunner.Run(config: DefaultConfig.Instance.WithOption(ConfigOptions.DisableOptimizationsValidator, true)); +BenchmarkRunner.Run(config: DefaultConfig.Instance.WithOption(ConfigOptions.DisableOptimizationsValidator, true)); #endif diff --git a/benchmarks/OneBillionDatomsTest/OneBillionDatomsTest.csproj b/benchmarks/OneBillionDatomsTest/OneBillionDatomsTest.csproj index de2292fc..50f585ce 100644 --- a/benchmarks/OneBillionDatomsTest/OneBillionDatomsTest.csproj +++ b/benchmarks/OneBillionDatomsTest/OneBillionDatomsTest.csproj @@ -12,7 +12,6 @@ - diff --git a/src/NexusMods.MnemonicDB.Abstractions/IConnection.cs b/src/NexusMods.MnemonicDB.Abstractions/IConnection.cs index 12abef7c..78d004e8 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/IConnection.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/IConnection.cs @@ -43,7 +43,7 @@ public interface IConnection /// /// A sequential stream of database revisions. /// - public IObservable Revisions { get; } + public IObservable Revisions { get; } /// /// A service provider that entities can use to resolve their values diff --git a/src/NexusMods.MnemonicDB.Abstractions/IDatomStore.cs b/src/NexusMods.MnemonicDB.Abstractions/IDatomStore.cs index 57b5f999..dc2f5699 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/IDatomStore.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/IDatomStore.cs @@ -17,7 +17,7 @@ public interface IDatomStore : IDisposable /// An observable of the transaction log, for getting the latest changes to the store. This observable /// will always start with the most recent value, so there is no reason to use `StartWith` or `Replay` on it. /// - public IObservable<(TxId TxId, ISnapshot Snapshot)> TxLog { get; } + public IObservable TxLog { get; } /// /// Gets the latest transaction id found in the log. @@ -32,20 +32,13 @@ public interface IDatomStore : IDisposable /// /// Transacts (adds) the given datoms into the store. /// - public Task TransactAsync(IndexSegment datoms, HashSet? txFunctions = null, Func? databaseFactory = null); + public Task<(StoreResult, IDb)> TransactAsync(IndexSegment datoms, HashSet? txFunctions = null); /// /// Transacts (adds) the given datoms into the store. /// - public StoreResult Transact(IndexSegment datoms, HashSet? txFunctions = null, Func? databaseFactory = null); - - /// - /// Executes an empty transaction. Returns a StoreResult valid asof the latest - /// transaction - /// - /// - public Task Sync(); + public (StoreResult, IDb) Transact(IndexSegment datoms, HashSet? txFunctions = null); /// /// Registers new attributes with the store. diff --git a/src/NexusMods.MnemonicDB.Abstractions/IDb.cs b/src/NexusMods.MnemonicDB.Abstractions/IDb.cs index a2b52368..38e98063 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/IDb.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/IDb.cs @@ -24,6 +24,11 @@ public interface IDb : IEquatable /// The connection that this database is using for its state. /// IConnection Connection { get; } + + /// + /// The datoms that were added in the most recent transaction (indicated by the basis TxId). + /// + IndexSegment RecentlyAdded { get; } /// /// The snapshot that this database is based on. @@ -40,13 +45,6 @@ public interface IDb : IEquatable /// public IndexSegment Get(EntityId entityId); - /// - /// Gets a read model for every enitity that references the given entity id - /// with the given attribute. - /// - public Entities GetReverse(EntityId id, Attribute attribute) - where TModel : IReadOnlyModel; - /// /// Get all the datoms for the given entity id. /// diff --git a/src/NexusMods.MnemonicDB.Abstractions/IndexType.cs b/src/NexusMods.MnemonicDB.Abstractions/IndexType.cs index f678cc2c..8017c9bd 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/IndexType.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/IndexType.cs @@ -5,7 +5,7 @@ namespace NexusMods.MnemonicDB.Abstractions; -public enum IndexType +public enum IndexType : byte { // Transaction log, the final source of truth, used // for replaying the database diff --git a/src/NexusMods.MnemonicDB.Abstractions/Query/ObservableDatoms.cs b/src/NexusMods.MnemonicDB.Abstractions/Query/ObservableDatoms.cs index b3f067c6..6ce42cf8 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/Query/ObservableDatoms.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/Query/ObservableDatoms.cs @@ -44,19 +44,19 @@ public static IObservable> ObserveDatoms(this IConnection conn var lastTxId = TxId.From(0); return conn.Revisions - .Where(rev => rev.AddedDatoms.Valid) + .Where(rev => rev.RecentlyAdded.Count > 0) .Select((rev, idx) => { lock (set) { - if (rev.Database.BasisTxId <= lastTxId) + if (rev.BasisTxId <= lastTxId) return ChangeSet.Empty; - lastTxId = rev.Database.BasisTxId; + lastTxId = rev.BasisTxId; if (idx == 0) - return Setup(set, rev.Database, descriptor); - return Diff(conn.Registry, set, rev.AddedDatoms, descriptor, equality); + return Setup(set, rev, descriptor); + return Diff(conn.Registry, set, rev.RecentlyAdded, descriptor, equality); } }); } diff --git a/src/NexusMods.MnemonicDB.Storage/NexusMods.MnemonicDB.Storage.csproj b/src/NexusMods.MnemonicDB.Storage/NexusMods.MnemonicDB.Storage.csproj deleted file mode 100644 index 76586016..00000000 --- a/src/NexusMods.MnemonicDB.Storage/NexusMods.MnemonicDB.Storage.csproj +++ /dev/null @@ -1,23 +0,0 @@ - - - - net8.0 - enable - enable - true - - - - - - - - - - - - - - - - diff --git a/src/NexusMods.MnemonicDB.Storage/Services.cs b/src/NexusMods.MnemonicDB.Storage/Services.cs deleted file mode 100644 index 719907d7..00000000 --- a/src/NexusMods.MnemonicDB.Storage/Services.cs +++ /dev/null @@ -1,45 +0,0 @@ -using Microsoft.Extensions.DependencyInjection; -using NexusMods.MnemonicDB.Abstractions; -using NexusMods.MnemonicDB.Abstractions.BuiltInEntities; -using NexusMods.MnemonicDB.Storage.Abstractions; -using NexusMods.MnemonicDB.Storage.RocksDbBackend; - -namespace NexusMods.MnemonicDB.Storage; - -/// -/// DI services for the MnemonicDB storage. -/// -public static class Services -{ - /// - /// Adds the MnemonicDB storage services to the service collection. - /// - public static IServiceCollection AddMnemonicDBStorage(this IServiceCollection services) - { - services.AddAttributeCollection(typeof(AttributeDefinition)); - services.AddAttributeCollection(typeof(Transaction)); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(s => (DatomStore)s.GetRequiredService()); - return services; - } - - /// - /// Adds the MnemonicDB storage settings to the service collection. - /// - public static IServiceCollection AddDatomStoreSettings(this IServiceCollection services, - DatomStoreSettings settings) - { - services.AddSingleton(settings); - return services; - } - - /// - /// Adds the RocksDB backend to the service collection. - /// - public static IServiceCollection AddRocksDbBackend(this IServiceCollection services) - { - services.AddSingleton(); - return services; - } -} diff --git a/src/NexusMods.MnemonicDB/Caching/IndexSegmentCache.cs b/src/NexusMods.MnemonicDB/Caching/IndexSegmentCache.cs new file mode 100644 index 00000000..8ac60a4b --- /dev/null +++ b/src/NexusMods.MnemonicDB/Caching/IndexSegmentCache.cs @@ -0,0 +1,280 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Diagnostics; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Threading; +using Microsoft.Extensions.Logging; +using NexusMods.MnemonicDB.Abstractions; +using NexusMods.MnemonicDB.Abstractions.ElementComparers; +using NexusMods.MnemonicDB.Abstractions.IndexSegments; +using NexusMods.MnemonicDB.Abstractions.Internals; +using NexusMods.MnemonicDB.Abstractions.Query; +using NexusMods.MnemonicDB.Storage; +using NexusMods.Paths; + +namespace NexusMods.MnemonicDB.Caching; + +[StructLayout(LayoutKind.Explicit, Size = 12)] +public readonly struct CacheKey : IEquatable +{ + [FieldOffset(0)] + public readonly IndexType IndexType; + + [FieldOffset(1)] + public readonly AttributeId AttributeId; + + [FieldOffset(4)] + public readonly EntityId EntityId; + + public CacheKey(IndexType indexType, AttributeId attributeId, EntityId entityId) + { + IndexType = indexType; + AttributeId = attributeId; + EntityId = entityId; + } + + public static CacheKey Create(IndexType indexType, AttributeId attributeId, EntityId entityId) => new(indexType, attributeId, entityId); + public static CacheKey Create(IndexType indexType, EntityId entityId) => new(indexType, AttributeId.From(0), entityId); + + /// + public bool Equals(CacheKey other) + { + return IndexType == other.IndexType && AttributeId.Equals(other.AttributeId) && EntityId.Equals(other.EntityId); + } + + /// + public override bool Equals(object? obj) + { + return obj is CacheKey other && Equals(other); + } + + /// + public override int GetHashCode() + { + return HashCode.Combine((int)IndexType, AttributeId, EntityId); + } +} + +public struct CacheValue : IEquatable +{ + public long LastAccessed; + public readonly IndexSegment Segment; + + public CacheValue(long lastAccessed, IndexSegment segment) + { + LastAccessed = lastAccessed; + Segment = segment; + } + + /// + /// Update the last accessed time to now. + /// + public void Hit() + { + LastAccessed = DateTime.UtcNow.Ticks; + } + + /// + public bool Equals(CacheValue other) + { + return LastAccessed == other.LastAccessed && Segment.Equals(other.Segment); + } + + /// + public override bool Equals(object? obj) + { + return obj is CacheValue other && Equals(other); + } + + /// + public override int GetHashCode() + { + return HashCode.Combine(LastAccessed, Segment); + } +} + +/// +/// Immutable cache root +/// +public class CacheRoot +{ + private readonly ImmutableDictionary _entries; + + private CacheRoot() + { + throw new NotSupportedException(); + } + + internal CacheRoot(ImmutableDictionary entries) + { + _entries = entries; + } + + /// + /// Get the index segment for the given key, if it exists. + /// + public bool TryGetValue(CacheKey key, out IndexSegment segment) + { + if (_entries.TryGetValue(key, out var value)) + { + value.Hit(); + segment = value.Segment; + return true; + } + segment = default; + return false; + } + + /// + /// Create a new cache root with the given segment added. + /// + public CacheRoot With(CacheKey key, IndexSegment segment, IndexSegmentCache cache) + { + var newEntries = _entries.SetItem(key, new CacheValue(DateTime.UtcNow.Ticks, segment)); + if (newEntries.Count > cache._entryCapacity) + { + newEntries = PurgeEntries(newEntries, newEntries.Count / 10); + } + + return new CacheRoot(newEntries); + } + + /// + /// Purge the `toPurge` oldest entries from the cache. + /// + private ImmutableDictionary PurgeEntries(ImmutableDictionary newEntries, int toPurge) + { + var toDrop = newEntries.OrderBy(kv => kv.Value.LastAccessed).Take(toPurge); + + var builder = newEntries.ToBuilder(); + var droppedSize = Size.Zero; + foreach (var kv in toDrop) + { + builder.Remove(kv.Key); + droppedSize += kv.Value.Segment.DataSize; + } + + return builder.ToImmutable(); + } + + /// + /// Evict cache entries for datoms in the given transaction log. + /// + public CacheRoot EvictNew(StoreResult result, IAttributeRegistry registry, out IndexSegment newDatoms) + { + newDatoms = result.Snapshot.Datoms(SliceDescriptor.Create(result.AssignedTxId, registry)); + + var editable = _entries.ToBuilder(); + foreach (var datom in newDatoms) + { + var eavtKey = CacheKey.Create(IndexType.EAVTCurrent, datom.E); + editable.Remove(eavtKey); + + if (datom.Prefix.ValueTag != ValueTags.Reference) + continue; + + var vaetKey = CacheKey.Create(IndexType.VAETCurrent, datom.A, MemoryMarshal.Read(datom.ValueSpan)); + editable.Remove(vaetKey); + + var referencesKey = CacheKey.Create(IndexType.VAETCurrent, MemoryMarshal.Read(datom.ValueSpan)); + editable.Remove(referencesKey); + } + return new(editable.ToImmutable()); + } + + /// + public override string ToString() + { + return $"CacheRoot: {_entries.Count} entries,"; + } +} + +/// +/// A cache of index segments, implemented as an immutable LRU cache, immutable so that each subsequent +/// Db instance can reuse the cache and all its contents. +/// +public class IndexSegmentCache +{ + private CacheRoot _root; + internal readonly int _entryCapacity; + internal readonly Size _maxSize; + + /// + /// Create a new index segment cache. + /// + public IndexSegmentCache() + { + _root = new CacheRoot(ImmutableDictionary.Empty); + _entryCapacity = 1_000_000; + } + + /// + /// Get the index segment for the given entity id, if it is not in the cache, cache it, then update the cache at the + /// given location so that it contains the new segment. + /// + public IndexSegment Get(EntityId entityId, IDb db) + { + var key = CacheKey.Create(IndexType.EAVTCurrent, entityId); + if (_root.TryGetValue(key, out var segment)) + return segment; + + segment = db.Snapshot.Datoms(SliceDescriptor.Create(entityId, db.Registry)); + UpdateEntry(key, segment); + return segment; + } + + /// + /// Get a segment for all the datoms that point to the given entity id via their value for the given attribute. + /// + public IndexSegment GetReverse(AttributeId attributeId, EntityId entityId, IDb db) + { + var key = CacheKey.Create(IndexType.VAETCurrent, attributeId, entityId); + if (_root.TryGetValue(key, out var segment)) + return segment; + + segment = db.Snapshot.Datoms(SliceDescriptor.Create(attributeId, entityId, db.Registry)); + UpdateEntry(key, segment); + return segment; + } + + /// + /// Get a segment for all the datoms that point to the given entity id. + /// + public IndexSegment GetReferences(EntityId entityId, IDb db) + { + var key = CacheKey.Create(IndexType.VAETCurrent, entityId); + if (_root.TryGetValue(key, out var segment)) + return segment; + + segment = db.Snapshot.Datoms(SliceDescriptor.CreateReferenceTo(entityId, db.Registry)); + UpdateEntry(key, segment); + return segment; + } + + /// + /// Creates a copy of the cache with the given datoms evicted, and the new datoms added. + /// + public IndexSegmentCache ForkAndEvict(StoreResult result, IAttributeRegistry registry, out IndexSegment newDatoms) + { + var newRoot = _root.EvictNew(result, registry, out newDatoms); + return new IndexSegmentCache { _root = newRoot }; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void UpdateEntry(CacheKey key, IndexSegment segment) + { + while (true) + { + var oldRoot = _root; + var newRoot = oldRoot.With(key, segment, this); + var result = Interlocked.CompareExchange(ref _root, newRoot, oldRoot); + + if (ReferenceEquals(result, oldRoot)) + return; + } + } + +} diff --git a/src/NexusMods.MnemonicDB/Connection.cs b/src/NexusMods.MnemonicDB/Connection.cs index 1e795b5e..f10a857c 100644 --- a/src/NexusMods.MnemonicDB/Connection.cs +++ b/src/NexusMods.MnemonicDB/Connection.cs @@ -25,7 +25,7 @@ public class Connection : IConnection private readonly Dictionary _declaredAttributes; private readonly ILogger _logger; - private BehaviorSubject _dbStream; + private BehaviorSubject _dbStream; private IDisposable? _dbStreamDisposable; /// @@ -37,27 +37,26 @@ public Connection(ILogger logger, IDatomStore store, IServiceProvide _logger = logger; _declaredAttributes = declaredAttributes.ToDictionary(a => a.Id); _store = store; - _dbStream = new BehaviorSubject(default!); + _dbStream = new BehaviorSubject(default!); Bootstrap(); } /// /// Scrubs the transaction stream so that we only ever move forward and never repeat transactions /// - private static IObservable<(TxId TxId, ISnapshot Snapshot)> ForwardOnly(IObservable<(TxId txId, ISnapshot snapshot)> dbStream) + private static IObservable ForwardOnly(IObservable dbStream) { TxId? prev = null; - return Observable.Create((IObserver<(TxId txId, ISnapshot snapshot)> observer) => + return Observable.Create((IObserver observer) => { - return dbStream.Subscribe((nextItem) => + return dbStream.Subscribe(nextItem => { - var (nextTxId, _) = nextItem; - if (prev != null && prev.Value >= nextTxId) + if (prev != null && prev.Value >= nextItem.BasisTxId) return; - observer.OnNext(nextItem); - prev = nextTxId; + observer.OnNext((Db)nextItem); + prev = nextItem.BasisTxId; }, observer.OnError, observer.OnCompleted); }); } @@ -74,7 +73,7 @@ public IDb Db // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract if (val == null) ThrowNullDb(); - return val!.Value.Database; + return val!.Value; } } @@ -94,7 +93,10 @@ private static void ThrowNullDb() public IDb AsOf(TxId txId) { var snapshot = new AsOfSnapshot(_store.GetSnapshot(), txId, (AttributeRegistry)_store.Registry); - return new Db(snapshot, this, txId, (AttributeRegistry)_store.Registry); + return new Db(snapshot, txId, (AttributeRegistry)_store.Registry) + { + Connection = this + }; } /// @@ -104,7 +106,7 @@ public ITransaction BeginTransaction() } /// - public IObservable Revisions + public IObservable Revisions { get { @@ -144,7 +146,10 @@ private void AddMissingAttributes(IEnumerable declaredAttributes) private IEnumerable ExistingAttributes() { - var db = new Db(_store.GetSnapshot(), this, TxId, (AttributeRegistry)_store.Registry); + var db = new Db(_store.GetSnapshot(), TxId, (AttributeRegistry)_store.Registry) + { + Connection = this + }; foreach (var attribute in AttributeDefinition.All(db)) { @@ -156,14 +161,11 @@ private IEnumerable ExistingAttributes() internal async Task Transact(IndexSegment datoms, HashSet? txFunctions) { StoreResult newTx; + IDb newDb; - if (txFunctions == null) - newTx = await _store.TransactAsync(datoms, txFunctions); - else - newTx = await _store.TransactAsync(datoms, txFunctions, snapshot => new Db(snapshot, this, TxId, (AttributeRegistry)_store.Registry)); - - var result = new CommitResult(new Db(newTx.Snapshot, this, newTx.AssignedTxId, (AttributeRegistry)_store.Registry) - , newTx.Remaps); + (newTx, newDb) = await _store.TransactAsync(datoms, txFunctions); + ((Db)newDb).Connection = this; + var result = new CommitResult(newDb, newTx.Remaps); return result; } @@ -174,15 +176,10 @@ private void Bootstrap() AddMissingAttributes(_declaredAttributes.Values); _dbStreamDisposable = ForwardOnly(_store.TxLog) - .Select(log => + .Select(db => { - var db = new Db(log.Snapshot, this, log.TxId, (AttributeRegistry)_store.Registry); - var addedItems = db.Datoms(SliceDescriptor.Create(db.BasisTxId, _store.Registry)); - return new Revision - { - Database = db, - AddedDatoms = addedItems - }; + db.Connection = this; + return db; }) .Subscribe(_dbStream); } diff --git a/src/NexusMods.MnemonicDB/Db.cs b/src/NexusMods.MnemonicDB/Db.cs index c6bf8cec..d4922617 100644 --- a/src/NexusMods.MnemonicDB/Db.cs +++ b/src/NexusMods.MnemonicDB/Db.cs @@ -1,51 +1,71 @@ using System; using System.Collections.Generic; +using System.Collections.Immutable; using System.Diagnostics; using System.Linq; -using System.Runtime.InteropServices; -using Microsoft.Extensions.DependencyInjection; using NexusMods.MnemonicDB.Abstractions; using NexusMods.MnemonicDB.Abstractions.Attributes; -using NexusMods.MnemonicDB.Abstractions.DatomIterators; -using NexusMods.MnemonicDB.Abstractions.ElementComparers; using NexusMods.MnemonicDB.Abstractions.IndexSegments; using NexusMods.MnemonicDB.Abstractions.Internals; using NexusMods.MnemonicDB.Abstractions.Models; using NexusMods.MnemonicDB.Abstractions.Query; +using NexusMods.MnemonicDB.Caching; using NexusMods.MnemonicDB.Storage; -using Reloaded.Memory.Extensions; namespace NexusMods.MnemonicDB; internal class Db : IDb { - private readonly Connection _connection; private readonly AttributeRegistry _registry; - - private readonly IndexSegmentCache _entityCache; - private readonly IndexSegmentCache<(EntityId, AttributeId)> _reverseCache; - private readonly IndexSegmentCache _referencesCache; + private readonly IndexSegmentCache _cache; private readonly RegistryId _registryId; - + + /// + /// The connection is used by several methods to navigate the graph of objects of Db, Connection, Datom Store, and + /// Attribute Registry. However, we want the Datom Store and Connection to be decoupled, so the Connection starts null + /// and is set by the Connection class after the Datom Store has pushed the Db object to it. + /// + private IConnection? _connection; public ISnapshot Snapshot { get; } public IAttributeRegistry Registry => _registry; - public Db(ISnapshot snapshot, Connection connection, TxId txId, AttributeRegistry registry) + public IndexSegment RecentlyAdded { get; } + + public Db(ISnapshot snapshot, TxId txId, AttributeRegistry registry) { Debug.Assert(snapshot != null, $"{nameof(snapshot)} cannot be null"); _registryId = registry.Id; _registry = registry; + _cache = new IndexSegmentCache(); + Snapshot = snapshot; + BasisTxId = txId; + RecentlyAdded = new IndexSegment(); + } + + private Db(ISnapshot snapshot, TxId txId, AttributeRegistry registry, RegistryId registryId, IConnection connection, IndexSegmentCache newCache, IndexSegment recentlyAdded) + { + _registry = registry; + _registryId = registryId; + _cache = newCache; _connection = connection; - _entityCache = new IndexSegmentCache(EntityDatoms, registry); - _reverseCache = new IndexSegmentCache<(EntityId, AttributeId)>(ReverseDatoms, registry); - _referencesCache = new IndexSegmentCache(ReferenceDatoms, registry); Snapshot = snapshot; BasisTxId = txId; + RecentlyAdded = recentlyAdded; + } + + /// + /// Create a new Db instance with the given store result and transaction id integrated, will evict old items + /// from the cache, and update the cache with the new datoms. + /// + internal Db WithNext(StoreResult storeResult, TxId txId) + { + var newCache = _cache.ForkAndEvict(storeResult, _registry, out var newDatoms); + return new Db(storeResult.Snapshot, txId, _registry, _registryId, _connection!, newCache, newDatoms); } - private static IndexSegment EntityDatoms(IDb db, EntityId id) + private IndexSegment EntityDatoms(IDb db, EntityId id) { - return db.Snapshot.Datoms(SliceDescriptor.Create(id, db.Registry)); + return _cache.Get(id, db); } private static IndexSegment ReverseDatoms(IDb db, (EntityId, AttributeId) key) @@ -54,43 +74,40 @@ private static IndexSegment ReverseDatoms(IDb db, (EntityId, AttributeId) key) } public TxId BasisTxId { get; } - - public IConnection Connection => _connection; - + + public IConnection Connection + { + get + { + Debug.Assert(_connection != null, "Connection is not set"); + return _connection!; + } + set + { + Debug.Assert(_connection == null || ReferenceEquals(_connection, value), "Connection is already set"); + _connection = value; + } + } + /// /// Gets the IndexSegment for the given entity id. /// public IndexSegment Get(EntityId entityId) { - return _entityCache.Get(this, entityId); + return Datoms(entityId); } public EntityIds GetBackRefs(ReferenceAttribute attribute, EntityId id) { - var segment = _reverseCache.Get(this, (id, attribute.GetDbId(_registryId))); + var segment = _cache.GetReverse(attribute.GetDbId(_registryId), id, this); return new EntityIds(segment, 0, segment.Count); } - - private static IndexSegment ReferenceDatoms(IDb db, EntityId eid) - { - return db.Snapshot.Datoms(SliceDescriptor.CreateReferenceTo(eid, db.Registry)); - } - + public IndexSegment ReferencesTo(EntityId id) { - return _referencesCache.Get(this, id); - } - - public IEnumerable GetAll(EntityId id, Attribute attribute) - { - var attrId = attribute.GetDbId(_registryId); - var results = _entityCache.Get(this, id) - .Where(d => d.A == attrId) - .Select(d => d.Resolve(attribute)); - - return results; + return _cache.GetReferences(id, this); } - + public IndexSegment Datoms(Attribute attribute, TValue value) { return Datoms(SliceDescriptor.Create(attribute, value, _registry)); @@ -100,24 +117,10 @@ public IndexSegment Datoms(ReferenceAttribute attribute, EntityId value) { return Datoms(SliceDescriptor.Create(attribute, value, _registry)); } - - public TModel Get(EntityId id) - where TModel : IHasEntityIdAndDb - { - return EntityConstructors.Constructor(id, this); - } - - public Entities GetReverse(EntityId id, Attribute attribute) - where TModel : IReadOnlyModel - { - var segment = _reverseCache.Get(this, (id, attribute.GetDbId(_registryId))); - var ids = new EntityIds(segment, 0, segment.Count); - return new Entities(ids, this); - } public IndexSegment Datoms(EntityId entityId) { - return _entityCache.Get(this, entityId); + return _cache.Get(entityId, this); } public IndexSegment Datoms(IAttribute attribute) diff --git a/src/NexusMods.MnemonicDB/NexusMods.MnemonicDB.csproj b/src/NexusMods.MnemonicDB/NexusMods.MnemonicDB.csproj index 4adfd9d4..40e2afb9 100644 --- a/src/NexusMods.MnemonicDB/NexusMods.MnemonicDB.csproj +++ b/src/NexusMods.MnemonicDB/NexusMods.MnemonicDB.csproj @@ -8,13 +8,13 @@ + - diff --git a/src/NexusMods.MnemonicDB/Services.cs b/src/NexusMods.MnemonicDB/Services.cs index 8ccc52f2..6cd24d20 100644 --- a/src/NexusMods.MnemonicDB/Services.cs +++ b/src/NexusMods.MnemonicDB/Services.cs @@ -1,5 +1,9 @@ using Microsoft.Extensions.DependencyInjection; using NexusMods.MnemonicDB.Abstractions; +using NexusMods.MnemonicDB.Abstractions.BuiltInEntities; +using NexusMods.MnemonicDB.Storage; +using NexusMods.MnemonicDB.Storage.Abstractions; +using NexusMods.MnemonicDB.Storage.RocksDbBackend; namespace NexusMods.MnemonicDB; @@ -14,7 +18,40 @@ public static class Services public static IServiceCollection AddMnemonicDB(this IServiceCollection services) { services.AddSingleton(); + services.AddMnemonicDBStorage(); return services; } + + /// + /// Adds the MnemonicDB storage services to the service collection. + /// + public static IServiceCollection AddMnemonicDBStorage(this IServiceCollection services) + { + services.AddAttributeDefinitionModel() + .AddTransactionModel() + .AddSingleton() + .AddSingleton() + .AddSingleton(s => (DatomStore)s.GetRequiredService()); + return services; + } + + /// + /// Adds the MnemonicDB storage settings to the service collection. + /// + public static IServiceCollection AddDatomStoreSettings(this IServiceCollection services, + DatomStoreSettings settings) + { + services.AddSingleton(settings); + return services; + } + + /// + /// Adds the RocksDB backend to the service collection. + /// + public static IServiceCollection AddRocksDbBackend(this IServiceCollection services) + { + services.AddSingleton(); + return services; + } } diff --git a/src/NexusMods.MnemonicDB.Storage/Abstractions/AIndex.cs b/src/NexusMods.MnemonicDB/Storage/Abstractions/AIndex.cs similarity index 100% rename from src/NexusMods.MnemonicDB.Storage/Abstractions/AIndex.cs rename to src/NexusMods.MnemonicDB/Storage/Abstractions/AIndex.cs diff --git a/src/NexusMods.MnemonicDB.Storage/Abstractions/IIndex.cs b/src/NexusMods.MnemonicDB/Storage/Abstractions/IIndex.cs similarity index 100% rename from src/NexusMods.MnemonicDB.Storage/Abstractions/IIndex.cs rename to src/NexusMods.MnemonicDB/Storage/Abstractions/IIndex.cs diff --git a/src/NexusMods.MnemonicDB.Storage/Abstractions/IIndexStore.cs b/src/NexusMods.MnemonicDB/Storage/Abstractions/IIndexStore.cs similarity index 100% rename from src/NexusMods.MnemonicDB.Storage/Abstractions/IIndexStore.cs rename to src/NexusMods.MnemonicDB/Storage/Abstractions/IIndexStore.cs diff --git a/src/NexusMods.MnemonicDB.Storage/Abstractions/IStoreBackend.cs b/src/NexusMods.MnemonicDB/Storage/Abstractions/IStoreBackend.cs similarity index 100% rename from src/NexusMods.MnemonicDB.Storage/Abstractions/IStoreBackend.cs rename to src/NexusMods.MnemonicDB/Storage/Abstractions/IStoreBackend.cs diff --git a/src/NexusMods.MnemonicDB.Storage/Abstractions/IWriteBatch.cs b/src/NexusMods.MnemonicDB/Storage/Abstractions/IWriteBatch.cs similarity index 100% rename from src/NexusMods.MnemonicDB.Storage/Abstractions/IWriteBatch.cs rename to src/NexusMods.MnemonicDB/Storage/Abstractions/IWriteBatch.cs diff --git a/src/NexusMods.MnemonicDB.Storage/Abstractions/KeyPrefix.cs b/src/NexusMods.MnemonicDB/Storage/Abstractions/KeyPrefix.cs similarity index 100% rename from src/NexusMods.MnemonicDB.Storage/Abstractions/KeyPrefix.cs rename to src/NexusMods.MnemonicDB/Storage/Abstractions/KeyPrefix.cs diff --git a/src/NexusMods.MnemonicDB.Storage/AttributeRegistry.cs b/src/NexusMods.MnemonicDB/Storage/AttributeRegistry.cs similarity index 100% rename from src/NexusMods.MnemonicDB.Storage/AttributeRegistry.cs rename to src/NexusMods.MnemonicDB/Storage/AttributeRegistry.cs diff --git a/src/NexusMods.MnemonicDB.Storage/DatomStorageStructures/PendingTransaction.cs b/src/NexusMods.MnemonicDB/Storage/DatomStorageStructures/PendingTransaction.cs similarity index 70% rename from src/NexusMods.MnemonicDB.Storage/DatomStorageStructures/PendingTransaction.cs rename to src/NexusMods.MnemonicDB/Storage/DatomStorageStructures/PendingTransaction.cs index 28bddf66..e61eb0da 100644 --- a/src/NexusMods.MnemonicDB.Storage/DatomStorageStructures/PendingTransaction.cs +++ b/src/NexusMods.MnemonicDB/Storage/DatomStorageStructures/PendingTransaction.cs @@ -16,7 +16,7 @@ internal class PendingTransaction /// A completion source for the transaction, resolves when the transaction is commited to the /// transaction log and available to readers. /// - public TaskCompletionSource CompletionSource { get; } = new(); + public TaskCompletionSource<(StoreResult, IDb)> CompletionSource { get; } = new(); /// /// The data to be commited @@ -27,16 +27,10 @@ internal class PendingTransaction /// Tx functions to be applied to the transaction, if any /// public required HashSet? TxFunctions { get; init; } - - /// - /// A function for creating a new database instance from a given snapshot. Not required - /// if TxFunctions is null. - /// - public required Func? DatabaseFactory { get; init; } - public void Complete(StoreResult result) + public void Complete(StoreResult result, IDb db) { Data = new IndexSegment(); - Task.Run(() => CompletionSource.SetResult(result)); + Task.Run(() => CompletionSource.SetResult((result, db))); } } diff --git a/src/NexusMods.MnemonicDB.Storage/DatomStore.cs b/src/NexusMods.MnemonicDB/Storage/DatomStore.cs similarity index 92% rename from src/NexusMods.MnemonicDB.Storage/DatomStore.cs rename to src/NexusMods.MnemonicDB/Storage/DatomStore.cs index aa55f575..1c601fe8 100644 --- a/src/NexusMods.MnemonicDB.Storage/DatomStore.cs +++ b/src/NexusMods.MnemonicDB/Storage/DatomStore.cs @@ -39,7 +39,7 @@ public class DatomStore : IDatomStore private readonly BlockingCollection _pendingTransactions; private readonly IIndex _txLog; - private BehaviorSubject<(TxId TxId, ISnapshot snapshot)>? _updatesSubject; + private BehaviorSubject? _updatesSubject; private readonly IIndex _vaetCurrent; private readonly IIndex _vaetHistory; private readonly PooledMemoryBufferWriter _writer; @@ -47,6 +47,8 @@ public class DatomStore : IDatomStore private TxId _asOfTx = TxId.MinValue; private Task? _bootStrapTask = null; + + private IDb? _currentDb = null; private static readonly TimeSpan TransactionTimeout = TimeSpan.FromMinutes(120); @@ -106,7 +108,7 @@ public DatomStore(ILogger logger, AttributeRegistry registry, DatomS Bootstrap(); } - + /// public TxId AsOfTxId => _asOfTx; @@ -114,15 +116,13 @@ public DatomStore(ILogger logger, AttributeRegistry registry, DatomS public IAttributeRegistry Registry => _registry; /// - public async Task TransactAsync(IndexSegment datoms, HashSet? txFunctions = null, - Func? factoryFn = null) + public async Task<(StoreResult, IDb)> TransactAsync(IndexSegment datoms, HashSet? txFunctions = null) { var pending = new PendingTransaction { Data = datoms, - TxFunctions = txFunctions, - DatabaseFactory = factoryFn + TxFunctions = txFunctions }; _pendingTransactions.Add(pending); @@ -136,15 +136,14 @@ public async Task TransactAsync(IndexSegment datoms, HashSet? txFunctions = null, - Func? factoryFn = null) + /// + public (StoreResult, IDb) Transact(IndexSegment datoms, HashSet? txFunctions = null) { var pending = new PendingTransaction { Data = datoms, - TxFunctions = txFunctions, - DatabaseFactory = factoryFn + TxFunctions = txFunctions }; _pendingTransactions.Add(pending); @@ -157,15 +156,9 @@ public StoreResult Transact(IndexSegment datoms, HashSet? txFunctio throw new TimeoutException($"Transaction didn't complete after {TransactionTimeout}"); } - - /// - public async Task Sync() - { - return await TransactAsync(new IndexSegment()); - } - + /// - public IObservable<(TxId TxId, ISnapshot Snapshot)> TxLog + public IObservable TxLog { get { @@ -185,7 +178,7 @@ public void RegisterAttributes(IEnumerable newAttrs) foreach (var attribute in newAttrsArray) AttributeDefinition.Insert(internalTx, attribute.Attribute, attribute.AttrEntityId.Value); internalTx.ProcessTemporaryEntities(); - Transact(datoms.Build(), null, null); + Transact(datoms.Build()); _registry.Populate(newAttrsArray); } @@ -227,14 +220,17 @@ private void ConsumeTransactions() AssignedTxId = _nextIdCache.AsOfTxId, Snapshot = _backend.GetSnapshot(), }; - pendingTransaction.CompletionSource.TrySetResult(storeResult); + pendingTransaction.Complete(storeResult, _currentDb!); continue; } Log(pendingTransaction, out var result); - - _updatesSubject?.OnNext((result.AssignedTxId, result.Snapshot)); - pendingTransaction.Complete(result); + + var sw = Stopwatch.StartNew(); + FinishTransaction(result, pendingTransaction); + + if (_logger.IsEnabled(LogLevel.Debug)) + _logger.LogDebug("Transaction {TxId} post-processed in {Elapsed}ms", result.AssignedTxId, sw.ElapsedMilliseconds); } catch (Exception ex) { @@ -249,6 +245,16 @@ private void ConsumeTransactions() } } + /// + /// Given the new store result, process the new database state, complete the transaction and notify the observers + /// + private void FinishTransaction(StoreResult result, PendingTransaction pendingTransaction) + { + _currentDb = ((Db)_currentDb!).WithNext(result, result.AssignedTxId); + _updatesSubject?.OnNext(_currentDb!); + pendingTransaction.Complete(result, _currentDb); + } + /// /// Sets up the initial state of the store. /// @@ -269,8 +275,7 @@ private void Bootstrap() var pending = new PendingTransaction { Data = builder.Build(), - TxFunctions = null, - DatabaseFactory = null + TxFunctions = null }; // Call directly into `Log` as the transaction channel is not yet set up Log(pending, out _); @@ -289,8 +294,9 @@ private void Bootstrap() _logger.LogError(ex, "Failed to bootstrap the datom store"); throw; } - - _updatesSubject = new BehaviorSubject<(TxId TxId, ISnapshot snapshot)>((_asOfTx, _currentSnapshot)); + + _currentDb = new Db(_currentSnapshot, _asOfTx, _registry); + _updatesSubject = new BehaviorSubject(_currentDb); _loggerThread = new Thread(ConsumeTransactions) { IsBackground = true, @@ -344,13 +350,13 @@ private void Log(PendingTransaction pendingTransaction, out StoreResult result) var secondaryBuilder = new IndexSegmentBuilder(_registry); var txId = EntityId.From(thisTx.Value); - secondaryBuilder.Add(txId, Transaction.Timestamp, DateTime.UtcNow); + secondaryBuilder.Add(txId, MnemonicDB.Abstractions.BuiltInEntities.Transaction.Timestamp, DateTime.UtcNow); if (pendingTransaction.TxFunctions != null) { try { - var db = pendingTransaction.DatabaseFactory!(currentSnapshot); + var db = _currentDb!; var tx = new InternalTransaction(db, secondaryBuilder); foreach (var fn in pendingTransaction.TxFunctions) { diff --git a/src/NexusMods.MnemonicDB.Storage/InMemoryBackend/Backend.cs b/src/NexusMods.MnemonicDB/Storage/InMemoryBackend/Backend.cs similarity index 100% rename from src/NexusMods.MnemonicDB.Storage/InMemoryBackend/Backend.cs rename to src/NexusMods.MnemonicDB/Storage/InMemoryBackend/Backend.cs diff --git a/src/NexusMods.MnemonicDB.Storage/InMemoryBackend/Batch.cs b/src/NexusMods.MnemonicDB/Storage/InMemoryBackend/Batch.cs similarity index 100% rename from src/NexusMods.MnemonicDB.Storage/InMemoryBackend/Batch.cs rename to src/NexusMods.MnemonicDB/Storage/InMemoryBackend/Batch.cs diff --git a/src/NexusMods.MnemonicDB.Storage/InMemoryBackend/IInMemoryIndex.cs b/src/NexusMods.MnemonicDB/Storage/InMemoryBackend/IInMemoryIndex.cs similarity index 100% rename from src/NexusMods.MnemonicDB.Storage/InMemoryBackend/IInMemoryIndex.cs rename to src/NexusMods.MnemonicDB/Storage/InMemoryBackend/IInMemoryIndex.cs diff --git a/src/NexusMods.MnemonicDB.Storage/InMemoryBackend/Index.cs b/src/NexusMods.MnemonicDB/Storage/InMemoryBackend/Index.cs similarity index 100% rename from src/NexusMods.MnemonicDB.Storage/InMemoryBackend/Index.cs rename to src/NexusMods.MnemonicDB/Storage/InMemoryBackend/Index.cs diff --git a/src/NexusMods.MnemonicDB.Storage/InMemoryBackend/IndexStore.cs b/src/NexusMods.MnemonicDB/Storage/InMemoryBackend/IndexStore.cs similarity index 100% rename from src/NexusMods.MnemonicDB.Storage/InMemoryBackend/IndexStore.cs rename to src/NexusMods.MnemonicDB/Storage/InMemoryBackend/IndexStore.cs diff --git a/src/NexusMods.MnemonicDB.Storage/InMemoryBackend/Snapshot.cs b/src/NexusMods.MnemonicDB/Storage/InMemoryBackend/Snapshot.cs similarity index 100% rename from src/NexusMods.MnemonicDB.Storage/InMemoryBackend/Snapshot.cs rename to src/NexusMods.MnemonicDB/Storage/InMemoryBackend/Snapshot.cs diff --git a/src/NexusMods.MnemonicDB.Storage/InternalTransaction.cs b/src/NexusMods.MnemonicDB/Storage/InternalTransaction.cs similarity index 100% rename from src/NexusMods.MnemonicDB.Storage/InternalTransaction.cs rename to src/NexusMods.MnemonicDB/Storage/InternalTransaction.cs diff --git a/src/NexusMods.MnemonicDB.Storage/NextIdCache.cs b/src/NexusMods.MnemonicDB/Storage/NextIdCache.cs similarity index 100% rename from src/NexusMods.MnemonicDB.Storage/NextIdCache.cs rename to src/NexusMods.MnemonicDB/Storage/NextIdCache.cs diff --git a/src/NexusMods.MnemonicDB.Storage/RocksDbBackend/Backend.cs b/src/NexusMods.MnemonicDB/Storage/RocksDbBackend/Backend.cs similarity index 100% rename from src/NexusMods.MnemonicDB.Storage/RocksDbBackend/Backend.cs rename to src/NexusMods.MnemonicDB/Storage/RocksDbBackend/Backend.cs diff --git a/src/NexusMods.MnemonicDB.Storage/RocksDbBackend/Batch.cs b/src/NexusMods.MnemonicDB/Storage/RocksDbBackend/Batch.cs similarity index 100% rename from src/NexusMods.MnemonicDB.Storage/RocksDbBackend/Batch.cs rename to src/NexusMods.MnemonicDB/Storage/RocksDbBackend/Batch.cs diff --git a/src/NexusMods.MnemonicDB.Storage/RocksDbBackend/IRocksDBIndexStore.cs b/src/NexusMods.MnemonicDB/Storage/RocksDbBackend/IRocksDBIndexStore.cs similarity index 100% rename from src/NexusMods.MnemonicDB.Storage/RocksDbBackend/IRocksDBIndexStore.cs rename to src/NexusMods.MnemonicDB/Storage/RocksDbBackend/IRocksDBIndexStore.cs diff --git a/src/NexusMods.MnemonicDB.Storage/RocksDbBackend/IRocksDbIndex.cs b/src/NexusMods.MnemonicDB/Storage/RocksDbBackend/IRocksDbIndex.cs similarity index 100% rename from src/NexusMods.MnemonicDB.Storage/RocksDbBackend/IRocksDbIndex.cs rename to src/NexusMods.MnemonicDB/Storage/RocksDbBackend/IRocksDbIndex.cs diff --git a/src/NexusMods.MnemonicDB.Storage/RocksDbBackend/Index.cs b/src/NexusMods.MnemonicDB/Storage/RocksDbBackend/Index.cs similarity index 100% rename from src/NexusMods.MnemonicDB.Storage/RocksDbBackend/Index.cs rename to src/NexusMods.MnemonicDB/Storage/RocksDbBackend/Index.cs diff --git a/src/NexusMods.MnemonicDB.Storage/RocksDbBackend/IndexStore.cs b/src/NexusMods.MnemonicDB/Storage/RocksDbBackend/IndexStore.cs similarity index 100% rename from src/NexusMods.MnemonicDB.Storage/RocksDbBackend/IndexStore.cs rename to src/NexusMods.MnemonicDB/Storage/RocksDbBackend/IndexStore.cs diff --git a/src/NexusMods.MnemonicDB.Storage/RocksDbBackend/Snapshot.cs b/src/NexusMods.MnemonicDB/Storage/RocksDbBackend/Snapshot.cs similarity index 100% rename from src/NexusMods.MnemonicDB.Storage/RocksDbBackend/Snapshot.cs rename to src/NexusMods.MnemonicDB/Storage/RocksDbBackend/Snapshot.cs diff --git a/tests/NexusMods.MnemonicDB.Storage.Tests/ABackendTest.cs b/tests/NexusMods.MnemonicDB.Storage.Tests/ABackendTest.cs index 5dc65ddc..2213757a 100644 --- a/tests/NexusMods.MnemonicDB.Storage.Tests/ABackendTest.cs +++ b/tests/NexusMods.MnemonicDB.Storage.Tests/ABackendTest.cs @@ -74,7 +74,7 @@ public async Task CanStoreDataInBlobs(IndexType type) var entityId = NextTempId(); segment.Add(entityId, Blobs.InKeyBlob, smallData); segment.Add(entityId, Blobs.InValueBlob, largeData); - var result = await DatomStore.TransactAsync(segment.Build()); + var (result, _) = await DatomStore.TransactAsync(segment.Build()); ids.Add(result.Remaps[entityId]); } @@ -179,7 +179,7 @@ private async Task GenerateData() segment.Add(collectionId, Collection.ModIds, modId2); - tx = await DatomStore.TransactAsync(segment.Build()); + (tx, _) = await DatomStore.TransactAsync(segment.Build()); } id1 = tx.Remaps[id1]; @@ -192,7 +192,7 @@ private async Task GenerateData() segment.Add(id2, File.Path, "/foo/qux"); segment.Add(id1, File.ModId, modId2); segment.Add(collectionId, Collection.ModIds, modId2, true); - tx = await DatomStore.TransactAsync(segment.Build()); + (tx, _) = await DatomStore.TransactAsync(segment.Build()); } return tx; } @@ -222,7 +222,7 @@ public async Task RetractedValuesAreSupported(IndexType type) segment.Add(id, File.Size, Size.From(42)); segment.Add(id, File.ModId, modId); - tx1 = await DatomStore.TransactAsync(segment.Build()); + (tx1, _) = await DatomStore.TransactAsync(segment.Build()); } id = tx1.Remaps[id]; @@ -236,7 +236,7 @@ public async Task RetractedValuesAreSupported(IndexType type) segment.Add(id, File.Size, Size.From(42), true); segment.Add(id, File.ModId, modId, true); - tx2 = await DatomStore.TransactAsync(segment.Build()); + (tx2, _) = await DatomStore.TransactAsync(segment.Build()); } diff --git a/tests/NexusMods.MnemonicDB.Storage.Tests/AStorageTest.cs b/tests/NexusMods.MnemonicDB.Storage.Tests/AStorageTest.cs index d8c2246c..11c8afae 100644 --- a/tests/NexusMods.MnemonicDB.Storage.Tests/AStorageTest.cs +++ b/tests/NexusMods.MnemonicDB.Storage.Tests/AStorageTest.cs @@ -53,7 +53,7 @@ protected AStorageTest(IServiceProvider provider, Func>(), Registry, DatomStoreSettings, backendFn(Registry)); - + Logger = provider.GetRequiredService>(); } public void Dispose() diff --git a/tests/NexusMods.MnemonicDB.Storage.Tests/NexusMods.MnemonicDB.Storage.Tests.csproj b/tests/NexusMods.MnemonicDB.Storage.Tests/NexusMods.MnemonicDB.Storage.Tests.csproj index 09d62d98..4d7ce30b 100644 --- a/tests/NexusMods.MnemonicDB.Storage.Tests/NexusMods.MnemonicDB.Storage.Tests.csproj +++ b/tests/NexusMods.MnemonicDB.Storage.Tests/NexusMods.MnemonicDB.Storage.Tests.csproj @@ -18,7 +18,7 @@ - + diff --git a/tests/NexusMods.MnemonicDB.Storage.Tests/NullConnection.cs b/tests/NexusMods.MnemonicDB.Storage.Tests/NullConnection.cs new file mode 100644 index 00000000..f64ced74 --- /dev/null +++ b/tests/NexusMods.MnemonicDB.Storage.Tests/NullConnection.cs @@ -0,0 +1,22 @@ +using NexusMods.MnemonicDB.Abstractions; +using NexusMods.MnemonicDB.Abstractions.Internals; + +namespace NexusMods.MnemonicDB.Storage.Tests; + +public class NullConnection : IConnection +{ + public IDb Db => throw new NotSupportedException(); + public IAttributeRegistry Registry => throw new NotSupportedException(); + public TxId TxId => throw new NotSupportedException(); + public IObservable Revisions => throw new NotSupportedException(); + public IServiceProvider ServiceProvider => throw new NotSupportedException(); + public IDb AsOf(TxId txId) + { + throw new NotSupportedException(); + } + + public ITransaction BeginTransaction() + { + throw new NotSupportedException(); + } +} diff --git a/tests/NexusMods.MnemonicDB.Tests/DbTests.cs b/tests/NexusMods.MnemonicDB.Tests/DbTests.cs index 4be4a482..f829c0b4 100644 --- a/tests/NexusMods.MnemonicDB.Tests/DbTests.cs +++ b/tests/NexusMods.MnemonicDB.Tests/DbTests.cs @@ -204,8 +204,8 @@ public async Task CanGetCommitUpdates() Connection.Revisions.Subscribe(update => { // Only Txes we care about - if (update.AddedDatoms.Any(d => d.E == realId)) - updates.Add(update.AddedDatoms.Select(d => d.Resolved).ToArray()); + if (update.RecentlyAdded.Any(d => d.E == realId)) + updates.Add(update.RecentlyAdded.Select(d => d.Resolved).ToArray()); }); for (var idx = 0; idx < 4; idx++)