diff --git a/src/NexusMods.MnemonicDB.Abstractions/IAnalyzer.cs b/src/NexusMods.MnemonicDB.Abstractions/IAnalyzer.cs new file mode 100644 index 0000000..e32e3e4 --- /dev/null +++ b/src/NexusMods.MnemonicDB.Abstractions/IAnalyzer.cs @@ -0,0 +1,22 @@ +namespace NexusMods.MnemonicDB.Abstractions; + +/// +/// Interface for a transaction analyzer. These can be injected via DI and they will then be fed each database transaction +/// to analyze and produce a result. +/// +public interface IAnalyzer +{ + /// + /// Analyze the database and produce a result. + /// + public object Analyze(IDb db); +} + +/// +/// Typed version of that specifies the type of the result. +/// +public interface IAnalyzer : IAnalyzer +{ + +} + diff --git a/src/NexusMods.MnemonicDB.Abstractions/IConnection.cs b/src/NexusMods.MnemonicDB.Abstractions/IConnection.cs index 78d004e..e3645fe 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/IConnection.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/IConnection.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using NexusMods.MnemonicDB.Abstractions.IndexSegments; using NexusMods.MnemonicDB.Abstractions.Internals; @@ -60,4 +61,9 @@ public interface IConnection /// /// public ITransaction BeginTransaction(); + + /// + /// The analyzers that are available for this connection + /// + public IAnalyzer[] Analyzers { get; } } diff --git a/src/NexusMods.MnemonicDB.Abstractions/IDb.cs b/src/NexusMods.MnemonicDB.Abstractions/IDb.cs index 38e9806..d959e56 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/IDb.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/IDb.cs @@ -84,4 +84,10 @@ public interface IDb : IEquatable /// Returns an index segment of all the datoms that are a reference pointing to the given entity id. /// IndexSegment ReferencesTo(EntityId eid); + + /// + /// Get the cached data for the given analyzer. + /// + TReturn AnalyzerData() + where TAnalyzer : IAnalyzer; } diff --git a/src/NexusMods.MnemonicDB/Connection.cs b/src/NexusMods.MnemonicDB/Connection.cs index f10a857..c4b5904 100644 --- a/src/NexusMods.MnemonicDB/Connection.cs +++ b/src/NexusMods.MnemonicDB/Connection.cs @@ -27,24 +27,26 @@ public class Connection : IConnection private BehaviorSubject _dbStream; private IDisposable? _dbStreamDisposable; + private readonly IAnalyzer[] _analyzers; /// /// Main connection class, co-ordinates writes and immutable reads /// - public Connection(ILogger logger, IDatomStore store, IServiceProvider provider, IEnumerable declaredAttributes) + public Connection(ILogger logger, IDatomStore store, IServiceProvider provider, IEnumerable declaredAttributes, IEnumerable analyzers) { ServiceProvider = provider; _logger = logger; _declaredAttributes = declaredAttributes.ToDictionary(a => a.Id); _store = store; _dbStream = new BehaviorSubject(default!); + _analyzers = analyzers.ToArray(); Bootstrap(); } /// /// Scrubs the transaction stream so that we only ever move forward and never repeat transactions /// - private static IObservable ForwardOnly(IObservable dbStream) + private IObservable ProcessUpdate(IObservable dbStream) { TxId? prev = null; @@ -52,9 +54,26 @@ private static IObservable ForwardOnly(IObservable dbStream) { return dbStream.Subscribe(nextItem => { + if (prev != null && prev.Value >= nextItem.BasisTxId) return; + var db = (Db)nextItem; + db.Connection = this; + + foreach (var analyzer in _analyzers) + { + try + { + var result = analyzer.Analyze(nextItem); + db.AnalyzerData.Add(analyzer.GetType(), result); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to analyze with {Analyzer}", analyzer.GetType().Name); + } + } + observer.OnNext((Db)nextItem); prev = nextItem.BasisTxId; }, observer.OnError, observer.OnCompleted); @@ -105,6 +124,9 @@ public ITransaction BeginTransaction() return new Transaction(this, _store.Registry); } + /// + public IAnalyzer[] Analyzers => _analyzers; + /// public IObservable Revisions { @@ -175,12 +197,7 @@ private void Bootstrap() { AddMissingAttributes(_declaredAttributes.Values); - _dbStreamDisposable = ForwardOnly(_store.TxLog) - .Select(db => - { - db.Connection = this; - return db; - }) + _dbStreamDisposable = ProcessUpdate(_store.TxLog) .Subscribe(_dbStream); } catch (Exception ex) diff --git a/src/NexusMods.MnemonicDB/Db.cs b/src/NexusMods.MnemonicDB/Db.cs index a54d94a..46c4c72 100644 --- a/src/NexusMods.MnemonicDB/Db.cs +++ b/src/NexusMods.MnemonicDB/Db.cs @@ -30,6 +30,8 @@ internal class Db : IDb public IAttributeRegistry Registry => _registry; public IndexSegment RecentlyAdded { get; } + + internal Dictionary AnalyzerData { get; } = new(); public Db(ISnapshot snapshot, TxId txId, AttributeRegistry registry) { @@ -107,7 +109,14 @@ public IndexSegment ReferencesTo(EntityId id) { return _cache.GetReferences(id, this); } - + + TReturn IDb.AnalyzerData() + { + if (AnalyzerData.TryGetValue(typeof(TAnalyzer), out var value)) + return (TReturn)value; + throw new KeyNotFoundException($"Analyzer {typeof(TAnalyzer).Name} not found"); + } + public IndexSegment Datoms(Attribute attribute, TValue value) { return Datoms(SliceDescriptor.Create(attribute, value, _registry)); diff --git a/tests/NexusMods.MnemonicDB.Storage.Tests/NullConnection.cs b/tests/NexusMods.MnemonicDB.Storage.Tests/NullConnection.cs index f64ced7..0417f03 100644 --- a/tests/NexusMods.MnemonicDB.Storage.Tests/NullConnection.cs +++ b/tests/NexusMods.MnemonicDB.Storage.Tests/NullConnection.cs @@ -19,4 +19,6 @@ public ITransaction BeginTransaction() { throw new NotSupportedException(); } + + public IAnalyzer[] Analyzers => throw new NotSupportedException(); } diff --git a/tests/NexusMods.MnemonicDB.TestModel/Analyzers/AttributesAnalyzer.cs b/tests/NexusMods.MnemonicDB.TestModel/Analyzers/AttributesAnalyzer.cs new file mode 100644 index 0000000..85ad10d --- /dev/null +++ b/tests/NexusMods.MnemonicDB.TestModel/Analyzers/AttributesAnalyzer.cs @@ -0,0 +1,21 @@ +using NexusMods.MnemonicDB.Abstractions; + +namespace NexusMods.MnemonicDB.TestModel.Analyzers; + +/// +/// Records all the attributes in each transaction +/// +public class AttributesAnalyzer : IAnalyzer> +{ + public object Analyze(IDb db) + { + var hashSet = new HashSet(); + var registry = db.Registry; + foreach (var datom in db.RecentlyAdded) + { + hashSet.Add(registry.GetAttribute(datom.A)); + } + + return hashSet; + } +} diff --git a/tests/NexusMods.MnemonicDB.TestModel/Analyzers/DatomCountAnalyzer.cs b/tests/NexusMods.MnemonicDB.TestModel/Analyzers/DatomCountAnalyzer.cs new file mode 100644 index 0000000..ae29ddc --- /dev/null +++ b/tests/NexusMods.MnemonicDB.TestModel/Analyzers/DatomCountAnalyzer.cs @@ -0,0 +1,14 @@ +using NexusMods.MnemonicDB.Abstractions; + +namespace NexusMods.MnemonicDB.TestModel.Analyzers; + +/// +/// Counts the number of dataoms in each transaction +/// +public class DatomCountAnalyzer : IAnalyzer +{ + public object Analyze(IDb db) + { + return db.RecentlyAdded.Count; + } +} diff --git a/tests/NexusMods.MnemonicDB.Tests/AMnemonicDBTest.cs b/tests/NexusMods.MnemonicDB.Tests/AMnemonicDBTest.cs index 8966c1b..ecb097c 100644 --- a/tests/NexusMods.MnemonicDB.Tests/AMnemonicDBTest.cs +++ b/tests/NexusMods.MnemonicDB.Tests/AMnemonicDBTest.cs @@ -7,6 +7,7 @@ using NexusMods.MnemonicDB.TestModel.Helpers; using NexusMods.Hashing.xxHash64; using NexusMods.MnemonicDB.TestModel; +using NexusMods.MnemonicDB.TestModel.Analyzers; using NexusMods.Paths; using Xunit.Sdk; using File = NexusMods.MnemonicDB.TestModel.File; @@ -23,6 +24,7 @@ public class AMnemonicDBTest : IDisposable private DatomStore _store; protected IConnection Connection; protected ILogger Logger; + private readonly IAnalyzer[] _analyzers; protected AMnemonicDBTest(IServiceProvider provider) @@ -40,7 +42,14 @@ protected AMnemonicDBTest(IServiceProvider provider) _backend = new Backend(_registry); _store = new DatomStore(provider.GetRequiredService>(), _registry, Config, _backend); - Connection = new Connection(provider.GetRequiredService>(), _store, provider, _attributes); + + _analyzers = + new IAnalyzer[]{ + new DatomCountAnalyzer(), + new AttributesAnalyzer(), + }; + + Connection = new Connection(provider.GetRequiredService>(), _store, provider, _attributes, _analyzers); Logger = provider.GetRequiredService>(); } @@ -130,7 +139,7 @@ protected async Task RestartDatomStore() _registry = new AttributeRegistry(_attributes); _store = new DatomStore(_provider.GetRequiredService>(), _registry, Config, _backend); - Connection = new Connection(_provider.GetRequiredService>(), _store, _provider, _attributes); + Connection = new Connection(_provider.GetRequiredService>(), _store, _provider, _attributes, _analyzers); } } diff --git a/tests/NexusMods.MnemonicDB.Tests/DbTests.cs b/tests/NexusMods.MnemonicDB.Tests/DbTests.cs index 0ba7a38..91ad931 100644 --- a/tests/NexusMods.MnemonicDB.Tests/DbTests.cs +++ b/tests/NexusMods.MnemonicDB.Tests/DbTests.cs @@ -8,6 +8,7 @@ using NexusMods.MnemonicDB.Abstractions.Query; using NexusMods.MnemonicDB.Abstractions.TxFunctions; using NexusMods.MnemonicDB.TestModel; +using NexusMods.MnemonicDB.TestModel.Analyzers; using NexusMods.Paths; using File = NexusMods.MnemonicDB.TestModel.File; @@ -800,6 +801,33 @@ public async Task CanWriteTupleAttributes() var avet = Connection.Db.Datoms(SliceDescriptor.Create(File.TuplePath, (EntityId.From(0), ""), (EntityId.MaxValueNoPartition, ""), Connection.Db.Registry)); await VerifyTable(avet.Resolved()); + } + + [Fact] + public async Task CanGetAnalyzerData() + { + using var tx = Connection.BeginTransaction(); + + var loadout1 = new Loadout.New(tx) + { + Name = "Test Loadout" + }; + + var mod = new Mod.New(tx) + { + Name = "Test Mod", + Source = new Uri("http://test.com"), + LoadoutId = loadout1 + }; + + var result = await tx.Commit(); + + result.Db.Should().Be(Connection.Db); + + var countData = Connection.Db.AnalyzerData(); + countData.Should().Be(result.Db.RecentlyAdded.Count); + var attrs = Connection.Db.AnalyzerData>(); + attrs.Should().NotBeEmpty(); } }