generated from Nexus-Mods/NexusMods.App.Template
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
LMDB implemented, but it doesn't work due to the size bug thing (again)
- Loading branch information
Showing
10 changed files
with
247 additions
and
22 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
74 changes: 74 additions & 0 deletions
74
benchmarks/NexusMods.EventSourcing.Benchmarks/EventStoreBenchmarks.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,74 @@ | ||
| using System; | ||
| using System.Linq; | ||
| using BenchmarkDotNet.Attributes; | ||
| using Microsoft.Extensions.Logging; | ||
| using NexusMods.EventSourcing.Abstractions; | ||
| using NexusMods.EventSourcing.RocksDB; | ||
| using NexusMods.EventSourcing.TestModel; | ||
| using NexusMods.EventSourcing.TestModel.Events; | ||
| using NexusMods.EventSourcing.TestModel.Model; | ||
|
|
||
| namespace NexusMods.EventSourcing.Benchmarks; | ||
|
|
||
| [MemoryDiagnoser] | ||
| public class EventStoreBenchmarks : ABenchmark | ||
| { | ||
| private readonly EntityId<Loadout>[] _ids; | ||
| private readonly RenameLoadout[] _events; | ||
|
|
||
| public EventStoreBenchmarks() | ||
| { | ||
| _ids = Enumerable.Range(0, 100) | ||
| .Select(_ => EntityId<Loadout>.NewId()) | ||
| .ToArray(); | ||
|
|
||
| _events = _ids | ||
| .Select(id => new RenameLoadout(id, "Loadout")) | ||
| .ToArray(); | ||
|
|
||
| } | ||
|
|
||
|
|
||
| [Params(typeof(InMemoryEventStore<EventSerializer>), | ||
| //typeof(FasterKVEventStore<EventSerializer>), | ||
| typeof(RocksDBEventStore<EventSerializer>))] | ||
| public Type EventStoreType { get; set; } = null!; | ||
|
|
||
| [GlobalSetup] | ||
| public void Setup() | ||
| { | ||
| MakeStore(EventStoreType); | ||
|
|
||
| foreach (var evEvent in _events) | ||
| { | ||
| EventStore.Add(evEvent); | ||
| } | ||
| } | ||
|
|
||
| [Benchmark] | ||
| public void AddEvent() | ||
| { | ||
| var rndEvent = _events[Random.Shared.Next(0, _events.Length)]; | ||
| EventStore.Add(rndEvent); | ||
| } | ||
|
|
||
|
|
||
| [Benchmark] | ||
| public void ReadEvents() | ||
| { | ||
| var ingester = new EventCounter(); | ||
| EventStore.EventsForEntity(_ids[Random.Shared.Next(0, _ids.Length)].Value, ingester); | ||
| } | ||
|
|
||
| private struct EventCounter : IEventIngester | ||
| { | ||
| public int Count { get; private set; } | ||
|
|
||
| public void Ingest(IEvent @event) | ||
| { | ||
| Count++; | ||
| } | ||
| } | ||
|
|
||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,99 @@ | ||
| using System; | ||
| using System.Buffers.Binary; | ||
| using LightningDB; | ||
| using NexusMods.EventSourcing.Abstractions; | ||
|
|
||
| namespace NexusMods.EventSourcing.LMDB; | ||
|
|
||
| public class LMDBEventStore<TSerializer> : IEventStore | ||
| where TSerializer : IEventSerializer | ||
| { | ||
| private readonly TSerializer _serializer; | ||
| private readonly Settings _settings; | ||
| private readonly LightningEnvironment _env; | ||
| private TransactionId _tx; | ||
|
|
||
| public LMDBEventStore(TSerializer serializer, Settings settings) | ||
| { | ||
| _serializer = serializer; | ||
| _settings = settings; | ||
|
|
||
| _env = new LightningEnvironment(_settings.StorageLocation.ToString()); | ||
| _env.MapSize = 1024L * 1024L; // 1 TiB | ||
| _env.Open(); | ||
| _tx = TransactionId.From(0); | ||
| } | ||
|
|
||
| public TransactionId Add<T>(T eventValue) where T : IEvent | ||
| { | ||
| using var tx = _env.BeginTransaction(); | ||
|
|
||
| lock (this) | ||
| { | ||
| _tx = _tx.Next(); | ||
|
|
||
| { | ||
| using var db = tx.OpenDatabase("events"); | ||
| Span<byte> keySpan = stackalloc byte[8]; | ||
| BinaryPrimitives.WriteUInt64BigEndian(keySpan, _tx.Value); | ||
| var serialized = _serializer.Serialize(eventValue); | ||
| tx.Put(db, keySpan, serialized); | ||
| } | ||
|
|
||
| { | ||
| using var db = tx.OpenDatabase("entityIndex"); | ||
| var ingester = new ModifiedEntitiesIngester(); | ||
| eventValue.Apply(ingester); | ||
| Span<byte> keySpan = stackalloc byte[24]; | ||
| BinaryPrimitives.WriteUInt64BigEndian(keySpan[16..], _tx.Value); | ||
| foreach (var entityId in ingester.Entities) | ||
| { | ||
| entityId.Value.TryWriteBytes(keySpan); | ||
| tx.Put(db, keySpan, keySpan); | ||
| } | ||
| } | ||
| tx.Commit(); | ||
| return _tx; | ||
| } | ||
| } | ||
|
|
||
| public void EventsForEntity<TIngester>(EntityId entityId, TIngester ingester) where TIngester : IEventIngester | ||
| { | ||
| using var tx = _env.BeginTransaction(TransactionBeginFlags.ReadOnly); | ||
| using var dbEIdx = tx.OpenDatabase("entityIndex"); | ||
| using var dbEvents = tx.OpenDatabase("events"); | ||
|
|
||
| using var cursor = tx.CreateCursor(dbEIdx); | ||
|
|
||
| Span<byte> startKey = stackalloc byte[24]; | ||
| entityId.Value.TryWriteBytes(startKey); | ||
| Span<byte> endKey = stackalloc byte[24]; | ||
| entityId.Value.TryWriteBytes(endKey); | ||
| BinaryPrimitives.WriteUInt64BigEndian(endKey[16..], ulong.MaxValue); | ||
|
|
||
| cursor.SetRange(startKey); | ||
|
|
||
| while (true) | ||
| { | ||
| var (result, key, _) = cursor.GetCurrent(); | ||
| if (result != MDBResultCode.Success) | ||
| { | ||
| break; | ||
| } | ||
|
|
||
| var id = EntityId.From(new Guid(key.AsSpan()[..16])); | ||
| if (id != entityId) | ||
| { | ||
| break; | ||
| } | ||
|
|
||
| var eventData = tx.Get(dbEvents, key.AsSpan()[16..]); | ||
| if (eventData.resultCode != MDBResultCode.Success) | ||
| { | ||
| break; | ||
| } | ||
| var evt = _serializer.Deserialize(eventData.value.AsSpan()); | ||
| ingester.Ingest(evt); | ||
| } | ||
| } | ||
| } |
19 changes: 19 additions & 0 deletions
19
src/NexusMods.EventSourcing.LMDB/NexusMods.EventSourcing.LMDB.csproj
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
|
|
||
| <PropertyGroup> | ||
| <TargetFramework>net8.0</TargetFramework> | ||
| <ImplicitUsings>enable</ImplicitUsings> | ||
| <Nullable>enable</Nullable> | ||
| </PropertyGroup> | ||
|
|
||
| <ItemGroup> | ||
| <PackageReference Include="LightningDB" Version="0.16.0" /> | ||
| <PackageReference Include="NexusMods.Paths" Version="0.4.0" /> | ||
| </ItemGroup> | ||
|
|
||
| <ItemGroup> | ||
| <ProjectReference Include="..\NexusMods.EventSourcing.Abstractions\NexusMods.EventSourcing.Abstractions.csproj" /> | ||
| <ProjectReference Include="..\NexusMods.EventSourcing\NexusMods.EventSourcing.csproj" /> | ||
| </ItemGroup> | ||
|
|
||
| </Project> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,8 @@ | ||
| using NexusMods.Paths; | ||
|
|
||
| namespace NexusMods.EventSourcing.LMDB; | ||
|
|
||
| public class Settings | ||
| { | ||
| public AbsolutePath StorageLocation { get; set; } = default!; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
12 changes: 12 additions & 0 deletions
12
tests/NexusMods.EventSourcing.Tests/EventStoreTests/LMDBEventStoreTests.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| using NexusMods.EventSourcing.LMDB; | ||
| using NexusMods.Paths; | ||
| using Settings = NexusMods.EventSourcing.RocksDB.Settings; | ||
|
|
||
| namespace NexusMods.EventSourcing.Tests.EventStoreTests; | ||
|
|
||
| public class LMDBEventStoreTests(EventSerializer serializer) : AEventStoreTest<LMDBEventStore<EventSerializer>>( | ||
| new LMDBEventStore<EventSerializer>(serializer, new LMDB.Settings | ||
| { | ||
| StorageLocation = FileSystem.Shared.GetKnownPath(KnownPath.EntryDirectory) | ||
| .Combine("FasterKV.EventStore" + Guid.NewGuid()) | ||
| })); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters