From 25d77ca2b46761ab51dc3246fcd9524a9b8bee6f Mon Sep 17 00:00:00 2001 From: Martin Regen Date: Tue, 1 Oct 2024 18:49:25 +0200 Subject: [PATCH] more benchmarks --- Tests/Common/Main.cs | 60 +++- .../Opc.Ua.Core.Tests.csproj | 2 +- .../Types/Utils/DictionaryKeyBenchmark.cs | 313 +++++++++++++++++- .../Types/Utils/HiResClock.cs | 6 +- 4 files changed, 346 insertions(+), 35 deletions(-) diff --git a/Tests/Common/Main.cs b/Tests/Common/Main.cs index ad0175eed..da1588d60 100644 --- a/Tests/Common/Main.cs +++ b/Tests/Common/Main.cs @@ -28,11 +28,14 @@ * ======================================================================*/ using System; +using System.Linq; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Running; using BenchmarkDotNet.Validators; +using Opc.Ua; [assembly: Config(typeof(BenchmarksDefaultConfig))] @@ -40,22 +43,51 @@ class BenchmarksDefaultConfig : ManualConfig { public BenchmarksDefaultConfig() { - AddJob(Job.Dry); - AddLogger(BenchmarkDotNet.Loggers.ConsoleLogger.Default); - AddValidator(JitOptimizationsValidator.DontFailOnError); + if (Program.CmdLineUsed) + { + var defaults = DefaultConfig.Instance; + foreach (var exporter in defaults.GetExporters()) + { + AddExporter(exporter); + } + foreach (var logger in defaults.GetLoggers()) + { + AddLogger(logger); + } + foreach (var analyser in defaults.GetAnalysers()) + { + AddAnalyser(analyser); + } + foreach (var validator in defaults.GetValidators()) + { + AddValidator(validator); + } + WithOptions(ConfigOptions.DisableOptimizationsValidator); + } + else + { + AddJob(Job.Dry); + AddLogger(ConsoleLogger.Default); + AddValidator(JitOptimizationsValidator.DontFailOnError); + } } -} -static class Program -{ - // Main Method - public static void Main(String[] args) + static class Program { - IConfig config = ManualConfig.Create(DefaultConfig.Instance) - // need this option because of reference to nunit.framework - .WithOptions(ConfigOptions.DisableOptimizationsValidator) - ; - BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args, config); + /// + /// Whether command line was used to start. + /// + public static bool CmdLineUsed = false; + + // Main Method + public static void Main(string[] args) + { + IConfig config = ManualConfig.Create(DefaultConfig.Instance) + // need this option because of reference to nunit.framework + .WithOptions(ConfigOptions.DisableOptimizationsValidator) + ; + CmdLineUsed = true; + BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args); + } } } - diff --git a/Tests/Opc.Ua.Core.Tests/Opc.Ua.Core.Tests.csproj b/Tests/Opc.Ua.Core.Tests/Opc.Ua.Core.Tests.csproj index 1740d187a..a7ed8a259 100644 --- a/Tests/Opc.Ua.Core.Tests/Opc.Ua.Core.Tests.csproj +++ b/Tests/Opc.Ua.Core.Tests/Opc.Ua.Core.Tests.csproj @@ -32,7 +32,7 @@ - + diff --git a/Tests/Opc.Ua.Core.Tests/Types/Utils/DictionaryKeyBenchmark.cs b/Tests/Opc.Ua.Core.Tests/Types/Utils/DictionaryKeyBenchmark.cs index 077248241..30bdf9e5f 100644 --- a/Tests/Opc.Ua.Core.Tests/Types/Utils/DictionaryKeyBenchmark.cs +++ b/Tests/Opc.Ua.Core.Tests/Types/Utils/DictionaryKeyBenchmark.cs @@ -42,6 +42,7 @@ using Opc.Ua; using Opc.Ua.Test; using System.Collections.Concurrent; +using BenchmarkDotNet.Configs; #if NET8_0_OR_GREATER using System.Collections.Frozen; #endif @@ -56,6 +57,9 @@ namespace Opc.Ua.Core.Tests.Types.UtilsTests [Parallelizable] [MemoryDiagnoser] [Orderer(SummaryOrderPolicy.FastestToSlowest)] + [GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)] + [BenchmarkCategory("uint")] + [CategoriesColumn] public class DictionaryUIntKeyBenchmark : DictionaryKeyBenchmark { } @@ -68,6 +72,9 @@ public class DictionaryUIntKeyBenchmark : DictionaryKeyBenchmark [Parallelizable] [MemoryDiagnoser] [Orderer(SummaryOrderPolicy.FastestToSlowest)] + [GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)] + [BenchmarkCategory(nameof(NodeId))] + [CategoriesColumn] public class DictionaryNodeIdKeyBenchmark : DictionaryKeyBenchmark { [Params(IdType.Numeric, IdType.String, IdType.Guid, IdType.Opaque, IdType.Opaque + 1)] @@ -79,7 +86,8 @@ public class DictionaryNodeIdKeyBenchmark : DictionaryKeyBenchmark /// Tests performance and memory usage of ImmutableDictionary. /// [Test] - [Benchmark] + [BenchmarkCategory("TryGet")] + [Benchmark()] public void TestNodeIdDictionary() { foreach (NodeId key in m_lookup) @@ -124,18 +132,6 @@ protected override void Initialize() } } - /// - /// Performance tests for ExpandedNodeId key dictionaries. - /// - [TestFixture, Category("Dictionary")] - [SetCulture("en-us"), SetUICulture("en-us")] - [Parallelizable] - [MemoryDiagnoser] - [Orderer(SummaryOrderPolicy.FastestToSlowest)] - public class DictionaryExpandedNodeIdKeyBenchmark : DictionaryKeyBenchmark - { - } - /// /// Performance tests for T key dictionaries. Abstract to exclude from tests. /// @@ -158,6 +154,14 @@ public abstract class DictionaryKeyBenchmark private FrozenDictionary m_frozenDictionary = null; #endif + // worker dictionaries for add/remove + private Dictionary m_regularWorker = null; + private ConcurrentDictionary m_concurrentWorker = null; + private SortedDictionary m_sortedWorker = null; + + // synchronization + private object m_lock; + #region Test Setup [OneTimeSetUp] public void OneTimeSetUp() @@ -190,11 +194,13 @@ public void GlobalCleanup() } #endregion + #region TryGetValue /// /// Tests performance and memory usage of ImmutableDictionary. /// [Test] - [Benchmark] + [Benchmark(Description = "Immutable")] + [BenchmarkCategory("TryGet")] public void TestImmutableDictionary() { foreach (T key in m_lookup) @@ -207,7 +213,8 @@ public void TestImmutableDictionary() /// Tests performance and memory usage of SortedDictionary. /// [Test] - [Benchmark] + [Benchmark(Description = "Concurrent")] + [BenchmarkCategory("TryGet")] public void TestConcurrentDictionary() { foreach (T key in m_lookup) @@ -221,6 +228,7 @@ public void TestConcurrentDictionary() /// [Test] [Benchmark] + [BenchmarkCategory("TryGet")] public void TestSortedDictionary() { foreach (T key in m_lookup) @@ -233,7 +241,8 @@ public void TestSortedDictionary() /// Tests performance and memory usage of ReadonlyDictionary. /// [Test] - [Benchmark] + [Benchmark(Description = "Readonly")] + [BenchmarkCategory("TryGet")] public void TestReadonlyDictionary() { foreach (T key in m_lookup) @@ -246,7 +255,8 @@ public void TestReadonlyDictionary() /// Tests performance and memory usage of Dictionary. /// [Test] - [Benchmark(Baseline = true)] + [Benchmark(Description = "Regular", Baseline = true)] + [BenchmarkCategory("TryGet")] public void TestRegularDictionary() { foreach (T key in m_lookup) @@ -259,7 +269,8 @@ public void TestRegularDictionary() /// Tests performance and memory usage of FrozenDictionary. /// [Test] - [Benchmark] + [Benchmark(Description = "Frozen")] + [BenchmarkCategory("TryGet")] public void TestFrozenDictionary() { #if NET8_0_OR_GREATER @@ -271,7 +282,271 @@ public void TestFrozenDictionary() TestRegularDictionary(); #endif } + #endregion + + #region RemoveValue + [IterationSetup(Target = nameof(TryRemoveConcurrentDictionary))] + public void PrepareTryRemoveConcurrentDictionary() + { + m_concurrentWorker = new ConcurrentDictionary(m_regularDictionary); + } + + [Test] + [Benchmark(Description = "Concurrent")] + [BenchmarkCategory("Remove")] + public void TryRemoveConcurrentDictionary() + { + foreach (T key in m_lookup) + { + _ = m_concurrentWorker.TryRemove(key, out _); + } + } + + [IterationSetup(Target = nameof(TryRemoveSortedDictionary))] + public void PrepareTryRemoveSortedDictionary() + { + m_sortedWorker = new SortedDictionary(m_regularDictionary); + } + + [Test] + [Benchmark] + [BenchmarkCategory("Remove")] + public void TryRemoveSortedDictionary() + { + foreach (T key in m_lookup) + { + lock (m_lock) + { + _ = m_sortedWorker.Remove(key); + } + } + } + + [IterationSetup(Target = nameof(TryRemoveRegularDictionary))] + public void PrepareTryRemoveDictionary() + { + m_regularWorker = new Dictionary(m_regularDictionary); + } + + /// + /// Tests performance and memory usage of Dictionary. + /// + [Test] + [Benchmark(Description = "Regular", Baseline = true)] + [BenchmarkCategory("Remove")] + public void TryRemoveRegularDictionary() + { + foreach (T key in m_lookup) + { + lock (m_lock) + { + _ = m_regularWorker.Remove(key); + } + } + } + #endregion + + #region Iterate + /// + /// Tests performance and memory usage of ImmutableDictionary. + /// + [Test] + [Benchmark(Description = "Immutable")] + [BenchmarkCategory("Iterate")] + public void IterateImmutableDictionary() + { + foreach (var keyPair in m_immutableDictionary) + { + _ = keyPair.Key; + _ = keyPair.Value; + } + } + + /// + /// Tests performance and memory usage of SortedDictionary. + /// + [Test] + [Benchmark(Description = "Concurrent")] + [BenchmarkCategory("Iterate")] + public void IterateConcurrentDictionary() + { + foreach (var keyPair in m_concurrentDictionary) + { + _ = keyPair.Key; + _ = keyPair.Value; + } + } + + /// + /// Tests performance and memory usage of SortedDictionary. + /// + [Test] + [Benchmark(Description = "ConcurrentToArray")] + [BenchmarkCategory("Iterate")] + public void IterateToArrayConcurrentDictionary() + { + foreach (var keyPair in m_concurrentDictionary.ToArray()) + { + _ = keyPair.Key; + _ = keyPair.Value; + } + } + + /// + /// Tests performance and memory usage of SortedDictionary. + /// + [Test] + [Benchmark] + [BenchmarkCategory("Iterate")] + public void IterateSortedDictionary() + { + foreach (var keyPair in m_sortedDictionary) + { + _ = keyPair.Key; + _ = keyPair.Value; + } + } + + /// + /// Tests performance and memory usage of ReadonlyDictionary. + /// + [Test] + [Benchmark(Description = "Readonly")] + [BenchmarkCategory("Iterate")] + public void IterateReadonlyDictionary() + { + foreach (var keyPair in m_readonlyDictionary) + { + _ = keyPair.Key; + _ = keyPair.Value; + } + } + + /// + /// Tests performance and memory usage of Dictionary. + /// + [Test] + [Benchmark(Description = "Regular", Baseline = true)] + [BenchmarkCategory("Iterate")] + public void IterateRegularDictionary() + { + foreach (var keyPair in m_regularDictionary) + { + _ = keyPair.Key; + _ = keyPair.Value; + } + } + + /// + /// Tests performance and memory usage of FrozenDictionary. + /// + [Test] + [Benchmark(Description = "Frozen")] + [BenchmarkCategory("Iterate")] + public void IterateFrozenDictionary() + { +#if NET8_0_OR_GREATER + foreach (var keyPair in m_frozenDictionary) + { + _ = keyPair.Key; + _ = keyPair.Value; + } +#else + IterateRegularDictionary(); +#endif + } + #endregion + + #region Count + public const int CountRepeats = 1000; + /// + /// Tests performance and memory usage of ImmutableDictionary. + /// + [Test] + [Benchmark(Description = "Immutable")] + [BenchmarkCategory("Count")] + public void CountImmutableDictionary() + { + for (int i = 0; i < CountRepeats; i++) + { + _ = m_immutableDictionary.Count; + } + } + /// + /// Tests performance and memory usage of ConcurrentDictionary. + /// + [Test] + [Benchmark(Description = "Concurrent")] + [BenchmarkCategory("Count")] + public void CountConcurrentDictionary() + { + for (int i = 0; i < CountRepeats; i++) + { + _ = m_concurrentDictionary.Count; + } + } + + /// + /// Tests performance and memory usage of SortedDictionary. + /// + [Test] + [Benchmark(Description = "Sorted")] + [BenchmarkCategory("Count")] + public void CountSortedDictionary() + { + for (int i = 0; i < CountRepeats; i++) + { + _ = m_sortedDictionary.Count; + } + } + + /// + /// Tests performance and memory usage of ReadonlyDictionary. + /// + [Test] + [Benchmark(Description = "Readonly")] + [BenchmarkCategory("Count")] + public void CountReadonlyDictionary() + { + for (int i = 0; i < CountRepeats; i++) + { + _ = m_readonlyDictionary.Count; + } + } + + /// + /// Tests performance and memory usage of Dictionary. + /// + [Test] + [Benchmark(Description = "Regular", Baseline = true)] + [BenchmarkCategory("Count")] + public void CountRegularDictionary() + { + for (int i = 0; i < CountRepeats; i++) + { + _ = m_regularDictionary.Count; + } + } + + /// + /// Tests performance and memory usage of FrozenDictionary. + /// + [Test] + [Benchmark(Description = "Frozen")] + [BenchmarkCategory("Count")] + public void CountFrozenDictionary() + { +#if NET8_0_OR_GREATER + for (int i = 0; i < CountRepeats; i++) + { + _ = m_frozenDictionary.Count; + } +#endif + } + #endregion + + #region Helper Methods protected virtual T GetRandomKey(Random random, DataGenerator generator) { if (typeof(T) == typeof(uint)) @@ -319,6 +594,8 @@ protected virtual void Initialize() #if NET8_0_OR_GREATER m_frozenDictionary = m_regularDictionary.ToFrozenDictionary(); #endif + m_lock = new object(); } + #endregion } } diff --git a/Tests/Opc.Ua.Core.Tests/Types/Utils/HiResClock.cs b/Tests/Opc.Ua.Core.Tests/Types/Utils/HiResClock.cs index 342be1fc1..3aa8c8da7 100644 --- a/Tests/Opc.Ua.Core.Tests/Types/Utils/HiResClock.cs +++ b/Tests/Opc.Ua.Core.Tests/Types/Utils/HiResClock.cs @@ -153,7 +153,8 @@ public void HiResUtcNowTickCount(bool disabled) long lastTickCount = HiResClock.UtcNow.Ticks; long firstTickCount = lastTickCount; int counts = 0; - while (stopWatch.ElapsedMilliseconds <= HiResClockTestDuration) + long elapsedMilliseconds = stopWatch.ElapsedMilliseconds; + while (elapsedMilliseconds <= HiResClockTestDuration) { long tickCount; do @@ -164,6 +165,7 @@ public void HiResUtcNowTickCount(bool disabled) Assert.LessOrEqual(lastTickCount, tickCount); lastTickCount = tickCount; counts++; + elapsedMilliseconds = stopWatch.ElapsedMilliseconds; } if (!disabled) { @@ -171,7 +173,7 @@ public void HiResUtcNowTickCount(bool disabled) } stopWatch.Stop(); long elapsed = (lastTickCount - firstTickCount) / TimeSpan.TicksPerMillisecond; - TestContext.Out.WriteLine("HiResClock counts: {0} resolution: {1}µs", counts, stopWatch.ElapsedMilliseconds * 1000 / counts); + TestContext.Out.WriteLine("HiResClock counts: {0} resolution: {1}µs", counts, (elapsedMilliseconds * 1000.0 / counts)); // test accuracy of counter vs. stop watch try {