Merge branch 'develop'

SMI · Sep 9, 2022 · b05cc2e · b05cc2e
2 parents 3815914 + dcbe2a1
commit b05cc2e
Show file tree

Hide file tree

Showing 44 changed files with 1,642 additions and 282 deletions.
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -2,6 +2,7 @@ version: 2
 updates:
 - package-ecosystem: "github-actions"
   directory: "/"
+  target-branch: develop
   schedule:
     interval: daily
 - package-ecosystem: nuget

diff --git a/.github/workflows/dotnet-core.yml b/.github/workflows/dotnet-core.yml
@@ -12,12 +12,6 @@ jobs:
     runs-on: windows-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/cache@v3
-        with:
-          path: ${{ github.workspace }}/.nuget/packages
-          key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj') }}
-          restore-keys: |
-            ${{ runner.os }}-nuget-
       - uses: actions/setup-dotnet@v2
         with:
           dotnet-version: 6.0.x
@@ -28,12 +22,13 @@ jobs:
           for /f tokens^=1^,2^ delims^=^" %%i in (SharedAssemblyInfo.cs) do @if "%%i"=="[assembly: AssemblyVersion(" echo ::set-output name=version::%%j
           for /f tokens^=2^,4^ delims^=^" %%i in (Rdmp.Dicom/Rdmp.Dicom.csproj) do @if "%%i"=="HIC.RDMP.Plugin" echo ::set-output name=rdmpversion::%%j
       - name: Install MS SQL 2019
-        uses: crazy-max/ghaction-chocolatey@v1
+        uses: crazy-max/ghaction-chocolatey@v2
         with:
           args: install sqllocaldb
       - name: Set up database
         run: |
-          Invoke-WebRequest -Uri https://github.com/HicServices/RDMP/releases/download/v${{ steps.version.outputs.rdmpversion }}/rdmp-cli-win-x64.zip -OutFile rdmp.zip
+          SqlLocalDB.exe create MSSQLLocalDB -s
+          Invoke-WebRequest -Uri https://github.com/HicServices/RDMP/releases/download/v${{ steps.version.outputs.rdmpversion }}/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-win-x64.zip -OutFile rdmp.zip
           Expand-Archive -Path rdmp.zip -DestinationPath .\rdmpcli
           sqlcmd -l 180 -S "(localdb)\MSSQLLocalDB" -Q "SELECT @@VERSION;"
           cmd.exe /c rdmpcli\rdmp.exe install --createdatabasetimeout 180  "(localdb)\MSSQLLocalDB" TEST_
@@ -43,30 +38,31 @@ jobs:
       - name: Build
         run: |
           cd Plugin/windows
-          dotnet publish --runtime win-x64 -c Release --self-contained false
+          dotnet publish --runtime win-x64 -c Release --self-contained false -nologo
           cd ../main
-          dotnet publish --runtime win-x64 -c Release --self-contained false
-          dotnet publish --runtime linux-x64 -c Release --self-contained false
+          dotnet publish --runtime win-x64 -c Release --self-contained false -nologo
+          dotnet publish --runtime linux-x64 -c Release --self-contained false -nologo
           cd ../..
           nuget pack ./Rdmp.Dicom.nuspec -Properties Configuration=Release -IncludeReferencedProjects -Symbols -Version ${{ steps.version.outputs.version }}
           nuget pack ./Rdmp.Dicom.Library.nuspec -Properties Configuration=Release -IncludeReferencedProjects -Symbols -Version ${{ steps.version.outputs.version }}
           
           cp Databases.Integration.yaml .\rdmpcli\Databases.yaml
           cmd /c rdmpcli\rdmp.exe pack --file Rdmp.Dicom.${{ steps.version.outputs.version }}.nupkg
           cmd /c rdmpcli\rdmp.exe cmd listsupportedcommands
-          cmd /c rdmpcli\rdmp.exe cmd describecommand CreateNewImagingDatasetSuite
+          cmd /c rdmpcli\rdmp.exe cmd describe CreateNewImagingDatasetSuite
       - name: Get Imaging Template
         run: |
           Invoke-WebRequest -Uri https://raw.githubusercontent.com/HicServices/DicomTypeTranslation/main/Templates/CT.it -OutFile D:\a\RdmpDicom\CT.it
       - name: Integration Tests
         run: |
           cmd /c rdmpcli\rdmp.exe cmd NewObject ConnectionStringKeyword MicrosoftSqlServer TrustServerCertificate yes
           cmd /c rdmpcli\rdmp.exe cmd NewObject ConnectionStringKeyword MicrosoftSqlServer Encrypt yes
-          cmd /c rdmpcli\rdmp.exe cmd CreateNewImagingDatasetSuite "DatabaseType:MicrosoftSqlServer:Name:ImagingTest:Server=(localdb)\MSSQLLocalDB;Integrated Security=true;Encrypt=yes;TrustServerCertificate=true" ./data DicomFileCollectionSource CT_ ../CT.it false false
+          cmd /c rdmpcli\rdmp.exe cmd CreateNewImagingDatasetSuite "DatabaseType:MicrosoftSqlServer:Name:ImagingTest:Server=(localdb)\MSSQLLocalDB;Integrated Security=true;Encrypt=yes;TrustServerCertificate=true" ./data DicomFileCollectionSource CT_ ../CT.it false true
           cmd /c rdmpcli\rdmp.exe cmd AddTag Catalogue:CT_ImageTable StudyDate null
           cmd /c rdmpcli\rdmp.exe cmd AddTag Catalogue:CT_*Table SeriesDate null
           cmd /c rdmpcli\rdmp.exe cmd AddTag Catalogue:CT_*Table SeriesDate null
-          cmd /c rdmpcli\rdmp.exe cmd AddTag Catalogue:CT_*Table SeriesDate null
+          cmd /c rdmpcli\rdmp.exe cmd AddTag Catalogue:CT_*Table SeriesDate null          
+          cmd /c rdmpcli\rdmp.exe cmd CreateNewImagingDatasetSuite "DatabaseType:MicrosoftSqlServer:Name:ImagingTest2:Server=(localdb)\MSSQLLocalDB;Integrated Security=true;Encrypt=yes;TrustServerCertificate=true" ./data DicomFileCollectionSource CT_ ../CT.it false true
       - name: Nuget push
         if: contains(github.ref,'refs/tags/')
         run: nuget push HIC.Rdmp.Dicom.${{ steps.version.outputs.version }}.nupkg -Source https://api.nuget.org/v3/index.json -ApiKey ${{ secrets.NUGET_KEY }} -SkipDuplicate

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [5.0.12] 2022-09-09
+
+- Fixed API for new `Bitmap` substitute (see https://github.com/HicServices/RDMP/pull/1355)
+
+## [5.0.11] 2022-08-24
+
+### Added
+
+- Added MetadataOnly property to FoDicomAnonymiser [#64](https://github.com/HicServices/RdmpDicom/issues/64)
+
+## [5.0.10] 2022-06-21
+
+- Refactored FoDicomAnonymiser to be easier for downstream consumption
+- Added HTTP Basic Authentication support to SemEHR API endpoint connection.
+
+## [5.0.9] 2022-05-27
+
+- Added SemEHR Request Timeout property so that the timeout setting can be changed as needed for longer running queries.
+
 ## [5.0.8] 2022-05-04
 
 ### Changed
@@ -355,7 +374,11 @@ Updated to be compatible with RDMP 4.2
 
 - Initial commit from private repo
 
-[Unreleased]: https://github.com/HicServices/RdmpDicom/compare/v5.0.8...develop
+[Unreleased]: https://github.com/HicServices/RdmpDicom/compare/v5.0.12...develop
+[5.0.12]: https://github.com/HicServices/RdmpDicom/compare/v5.0.11...v5.0.12
+[5.0.11]: https://github.com/HicServices/RdmpDicom/compare/v5.0.10...v5.0.11
+[5.0.10]: https://github.com/HicServices/RdmpDicom/compare/v5.0.9...v5.0.10
+[5.0.9]: https://github.com/HicServices/RdmpDicom/compare/v5.0.8...v5.0.9
 [5.0.8]: https://github.com/HicServices/RdmpDicom/compare/v5.0.7...v5.0.8
 [5.0.7]: https://github.com/HicServices/RdmpDicom/compare/v5.0.6...v5.0.7
 [5.0.6]: https://github.com/HicServices/RdmpDicom/compare/v5.0.5...v5.0.6

diff --git a/Packages.md b/Packages.md
@@ -9,6 +9,7 @@
 
 | Package | Source Code | Version | License | Purpose | Additional Risk Assessment |
 | ------- | ------------| --------| ------- | ------- | -------------------------- |
-| HIC.DicomTypeTranslation | [GitHub](https://github.com/HicServices/DicomTypeTranslation) | [4.0.0](https://www.nuget.org/packages/HIC.DicomTypeTranslation/4.0.0) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Translate dicom types into C# / database types | |
-| HIC.RDMP.Plugin | [GitHub](https://github.com/HicServices/RDMP) | [7.0.11](https://www.nuget.org/packages/HIC.RDMP.Plugin/7.0.11) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Interact with RDMP objects, base classes for plugin components etc | |
-| SharpCompress | [GitHub](https://github.com/adamhathcock/sharpcompress) | [0.31.0](https://www.nuget.org/packages/SharpCompress/0.31.0) | [MIT] | Access diverse archive formats including RAR, 7zip | |
+| HIC.DicomTypeTranslation | [GitHub](https://github.com/HicServices/DicomTypeTranslation) | [4.0.1](https://www.nuget.org/packages/HIC.DicomTypeTranslation/4.0.1) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Translate dicom types into C# / database types | |
+| HIC.RDMP.Plugin | [GitHub](https://github.com/HicServices/RDMP) | [7.0.20](https://www.nuget.org/packages/HIC.RDMP.Plugin/7.0.20) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Interact with RDMP objects, base classes for plugin components etc | |
+| SharpCompress | [GitHub](https://github.com/adamhathcock/sharpcompress) | [0.32.2](https://www.nuget.org/packages/SharpCompress/0.32.2) | [MIT] | Access diverse archive formats including RAR, 7zip | |
+| Terminal.Gui | [GitHub](https://github.com/migueldeicaza/gui.cs) | [1.7.2](https://www.nuget.org/packages/Terminal.Gui/1.7.2) | [MIT] | UIs compatible with RDMP gui command line | |
diff --git a/Rdmp.Dicom.Library.nuspec b/Rdmp.Dicom.Library.nuspec
@@ -13,9 +13,10 @@
         <description>Rdmp library containing Dicom attachers, Dicom server cache sources etc.</description>
         <copyright>Copyright 2018-2019</copyright>
         <dependencies>
-          <dependency id="HIC.DicomTypeTranslation" version="4.0.0" />
-          <dependency id="HIC.RDMP.Plugin" version="7.0.11" />
-          <dependency id="SharpCompress" version="0.31.0" />
+          <dependency id="HIC.DicomTypeTranslation" version="4.0.1" />
+          <dependency id="HIC.RDMP.Plugin" version="7.0.20" />
+          <dependency id="SharpCompress" version="0.32.2" />
+		  <dependency id="Terminal.Gui" version="1.7.2" />
         </dependencies>
     </metadata>
   <files>

diff --git a/Rdmp.Dicom.Tests/Integration/FoDicomAnonymiserStandaloneTests.cs b/Rdmp.Dicom.Tests/Integration/FoDicomAnonymiserStandaloneTests.cs
@@ -0,0 +1,44 @@
+using NUnit.Framework;
+using Rdmp.Dicom.Extraction.FoDicomBased;
+using Rdmp.Dicom.Extraction.FoDicomBased.DirectoryDecisions;
+using ReusableLibraryCode.Progress;
+using System.IO;
+
+namespace Rdmp.Dicom.Tests.Integration
+{
+    public class FoDicomAnonymiserStandaloneTests
+    {
+        [Test]
+        public void TestAnonymiseAFile()
+        {
+            var anon = new FoDicomAnonymiser();
+
+            var inPath = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.WorkDirectory, "in"));
+            var outPath = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.WorkDirectory, "out"));
+
+            if (inPath.Exists)
+                inPath.Delete(true);
+            inPath.Create();
+
+            if (outPath.Exists)
+                outPath.Delete(true);
+            outPath.Create();
+
+            // put a dicom file in the in dir
+            var testFile = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, @"TestData/IM-0001-0013.dcm"));
+            testFile.CopyTo(Path.Combine(inPath.FullName, "blah.dcm"),true);
+
+            anon.Initialize(1, outPath,null /*no UID mapping*/);
+
+            var putter = new PutInRoot();
+
+            anon.ProcessFile(
+                new AmbiguousFilePath(inPath.FullName, "blah.dcm"),
+                new ThrowImmediatelyDataLoadEventListener(),
+                new ZipPool(),
+                "fffff",
+                putter, null);
+        }
+
+    }
+}
diff --git a/Rdmp.Dicom.Tests/Integration/FoDicomAnonymiserTests.cs b/Rdmp.Dicom.Tests/Integration/FoDicomAnonymiserTests.cs
@@ -16,11 +16,13 @@
 using System.Collections.Generic;
 using System.Data;
 using System.IO;
+using System.Linq;
 using Tests.Common;
 using DatabaseType = FAnsi.DatabaseType;
 
 namespace Rdmp.Dicom.Tests.Integration
 {
+
     public class FoDicomAnonymiserTests:DatabaseTests
     {
         [OneTimeSetUp]
@@ -194,7 +196,7 @@ public void TestAnonymisingDataset(Type putterType,bool keepDates)
                 Assert.AreEqual(value, anoDicom.Dataset.GetValue<string>(key, 0));
         }
 
-        [Test]
+        [TestCase()]
         public void TestSkipAnonymisationOnStructuredReports()
         {
             var uidMapDb = GetCleanedServer(DatabaseType.MicrosoftSQLServer, "TESTUIDMapp");
@@ -392,6 +394,121 @@ public void TestUIDTableExists()
 
         }
 
+        [TestCase(typeof(PutInReleaseIdentifierSubfolders))]
+        [TestCase(typeof(PutInUidSeriesFolders))]
+        [TestCase(typeof(PutInRoot))]
+        public void TestAnonymisingDataset_MetadataOnlyVsReal(Type putterType)
+        {
+            var uidMapDb = GetCleanedServer(DatabaseType.MicrosoftSQLServer, "TESTUIDMapp");
+
+            MasterDatabaseScriptExecutor e = new(uidMapDb);
+            var patcher = new SMIDatabasePatcher();
+            e.CreateAndPatchDatabase(patcher, new AcceptAllCheckNotifier());
+
+            var eds = new ExternalDatabaseServer(CatalogueRepository, "eds", patcher);
+            eds.SetProperties(uidMapDb);
+
+            Dictionary<DicomTag, string> thingThatShouldDisappear = new()
+            {
+                //Things we would want to disappear
+                {DicomTag.PatientName,"Moscow"},
+                {DicomTag.PatientBirthDate,"20010101"},
+                {DicomTag.StudyDescription,"Frank has lots of problems, he lives at 60 Pancake road"},
+                {DicomTag.SeriesDescription,"Coconuts"},
+                {DicomTag.AlgorithmName,"Chessnuts"}, // would not normally be dropped by anonymisation
+                {DicomTag.StudyDate,"20020101"}
+            };
+
+            Dictionary<DicomTag, string> thingsThatShouldRemain = new()
+            {
+                //Things we would want to remain
+                //{DicomTag.SmokingStatus,"YES"},
+            };
+
+            var dicom = new DicomDataset
+            {
+                {DicomTag.SOPInstanceUID, "123.4.4"},
+                {DicomTag.SeriesInstanceUID, "123.4.5"},
+                {DicomTag.StudyInstanceUID, "123.4.6"},
+                {DicomTag.SOPClassUID,"1"}
+            };
+
+            foreach (var (key, value) in thingThatShouldDisappear)
+                dicom.AddOrUpdate(key, value);
+
+            foreach (var (key, value) in thingsThatShouldRemain)
+                dicom.AddOrUpdate(key, value);
+
+            dicom.AddOrUpdate(DicomTag.StudyDate, new DateTime(2002, 01, 01));
+
+            var fi = new FileInfo(Path.Combine(TestContext.CurrentContext.WorkDirectory, "madness.dcm"));
+
+            DicomFile df = new(dicom);
+            df.Save(fi.FullName);
+
+            DataTable dtFirstTime = null;
+
+            for (int i = 0; i < 2; i++)
+            {
+                var dt = new DataTable();
+                dt.Columns.Add("Filepath");
+                dt.Columns.Add("SOPInstanceUID");
+                dt.Columns.Add("SeriesInstanceUID");
+                dt.Columns.Add("StudyInstanceUID");
+                dt.Columns.Add("Pat");
+                //note we don't have series
+
+                dt.Rows.Add(fi.Name, "123.4.4", "123.4.5", "123.4.6", "Hank");
+
+                var anonymiser = new FoDicomAnonymiser();
+
+                IExtractCommand cmd = MockExtractionCommand();
+
+                //give the mock to anonymiser
+                anonymiser.PreInitialize(cmd, new ThrowImmediatelyDataLoadEventListener());
+
+                anonymiser.PutterType = putterType;
+                anonymiser.ArchiveRootIfAny = TestContext.CurrentContext.WorkDirectory;
+                anonymiser.RelativeArchiveColumnName = "Filepath";
+                anonymiser.UIDMappingServer = eds;
+                anonymiser.DeleteTags = "AlgorithmName";
+
+                // the thing we are actually testing
+                anonymiser.MetadataOnly = i == 0;
+
+                var anoDt = anonymiser.ProcessPipelineData(dt, new ThrowImmediatelyDataLoadEventListener(), new());
+
+                Assert.AreEqual(1, anoDt.Rows.Count);
+
+                //Data table should contain new UIDs
+                Assert.AreNotEqual("123.4.4", anoDt.Rows[0]["SOPInstanceUID"]);
+                Assert.AreEqual(56, anoDt.Rows[0]["SOPInstanceUID"].ToString().Length);
+
+                Assert.AreNotEqual("123.4.6", anoDt.Rows[0]["StudyInstanceUID"]);
+                Assert.AreEqual(56, anoDt.Rows[0]["StudyInstanceUID"].ToString().Length);
+
+                // second time
+                if(dtFirstTime != null)
+                {
+                    // rows should be the same whether or not we are doing Metadata only extraction
+                    foreach(DataRow row in dtFirstTime.Rows)
+                    {
+                        AssertContains(dt, row.ItemArray);
+                    }
+                }
+
+                dtFirstTime = dt;
+            }
+        }
+
+        private void AssertContains(DataTable dt, params object[] rowValues)
+        {
+            Assert.IsTrue(dt.Rows.Cast<DataRow>().Any(r =>
+                    rowValues.All(v => r.ItemArray.Contains(v))), "Did not find expected row " + string.Join(",", rowValues)
+                                                          + Environment.NewLine + "Rows seen were:" +
+                                                          string.Join(Environment.NewLine,
+                                                              dt.Rows.Cast<DataRow>().Select(r => string.Join(",", r.ItemArray))));
+        }
         private IExtractDatasetCommand MockExtractionCommand()
         {
             //Setup Mocks

diff --git a/Rdmp.Dicom.Tests/Rdmp.Dicom.Tests.csproj b/Rdmp.Dicom.Tests/Rdmp.Dicom.Tests.csproj
@@ -40,9 +40,9 @@
     <ProjectReference Include="..\Rdmp.Dicom\Rdmp.Dicom.csproj" />
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="HIC.BadMedicine.Dicom" Version="0.0.11" />
-    <PackageReference Include="HIC.RDMP.Plugin.Test" Version="7.0.11" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
+    <PackageReference Include="HIC.BadMedicine.Dicom" Version="0.0.14" />
+    <PackageReference Include="HIC.RDMP.Plugin.Test" Version="7.0.20" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.3.1" />
     <PackageReference Include="NUnit" Version="3.13.3" />
     <PackageReference Include="NUnit3TestAdapter" Version="4.2.1" />
     <PackageReference Include="NunitXml.TestLogger" Version="3.0.117" />

diff --git a/Rdmp.Dicom.Tests/Unit/ConnectToOrthancTest.cs b/Rdmp.Dicom.Tests/Unit/ConnectToOrthancTest.cs
@@ -16,9 +16,7 @@ internal class PublicPacsTest
         public void EchoTest(string host, int port)
         {
             var success = false;
-            var client = new DicomClient(host, port, false, LocalAetTitle, RemoteAetTitle, new DicomClientOptions(),
-                new DicomServiceOptions(), new DesktopNetworkManager(), new ConsoleLogManager(),
-                new DefaultTranscoderManager());
+            var client = DicomClientFactory.Create(host, port, false, LocalAetTitle, RemoteAetTitle);
             client.AddRequestAsync(new DicomCEchoRequest
             {
                 OnResponseReceived = (req,res) => {

diff --git a/Rdmp.Dicom.Tests/Unit/DICOMFileCollectionSourceTests.cs b/Rdmp.Dicom.Tests/Unit/DICOMFileCollectionSourceTests.cs
@@ -78,7 +78,7 @@ public void Test_ZipFile(bool expressRelative)
 
             //generate some random dicoms
             var r = new Random(999);
-            DicomDataGenerator generator = new(r, dirToLoad, "CT") {MaximumImages = 5};
+            DicomDataGenerator generator = new(r, dirToLoad.FullName, "CT") {MaximumImages = 5};
             var people = new PersonCollection();
             people.GeneratePeople(1,r);
             generator.GenerateTestDataFile(people,new("./inventory.csv"),1);
@@ -175,7 +175,7 @@ public void Test_ZipFileNotation(bool expressRelative)
 
             //generate some random dicoms
             var r = new Random(999);
-            DicomDataGenerator generator = new(r, dirToLoad, "CT") {MaximumImages = 5};
+            DicomDataGenerator generator = new(r, dirToLoad.FullName, "CT") {MaximumImages = 5};
             var people = new PersonCollection();
             people.GeneratePeople(1,r);
             generator.GenerateTestDataFile(people,new("./inventory.csv"),1);

diff --git a/Rdmp.Dicom.Tests/Unit/PacsFetch.cs b/Rdmp.Dicom.Tests/Unit/PacsFetch.cs
@@ -5,6 +5,7 @@
 using FellowOakDicom;
 using FellowOakDicom.Imaging.Codec;
 using FellowOakDicom.Log;
+using FellowOakDicom.Memory;
 using FellowOakDicom.Network;
 using FellowOakDicom.Network.Client;
 using NUnit.Framework;
@@ -16,7 +17,9 @@ internal class PacsFetch
     class QRService : DicomService, IDicomServiceProvider, IDicomCFindProvider, IDicomCEchoProvider,
         IDicomCMoveProvider
     {
-        public QRService(INetworkStream stream, Encoding fallbackEncoding, Logger log) : base(stream, fallbackEncoding, log, new ConsoleLogManager(), new DesktopNetworkManager(), new DefaultTranscoderManager())
+        private static readonly DicomServiceDependencies Deps = new(new ConsoleLogManager(),
+            new DesktopNetworkManager(), new DefaultTranscoderManager(), new ArrayPoolMemoryProvider());
+        public QRService(INetworkStream stream, Encoding fallbackEncoding, Logger log) : base(stream, fallbackEncoding, log,Deps)
         {
         }