From bad1e17f7bbb12df79ce048ef1725be61cbbf147 Mon Sep 17 00:00:00 2001 From: Dan Field Date: Thu, 16 Feb 2023 15:47:43 +0000 Subject: [PATCH] added test for DISABLED digest type to FoxML digest Validation (#190) * added test for DISABLED digest type * Adding Integration Test and a new exceptin for disabled digests * checkstyle fixes * changing index to match inline-it et al * seperating test index in case of clash * fixing issues with the disabled integration test * removing superflous log message as we throw an exception now * git wasn't adding an empty dir. used .gitkeep and now the test should parse tge datastream dir * .gitkeep broke the indexer. copying a datastream from another test * reverting to logging rather than throwing exception * improved DISABLED fixity Integration Test to use fcrepo-storage-ocfl session object * Switch from assembly to shade plugin to correct Jena re-packaging (#189) Resolves: https://fedora-repository.atlassian.net/browse/FCREPO-3836 * Alter test assertion to allow for ocfl 1.0 and 1.1 (#192) Resolves: https://fedora-repository.atlassian.net/browse/FCREPO-3863 * Bump woodstox-core from 6.2.3 to 6.4.0 (#191) Bumps [woodstox-core](https://github.com/FasterXML/woodstox) from 6.2.3 to 6.4.0. - [Release notes](https://github.com/FasterXML/woodstox/releases) - [Commits](https://github.com/FasterXML/woodstox/compare/woodstox-core-6.2.3...woodstox-core-6.4.0) --- updated-dependencies: - dependency-name: com.fasterxml.woodstox:woodstox-core dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Increment version to 6.3.1-SNAPSHOT * fixing issues with the disabled integration test * removing superflous log message as we throw an exception now * git wasn't adding an empty dir. used .gitkeep and now the test should parse tge datastream dir * .gitkeep broke the indexer. copying a datastream from another test * reverting to logging rather than throwing exception * improved DISABLED fixity Integration Test to use fcrepo-storage-ocfl session object * fixed test to look for a DISABLED digest --------- Signed-off-by: dependabot[bot] Co-authored-by: Dan Field Co-authored-by: Dan Field Co-authored-by: Jared Whiklo Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Daniel Bernstein --- ...FoxmlInputStreamFedoraObjectProcessor.java | 14 +- .../handlers/ocfl/ArchiveGroupHandler.java | 13 +- .../fcrepo/migration/DisabledDigestIT.java | 74 ++ .../org/fcrepo/migration/InlineXmlIT.java | 1 + ...en-access-UW-Madison-all%2FXACML%2FXACML.0 | 30 + .../objects/2013/0418/22/15/llgc-id_1591190 | 706 ++++++++++++++++++ .../spring/inline-disabled-it-setup.xml | 89 +++ 7 files changed, 922 insertions(+), 5 deletions(-) create mode 100644 src/test/java/org/fcrepo/migration/DisabledDigestIT.java create mode 100644 src/test/resources/inline-disabled-akubra/datastreams/dlmap/e/9e/b4/info%3Afedora%2F1711.dl%3AAccess-policy-open-access-UW-Madison-all%2FXACML%2FXACML.0 create mode 100644 src/test/resources/inline-disabled-akubra/objects/2013/0418/22/15/llgc-id_1591190 create mode 100644 src/test/resources/spring/inline-disabled-it-setup.xml diff --git a/src/main/java/org/fcrepo/migration/foxml/FoxmlInputStreamFedoraObjectProcessor.java b/src/main/java/org/fcrepo/migration/foxml/FoxmlInputStreamFedoraObjectProcessor.java index 0a8fc6ec..5137e7a7 100644 --- a/src/main/java/org/fcrepo/migration/foxml/FoxmlInputStreamFedoraObjectProcessor.java +++ b/src/main/java/org/fcrepo/migration/foxml/FoxmlInputStreamFedoraObjectProcessor.java @@ -193,7 +193,13 @@ public void processObject(final StreamingFedoraObjectHandler handler) { dsInfo = new Foxml11DatastreamInfo(objectInfo, reader); } else if (reader.getLocalName().equals("datastreamVersion")) { final var v = new Foxml11DatastreamVersion(dsInfo, reader); - v.validateInlineXml(); + try { + v.validateInlineXml(); + } catch (RuntimeException e) { + // do we need to do anyting with disabled digests? + LOG.error("Inline Validation failed", e); + throw new RuntimeException(e); + } handler.processDatastreamVersion(v); } else { throw new RuntimeException("Unexpected element! \"" + reader.getLocalName() + "\"!"); @@ -468,6 +474,12 @@ private String extractInlineXml() throws XMLStreamException { private void validateInlineXml() { if (isInlineXml && contentDigest != null && StringUtils.isNotBlank(contentDigest.getDigest())) { + + if (StringUtils.equals(contentDigest.getType(), "DISABLED")) { + LOG.warn("Datastream Digest DISABLED. Skipping digest validation"); + return; + } + final var transformedXml = transformInlineXmlForChecksum(); final var digest = DigestUtils.getDigest(contentDigest.getType()); final var digestBytes = DigestUtils.digest(digest, transformedXml); diff --git a/src/main/java/org/fcrepo/migration/handlers/ocfl/ArchiveGroupHandler.java b/src/main/java/org/fcrepo/migration/handlers/ocfl/ArchiveGroupHandler.java index f510adab..4e2cc466 100644 --- a/src/main/java/org/fcrepo/migration/handlers/ocfl/ArchiveGroupHandler.java +++ b/src/main/java/org/fcrepo/migration/handlers/ocfl/ArchiveGroupHandler.java @@ -654,10 +654,15 @@ private ResourceHeaders createDatastreamHeaders(final DatastreamVersion dv, } if (dv.getContentDigest() != null && !Strings.isNullOrEmpty(dv.getContentDigest().getDigest())) { - final var digest = dv.getContentDigest(); - final var digests = new ArrayList(); - digests.add(URI.create("urn:" + digest.getType().toLowerCase() + ":" + digest.getDigest().toLowerCase())); - headers.withDigests(digests); + if (!dv.getContentDigest().getDigest().equals("none")) { + final var digest = dv.getContentDigest(); + final var digests = new ArrayList(); + digests.add(URI.create("urn:" + digest.getType().toLowerCase() + ":" + + digest.getDigest().toLowerCase())); + headers.withDigests(digests); + } else { + LOGGER.warn("Digest content 'none' found. Not adding to header"); + } } headers.withMimeType(mime); diff --git a/src/test/java/org/fcrepo/migration/DisabledDigestIT.java b/src/test/java/org/fcrepo/migration/DisabledDigestIT.java new file mode 100644 index 00000000..d231337f --- /dev/null +++ b/src/test/java/org/fcrepo/migration/DisabledDigestIT.java @@ -0,0 +1,74 @@ +/** + * Copyright 2015 DuraSpace, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.fcrepo.migration; + +import org.apache.commons.io.FileUtils; +import org.junit.After; +import org.junit.Test; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.support.ClassPathXmlApplicationContext; +import org.fcrepo.storage.ocfl.OcflObjectSessionFactory; + +import java.nio.file.Files; +import java.nio.file.Paths; + +import static org.junit.Assert.assertTrue; + +/** + * @author Dan Field + */ +public class DisabledDigestIT { + + private ConfigurableApplicationContext context; + private Migrator migrator; + private OcflObjectSessionFactory sessionFactory; + + @After + public void tearDown() { + if (context != null) { + context.close(); + } + } + + private void setup(final String name) throws Exception { + final var storage = Paths.get(String.format("target/test/ocfl/%s/storage", name)); + final var staging = Paths.get(String.format("target/test/ocfl/%s/staging", name)); + + if (Files.exists(storage)) { + FileUtils.forceDelete(storage.toFile()); + } + if (Files.exists(staging)) { + FileUtils.forceDelete(staging.toFile()); + } + + Files.createDirectories(storage); + Files.createDirectories(staging); + + context = new ClassPathXmlApplicationContext(String.format("spring/%s-setup.xml", name)); + migrator = (Migrator) context.getBean("migrator"); + sessionFactory = (OcflObjectSessionFactory) context.getBean("ocflSessionFactory"); + } + + @Test + public void testMigrateObjectWithExternalDatastreamAndDisabledDigest() throws Exception { + setup("inline-disabled-it"); + migrator.run(); + final var session = sessionFactory.newSession("info:fedora/llgc-id:1591190"); + assertTrue(session.containsResource("info:fedora/llgc-id:1591190/POLICY")); + } + +} diff --git a/src/test/java/org/fcrepo/migration/InlineXmlIT.java b/src/test/java/org/fcrepo/migration/InlineXmlIT.java index f1767efc..da49c541 100644 --- a/src/test/java/org/fcrepo/migration/InlineXmlIT.java +++ b/src/test/java/org/fcrepo/migration/InlineXmlIT.java @@ -115,4 +115,5 @@ public void failMigrationWhenInlineXmlDoesNotMatchDigest() throws Exception { } } + } diff --git a/src/test/resources/inline-disabled-akubra/datastreams/dlmap/e/9e/b4/info%3Afedora%2F1711.dl%3AAccess-policy-open-access-UW-Madison-all%2FXACML%2FXACML.0 b/src/test/resources/inline-disabled-akubra/datastreams/dlmap/e/9e/b4/info%3Afedora%2F1711.dl%3AAccess-policy-open-access-UW-Madison-all%2FXACML%2FXACML.0 new file mode 100644 index 00000000..49e89aee --- /dev/null +++ b/src/test/resources/inline-disabled-akubra/datastreams/dlmap/e/9e/b4/info%3Afedora%2F1711.dl%3AAccess-policy-open-access-UW-Madison-all%2FXACML%2FXACML.0 @@ -0,0 +1,30 @@ + + + + Open Access: any user can view, listen to, and/or download any representation of the object, + as long as they're associated with or present at UW-Madison. + + + + + + + + + + http://digital.library.wisc.edu/1711.dl/vocabulary/access/population#uwmadison + + + + + diff --git a/src/test/resources/inline-disabled-akubra/objects/2013/0418/22/15/llgc-id_1591190 b/src/test/resources/inline-disabled-akubra/objects/2013/0418/22/15/llgc-id_1591190 new file mode 100644 index 00000000..ba7e0165 --- /dev/null +++ b/src/test/resources/inline-disabled-akubra/objects/2013/0418/22/15/llgc-id_1591190 @@ -0,0 +1,706 @@ + + + + + + + + + + + + + + + + + + + ingestObject + + fedoraAdmin + 2013-04-18T21:15:50.582Z + Ingested by NLW ingest program + + + + + + + + + + + + + oai:llgc.org.uk:llgc-id:1591190 + AOAAA00079444 + 10107/1591190 + (WlAbNL)006509574 + 0047235 + + + + + + + + + + + + + + + + + + + FfIG Group + Discussions November/December 2008 + + + National Library of Wales Fedora ingest program + Version 1.0 + + + Llyfrgell Genedlaethol Cymru : The National Library of Wales + + + Llyfrgell Genedlaethol Cymru : The National Library of Wales + + + + + + + + Dei Tomos: Glannau Shir Gar + + Traethau gwag, cestyll, pysgotwyr a hanes torri record byd mewn car ac mewn awyren. Dei Tomos' travels along the shores of Carmarthenshire. + + BBC Radio Cymru + 2013-01-01T17:28:00 + 2013-01-01T18:35:00 + 201301011728 + + wlk + + + sound recording + 166 Kbps + + 01:07:00 + access +
electronic
+ + + WlAbNL + 130417 + 20130417194300.0 + vtls006509574 + Converted from MARCXML to MODS version 3.3 using MARC21slim2MODS3-3.xsl + (Revision 1.51) + + eng + + 10107/1591190 + llgc-id:1591190 + 0047235 + + I wylio/gwrando ar y recordiad hwn, gwnewch gais a chysylltwch ag aelod o staff, os gwelwch yn dda. = To consult this recording, please place a request and contact a member of staff. + Recorded 1 January 2013 17:28:00. + Recorded off-air by NLW under the terms of Statutory Instrument 1991 No. 1116. Freeview data stream and associated metadata captured by BoB (Box of Broadcasts), supplied by Cambridge Imaging Systems Ltd. + Content not verified. + Radio programs. + + BBC Radio Cymru + + + + + + + + + + + + + + + + + WlAbNL_METS_AOAAA00079444 + BoB1 + + file + aoaaa00079444 + + + + + + + + + + + + + + + lossy + + 128 + Fixed + MPEG Audio + profile: Layer 2 + 1 + 48.0 + + + 01:07:00 + Mode: Joint stereo + data_rate element above is in Kbps + sampling_frequency element above is in KHz + Stream_size: 62.3 MiB (77%) + Language: Welsh + 2 + + + + + + + + + + + + WlAbNL_METS_AOAAA00079444 + bob1 + + file + + 0 + + MD5 + 9c188cb6ec979097c0927d632baf8745 + + + SHA-1 + a8466bd11039eda5c667f202aa72867a + + + + MPEG2 Transport Stream + ISO/IEC 13818-1, Second edition 2000-12-01, http://neuron2.net/library/mpeg2/iso13818-1.pdf + + + + + + + + + + + + + + + WlAbNL + R-001 + + all + + Copyright (Archiving Broadcasts Designated + Bodies), Order, 18/10/1993, No. 74 + +

The Library is a designated body under the Order made under Section 75 of the + Copyright, Designs and Patents Act 1988 which provides that copies of broadcasts + of a designated class may be made for the purpose of being placed in an archive + maintained by a designated body without infringement of copyright.

+ + + + all + + 2008-12-01 + 9999-01-01 + + + + + + + + + + + + + + + + WlAbNL + V-001 + + format validation + 2013-02-13T12:02:02 + + C-001 + + + WlAbNL + media_info + + + + + + + + + + + WlAbNL + FC-001 + + fixity check + 2008-11-21T10:00:00 + + C-001 + + + WlAbNL + UNIX_TOOLS-001 + + + + + + + + + + + WlAbNL + REC-001 + + off-air recording + 2013-01-01T17:28:00 + + WlAbNL + HP-01 + + + WlAbNL + HP-02 + + + WlAbNL + INFORTREND-01 + + + WlAbNL + CIS-01 + + + WlAbNL + CIS-02 + + + WlAbNL + CIS-03 + + + WlAbNL + CIS-04 + + + WlAbNL + CIS-05 + + + WlAbNL + IMAGEN-01 + + + WlAbNL + IMAGEN-02 + + + WlAbNL + IMAGEN-03 + + + WlAbNL + IMAGEN-04 + + + WlAbNL + IMAGEN-05 + + + WlAbNL + IMAGEN-06 + + + WlAbNL + PHP-01 + + + WlAbNL + POSTGRE-01 + + + WlAbNL + MEDIAPLAN-01 + + + WlAbNL + VIDEOARCHIVE-01 + + + + + + + + + + + WlAbNL + media_info + + media_info version 0.7.60 + software + + + + + + + + + + WlAbNL +UNIX_TOOLS-001 + + UNIX checksum tools (/usr/bin/md5sum, /usr/bin/gpg --print-md sha1), + see dev.llgc.org.uk + software + + + + + + + + + + WlAbNL + CIS-01 + + CIS Capture Server version 1.5.0.14 + software + + + + + + + + + + WlAbNL + CIS-02 + + CIS Feed Server 2.5.129.2 + software + + + + + + + + + + WlAbNL + CIS-03 + + CIS Transcoder version 1.9.0.23 + software + + + + + + + + + + WlAbNL + IMAGEN-01 + + Imagen Server version 1.1.0.69 + software + + + + + + + + + + WlAbNL + IMAGEN-02 + + Imagen Server Soap Interface version 1.0.0.30 + software + + + + + + + + + + WlAbNL + IMAGEN-03 + + Imagen Client version 2.1.0.179 + software + + + + + + + + + + WlAbNL + IMAGEN-04 + + Imagen Media Control Centre version 1.0.0.48 + software + + + + + + + + + + WlAbNL + PHP-01 + + PHP version 5.0.5.5 + software + + + + + + + + + + WlAbNL + POSTGRE-01 + + PostrgreSQL version 8.1 + software + + + + + + + + + + WlAbNL + CIS-04 + + CIS Web Site version 1.6 + software + + + + + + + + + + WlAbNL + CIS-05 + + CIS Encoder Service version 1.0.0.22 + software + + + + + + + + + + WlAbNL + IMAGEN-05 + + Imagen Ingest GUI version 1.0.0.39 + software + + + + + + + + + + WlAbNL + IMAGEN-06 + + Imagen Client 2.1.0.174 + software + + + + + + + + + + WlAbNL + MEDIAPLAN-01 + + Media Plan version 1.3.0.36 + software + + + + + + + + + + WlAbNL + VIDEOARCHIVE-01 + + Video Archive.NET version 1.4.1.34 + software + + + + + + + + + + WlAbNL + HP-01 + + 2 x HP DL360 G4p servers each with 2xSCSI hard disk drives, with RAID1 + mirror configuration + hardware + + + + + + + + + + WlAbNL + INFORTREND-01 + + 2x Infortrend 24-Bay SCSI to SATA RAID controllers + hardware + + + + + + + + + + WlAbNL + HP-02 + + 1x HP DC7700 workstation with 6Mbps encoder card + hardware + + + + + + + + + + + + + + + + + + + + + + + + + + + Dei Tomos: Glannau Shir Gar + BBC Radio Cymru + Traethau gwag, cestyll, pysgotwyr a hanes torri record byd mewn car ac mewn awyren. Dei Tomos' travels along the shores of Carmarthenshire. + Recorded 1 January 2013 17:28:00. + [Wales] : BBC Radio Cymru, + 2013-January-01 17:28:00 + sound recording + Radio programs. + llgc-id:1591190 + ||| + + Nid oes modd gwneud copïau o'r deunydd hwn heb ganiatâd deiliaid yr hawlfraint. = It is not possible to make copies of this material without permission from the copyright holders. + + + + + + + + + + + + + + + + + + + + + diff --git a/src/test/resources/spring/inline-disabled-it-setup.xml b/src/test/resources/spring/inline-disabled-it-setup.xml new file mode 100644 index 00000000..182975b4 --- /dev/null +++ b/src/test/resources/spring/inline-disabled-it-setup.xml @@ -0,0 +1,89 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +