Skip to content

Commit

Permalink
CMR-8506 Adding Bulk granule update for ECHO10 OnlineAccessURLs (#1843)
Browse files Browse the repository at this point in the history
* CMR-8506 Adding Bulk granule update for ECHO10 OnlineAccessURLs

* Update ingest-app/resources/granule_bulk_update_schema.json

Co-authored-by: K. Lucia Z. <[email protected]>

* Update ingest-app/resources/granule_bulk_update_schema.json

Co-authored-by: K. Lucia Z. <[email protected]>

* Updated to fix comment.

---------

Co-authored-by: K. Lucia Z. <[email protected]>
  • Loading branch information
eereiter and zimzoom authored Feb 1, 2023
1 parent 86a118b commit 21d4bab
Show file tree
Hide file tree
Showing 6 changed files with 238 additions and 360 deletions.
39 changes: 23 additions & 16 deletions ingest-app/resources/granule_bulk_update_schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
"Checksum",
"Format",
"MimeType",
"OnlineAccessURL",
"OnlineResourceURL",
"OPeNDAPLink",
"S3Link",
Expand Down Expand Up @@ -49,6 +50,24 @@
}
}
},
"UpdateURLType": {
"type": "object",
"additionalProperties": false,
"description": "Describes an existing URL and replacement URL.",
"properties": {
"from": {
"description": "The existing URL to be updated.",
"type": "string",
"minLength": 1
},
"to": {
"description": "The new URL to be substituted",
"type": "string",
"minLength": 1
}
},
"required": [ "from", "to" ]
},
"FileType": {
"type": "object",
"additionalProperties": false,
Expand Down Expand Up @@ -82,23 +101,11 @@
"MimeType": {
"type": "string"
},
"OnlineAccessURL": {
"$ref": "#/definitions/UpdateURLType"
},
"OnlineResourceURL": {
"type": "object",
"additionalProperties": false,
"description": "Describes an OnlineResource existing URL and replacment URL.",
"properties": {
"from": {
"description": "The existing URL to updated.",
"type": "string",
"minLength": 1
},
"to": {
"description": "The new URL to be substituted",
"type": "string",
"minLength": 1
}
},
"required": [ "from", "to" ]
"$ref": "#/definitions/UpdateURLType"
},
"Checksum": {
"type": "object",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,67 +5,10 @@
[clojure.data.xml :as xml]
[clojure.set :as set]
[clojure.string :as string]
[clojure.zip :as zip]
[cmr.common.services.errors :as errors]
[cmr.common.xml :as cx]
[cmr.ingest.services.granule-bulk-update.utils.echo10 :as echo10-utils]))

(defn- xml-elem->online-access-url
"Parses and returns XML element for OnlineAccessURL"
[elem]
(let [url (cx/string-at-path elem [:URL])
description (cx/string-at-path elem [:URLDescription])
mime-type (cx/string-at-path elem [:MimeType])]
{:url url
:url-description description
:mime-type mime-type}))

(defn- update-accesses
"Constructs the new OnlineAccessURLs node in zipper representation"
[online-access-urls url-map]
(let [edn-access-urls (map xml-elem->online-access-url online-access-urls)
access-urls (map #(merge %
(when-let [mime-type (get url-map (:url %))]
{:mime-type mime-type}))
edn-access-urls)]
(xml/element
:OnlineAccessURLs {}
(for [r access-urls]
(let [{:keys [url url-description mime-type]} r]
(xml/element :OnlineAccessURL {}
(xml/element :URL {} url)
(when url-description (xml/element :URLDescription {} url-description))
(when mime-type (xml/element :MimeType {} mime-type))))))))

(defn- replace-in-tree
"Take a parsed granule xml, replace the given node with the provided replacement
Returns the zipper representation of the updated xml."
[tree element-tag replacement]
(let [zipper (zip/xml-zip tree)
start-loc (zip/down zipper)]
(loop [loc start-loc done false]
(if done
(zip/root loc)
(if-let [right-loc (zip/right loc)]
(cond
;; at an OnlineResources element, replace the node with updated value
(= element-tag (-> right-loc zip/node :tag))
(recur (zip/replace right-loc replacement) true)

;; no action needs to be taken, move to the next node
:else
(recur right-loc false))
(recur loc true))))))

(defn- update-online-access-urls
"Return an OnlineAccessURLs node where MimeType is updated where the URL has a matching entry in the url-map."
[tree url-map]
(let [online-access-urls (cx/elements-at-path
tree
[:OnlineAccessURLs :OnlineAccessURL])]
(when (seq online-access-urls)
(update-accesses online-access-urls url-map))))

(defn update-mime-type
"Update the the MimeType for elements within OnlineResources and OnlineAccess in echo10
granule metadata and return granule."
Expand Down Expand Up @@ -93,10 +36,10 @@
"]")]))

online-resources (echo10-utils/update-online-resources parsed :url :mime-type url-map)
access-urls (update-online-access-urls parsed url-map)
access-urls (echo10-utils/update-online-access-urls parsed :url :mime-type url-map)

updated-metadata (-> parsed
(replace-in-tree :OnlineResources online-resources)
(replace-in-tree :OnlineAccessURLs access-urls)
(echo10-utils/replace-in-tree :OnlineResources online-resources)
(echo10-utils/replace-in-tree :OnlineAccessURLs access-urls)
xml/indent-str)]
(assoc concept :metadata updated-metadata)))
Original file line number Diff line number Diff line change
Expand Up @@ -5,32 +5,10 @@
[clojure.data.xml :as xml]
[clojure.set :as set]
[clojure.string :as string]
[clojure.zip :as zip]
[cmr.common.log :refer [info debug]]
[cmr.common.services.errors :as errors]
[cmr.common.xml :as cx]
[cmr.ingest.services.granule-bulk-update.utils.echo10 :as echo10-utils]))

(defn- replace-in-tree
"Take a parsed granule xml, replace the given node with the provided replacement
Returns the zipper representation of the updated xml."
[tree element-tag replacement]
(let [zipper (zip/xml-zip tree)
start-loc (zip/down zipper)]
(loop [loc start-loc done false]
(if done
(zip/root loc)
(if-let [right-loc (zip/right loc)]
(cond
;; at an OnlineResources element, replace the node with updated value
(= element-tag (-> right-loc zip/node :tag))
(recur (zip/replace right-loc replacement) true)

;; no action needs to be taken, move to the next node
:else
(recur right-loc false))
(recur loc true))))))

(defn update-online-resource-url
"Update the the URL for elements within OnlineResources in echo10
granule metadata and return metadata."
Expand All @@ -57,5 +35,34 @@

online-resources (echo10-utils/update-online-resources parsed :url :url url-map)]
(-> parsed
(replace-in-tree :OnlineResources online-resources)
(echo10-utils/replace-in-tree :OnlineResources online-resources)
xml/indent-str)))

(defn update-online-access-url
"Update the the URL for elements within OnlineAccesses in echo10
granule metadata and return metadata."
[concept links]
(let [parsed (xml/parse-str (:metadata concept))
url-map (apply merge (map #(hash-map (:from %) (:to %)) links))

existing-urls (map #(cx/string-at-path % [:URL])
(cx/elements-at-path parsed [:OnlineAccessURLs :OnlineAccessURL]))
_ (when-not (set/superset? (set existing-urls) (set (keys url-map)))
(errors/throw-service-errors
:invalid-data
[(str "Update failed - please only specify URLs contained in the"
" existing granule OnlineAccessURLs ["
(string/join ", " (set/difference (set (keys url-map)) (set existing-urls)))
"] were not found")]))

_ (when-not (= (count (set (keys url-map))) (count links))
(errors/throw-service-errors
:invalid-data
[(str "Update failed - duplicate URLs provided for granule update ["
(string/join ", " (for [[v freq] (frequencies (map :URL links)) :when (> freq 1)] v))
"]")]))

online-access-urls (echo10-utils/update-online-access-urls parsed :url :url url-map)]
(-> parsed
(echo10-utils/replace-in-tree :OnlineAccessURLs online-access-urls)
xml/indent-str)))
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
"Contains functions for updating ECHO10 granule xml metadata."
(:require
[clojure.data.xml :as xml]
[clojure.zip :as zip]
[cmr.common.xml :as cx]))

(defn xml-elem->online-resource
Expand Down Expand Up @@ -43,3 +44,60 @@
[:OnlineResources :OnlineResource])]
(when (seq online-resources)
(update-resources online-resources locator-field update-field value-map))))

(defn- xml-elem->online-access-url
"Parses and returns XML element for OnlineAccessURL"
[elem]
(let [url (cx/string-at-path elem [:URL])
description (cx/string-at-path elem [:URLDescription])
mime-type (cx/string-at-path elem [:MimeType])]
{:url url
:url-description description
:mime-type mime-type}))

(defn- update-accesses
"Constructs the new OnlineAccessURLs node in zipper representation"
[online-access-urls locator-field value-field value-map]
(let [edn-access-urls (map xml-elem->online-access-url online-access-urls)
access-urls (map #(merge %
(when-let [replacement (get value-map (get % locator-field))]
(hash-map value-field replacement)))
edn-access-urls)]
(xml/element
:OnlineAccessURLs {}
(for [r access-urls]
(let [{:keys [url url-description mime-type]} r]
(xml/element :OnlineAccessURL {}
(xml/element :URL {} url)
(when url-description (xml/element :URLDescription {} url-description))
(when mime-type (xml/element :MimeType {} mime-type))))))))

(defn update-online-access-urls
"Return an OnlineAccessURLs node where UPDATE-FIELD is updated where the
LOCATOR-FIELD has a matching key in the VALUE-MAP."
[xml-tree locator-field value-field value-map]
(let [online-access-urls (cx/elements-at-path
xml-tree
[:OnlineAccessURLs :OnlineAccessURL])]
(when (seq online-access-urls)
(update-accesses online-access-urls locator-field value-field value-map))))

(defn replace-in-tree
"Take a parsed granule xml, replace the given node with the provided replacement
Returns the zipper representation of the updated xml."
[tree element-tag replacement]
(let [zipper (zip/xml-zip tree)
start-loc (zip/down zipper)]
(loop [loc start-loc done false]
(if done
(zip/root loc)
(if-let [right-loc (zip/right loc)]
(cond
;; at an OnlineResources element, replace the node with updated value
(= element-tag (-> right-loc zip/node :tag))
(recur (zip/replace right-loc replacement) true)

;; no action needs to be taken, move to the next node
:else
(recur right-loc false))
(recur loc true))))))
Loading

0 comments on commit 21d4bab

Please sign in to comment.