Merge pull request #191 from NCI-GDC/feat/DEV-2143-diff-and-merge-map…

…ping-functionality

DEV-2143: add mapping diff/merge functionality

.gitignore

@@ -40,7 +40,7 @@ coverage.xml
 .pytest_cache/
 pytestdebug.log
 
-# Jenkins 
+# Jenkins
 .local/
 .eggs/
 .ssh/
@@ -81,3 +81,4 @@ docs/_build
 .appveyor.token
 *.bak
 *.log
+test-reports/*

.travis.yml

@@ -3,7 +3,6 @@ language: python
 env:
   - ES_VERSION="7.7.1"
 python:
-  - "3.6"
   - "3.7"
 
 before_install:

README.md

@@ -1,5 +1,5 @@
 [![Build Status](https://travis-ci.org/NCI-GDC/gdc-models.svg)](https://travis-ci.org/NCI-GDC/gdc-models)
-[![Python 3.6](https://img.shields.io/badge/python-3.6-blue.svg)](https://www.python.org/downloads/release/python-360/)
+[![Python 3.7](https://img.shields.io/badge/python-3.7-blue.svg)](https://www.python.org/downloads/release/python-370/)
 [![Codacy Badge](https://api.codacy.com/project/badge/Grade/f71223e269e64eaaa9f6069ceab526c2)](https://www.codacy.com/manual/NCI-GDC/gdc-models?utm_source=github.com&utm_medium=referral&utm_content=NCI-GDC/gdc-models&utm_campaign=Badge_Grade)
 [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit)
 

dev-requirements.txt

@@ -1,12 +1,12 @@
 #
-# This file is autogenerated by pip-compile
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.7
+# by the following command:
 #
 #    pip-compile dev-requirements.in
 #
 appdirs==1.4.4
     # via virtualenv
-aspy.yaml==1.3.0
+aspy-yaml==1.3.0
     # via pre-commit
 atomicwrites==1.4.0
     # via pytest
@@ -22,12 +22,12 @@ coverage==5.2
     # via pytest-cov
 distlib==0.3.1
     # via virtualenv
-elasticsearch-dsl==7.4.0
-    # via -r dev-requirements.in
 elasticsearch==7.12.0
     # via
     #   -c requirements.txt
     #   elasticsearch-dsl
+elasticsearch-dsl==7.4.0
+    # via -r dev-requirements.in
 filelock==3.0.12
     # via virtualenv
 identify==1.4.24
@@ -38,18 +38,12 @@ importlib-metadata==1.7.0
     #   pre-commit
     #   pytest
     #   virtualenv
-importlib-resources==3.2.1
-    # via
-    #   pre-commit
-    #   virtualenv
 more-itertools==5.0.0
     # via pytest
 nodeenv==1.4.0
     # via pre-commit
 packaging==20.4
     # via pytest
-pathlib2==2.3.7.post1
-    # via pytest
 pluggy==0.13.1
     # via pytest
 pre-commit==1.21.0
@@ -58,26 +52,25 @@ py==1.9.0
     # via pytest
 pyparsing==2.4.7
     # via packaging
-pytest-cov==2.10.0
-    # via -r dev-requirements.in
 pytest==4.6.11
     # via
     #   -r dev-requirements.in
     #   pytest-cov
+pytest-cov==2.10.0
+    # via -r dev-requirements.in
 python-dateutil==2.8.2
     # via elasticsearch-dsl
 pyyaml==5.3.1
     # via
     #   -c requirements.txt
-    #   aspy.yaml
+    #   aspy-yaml
     #   pre-commit
 six==1.15.0
     # via
     #   cfgv
     #   elasticsearch-dsl
     #   more-itertools
     #   packaging
-    #   pathlib2
     #   pre-commit
     #   pytest
     #   python-dateutil
@@ -93,6 +86,4 @@ virtualenv==20.0.27
 wcwidth==0.2.5
     # via pytest
 zipp==1.2.0
-    # via
-    #   importlib-metadata
-    #   importlib-resources
+    # via importlib-metadata

gdcmodels/mapping_utils.py

@@ -0,0 +1,14 @@
+from typing import Dict
+
+import deepdiff
+import mergedeep
+
+
+def deep_merge_mapping_files(one: Dict, two: Dict) -> Dict:
+    return mergedeep.merge(one, two, strategy=mergedeep.Strategy.ADDITIVE)
+
+
+def deep_diff_mapping_files(new: Dict, old: Dict) -> Dict:
+    """Return the dictionary of things that are in the old dict and not in the new"""
+    diff = deepdiff.DeepDiff(new, old)
+    return {} + deepdiff.Delta(diff, force=True)

requirements.txt

@@ -1,13 +1,19 @@
 #
-# This file is autogenerated by pip-compile
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.7
+# by the following command:
 #
 #    pip-compile
 #
 certifi==2020.12.5
     # via elasticsearch
+deepdiff==6.5.0
+    # via gdcmodels (setup.py)
 elasticsearch==7.12.0
     # via gdcmodels (setup.py)
+mergedeep==1.3.4
+    # via gdcmodels (setup.py)
+ordered-set==4.1.0
+    # via deepdiff
 pyyaml==5.3.1
     # via gdcmodels (setup.py)
 urllib3==1.26.4

setup.py

@@ -1,5 +1,4 @@
-from setuptools import setup, find_packages
-
+from setuptools import find_packages, setup
 
 setup(
     name="gdcmodels",
@@ -11,11 +10,10 @@
     install_requires=[
         "PyYAML>=3.11,<6",
         "elasticsearch>=7.0.0,<8.0.0",
+        "deepdiff~=6.5.0",
+        "mergedeep~=1.3.4",
     ],
-    package_data={
-        "gdcmodels": ["es-models/*/*.yaml"]
-    },
-    entry_points={
-        "console_scripts": ["init_index=gdcmodels.init_index:main"]
-    },
+    python_requires=">=3.7",
+    package_data={"gdcmodels": ["es-models/*/*.yaml"]},
+    entry_points={"console_scripts": ["init_index=gdcmodels.init_index:main"]},
 )

tests/test_mapping_utils.py

@@ -0,0 +1,40 @@
+import pytest
+from gdcmodels import mapping_utils
+
+
+@pytest.mark.parametrize(
+    "one,two,expected",
+    [
+        (
+            {"a": {"a": None}},
+            {"a": {"b": None}},
+            {"a": {"a": None, "b": None}},
+        ),
+        (
+            {"a": {"a": {"a": None}}},
+            {"a": {"b": None}},
+            {"a": {"a": {"a": None}, "b": None}},
+        ),
+    ],
+)
+def test_deep_merge_mapping_files(one, two, expected):
+    assert mapping_utils.deep_merge_mapping_files(one, two) == expected
+
+
+@pytest.mark.parametrize(
+    "old,new,difference",
+    [
+        (
+            {"a": {"a": None, "b": None}},
+            {"a": {"a": None}},
+            {"a": {"b": None}},
+        ),
+        (
+            {"a": {"a": {"a": None}, "b": None}},
+            {"a": {"a": {"a": None}}},
+            {"a": {"b": None}},
+        ),
+    ],
+)
+def test_deep_diff_mapping_files(old, new, difference):
+    assert mapping_utils.deep_diff_mapping_files(new, old) == difference