Import @64490 2019.01.1

mule/LICENCE.txt

@@ -1,5 +1,5 @@
 !-------------------------------------------------------------------------------!
-! (C) Crown Copyright 2018, Met Office. All rights reserved.                    !
+! (C) Crown Copyright 2019, Met Office. All rights reserved.                    !
 !                                                                               !
 ! Redistribution and use in source and binary forms, with or without            !
 ! modification, are permitted provided that the following conditions are met:   !

mule/docs/source/conf.py

@@ -48,16 +48,16 @@
 
 # General information about the project.
 project = u'Mule'
-copyright = u'2018, UM Systems Team'
+copyright = u'2019, UM Systems Team'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '2018.07.1'
+version = '2019.01.1'
 # The full version, including alpha/beta/rc tags.
-release = '2018.07.1'
+release = '2019.01.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

mule/lib/mule/__init__.py

@@ -57,7 +57,7 @@
 from contextlib import contextmanager
 from mule.stashmaster import STASHmaster
 
-__version__ = "2018.07.1"
+__version__ = "2019.01.1"
 
 # UM fixed length header names and positions
 _UM_FIXED_LENGTH_HEADER = [
@@ -744,23 +744,31 @@ def _get_raw_payload_bytes(self):
             data = self._data_provider._read_bytes()
         return data
 
-    def _can_copy_deferred_data(self, required_lbpack, required_bacc):
+    def _can_copy_deferred_data(self, required_lbpack, required_bacc,
+                                required_word):
         """
         Return whether or not it is possible to simply re-use the bytes
         making up the field; for this to be possible the data must be
-        unmodified, and the requested output packing must be the same
-        as the input packing.
+        unmodified, and the requested output packing and disk word size must
+        be the same as the input.
 
         """
         # Whether or not this is possible depends on if the Field's
         # data provider has been wrapped in any operations
         compatible = hasattr(self._data_provider, "_read_bytes")
         if compatible:
+            # Is the packing code the same
             src_lbpack = self._data_provider.source.lbpack
-            src_bacc = self._data_provider.source.bacc
-            # The packing words are compatible if nothing else is different.
-            compatible = (required_lbpack == src_lbpack and
-                          required_bacc == src_bacc)
+            compatible = required_lbpack == src_lbpack
+
+            # If it's WGDOS packing, the accuracy matters too
+            if src_lbpack == 1:
+                src_bacc = self._data_provider.source.bacc
+                compatible = compatible and required_bacc == src_bacc
+            else:
+                # Otherwise the disk size matters
+                src_word = self._data_provider.DISK_RECORD_SIZE
+                compatible = compatible and required_word == src_word
 
         return compatible
 
@@ -1514,7 +1522,7 @@ def default_from_raw(values):
                     # Check number format is valid
                     if num_format not in (0, 2, 3):
                         msg = 'Unsupported number format (lbpack N4): {0}'
-                        raise ValueError(msg.format(format))
+                        raise ValueError(msg.format(num_format))
 
                     # With that check out of the way remove the N4 digit and
                     # proceed with the N1 - N3 digits
@@ -1688,14 +1696,27 @@ def _write_to_file(self, output_file):
                     if field.lbpack % 10 == 1 and int(field.bacc) == -99:
                         field.lbpack = 10*(field.lbpack//10)
 
-                    if field._can_copy_deferred_data(field.lbpack, field.bacc):
+                    if field._can_copy_deferred_data(
+                            field.lbpack, field.bacc, self.WORD_SIZE):
                         # The original, unread file data is encoded as wanted,
                         # so extract the raw bytes and write them back out
                         # again unchanged; however first trim off any existing
                         # padding to allow the code below to re-pad the output
                         data_bytes = field._get_raw_payload_bytes()
-                        data_bytes = data_bytes[:field.lblrec*self.WORD_SIZE]
+                        data_bytes = data_bytes[
+                            :field.lblrec *
+                            field._data_provider.DISK_RECORD_SIZE]
                         output_file.write(data_bytes)
+
+                        # Calculate lblrec and lbnrec based on what will be
+                        # written (just in case they are wrong or have come
+                        # from a pp file)
+                        field.lblrec = (
+                            field._data_provider.DISK_RECORD_SIZE *
+                            field.lblrec // self.WORD_SIZE)
+                        field.lbnrec = (
+                            field.lblrec -
+                            (field.lblrec % -self._WORDS_PER_SECTOR))
                     else:
 
                         # Strip just the n1-n3 digits from the lbpack value

mule/lib/mule/pp.py

@@ -50,12 +50,24 @@ class PPField3(PPField, mule.Field3):
 # Mapping to go from release number to field object
 FIELD_SELECT = {2: PPField2, 3: PPField3}
 
+
+# Similarly, adjust the record size for the read providers required here
+PP_WORD_SIZE = 4
+
+
+class _ReadPPProviderUnpacked(mule.ff._ReadFFProviderCray32Packed):
+    DISK_RECORD_SIZE = PP_WORD_SIZE
+
+
+class _ReadPPProviderWGDOSPacked(mule.ff._ReadFFProviderWGDOSPacked):
+    DISK_RECORD_SIZE = PP_WORD_SIZE
+
 # Create mappings for the lbpack n3-n1 digits (similar to how the mule file
 # classes contain mappings like these).  The only real difference is that the
 # "Unpacked" provider uses the 32-bit class (since PP files are 32-bit)
 _READ_PROVIDERS = {
-    "000": mule.ff._ReadFFProviderCray32Packed,
-    "001": mule.ff._ReadFFProviderWGDOSPacked,
+    "000": _ReadPPProviderUnpacked,
+    "001": _ReadPPProviderWGDOSPacked,
     }
 
 _WRITE_OPERATORS = {
@@ -150,7 +162,7 @@ def fields_from_pp_file(pp_file_obj_or_path):
         # This should be equivalent to lbnrec, but can sometimes be set to
         # zero... so to allow the existing provider to work add this value
         # to the reference field's headers
-        field_ref.lbnrec = reclen//4
+        field_ref.lbnrec = reclen//PP_WORD_SIZE
 
         # Associate the provider
         offset = pp_file.tell()
@@ -167,10 +179,16 @@ def fields_from_pp_file(pp_file_obj_or_path):
         provider = _READ_PROVIDERS[lbpack321](field_ref, pp_file, offset)
         field = type(field_ref)(ints, reals, provider)
 
+        # Change the DTYPE variables back to 64-bit - this is slightly hacky
+        # but *only* the UM File logic in the main part of Mule utilises this,
+        # and it will go wrong if it gets a PPField with it set to 32-bit
+        field.DTYPE_REAL = ">f8"
+        field.DTYPE_INT = ">i8"
+
         # Now check if the field contains extra data
         if field.lbext > 0:
             # Skip past the field data only (relative seek to avoid overflows)
-            pp_file.seek((field.lblrec - field.lbext)*4, 1)
+            pp_file.seek((field.lblrec - field.lbext)*PP_WORD_SIZE, 1)
 
             # Save the current file position
             start = pp_file.tell()
@@ -179,7 +197,7 @@ def fields_from_pp_file(pp_file_obj_or_path):
             # end of the record is reached
             vectors = {}
             ext_consumed = 0
-            while pp_file.tell() - start < field.lbext*4:
+            while pp_file.tell() - start < field.lbext*PP_WORD_SIZE:
 
                 # First read the code
                 vector_code = np.fromfile(pp_file, ">i4", 1)[0]
@@ -247,13 +265,11 @@ def fields_to_pp_file(pp_file_obj_or_path, field_or_fields, umfile=None):
 
         # Similar to the mule file classes, the unpacking of data can be
         # skipped if the packing and accuracy are unchanged and the fields
-        # were already PP fields
-        if (field._can_copy_deferred_data(field.lbpack, field.bacc) and
-           isinstance(field, PPField)):
-
+        # have the appropriate word size on disk
+        if (field._can_copy_deferred_data(
+                field.lbpack, field.bacc, PP_WORD_SIZE)):
             # Get the raw bytes containing the data
             data_bytes = field._get_raw_payload_bytes()
-
         else:
             # If the field has been modified follow a similar set of steps to
             # the mule file classes
@@ -266,8 +282,13 @@ def fields_to_pp_file(pp_file_obj_or_path, field_or_fields, umfile=None):
 
             data_bytes, _ = _WRITE_OPERATORS[lbpack321].to_bytes(field)
 
-            field.lblrec = len(data_bytes)//4
-            field.lbnrec = len(data_bytes)//4
+        # Either way, make sure the header addresses are correct for a pp file
+        # both lbegin and lblnrec are zeroed, as pp files don't require a
+        # direct access seek point (lbegin) and being sequential they also
+        # don't have field padding (lbnrec)
+        field.lbegin = 0
+        field.lbnrec = 0
+        field.lblrec = len(data_bytes)//PP_WORD_SIZE
 
         # If the field appears to be variable resolution, attach the
         # relevant extra data (requires that a UM file object was given)
@@ -302,7 +323,7 @@ def fields_to_pp_file(pp_file_obj_or_path, field_or_fields, umfile=None):
                 cdc = umfile.column_dependent_constants
 
                 # Calculate U vectors
-                if grid_type == 18:  # U points
+                if grid_type in (11, 18):  # U points
                     vector[1] = cdc.lambda_u
                     if stagger == "new_dynamics":
                         vector[12] = cdc.lambda_p
@@ -328,7 +349,7 @@ def fields_to_pp_file(pp_file_obj_or_path, field_or_fields, umfile=None):
                                                cdc.lambda_u[-1]])
 
                 # Calculate V vectors
-                if grid_type == 19:  # V points
+                if grid_type in (11, 19):  # V points
                     vector[2] = rdc.phi_v
                     if stagger == "new_dynamics":
                         vector[14] = rdc.phi_p
@@ -366,7 +387,8 @@ def fields_to_pp_file(pp_file_obj_or_path, field_or_fields, umfile=None):
 
         # Calculate the record length (pp files are not direct-access, so each
         # record begins and ends with its own length)
-        lookup_reclen = np.array((len(ints) + len(reals))*4).astype(">i4")
+        lookup_reclen = np.array(
+            (len(ints) + len(reals))*PP_WORD_SIZE).astype(">i4")
 
         # Write the first record (the field header)
         pp_file.write(lookup_reclen)
@@ -378,7 +400,7 @@ def fields_to_pp_file(pp_file_obj_or_path, field_or_fields, umfile=None):
         reclen = len(data_bytes)
 
         if vector:
-            reclen += extra_len*4
+            reclen += extra_len*PP_WORD_SIZE
             keys = [1, 2, 12, 13, 14, 15]
             sizes = ([field.lbnpt, field.lbrow] +
                      [field.lbnpt]*2 + [field.lbrow]*2)

mule/lib/mule/tests/__init__.py

@@ -31,6 +31,40 @@
 import unittest as tests
 
 from mule import Field
+import mule.stashmaster
+
+# Override the STASHmaster to pickup the one local to these tests
+mule.stashmaster.STASHMASTER_PATH_PATTERN = os.path.join(
+    os.path.dirname(__file__), "test_stashmaster")
+
+# Test for availability of mock (which is *not* a standard library at Python 2
+# (it was added to the standard library unittest at Python 3)
+try:
+    if six.PY2:
+        import mock
+    elif six.PY3:
+        import unittest.mock as mock
+except ImportError:
+    MOCK_AVAILABLE = False
+else:
+    MOCK_AVAILABLE = True
+
+
+def skip_mock(fn):
+    """
+    Decorator which can be used to skip a test if the mock library isn't
+    available.  This will completely disable the definition of a method or
+    class if it is decorated like this:
+
+    @skip_mock
+    def test_which_uses_mock(*args):
+        etc
+
+    """
+    skip = tests.skipIf(
+        condition=not MOCK_AVAILABLE,
+        reason="Test required 'mock'")
+    return skip(fn)
 
 
 def _testdata_path():

mule/lib/mule/tests/test_stashmaster

@@ -0,0 +1,44 @@
+H1| SUBMODEL_NUMBER=1
+H2| SUBMODEL_NAME=ATMOS
+H3| UM_VERSION=X.X
+#
+#|Model |Sectn | Item |Name                                |
+#|Space |Point | Time | Grid |LevelT|LevelF|LevelL|PseudT|PseudF|PseudL|LevCom|
+#| Option Codes                   | Version Mask         | Halo |
+#|DataT |DumpP | PC1  PC2  PC3  PC4  PC5  PC6  PC7  PC8  PC9  PCA |
+#|Rotate| PPF  | USER | LBVC | BLEV | TLEV |RBLEVV| CFLL | CFFF |
+#
+#===============================================================================
+#
+1|    1 |    3 |  236 |TEMPERATURE AT 1.5M                 |
+2|    0 |    0 |    1 |    1 |    5 |   -1 |   -1 |    0 |    0 |    0 |    0 |
+3| 000000000000000000000000000000 | 00000000000100000001 |    3 |
+4|    1 |    2 |  -6   -3   -3   -3  -12   21  -12  -99  -99  -99 |
+5|    0 |   16 |    0 |    1 |    0 |    0 |    0 | 9999 |   58 |
+#
+1|    1 |    8 |  225 |DEEP SOIL TEMP. AFTER HYDROLOGY DEGK|
+2|    0 |    0 |    1 |    2 |    6 |    8 |    9 |    0 |    0 |    0 |    1 |
+3| 000000000000000000000000000000 | 00000000000010000000 |    3 |
+4|    1 |    2 |  -3   -3   -3   -3   -6  -99   -6  -99  -99  -99 |
+5|    0 |   23 |    0 |    6 |    0 |    0 |    0 | 9999 |  344 |
+#
+1|    1 |    0 |   33 |OROGRAPHY (/STRAT LOWER BC)         |
+2|    2 |    0 |    1 |    1 |    5 |   -1 |   -1 |    0 |    0 |    0 |    0 |
+3| 000000000000010000000000000000 | 00000000000000000001 |    3 |
+4|    1 |    2 | -99   -3   -3   -3   -6   30   -3  -99  -99  -99 |
+5|    0 |    1 |    0 |  129 |    0 |    0 |    0 | 9999 |   73 |
+#
+1|    1 |   16 |    4 |TEMPERATURE ON THETA LEVELS         |
+2|    0 |    0 |    1 |    1 |    2 |    1 |    2 |    0 |    0 |    0 |    1 |
+3| 000000000000000000000000000000 | 00000000000000000001 |    3 |
+4|    1 |    2 |  -3  -10   -3   -3  -14   21   -3  -99  -99  -99 |
+5|    0 |   16 |    0 |   65 |    0 |    0 |    0 |    0 |    3 |
+#
+#===============================================================================
+#
+1|   -1 |   -1 |   -1 |END OF FILE MARK                    |
+2|    0 |    0 |    0 |    0 |    0 |    0 |    0 |    0 |    0 |    0 |    0 |
+3| 000000000000000000000000000000 | 00000000000000000000 |    0 |
+4|    0 |    0 | -99  -99  -99  -99  -30  -99  -99  -99  -99  -99 |
+5|    0 |    0 |    0 |    0 |    0 |    0 |    0 |    0 |    0 |
+#

mule/lib/mule/tests/unit/test_Field.py

@@ -29,12 +29,6 @@
 import mule.tests as tests
 from mule import Field, _NullReadProvider
 
-import six
-if six.PY2:
-    import mock
-elif six.PY3:
-    import unittest.mock as mock
-
 
 class Test_int_headers(tests.MuleTest):
     def test(self):
@@ -70,12 +64,16 @@ def _check_formats(self,
                        old_bacc=-6, new_bacc=-6,
                        absent_provider=False):
 
-        lookup_entry = mock.Mock(lbpack=old_lbpack, bacc=old_bacc)
+        class dummy_lookup(object):
+            lbpack = old_lbpack
+            bacc = old_bacc
+        lookup_entry = dummy_lookup()
+
         provider = _NullReadProvider(lookup_entry, None, None)
         if absent_provider:
             provider = None
         field = Field(list(range(45)), list(range(19)), provider)
-        return field._can_copy_deferred_data(new_lbpack, new_bacc)
+        return field._can_copy_deferred_data(new_lbpack, new_bacc, 8)
 
     def test_okay_simple(self):
         self.assertTrue(self._check_formats(1234, 1234))
@@ -87,7 +85,7 @@ def test_fail_nodata(self):
         self.assertFalse(self._check_formats(1234, 1234, absent_provider=True))
 
     def test_fail_different_bacc(self):
-        self.assertFalse(self._check_formats(1234, 1234, new_bacc=-8))
+        self.assertFalse(self._check_formats(1, 1, new_bacc=-8))
 
 
 if __name__ == '__main__':