From 0fafc1ec10cae2ef8b3c8d2d3ef0da04f9a899b6 Mon Sep 17 00:00:00 2001 From: Binh-Minh Ribler Date: Wed, 31 Jan 2024 20:26:45 -0600 Subject: [PATCH 1/4] Fix HDFFR-82 Description: A special bit, SZ_H4_REV_2, was used to indicate that the szip info was stored in a new way. This bit was set in the options_mask field of the szip info struct. As a result, the value of options_mask became incorrect. This PR removed the special bit from the options_mask before returning it to the application. This change does not affect the data format in the file. --- hdf/src/cszip.c | 45 ++++++++++++++++++++++- hdf/src/hchunks.c | 1 + hdf/src/hproto.h | 1 + mfhdf/dumper/testfiles/dumpsds-15szip.out | 2 +- mfhdf/libsrc/mfsd.c | 8 +++- mfhdf/test/tszip.c | 3 +- 6 files changed, 54 insertions(+), 6 deletions(-) diff --git a/hdf/src/cszip.c b/hdf/src/cszip.c index 7056820494..c361f45b0b 100644 --- a/hdf/src/cszip.c +++ b/hdf/src/cszip.c @@ -951,7 +951,7 @@ HCPcszip_endaccess(accrec_t *access_rec) HCPsetup_szip_parms -- Initialize SZIP parameters USAGE - intn HCPcszip_setup_parms( comp_info *c_info, int32 nt, int32 ndims, int32 *dims, int32 *cdims) + intn HCPsetup_szip_parms( comp_info *c_info, int32 nt, int32 ndims, int32 *dims, int32 *cdims) comp_info *c_info; IN/OUT: the szip compression params int32 nt; IN: the number type of the data int32 ncomp; IN: components in GR, 1 for SD @@ -969,7 +969,7 @@ HCPcszip_endaccess(accrec_t *access_rec) pixels_per_scanline bits_per_pixel - This is called from SDsetcompress, SDsetchunk, GRsetcompress, GRsetchunk + This is called from GRsetup_szip_parms and SDsetup_szip_parms GLOBAL VARIABLES COMMENTS, BUGS, ASSUMPTIONS @@ -1059,3 +1059,44 @@ HCPsetup_szip_parms(comp_info *c_info, int32 nt, int32 ncomp, int32 ndims, int32 return FAIL; #endif } + +/*-------------------------------------------------------------------------- + NAME + HCPrm_szip_special_bit -- Removes the special bit that signals szip revised format + + USAGE + intn HCPrm_szip_special_bit(comp_info *c_info) + comp_info *c_info; IN/OUT: the szip compression params + + RETURNS + Returns SUCCEED + + DESCRIPTION + + A special bit, SZ_H4_REV_2, was introduced to indicate that the szip info + was stored in a new way. This bit was set in the options_mask field + of the szip info struct. As a result, the value of options_mask became + incorrect when the special bit was not removed from the options_mask before + returning to the application. + + This is used in SDgetcompinfo and GRgetcompinfo. + +--------------------------------------------------------------------------*/ +intn +HCPrm_szip_special_bit(comp_info *c_info) +{ + int sz_newway = 0; /* indicates the special bit presents in the options_mask */ + intn ret_value = SUCCEED; + + if (c_info == NULL) + HRETURN_ERROR(DFE_INTERNAL, FAIL); + + /* if the special bit presents for SZIP compression, remove it to + return the correct options_mask */ + sz_newway = (int)c_info->szip.options_mask & SZ_H4_REV_2; + if (sz_newway) + c_info->szip.options_mask = c_info->szip.options_mask & ~SZ_H4_REV_2; + +done: + return SUCCEED; +} diff --git a/hdf/src/hchunks.c b/hdf/src/hchunks.c index 7a6979b05a..7edada53ce 100644 --- a/hdf/src/hchunks.c +++ b/hdf/src/hchunks.c @@ -1507,6 +1507,7 @@ HMCcreate(int32 file_id, /* IN: file to put chunked element in */ if (HCPdecode_header((uint8 *)info->comp_sp_tag_header, (comp_model_t *)&info->model_type, info->minfo, (comp_coder_t *)&info->comp_type, info->cinfo) == FAIL) HGOTO_ERROR(DFE_INTERNAL, FAIL); + break; default: /* Do nothing */ diff --git a/hdf/src/hproto.h b/hdf/src/hproto.h index e8a29fe29d..ddaa0d37fc 100644 --- a/hdf/src/hproto.h +++ b/hdf/src/hproto.h @@ -393,6 +393,7 @@ HDFLIBAPI intn HCPdecode_header(uint8 *p, comp_model_t *model_type, model_info * HDFLIBAPI intn HCPsetup_szip_parms(comp_info *c_info, int32 nt, int32 ncomp, int32 ndims, int32 *dims, int32 *cdims); +HDFLIBAPI intn HCPrm_szip_special_bit(comp_info *c_info); /* ** from hbuffer.c */ diff --git a/mfhdf/dumper/testfiles/dumpsds-15szip.out b/mfhdf/dumper/testfiles/dumpsds-15szip.out index 86aaf0cf9b..704fc37f03 100644 --- a/mfhdf/dumper/testfiles/dumpsds-15szip.out +++ b/mfhdf/dumper/testfiles/dumpsds-15szip.out @@ -79,7 +79,7 @@ Variable Name = SDSszip Type= 32-bit signed integer Ref. = 6 Compression method = SZIP - Option mask = H4_SZ_EC_OPTION_MASK|H4_SZ_RAW_OPTION_MASK (65668) + Option mask = H4_SZ_EC_OPTION_MASK|H4_SZ_RAW_OPTION_MASK (132) Pixels per block = 2 Pixels per scanline = 5 Bits per pixel = 32 diff --git a/mfhdf/libsrc/mfsd.c b/mfhdf/libsrc/mfsd.c index 0266e21c07..2b9acd908e 100644 --- a/mfhdf/libsrc/mfsd.c +++ b/mfhdf/libsrc/mfsd.c @@ -4098,10 +4098,16 @@ SDgetcompinfo(int32 sdsid, /* IN: dataset ID */ /* use lower-level routine to get the compression information */ status = HCPgetcompinfo(handle->hdf_file, var->data_tag, var->data_ref, comp_type, c_info); - if (status == FAIL) HGOTO_ERROR(DFE_INTERNAL, FAIL); + /* remove the szip special bit if necessary */ + if (*comp_type == COMP_CODE_SZIP) { + status = HCPrm_szip_special_bit(c_info); + if (status == FAIL) + HGOTO_ERROR(DFE_INTERNAL, FAIL); + } + done: return ret_value; } /* SDgetcompinfo */ diff --git a/mfhdf/test/tszip.c b/mfhdf/test/tszip.c index a78eba752f..c2d9bb9ad9 100644 --- a/mfhdf/test/tszip.c +++ b/mfhdf/test/tszip.c @@ -1089,8 +1089,7 @@ test_szip_unlimited() /* Initialize for SZIP */ c_info.szip.pixels_per_block = 2; - - c_info.szip.options_mask = SZ_EC_OPTION_MASK; + c_info.szip.options_mask = SZ_EC_OPTION_MASK; c_info.szip.options_mask |= SZ_RAW_OPTION_MASK; c_info.szip.bits_per_pixel = 0; c_info.szip.pixels = 0; From 6d55b11a29107fc000e219f9a24aebcb4a753c4a Mon Sep 17 00:00:00 2001 From: Binh-Minh Ribler Date: Wed, 31 Jan 2024 21:55:42 -0600 Subject: [PATCH 2/4] Added a missing file from previous commit --- mfhdf/dumper/testfiles/dumpsds-15.out | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mfhdf/dumper/testfiles/dumpsds-15.out b/mfhdf/dumper/testfiles/dumpsds-15.out index 45a9c046b6..5024ed018c 100644 --- a/mfhdf/dumper/testfiles/dumpsds-15.out +++ b/mfhdf/dumper/testfiles/dumpsds-15.out @@ -79,7 +79,7 @@ Variable Name = SDSszip Type= 32-bit signed integer Ref. = 6 Compression method = SZIP - Option mask = H4_SZ_EC_OPTION_MASK|H4_SZ_RAW_OPTION_MASK (65668) + Option mask = H4_SZ_EC_OPTION_MASK|H4_SZ_RAW_OPTION_MASK (132) Pixels per block = 2 Pixels per scanline = 5 Bits per pixel = 32 From 678b33aaae55792075649e5f6123c5f5771baf56 Mon Sep 17 00:00:00 2001 From: Binh-Minh Ribler Date: Thu, 1 Feb 2024 02:31:10 -0600 Subject: [PATCH 3/4] Clean-up deprecated functions Description: The functions SDgetcompress/GRgetcompress have incorrect behavior and were deprecated in favor of SDgetcompinfo/GRgetcompinfo several versions ago. They should not be used anymore. Removed the internal function HCPgetcompress and replaced the contents of SDgetcompress/GRgetcompress with a call to SDgetcompinfo/GRgetcompinfo. --- hdf/src/hcomp.c | 107 -------------------------------------------- hdf/src/hproto.h | 3 -- hdf/src/mfgr.c | 57 +++-------------------- mfhdf/libsrc/mfsd.c | 44 +++--------------- 4 files changed, 10 insertions(+), 201 deletions(-) diff --git a/hdf/src/hcomp.c b/hdf/src/hcomp.c index 4f3de208b7..f16bb5b7fe 100644 --- a/hdf/src/hcomp.c +++ b/hdf/src/hcomp.c @@ -808,102 +808,6 @@ HCcreate(int32 file_id, uint16 tag, uint16 ref, comp_model_t model_type, model_i return ret_value; } /* end HCcreate() */ -/*-------------------------------------------------------------------------- - NAME - HCPgetcompress -- Retrieves compression information of an element - USAGE - intn HCPgetcompress(aid, coder_type, c_info) - int32 aid; IN: access record ID - comp_coder_t* coder_type; OUT: the type of compression - comp_info* c_info; OUT: ptr to compression information - structure for storing the retrieved info - RETURNS - SUCCEED/FAIL - DESCRIPTION - This routine retrieves the compression type and the compression - information of the element, identified by 'aid'. The routine is - used by GRgetcompinfo and SDgetcompinfo at this time. - - GLOBAL VARIABLES - COMMENTS, BUGS, ASSUMPTIONS - EXAMPLES - REVISION LOG - July 2001: Added to fix bug #307 - BMR - Dec. 2004: Changed name to HCPgetcompress, to be consistent with other - practice. REM ---------------------------------------------------------------------------*/ -intn -HCPgetcompress(int32 file_id, uint16 data_tag, uint16 data_ref, - comp_coder_t *comp_type, /* OUT: compression type */ - comp_info *c_info) /* OUT: retrieved compression info */ -{ - int32 aid = 0, status; - accrec_t *access_rec = NULL; /* access element record */ - compinfo_t *info = NULL; /* compressed element information */ - model_info m_info; /* modeling information - dummy */ - int32 ret_value = SUCCEED; - - /* clear error stack */ - HEclear(); - - /* start read access on the access record of the data element, which - is being inquired for its compression information */ - aid = Hstartread(file_id, data_tag, data_ref); - - /* get the access_rec pointer */ - access_rec = HAatom_object(aid); - if (access_rec == NULL) - HGOTO_ERROR(DFE_ARGS, FAIL); - - /* if the element is compressed, get the compression info as requested*/ - if (access_rec->special == SPECIAL_COMP) { - info = (compinfo_t *)access_rec->special_info; - if (info == NULL) - HGOTO_ERROR(DFE_COMPINFO, FAIL); - - status = HCIread_header(access_rec, info, c_info, &m_info); - if (status == FAIL) - HGOTO_ERROR(DFE_COMPINFO, FAIL); - - /* get the compression type */ - *comp_type = info->cinfo.coder_type; - - } /* end if element is compressed */ - - /* if the element is chunked, call HMCgetcompress to get the - compression info as appropriate */ - else if (access_rec->special == SPECIAL_CHUNKED) { - status = HMCgetcompress(access_rec, comp_type, c_info); - if (status == FAIL) - HGOTO_ERROR(DFE_COMPINFO, FAIL); - } - - /* flag the error when attempting to get compression info on a - non-compressed element */ - else - /* EIP 9/16/03 Fail but return compression type COMP_CODE_NONE - instead of junk in this case. - */ - { - /*Mac OSX screams here (comp_coder_t)*comp_type = COMP_CODE_NONE; */ - *comp_type = COMP_CODE_NONE; - HGOTO_ERROR(DFE_ARGS, FAIL); - } - /* end access to the aid appropriately */ - if (Hendaccess(aid) == FAIL) - HGOTO_ERROR(DFE_CANTENDACCESS, FAIL); - -done: - if (ret_value == FAIL) { /* Error condition cleanup */ - /* end access to the aid if it's been accessed */ - if (aid != 0) - if (Hendaccess(aid) == FAIL) - HERROR(DFE_CANTENDACCESS); - } - - return ret_value; -} /* HCPgetcompress */ - /*-------------------------------------------------------------------------- NAME HCPgetcompinfo -- Retrieves compression information of an element @@ -920,17 +824,6 @@ HCPgetcompress(int32 file_id, uint16 data_tag, uint16 data_ref, information of the element, identified by 'aid'. The routine is used by GRgetcompinfo and SDgetcompinfo at this time. - GLOBAL VARIABLES - COMMENTS, BUGS, ASSUMPTIONS - EXAMPLES - REVISION LOG - July 2001: Added to fix bug #307 - BMR - Dec. 2004: Changed name to HCPgetcompress, to be consistent with other - practice. REM - Apr. 2005: HCPgetcompinfo was added to fix bugzilla #130 and may replace - HCPgetcompress in the future because HCPgetcompress did not - behave correctly. The revision logs above are carried over - from HCPgetcompress for the records. --------------------------------------------------------------------------*/ intn HCPgetcompinfo(int32 file_id, uint16 data_tag, uint16 data_ref, diff --git a/hdf/src/hproto.h b/hdf/src/hproto.h index ddaa0d37fc..b12cee5958 100644 --- a/hdf/src/hproto.h +++ b/hdf/src/hproto.h @@ -365,9 +365,6 @@ HDFLIBAPI intn HXsetdir(const char *dir); HDFLIBAPI int32 HCcreate(int32 file_id, uint16 tag, uint16 ref, comp_model_t model_type, model_info *m_info, comp_coder_t coder_type, comp_info *c_info); -HDFLIBAPI intn HCPgetcompress(int32 file_id, uint16 data_tag, uint16 data_ref, comp_coder_t *coder_type, - comp_info *c_info); - HDFLIBAPI intn HCPgetcompinfo(int32 file_id, uint16 data_tag, uint16 data_ref, comp_coder_t *coder_type, comp_info *c_info); diff --git a/hdf/src/mfgr.c b/hdf/src/mfgr.c index aa80364283..4068dd8e6a 100644 --- a/hdf/src/mfgr.c +++ b/hdf/src/mfgr.c @@ -4083,7 +4083,7 @@ GRsetcompress(int32 riid, comp_coder_t comp_type, comp_info *cinfo) /*-------------------------------------------------------------------------- NAME - GRgetcompress + GRgetcompress - Deprecated in favor of GRgetcompinfo PURPOSE Get the compression information of a raster image's data. @@ -4109,53 +4109,15 @@ GRsetcompress(int32 riid, comp_coder_t comp_type, comp_info *cinfo) mapped to a quantization table. Thus, only the correct compression type will be returned; cinfo will only contain 0s. - EXAMPLES - REVISION LOG - July 2001: Added to fix bug #307 -BMR - Apr 2005: Replaced by GRgetcompinfo due to deficiency in handling some - special elements. -BMR --------------------------------------------------------------------------*/ intn GRgetcompress(int32 riid, comp_coder_t *comp_type, comp_info *cinfo) { - ri_info_t *ri_ptr; /* ptr to the image to work with */ - int32 file_id; - uint16 scheme; /* compression scheme used for JPEG images */ - intn ret_value = SUCCEED; - - /* clear error stack and check validity of args */ - HEclear(); - - /* check the validity of the RI ID */ - if (HAatom_group(riid) != RIIDGROUP) - HGOTO_ERROR(DFE_ARGS, FAIL); - - /* and check the output arguments */ - if (comp_type == NULL || cinfo == NULL) - HGOTO_ERROR(DFE_ARGS, FAIL); - - /* locate RI's object in hash table */ - if (NULL == (ri_ptr = (ri_info_t *)HAatom_object(riid))) - HGOTO_ERROR(DFE_BADPTR, FAIL); - - file_id = ri_ptr->gr_ptr->hdf_file_id; /* temporary use */ + intn ret_value = SUCCEED; - /* If the compression scheme used was JPEG, return the compression type - and 0 for the 'quality' and 'force_baseline' parameters, because - these parameters are currently not possible to be retrieved. */ - scheme = ri_ptr->img_dim.comp_tag; - if (scheme == DFTAG_JPEG5 || scheme == DFTAG_GREYJPEG5 || scheme == DFTAG_JPEG || - scheme == DFTAG_GREYJPEG) { - *comp_type = COMP_CODE_JPEG; - cinfo->jpeg.quality = 0; - cinfo->jpeg.force_baseline = 0; - } - else { - /* use lower-level routine to get the compression information */ - ret_value = HCPgetcompress(file_id, ri_ptr->img_tag, ri_ptr->img_ref, comp_type, cinfo); - if (ret_value == FAIL) - HGOTO_ERROR(DFE_INTERNAL, FAIL); - } + ret_value = GRgetcompinfo(riid, comp_type, cinfo); + if (ret_value == FAIL) + HGOTO_ERROR(DFE_INTERNAL, FAIL); done: return ret_value; @@ -4293,15 +4255,6 @@ GRgetcomptype(int32 riid, comp_coder_t *comp_type) mapped to a quantization table. Thus, only the correct compression type will be returned; cinfo will only contain 0s. - EXAMPLES - REVISION LOG - July 2001: Added to fix bug #307 - BMR (from GRgetcompress) - Apr 2005: This function was actually created at this time, but it is - almost a duplicate of GRgetcompress, which is intended to be - removed in the future, due to its incorrect behavior. The - only difference is the call to the low-level routine, - HCPgetcompinfo, instead of HCPgetcompress. - --------------------------------------------------------------------------*/ intn GRgetcompinfo(int32 riid, comp_coder_t *comp_type, comp_info *cinfo) diff --git a/mfhdf/libsrc/mfsd.c b/mfhdf/libsrc/mfsd.c index 2b9acd908e..8c4339352b 100644 --- a/mfhdf/libsrc/mfsd.c +++ b/mfhdf/libsrc/mfsd.c @@ -3513,7 +3513,8 @@ SDgetexternalinfo(int32 id, /* IN: dataset ID */ /************************** Deprecated ****************************** NAME SDgetexternalfile -- retrieves external file information - (Deprecated) + (Deprecated in favor of SDgetexternalinfo) + USAGE int32 SDgetexternalfile(id, filename, offset) int32 id; @@ -3968,9 +3969,10 @@ SDsetcompress(int32 id, /* IN: dataset ID */ #ifndef H4_NO_DEPRECATED_SYMBOLS -/****************************************************************************** +/****************************** Deprecated *********************************** NAME SDgetcompress -- Retrieves compression information of a dataset + (Deprecated in favor of SDgetcompinfo) DESCRIPTION This routine uses HCPgetcompress to retrieve the compression type @@ -3979,11 +3981,6 @@ SDsetcompress(int32 id, /* IN: dataset ID */ RETURNS SUCCEED/FAIL - MODIFICATION - July 2001: Added to fix bug #307 - BMR - Apr 2005: This function has incorrect behavior and is replaced by - SDgetcompinfo. SDgetcompress will be removed in the future. - ******************************************************************************/ intn SDgetcompress( @@ -4001,30 +3998,7 @@ SDgetcompress( fprintf(stderr, "SDgetcompress: I've been called\n"); #endif /* SDDEBUG */ - /* clear error stack */ - HEclear(); - - if (comp_type == NULL || c_info == NULL) - HGOTO_ERROR(DFE_ARGS, FAIL); - - handle = SDIhandle_from_id(id, SDSTYPE); - if (handle == NULL || handle->file_type != HDF_FILE) - HGOTO_ERROR(DFE_ARGS, FAIL); - if (handle->vars == NULL) - HGOTO_ERROR(DFE_ARGS, FAIL); - - var = SDIget_var(handle, id); - if (var == NULL) - HGOTO_ERROR(DFE_ARGS, FAIL); - -#ifdef SDDEBUG - printf("SDgetcompress(): var->data_ref=%d, var->aid=%d\n", (int)var->data_ref, (int)var->aid); -#endif /* SDDEBUG */ - if (!var->data_ref) - HGOTO_ERROR(DFE_ARGS, FAIL); - - /* use lower-level routine to get the compression information */ - status = HCPgetcompress(handle->hdf_file, var->data_tag, var->data_ref, comp_type, c_info); + status = SDgetcompinfo(id, comp_type, c_info); if (status == FAIL) HGOTO_ERROR(DFE_INTERNAL, FAIL); @@ -4045,14 +4019,6 @@ SDgetcompress( RETURNS SUCCEED/FAIL - MODIFICATION - July 2001: Added to fix bug #307 - BMR (from SDgetcompress) - Apr 2005: This function was actually created at this time, but it is - almost a duplicate of SDgetcompress, which is intended to be - removed in the future, due to its incorrect behavior. The - only difference is the call to the low-level routine, - HCPgetcompinfo, instead of HCPgetcompress. - ******************************************************************************/ intn SDgetcompinfo(int32 sdsid, /* IN: dataset ID */ From 1188e45c31382f272290917e266fada035a4588f Mon Sep 17 00:00:00 2001 From: Binh-Minh Ribler Date: Mon, 5 Feb 2024 23:48:26 -0600 Subject: [PATCH 4/4] Uncommented the Fortran getcompress test --- mfhdf/fortran/mfsdf.c | 8 ++++---- mfhdf/fortran/tszip.f | 15 +++++---------- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/mfhdf/fortran/mfsdf.c b/mfhdf/fortran/mfsdf.c index 153df13a24..919a88aafa 100644 --- a/mfhdf/fortran/mfsdf.c +++ b/mfhdf/fortran/mfsdf.c @@ -1575,7 +1575,7 @@ nsfsflmd(intf *id, intf *fillmode) /*------------------------------------------------------------------------- * Name: scgichnk - * Puporse: Call SDgetchunkinfo + * Purpose: Call SDgetchunkinfo * Inputs: id: SDS access id * Outputs: dim_length: chunk dimensions * flags: -1 - SDS is nonchunked @@ -1750,7 +1750,7 @@ nscscchnk(intf *id, intf *maxcache, intf *flags) /*------------------------------------------------------------------------- * Name: scschnk - * Puporse: Call SDsetchunk + * Purpose: Call SDsetchunk * Inputs: id: SDS access id * dim_length: chunk dimensions * comp_type: type of compression @@ -1935,7 +1935,7 @@ nscwcchnk(intf *id, intf *start, _fcd char_data) /*------------------------------------------------------------------------- * Name: scscompress - * Puporse: Call SDsetcompress + * Purpose: Call SDsetcompress * Inputs: id: SDS access id * comp_type: type of compression * COMP_CODE_NONE = 0 @@ -2001,7 +2001,7 @@ nscscompress(intf *id, intf *comp_type, intf *comp_prm) /*------------------------------------------------------------------------- * Name: scgcompress - * Puporse: Call SDgetcompress + * Purpose: Call SDgetcompinfo * Inputs: id: SDS access id * Outputs: comp_type: type of compression * COMP_CODE_NONE = 0 diff --git a/mfhdf/fortran/tszip.f b/mfhdf/fortran/tszip.f index 28e75284da..231d535527 100644 --- a/mfhdf/fortran/tszip.f +++ b/mfhdf/fortran/tszip.f @@ -277,17 +277,12 @@ program test_szip . compressed dataset' err_szip = err_szip + 1 endif -C write(*,*) comp_arg(1), comp_prm_out(1) -C write(*,*) comp_arg(2), comp_prm_out(2) -C write(*,*) comp_prm_out(3) -C write(*,*) comp_prm_out(4) -C write(*,*) comp_prm_out(5) -C if ((comp_arg(1) .ne. comp_prm_out(1)) .or. -C . (comp_arg(2) .ne. comp_prm_out(2))) then -C print *, 'wrong compression parameter' -C err_szip = err_szip + 1 -C endif + if ((comp_arg(1) .ne. comp_prm_out(1)) .or. + . (comp_arg(2) .ne. comp_prm_out(2))) then + print *, 'wrong compression parameter' + err_szip = err_szip + 1 + endif C C Read part of the data back using sfrdata function