Skip to content

Commit

Permalink
Test
Browse files Browse the repository at this point in the history
  • Loading branch information
@jhendersonHDF
jhendersonHDF committed Jun 13, 2024
1 parent 539d175 commit bd2bcef
Show file tree
Hide file tree
Showing 47 changed files with 338 additions and 124 deletions.
8 changes: 8 additions & 0 deletions release_docs/RELEASE.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -756,6 +756,14 @@ New Features

Tools:
------
- Add option to adjust the page buffer size in tools

The page buffer cache size for a file can now be adjusted using the
--page-buffer-size=N
option in the h5repack, h5diff, h5dump, h5ls, and h5stat tools. This
will call the H5Pset_page_buffer_size() API function with the specified
size in bytes.

- Allow h5repack to reserve space for a user block without a file

This is useful for users who want to reserve space
Expand Down
70 changes: 48 additions & 22 deletions tools/lib/h5diff.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -624,46 +624,72 @@ h5diff(const char *fname1, const char *fname2, const char *objname1, const char
*-------------------------------------------------------------------------
*/
/* open file 1 */
if (opts->vfd_info[0].u.name) {
if ((fapl1_id = h5tools_get_fapl(H5P_DEFAULT, NULL, &(opts->vfd_info[0]))) < 0) {
parallel_print("h5diff: unable to create fapl for input file\n");
H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "unable to create input fapl\n");
if ((fapl1_id = h5tools_get_new_fapl(H5P_DEFAULT)) < 0) {
parallel_print("h5diff: unable to create fapl for input file\n");
H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "unable to create input fapl\n");
}

/* Set non-default virtual file driver, if requested */
if (opts->custom_vfd[0] && opts->vfd_info[0].u.name) {
if (h5tools_set_fapl_vfd(fapl1_id, &(opts->vfd_info[0])) < 0) {
parallel_print("h5diff: unable to set VFD on fapl for input file\n");
H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "failed to set VFD on FAPL\n");
}
}

/* Set non-default VOL connector, if requested */
if (opts->custom_vol[0]) {
if (h5tools_set_fapl_vol(fapl1_id, &(opts->vol_info[0])) < 0) {
parallel_print("h5diff: unable to set VOL on fapl for input file\n");
H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "failed to set VOL on FAPL\n");
}
}

if (opts->custom_vol[0] || opts->custom_vfd[0]) {
if ((fapl1_id = h5tools_get_fapl(fapl1_id, opts->custom_vol[0] ? &(opts->vol_info[0]) : NULL,
opts->custom_vfd[0] ? &(opts->vfd_info[0]) : NULL)) < 0) {
parallel_print("h5diff: unable to create fapl for input file\n");
H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "unable to create input fapl\n");
if (opts->page_cache > 0) {
if (H5Pset_page_buffer_size(fapl1_id, opts->page_cache, 0, 0) < 0) {
parallel_print("h5diff: unable to set page buffer cache size for fapl for input file\n");
H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "unable to set page buffer cache size on FAPL\n");
}
}

if ((file1_id = h5tools_fopen(fname1, H5F_ACC_RDONLY, fapl1_id, (fapl1_id != H5P_DEFAULT), NULL,
(size_t)0)) < 0) {
if ((file1_id = h5tools_fopen(fname1, H5F_ACC_RDONLY, fapl1_id,
(opts->custom_vol[0] || opts->custom_vfd[0]), NULL, (size_t)0)) < 0) {
parallel_print("h5diff: <%s>: unable to open file\n", fname1);
H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "<%s>: unable to open file\n", fname1);
}
H5TOOLS_DEBUG("file1_id = %s", fname1);

/* open file 2 */
if (opts->vfd_info[1].u.name) {
if ((fapl2_id = h5tools_get_fapl(H5P_DEFAULT, NULL, &(opts->vfd_info[1]))) < 0) {
parallel_print("h5diff: unable to create fapl for output file\n");
H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "unable to create output fapl\n");
if ((fapl2_id = h5tools_get_new_fapl(H5P_DEFAULT)) < 0) {
parallel_print("h5diff: unable to create fapl for output file\n");
H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "unable to create output fapl\n");
}

/* Set non-default virtual file driver, if requested */
if (opts->custom_vfd[1] && opts->vfd_info[1].u.name) {
if (h5tools_set_fapl_vfd(fapl2_id, &(opts->vfd_info[1])) < 0) {
parallel_print("h5diff: unable to set VFD on fapl for output file\n");
H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "failed to set VFD on FAPL\n");
}
}

/* Set non-default VOL connector, if requested */
if (opts->custom_vol[1]) {
if (h5tools_set_fapl_vol(fapl2_id, &(opts->vol_info[1])) < 0) {
parallel_print("h5diff: unable to set VOL on fapl for output file\n");
H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "failed to set VOL on FAPL\n");
}
}

if (opts->custom_vol[1] || opts->custom_vfd[1]) {
if ((fapl2_id = h5tools_get_fapl(fapl2_id, opts->custom_vol[1] ? &(opts->vol_info[1]) : NULL,
opts->custom_vfd[1] ? &(opts->vfd_info[1]) : NULL)) < 0) {
parallel_print("h5diff: unable to create fapl for output file\n");
H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "unable to create output fapl\n");
if (opts->page_cache > 0) {
if (H5Pset_page_buffer_size(fapl2_id, opts->page_cache, 0, 0) < 0) {
parallel_print("h5diff: unable to set page buffer cache size for fapl for output file\n");
H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "unable to set page buffer cache size for output fapl\n");
}
}

if ((file2_id = h5tools_fopen(fname2, H5F_ACC_RDONLY, fapl2_id, (fapl2_id != H5P_DEFAULT), NULL,
(size_t)0)) < 0) {
if ((file2_id = h5tools_fopen(fname2, H5F_ACC_RDONLY, fapl2_id,
(opts->custom_vol[1] || opts->custom_vfd[1]), NULL, (size_t)0)) < 0) {
parallel_print("h5diff: <%s>: unable to open file\n", fname2);
H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "<%s>: unable to open file\n", fname2);
}
Expand Down
1 change: 1 addition & 0 deletions tools/lib/h5diff.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ typedef struct {
h5tools_vfd_info_t vfd_info[2]; /* VFD information for input file, output file */
bool custom_vol[2]; /* Using a custom input, output VOL? */
bool custom_vfd[2]; /* Using a custom input, output VFD? */
size_t page_cache; /* Size to request for page buffer cache */
} diff_opt_t;

/*-------------------------------------------------------------------------
Expand Down
45 changes: 28 additions & 17 deletions tools/lib/h5tools.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -469,7 +469,7 @@ h5tools_set_error_file(const char *fname, int is_bin)
* negative - failed
*-------------------------------------------------------------------------
*/
static herr_t
herr_t
h5tools_set_fapl_vfd(hid_t fapl_id, h5tools_vfd_info_t *vfd_info)
{
herr_t ret_value = SUCCEED;
Expand Down Expand Up @@ -650,7 +650,7 @@ h5tools_set_fapl_vfd(hid_t fapl_id, h5tools_vfd_info_t *vfd_info)
* negative - failed
*-------------------------------------------------------------------------
*/
static herr_t
herr_t
h5tools_set_fapl_vol(hid_t fapl_id, h5tools_vol_info_t *vol_info)
{
htri_t connector_is_registered;
Expand Down Expand Up @@ -746,9 +746,9 @@ h5tools_set_fapl_vol(hid_t fapl_id, h5tools_vol_info_t *vol_info)
}

/*-------------------------------------------------------------------------
* Function: h5tools_get_fapl
* Function: h5tools_get_new_fapl
*
* Purpose: Copies an input fapl and then sets a VOL and/or a VFD on it.
* Purpose: Copies an input fapl.
*
* The returned fapl must be closed by the caller.
*
Expand All @@ -757,7 +757,7 @@ h5tools_set_fapl_vol(hid_t fapl_id, h5tools_vol_info_t *vol_info)
*-------------------------------------------------------------------------
*/
hid_t
h5tools_get_fapl(hid_t prev_fapl_id, h5tools_vol_info_t *vol_info, h5tools_vfd_info_t *vfd_info)
h5tools_get_new_fapl(hid_t prev_fapl_id)
{
hid_t new_fapl_id = H5I_INVALID_HID;
hid_t ret_value = H5I_INVALID_HID;
Expand All @@ -775,16 +775,6 @@ h5tools_get_fapl(hid_t prev_fapl_id, h5tools_vol_info_t *vol_info, h5tools_vfd_i
H5TOOLS_GOTO_ERROR(H5I_INVALID_HID, "H5Pcopy failed");
}

/* Set non-default VOL connector, if requested */
if (vol_info)
if (h5tools_set_fapl_vol(new_fapl_id, vol_info) < 0)
H5TOOLS_GOTO_ERROR(H5I_INVALID_HID, "failed to set VOL on FAPL");

/* Set non-default virtual file driver, if requested */
if (vfd_info)
if (h5tools_set_fapl_vfd(new_fapl_id, vfd_info) < 0)
H5TOOLS_GOTO_ERROR(H5I_INVALID_HID, "failed to set VFD on FAPL");

ret_value = new_fapl_id;

done:
Expand Down Expand Up @@ -1026,9 +1016,23 @@ h5tools_fopen(const char *fname, unsigned flags, hid_t fapl_id, bool use_specifi
vfd_info.u.name = drivernames[drivernum];

/* Get a fapl reflecting the selected VOL connector and VFD */
if ((tmp_fapl_id = h5tools_get_fapl(fapl_id, &vol_info, &vfd_info)) < 0)
if ((tmp_fapl_id = h5tools_get_new_fapl(fapl_id)) < 0)
continue;

if (h5tools_set_fapl_vol(tmp_fapl_id, &vol_info) < 0) {
/* Close the temporary fapl */
H5Pclose(tmp_fapl_id);
tmp_fapl_id = H5I_INVALID_HID;
continue;
}

if (h5tools_set_fapl_vfd(tmp_fapl_id, &vfd_info) < 0) {
/* Close the temporary fapl */
H5Pclose(tmp_fapl_id);
tmp_fapl_id = H5I_INVALID_HID;
continue;
}

/* Can we open the file with this combo? */
H5E_BEGIN_TRY
{
Expand All @@ -1051,9 +1055,16 @@ h5tools_fopen(const char *fname, unsigned flags, hid_t fapl_id, bool use_specifi
/* NOT the native VOL connector */

/* Get a FAPL for the current VOL connector */
if ((tmp_fapl_id = h5tools_get_fapl(fapl_id, &vol_info, NULL)) < 0)
if ((tmp_fapl_id = h5tools_get_new_fapl(fapl_id)) < 0)
continue;

if (h5tools_set_fapl_vol(tmp_fapl_id, &vol_info) < 0) {
/* Close the temporary fapl */
H5Pclose(tmp_fapl_id);
tmp_fapl_id = H5I_INVALID_HID;
continue;
}

/* Can we open the file with this connector? */
if ((fid = h5tools_fopen(fname, flags, tmp_fapl_id, true, drivername, drivername_size)) >= 0) {
used_fapl_id = tmp_fapl_id;
Expand Down
5 changes: 3 additions & 2 deletions tools/lib/h5tools.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -673,9 +673,10 @@ H5TOOLS_DLL int h5tools_set_input_file(const char *fname, int is_bin);
H5TOOLS_DLL int h5tools_set_output_file(const char *fname, int is_bin);
H5TOOLS_DLL int h5tools_set_error_file(const char *fname, int is_bin);

H5TOOLS_DLL hid_t h5tools_get_fapl(hid_t prev_fapl_id, h5tools_vol_info_t *vol_info,
h5tools_vfd_info_t *vfd_info);
H5TOOLS_DLL hid_t h5tools_get_new_fapl(hid_t prev_fapl_id);
H5TOOLS_DLL herr_t h5tools_get_vfd_name(hid_t fid, hid_t fapl_id, char *drivername, size_t drivername_size);
H5TOOLS_DLL herr_t h5tools_set_fapl_vfd(hid_t fapl_id, h5tools_vfd_info_t *vfd_info);
H5TOOLS_DLL herr_t h5tools_set_fapl_vol(hid_t fapl_id, h5tools_vol_info_t *vol_info);
H5TOOLS_DLL hid_t h5tools_fopen(const char *fname, unsigned flags, hid_t fapl, bool use_specific_driver,
char *drivername, size_t drivername_size);
H5TOOLS_DLL hid_t h5tools_get_little_endian_type(hid_t type);
Expand Down
11 changes: 7 additions & 4 deletions tools/libtest/h5tools_test_utils.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -927,7 +927,7 @@ test_populate_ros3_fa(void)
*
* Function: test_set_configured_fapl()
*
* Purpose: Verify `h5tools_get_fapl()` with ROS3 and HDFS VFDs
* Purpose: Verify `h5tools_get_new_fapl()` with ROS3 and HDFS VFDs
*
* Return: 0 if test passes
* 1 if failure
Expand Down Expand Up @@ -1111,12 +1111,15 @@ test_set_configured_fapl(void)
vfd_info.info = C.conf_fa;
vfd_info.u.name = C.vfdname;

if (C.expected == 1)
result = h5tools_get_fapl(H5P_DEFAULT, NULL, &vfd_info);
if (C.expected == 1) {
result = h5tools_get_new_fapl(H5P_DEFAULT);
result = h5tools_set_fapl_vfd(result, &vfd_info);
}
else {
H5E_BEGIN_TRY
{
result = h5tools_get_fapl(H5P_DEFAULT, NULL, &vfd_info);
result = h5tools_get_new_fapl(H5P_DEFAULT);
result = h5tools_set_fapl_vfd(result, &vfd_info);
}
H5E_END_TRY
}
Expand Down
12 changes: 11 additions & 1 deletion tools/src/h5diff/h5diff_common.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ static int check_d_input(const char *);
* Command-line options: The user can specify short or long-named
* parameters.
*/
static const char *s_opts = "cd:ehln:p:qrv*xA:CE:NS*V";
static const char *s_opts = "cd:ehln:p:qrv*xA:CE:K:NS*V";
static struct h5_long_options l_opts[] = {{"compare", no_arg, 'c'},
{"delta", require_arg, 'd'},
{"use-system-epsilon", no_arg, 'e'},
Expand All @@ -39,6 +39,7 @@ static struct h5_long_options l_opts[] = {{"compare", no_arg, 'c'},
{"exclude-attribute", require_arg, 'A'},
{"no-compact-subset", no_arg, 'C'},
{"exclude-path", require_arg, 'E'},
{"page-buffer-size", require_arg, 'K'},
{"nan", no_arg, 'N'},
{"enable-error-stack", optional_arg, 'S'},
{"version", no_arg, 'V'},
Expand Down Expand Up @@ -137,6 +138,9 @@ parse_command_line(int argc, const char *const *argv, const char **fname1, const
/**this is bad in mixing option with results**/
opts->not_cmp = 0;

/* init for page buffer cache size option */
opts->page_cache = 0;

/* init for exclude-path option */
exclude_head = NULL;

Expand Down Expand Up @@ -320,6 +324,10 @@ parse_command_line(int argc, const char *const *argv, const char **fname1, const
opts->use_system_epsilon = 1;
break;

case 'K':
opts->page_cache = strtoul(H5_optarg, NULL, 0);
break;

case '1':
opts->vol_info[0].type = VOL_BY_VALUE;
opts->vol_info[0].u.value = (H5VL_class_value_t)atoi(H5_optarg);
Expand Down Expand Up @@ -622,6 +630,8 @@ usage(void)
PRINTVALSTREAM(rawoutstream, " 3 : All level 2 information plus file names.\n");
PRINTVALSTREAM(rawoutstream, " -q, --quiet\n");
PRINTVALSTREAM(rawoutstream, " Quiet mode. Do not produce output.\n");
PRINTVALSTREAM(rawoutstream,
" --page-buffer-size=N Set the page buffer cache size, N=non-negative integers\n");
PRINTVALSTREAM(rawoutstream,
" --vol-value-1 Value (ID) of the VOL connector to use for opening the\n");
PRINTVALSTREAM(rawoutstream, " first HDF5 file specified\n");
Expand Down
40 changes: 34 additions & 6 deletions tools/src/h5dump/h5dump.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
/* Name of tool */
#define PROGRAMNAME "h5dump"

size_t page_cache = 0;
const char *outfname_g = NULL;
static bool doxml_g = false;
static bool useschema_g = true;
Expand Down Expand Up @@ -97,7 +98,7 @@ struct handler_t {
*/
/* The following initialization makes use of C language concatenating */
/* "xxx" "yyy" into "xxxyyy". */
static const char *s_opts = "a:b*c:d:ef:g:hik:l:m:n*o*pq:rs:t:uvw:xyz:A*BCD:E*F:G:HM:N:O*RS:VX:";
static const char *s_opts = "a:b*c:d:ef:g:hik:l:m:n*o*pq:rs:t:uvw:xyz:A*BCD:E*F:G:HK:M:N:O*RS:VX:";
static struct h5_long_options l_opts[] = {{"attribute", require_arg, 'a'},
{"binary", optional_arg, 'b'},
{"count", require_arg, 'c'},
Expand Down Expand Up @@ -132,6 +133,7 @@ static struct h5_long_options l_opts[] = {{"attribute", require_arg, 'a'},
{"form", require_arg, 'F'},
{"vds-gap-size", require_arg, 'G'},
{"header", no_arg, 'H'},
{"page-buffer-size", require_arg, 'K'},
{"packed-bits", require_arg, 'M'},
{"any_path", require_arg, 'N'},
{"ddl", optional_arg, 'O'},
Expand Down Expand Up @@ -198,6 +200,8 @@ usage(const char *prog)
PRINTVALSTREAM(rawoutstream, " -O F, --ddl=F Output ddl text into file F\n");
PRINTVALSTREAM(rawoutstream,
" Use blank(empty) filename F to suppress ddl display\n");
PRINTVALSTREAM(rawoutstream,
" --page-buffer-size=N Set the page buffer cache size, N=non-negative integers\n");
PRINTVALSTREAM(rawoutstream,
" --s3-cred=<cred> Supply S3 authentication information to \"ros3\" vfd.\n");
PRINTVALSTREAM(rawoutstream,
Expand Down Expand Up @@ -1019,6 +1023,9 @@ parse_command_line(int argc, const char *const *argv)
goto error;
}
break;
case 'K':
page_cache = strtoul(H5_optarg, NULL, 0);
break;

/** begin XML parameters **/
case 'x':
Expand Down Expand Up @@ -1368,10 +1375,30 @@ main(int argc, char *argv[])
/* Initialize indexing options */
h5trav_set_index(sort_by, sort_order);

if (use_custom_vol_g || use_custom_vfd_g) {
if ((fapl_id = h5tools_get_fapl(H5P_DEFAULT, use_custom_vol_g ? &vol_info_g : NULL,
use_custom_vfd_g ? &vfd_info_g : NULL)) < 0) {
error_msg("unable to create FAPL for file access\n");
if ((fapl_id = h5tools_get_new_fapl(H5P_DEFAULT)) < 0) {
error_msg("unable to create FAPL for file access\n");
h5tools_setstatus(EXIT_FAILURE);
goto done;
}
/* Set non-default VOL connector, if requested */
if (use_custom_vol_g) {
if (h5tools_set_fapl_vol(fapl_id, &vol_info_g) < 0) {
error_msg("unable to set VOL on fapl for file\n");
h5tools_setstatus(EXIT_FAILURE);
goto done;
}
}
/* Set non-default virtual file driver, if requested */
if (use_custom_vfd_g) {
if (h5tools_set_fapl_vfd(fapl_id, &vfd_info_g) < 0) {
error_msg("unable to set VFD on fapl for file\n");
h5tools_setstatus(EXIT_FAILURE);
goto done;
}
}
if (page_cache > 0) {
if (H5Pset_page_buffer_size(fapl_id, page_cache, 0, 0) < 0) {
error_msg("unable to set page buffer cache size for file access\n");
h5tools_setstatus(EXIT_FAILURE);
goto done;
}
Expand All @@ -1394,7 +1421,8 @@ main(int argc, char *argv[])
goto done;
}
else
fid = h5tools_fopen(fname, H5F_ACC_RDONLY, fapl_id, (fapl_id != H5P_DEFAULT), NULL, 0);
fid = h5tools_fopen(fname, H5F_ACC_RDONLY, fapl_id, (use_custom_vol_g || use_custom_vfd_g), NULL,
0);

if (fid < 0) {
error_msg("unable to open file \"%s\"\n", fname);
Expand Down
Loading

0 comments on commit bd2bcef

Please sign in to comment.