From 1e4677a0956a732c90dda25b069761d04af87614 Mon Sep 17 00:00:00 2001 From: Leonardo Bautista-Gomez Date: Thu, 25 Feb 2021 12:07:01 +0100 Subject: [PATCH 01/11] Version bump to 1.6 --- CHANGELOG | 80 ++++++++++++++++++++++++++++++++++++++++++++++++++ CMakeLists.txt | 2 +- VERSION | 2 +- 3 files changed, 82 insertions(+), 2 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 83072ec03..3974cb434 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,83 @@ +Version 1.6: + - Merge pull request #392 from leobago/develop + - Merge pull request #391 from leobago/O3 + - only gcc again + - Merge pull request #390 from leobago/hotfix_warnings + - Merge pull request #389 from leobago/h5_async_buffer + - activate all compilers (reverse after pipeline run) + - compiles everything with build type release (adds O3 flag) + - removed O1 + - removes compiler warnings + - free memory + - cleaned + - fixed vpr test + - itf checks vpr pass + - seems to work + - Merge pull request #388 from leobago/spack + - Merge remote-tracking branch 'origin/develop' into spack + - Merge pull request #387 from leobago/mpich_install + - pipeline only triggers PR-# branch checks and only triggers GCC for PR to develop. For PRs to master, the full pipeline is executed + - test pipeline + - test pipeline + - test pipeline + - test pipeline + - test pipeline + - test pipeline + - removed hdf5 check intel + - PGI without Fortran (GCC commented) + - PGI without Fortran + - escaped dollars + - added quotes to the cmd grep + - adjusted run wrapper + - tests intel mpich pgi and llvm + - prepared intel + - installed llvm compilers + - removes some artefacts and add a gitignore to include + - changes the install structure + - uncomment pgi stages + - installed pgi compilers + - fixes wrong type for blockMetaInfo_t + - added mpich + - quickfix + - source build script with a dot in shell env + - source build script with a dot in shell env + - source build script with a dot + - source build script + - uses another docker image + - added sionlib mpich + - installed hdf5 with mpich mpi + - export ld_library_path + - added mpich checks + - Merge pull request #386 from leobago/develop + - uncommented other checks + - corrected mount path + - update of PGI compilers to NVIDIA HPC SDK + - fixes Jenkinsfile for Clang and PGI CI + - Merge branch 'develop' of https://github.com/leobago/fti into develop + - Fix recover lL4 Warning, fast_forward bug and catch FTI_Init in heatDist + - Merge pull request #383 from leobago/bugfix/fix_osx_compile_error + - Update recover.c + - Renamed macros FREAD/FWRITE to FTI_EH_FREAD/FTI_EH_FWRITE/ + - Merge pull request #382 from leobago/refactor-ci + - fixed linking issue for cpp app + - edited APIs for c++ string operability + - added test suite for c++ string operability + - Merge pull request #381 from leobago/refactor-ci + - fixed erroneous pointer assignment + - enhanced check for implicit var declaration in loop+fixed files formatting + - Merge pull request #378 from leobago/refactor-ci + - split standard/diffsizes into two suites each + - updated test stats in documentation + - updated docu + set fast_forward for tests + - Merge pull request #377 from leobago/refactor-ci + - fixed faulty step + - fixing compilerName missing variable + - moved compilation inside steps + - fixing nested steps + - moved compilation step inside stages + - restructured steps/stages of Jenkins + - Added fast_forward to convenient itf tests + Version 1.5.1: - Merge branch 'master' of github.com:leobago/fti - Merge pull request #375 from leobago/develop diff --git a/CMakeLists.txt b/CMakeLists.txt index 9478f1bd9..78a1d4969 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,7 +32,7 @@ if(ENABLE_GPU) list(APPEND LANGUAGES "CUDA" "CXX") endif() -project("FTI" VERSION 1.5.1 LANGUAGES ${LANGUAGES}) +project("FTI" VERSION 1.6 LANGUAGES ${LANGUAGES}) # ----------------------- Global Properties and Set-up ------------------------ diff --git a/VERSION b/VERSION index 26ca59460..810ee4e91 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.5.1 +1.6 From f31a35d7655afb8e9a36deced2b366f7971a9ca6 Mon Sep 17 00:00:00 2001 From: Leo Date: Fri, 26 Feb 2021 10:57:25 +0100 Subject: [PATCH 02/11] Update AUTHORS --- AUTHORS | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/AUTHORS b/AUTHORS index fca3bfe6c..5131201d1 100644 --- a/AUTHORS +++ b/AUTHORS @@ -19,11 +19,11 @@ Contributors: Acknowledgements: -This work has been supported by EU H2020 ICT project LEGaTO, contract #780681, -the EU Horizon 2020 (H2020) funding framework under grant/award #824158; -Energy oriented Centre of Excellence II (EoCoE-II) and the EU H2020 +This work has been supported by EU H2020 ICT project LEGaTO contract #780681; +the EU Horizon 2020 (H2020) funding framework under grant/award #824158 +Energy oriented Centre of Excellence II (EoCoE-II); and the EU H2020 Mont-Blanc 2020 project under Grant Agreement #779877. -This scientific/academic work is financed from financial resources for science +This scientific/academic work is also financed from financial resources for science in the years 2016 - 2018 granted for the realization of the international project co-financed by Polish Ministry of Science and Higher Education. From 23504ccd2db7a8f1981add1163e63a1ece4b30d6 Mon Sep 17 00:00:00 2001 From: kellekai Date: Wed, 21 Jul 2021 09:31:32 +0200 Subject: [PATCH 03/11] changed types - NOT working for FTIFF level 3 --- examples/heatd2.c | 2 +- examples/heatdis.c | 1 - include/fti-intern.h | 64 +++--- include/fti.h | 4 +- src/IO/cuda-md5/md5Opt.c | 2 +- src/IO/ftiff-dcp.c | 34 +-- src/IO/ftiff-dcp.h | 14 +- src/IO/ftiff.c | 208 +++++++++--------- src/IO/ftiff.h | 14 +- src/IO/hdf5-fti.c | 28 +-- src/IO/hdf5-fti.h | 2 +- src/IO/posix-dcp.c | 68 +++--- src/IO/posix-dcp.h | 2 +- src/IO/posix.c | 6 +- src/api.c | 54 ++--- src/checkpoint.c | 14 +- src/meta.c | 64 +++--- src/meta.h | 8 +- src/postckpt.c | 45 ++-- src/postreco.c | 14 +- src/recover.c | 10 +- src/recover.h | 2 +- src/util/keymap.c | 8 +- src/util/keymap.h | 2 +- src/util/macros.h | 4 +- src/util/tools.c | 9 +- testing/suites/core/ckptDiffSizes/diffSizes.c | 5 +- testing/suites/core/multiLevelCkpt/check.c | 1 - .../features/cornerCases/ckptHierarchy.c | 2 +- .../suites/features/cornerCases/consistency.c | 10 +- .../features/differentialCkpt/checkDCPPosix.c | 3 +- .../features/differentialCkpt/diff_test.c | 5 +- .../features/differentialCkpt/diff_test.h | 7 +- .../differentialCkpt/diff_test_func.c | 10 +- testing/suites/features/getConfig/getConfig.c | 2 +- testing/suites/features/hdf5/hdf5Test.c | 1 - .../features/recoverName/checkRecoverName.c | 5 +- .../features/recoverVar/checkRecoverVar.c | 5 +- testing/suites/features/syncIntv/syncIntv.c | 2 - .../features/variateProcessorRestart/test.c | 6 +- 40 files changed, 383 insertions(+), 364 deletions(-) diff --git a/examples/heatd2.c b/examples/heatd2.c index e84dacaff..ef0419e1e 100644 --- a/examples/heatd2.c +++ b/examples/heatd2.c @@ -106,7 +106,7 @@ double doWork(int numprocs, int rank, int M, int nbLines, double *g, int main(int argc, char *argv[]) { - int rank, nbProcs, nbLines, i, j, N, M, res; + int rank, nbProcs, nbLines, i, N, M, res; double wtime, *h, *g, *grid, globalerror = 1; char fn[32]; diff --git a/examples/heatdis.c b/examples/heatdis.c index 46298a338..cf63ca3f6 100644 --- a/examples/heatdis.c +++ b/examples/heatdis.c @@ -119,7 +119,6 @@ int main(int argc, char *argv[]) { wtime = MPI_Wtime(); for (i = 0; i < ITER_TIMES; i++) { - int checkpointed = FTI_Snapshot(); localerror = doWork(nbProcs, rank, M, nbLines, g, h); if (((i%ITER_OUT) == 0) && (rank == 0)) { printf("Step : %d, error = %f\n", i, globalerror); diff --git a/include/fti-intern.h b/include/fti-intern.h index 1f9c6c371..8b16e4006 100644 --- a/include/fti-intern.h +++ b/include/fti-intern.h @@ -140,7 +140,7 @@ extern "C" { typedef struct FTIT_datasetInfo { int varID; - uint32_t varSize; + uint64_t varSize; } FTIT_datasetInfo; typedef struct FTIT_dcpConfigurationPosix { @@ -156,16 +156,16 @@ extern "C" { int nbLayerReco; int nbVarReco; unsigned int Counter; - uint32_t FileSize; - uint32_t dataSize; - uint32_t dcpSize; - uint32_t LayerSize[MAX_STACK_SIZE]; + uint64_t FileSize; + uint64_t dataSize; + uint64_t dcpSize; + uint64_t LayerSize[MAX_STACK_SIZE]; FTIT_datasetInfo datasetInfo[MAX_STACK_SIZE][FTI_BUFS]; char LayerHash[MAX_STACK_SIZE*MD5_DIGEST_STRING_LENGTH]; } FTIT_dcpExecutionPosix; typedef struct FTIT_dcpDatasetPosix { - uint32_t hashDataSize; + uint64_t hashDataSize; unsigned char* currentHashArray; unsigned char* oldHashArray; } FTIT_dcpDatasetPosix; @@ -229,15 +229,15 @@ extern "C" { char checksum[MD5_DIGEST_STRING_LENGTH]; /**< file hash without meta */ unsigned char myHash[MD5_DIGEST_LENGTH]; /**< hash of this struct */ int ckptId; /**< Checkpoint ID */ - size_t metaSize; /**< size of ckpt data */ - size_t ckptSize; /**< also file size TODO remove */ - size_t dataSize; /**< size of protected data without meta data */ - size_t pureDataSize; /**< total data size (for printouts) */ - size_t fs; /**< file size */ - size_t maxFs; /**< maximum file size in group */ - size_t ptFs; /**< partner copy file size */ - uint64_t timestamp; /**< time (ns) cp was created (CLOCK_REALTIME) */ - size_t dcpSize; /**< how much actually written by rank */ + uint64_t metaSize; /**< size of ckpt data */ + uint64_t ckptSize; /**< also file size TODO remove */ + uint64_t dataSize; /**< size of protected data without meta data */ + uint64_t pureDataSize; /**< total data size (for printouts) */ + uint64_t fs; /**< file size */ + uint64_t maxFs; /**< maximum file size in group */ + uint64_t ptFs; /**< partner copy file size */ + uint32_t timestamp; /**< time (ns) cp was created (CLOCK_REALTIME) */ + uint64_t dcpSize; /**< how much actually written by rank */ } FTIFF_metaInfo; /** @typedef FTIT_DataDiffHash @@ -253,7 +253,7 @@ extern "C" { uint32_t* bit32hash[2]; /**< CRC32 digest */ uint16_t* blockSize; /**< data block size */ bool* isValid; /**< indicates if data block is valid */ - int32_t nbHashes; /**< number of hashes for the data chunk */ + uint64_t nbHashes; /**< number of hashes for the data chunk */ int currentId; int creationType; int lifetime; @@ -275,8 +275,8 @@ extern "C" { bool hasCkpt; /**< indicates if container is stored in ckpt */ uintptr_t dptr; /**< data pointer offset */ uintptr_t fptr; /**< file pointer offset */ - int32_t chunksize; /**< chunk size of variable in this block */ - int32_t containersize;/**< cont size stored of variable in this block */ + uint64_t chunksize; /**< chunk size of variable in this block */ + uint64_t containersize;/**< cont size stored of variable in this block */ unsigned char hash[MD5_DIGEST_LENGTH]; /**< hash of variable chunk */ unsigned char myhash[MD5_DIGEST_LENGTH];/**< hash of this structure */ bool update; /**< TRUE if struct needs to be updated in ckpt file */ @@ -293,7 +293,7 @@ extern "C" { */ typedef struct FTIFF_db { int numvars; /**< number of protected variables in datablock */ - int32_t dbsize; /**< size of metadata + data for block in bytes */ + uint64_t dbsize; /**< size of metadata + data for block in bytes */ unsigned char myhash[MD5_DIGEST_LENGTH]; /**< hash of variable chunk */ bool update; /**< TRUE if struct needs to be updated in ckpt file */ bool finalized; /**< TRUE if block is stored in cp file */ @@ -366,7 +366,7 @@ extern "C" { */ typedef struct FTIT_Datatype { int id; /**< ID of the data type. */ - size_t size; /**< Size of the data type. */ + uint64_t size; /**< Size of the data type. */ FTIT_complexType* structure; /**< Logical structure for HDF5. */ FTIT_H5Group* h5group; /**< Group of this datatype. */ #ifdef ENABLE_HDF5 @@ -409,7 +409,7 @@ extern "C" { typedef struct FTIT_typeField { FTIT_Datatype *type; /**< FTI type ID of the field. */ int id; /**< Order of the field in the structure */ - size_t offset; /**< Offset of the field in structure. */ + uint64_t offset; /**< Offset of the field in structure. */ int rank; /**< Field rank (max. 32) */ int dimLength[32]; /**< Lenght of each dimention */ char name[FTI_BUFS]; /**< Name of the field */ @@ -444,15 +444,15 @@ extern "C" { */ typedef struct FTIT_dataset { int id; /**< ID to search/update dataset */ - int eleSize; /**< Element size for the dataset */ + size_t eleSize; /**< Element size for the dataset */ int rank; /**< Rank of dataset (for HDF5) */ int dimLength[32]; /**< Lenght of each dimention */ bool recovered; /**< True if metadata restored */ bool isDevicePtr; /**< True if on device memory */ - int32_t count; /**< nb of elements in dataset */ - int32_t size; /**< size of the data */ - int32_t sizeStored; /**< size of the data in last CP */ - size_t filePos; /**< offset of buffer in CP file */ + int64_t count; /**< nb of elements in dataset */ + int64_t size; /**< size of the data */ + int64_t sizeStored; /**< size of the data in last CP */ + uint64_t filePos; /**< offset of buffer in CP file */ FTIT_attribute attribute; FTIT_sharedData sharedData; /**< Info if dataset is subset */ FTIT_dcpDatasetPosix dcpInfoPosix; /**< dCP info for posix I/O */ @@ -473,9 +473,9 @@ extern "C" { int level; /**< checkpoint level */ int ckptId; /**< Current Ckpt ID */ int ckptIdL4; /**< Current L4 Ckpt ID */ - int32_t maxFs; /**< Maximum file size. */ - int32_t fs; /**< File size. */ - int32_t pfs; /**< Partner file size. */ + int64_t maxFs; /**< Maximum file size. */ + int64_t fs; /**< File size. */ + int64_t pfs; /**< Partner file size. */ char ckptFile[FTI_BUFS]; /**< Ckpt file name. [FTI_BUFS] */ } FTIT_metadata; @@ -522,7 +522,7 @@ extern "C" { char lTmpDir[FTI_BUFS]; /**< Local temporary directory. */ char gTmpDir[FTI_BUFS]; /**< Global temporary directory. */ char mTmpDir[FTI_BUFS]; /**< Metadata temporary directory. */ - size_t cHostBufSize; /**< Host buffer size for GPU data.*/ + uint64_t cHostBufSize; /**< Host buffer size for GPU data.*/ char suffix[4]; /** Suffix of the checkpoint files */ FTIT_dcpConfigurationPosix dcpInfoPosix; /**< dCP info for posix I/O */ // int fastForward; /**< Fast forward rate for ckpt intervals */ @@ -618,7 +618,7 @@ extern "C" { int(*WriteData) (FTIT_dataset * , void *write_info); int(*finCKPT) (void *fileDesc); - size_t(*getPos) (void *fileDesc); + uint64_t(*getPos) (void *fileDesc); void(*finIntegrity) (unsigned char *, void*); }FTIT_IO; @@ -678,7 +678,7 @@ extern "C" { unsigned int ckptId; /**< Checkpoint ID. */ unsigned int ckptNext; /**< Iteration for next CP. */ unsigned int ckptLast; /**< Iteration for last CP. */ - int32_t ckptSize; /**< Checkpoint size. */ + int64_t ckptSize; /**< Checkpoint size. */ unsigned int nbVar; /**< nb of protected variables */ unsigned int nbVarStored; /**< nb prot. var. stored in CP */ int nbGroup; /**< Number of protected groups. */ diff --git a/include/fti.h b/include/fti.h index 76050feb9..5fb58fac3 100644 --- a/include/fti.h +++ b/include/fti.h @@ -87,7 +87,7 @@ extern "C" { int FTI_InitGroup(FTIT_H5Group* h5group, const char* name, FTIT_H5Group* parent); int FTI_RenameGroup(FTIT_H5Group* h5group, const char* name); - int FTI_Protect(int id, void* ptr, int32_t count, fti_id_t tid); + int FTI_Protect(int id, void* ptr, uint64_t count, fti_id_t tid); int FTI_SetAttribute(int id, FTIT_attribute attribute, FTIT_attributeFlag flag); int FTI_DefineDataset(int id, int rank, int* dimLength, const char* name, @@ -102,7 +102,7 @@ extern "C" { int FTI_UpdateGlobalDataset(int id, int rank, FTIT_hsize_t* dimLength); int FTI_UpdateSubset(int id, int rank, FTIT_hsize_t* offset, FTIT_hsize_t* count, int did); - int32_t FTI_GetStoredSize(int id); + uint64_t FTI_GetStoredSize(int id); void* FTI_Realloc(int id, void* ptr); int FTI_BitFlip(int datasetID); int FTI_Checkpoint(int id, int level); diff --git a/src/IO/cuda-md5/md5Opt.c b/src/IO/cuda-md5/md5Opt.c index c5fd9ba6e..0deff8b3e 100644 --- a/src/IO/cuda-md5/md5Opt.c +++ b/src/IO/cuda-md5/md5Opt.c @@ -98,7 +98,7 @@ int FTI_initMD5(int32_t cSize, int32_t tempSize, FTIT_configuration *FTI_Conf) { **/ /*-------------------------------------------------------------------------*/ int MD5CPU(FTIT_dataset *data) { - uint32_t dataSize = data->size; + uint64_t dataSize = data->size; unsigned char block[md5ChunkSize]; size_t i; unsigned char *ptr = (unsigned char *) data->ptr; diff --git a/src/IO/ftiff-dcp.c b/src/IO/ftiff-dcp.c index 3b46ffc98..a64c5e96c 100644 --- a/src/IO/ftiff-dcp.c +++ b/src/IO/ftiff-dcp.c @@ -342,13 +342,13 @@ int FTI_GetDcpMode() { /** @brief Reallocate meta data related to dCP @param FTIT_DataDiffHash metadata to be reallocated. - @param int32_t number of hashes that i need to reallocate + @param uint64_t number of hashes that i need to reallocate @return integer FTI_SCES if successful. This function reallocates all the the metadata related to the dCP (isValid & blockSize); **/ /*-------------------------------------------------------------------------*/ -int FTI_ReallocateDataDiff(FTIT_DataDiffHash *dhash, int32_t nbHashes) { +int FTI_ReallocateDataDiff(FTIT_DataDiffHash *dhash, uint64_t nbHashes) { if (!dcpEnabled) return FTI_SCES; @@ -489,7 +489,7 @@ When the checkpoint will terminate the current hash will be freed, whereas the n will be used as current. Keep in mind that the next has the correct size **/ /*-------------------------------------------------------------------------*/ -int FTI_CollapseBlockHashArray(FTIT_DataDiffHash* hashes, int32_t chunkSize) { +int FTI_CollapseBlockHashArray(FTIT_DataDiffHash* hashes, uint64_t chunkSize) { if (!dcpEnabled) return FTI_SCES; @@ -498,8 +498,8 @@ int FTI_CollapseBlockHashArray(FTIT_DataDiffHash* hashes, int32_t chunkSize) { bool changeSize = true; - int32_t nbHashesOld = hashes->nbHashes; - int32_t newNumber = FTI_CalcNumHashes(chunkSize); + uint64_t nbHashesOld = hashes->nbHashes; + uint64_t newNumber = FTI_CalcNumHashes(chunkSize); // update to new number of hashes (which might be actually unchanged) hashes->nbHashes = newNumber; @@ -554,7 +554,7 @@ When the checkpoint will terminate the current hash will be freed, whereas the n will be used as current. Keep in mind that the next has the correct size **/ /*-------------------------------------------------------------------------*/ -int FTI_ExpandBlockHashArray(FTIT_DataDiffHash* dataHash, int32_t chunkSize) { +int FTI_ExpandBlockHashArray(FTIT_DataDiffHash* dataHash, uint64_t chunkSize) { if (!dcpEnabled) return FTI_SCES; @@ -562,10 +562,10 @@ int FTI_ExpandBlockHashArray(FTIT_DataDiffHash* dataHash, int32_t chunkSize) { return FTI_SCES; bool changeSize = true; - int32_t nbHashesOld = dataHash->nbHashes; + uint64_t nbHashesOld = dataHash->nbHashes; // - int32_t newNumber = FTI_CalcNumHashes(chunkSize); + uint64_t newNumber = FTI_CalcNumHashes(chunkSize); assert(nbHashesOld <= newNumber); if (newNumber == nbHashesOld) { @@ -609,13 +609,13 @@ int FTI_ExpandBlockHashArray(FTIT_DataDiffHash* dataHash, int32_t chunkSize) { /** @brief Computes number of hashblocks for chunk size. @param chunkSize chunk size of data chunk - @return int32_t FTI_SCES if successful. + @return int FTI_SCES if successful. This function computes the number of hash blocks according to the set dCP block size corresponding to chunkSize. **/ /*-------------------------------------------------------------------------*/ -int32_t FTI_CalcNumHashes(int32_t chunkSize) { +uint64_t FTI_CalcNumHashes(uint32_t chunkSize) { if ((chunkSize%((uint32_t)DCP_BLOCK_SIZE)) == 0) { return chunkSize/DCP_BLOCK_SIZE; } else { @@ -623,14 +623,14 @@ int32_t FTI_CalcNumHashes(int32_t chunkSize) { } } -void PrintDataHashInfo(FTIT_DataDiffHash* dataHash, int32_t chunkSize, int id) { +void PrintDataHashInfo(FTIT_DataDiffHash* dataHash, uint32_t chunkSize, int id) { char str[FTI_BUFS]; FTI_Print("+++++++++++++++ INFO IS +++++++++++++++", FTI_INFO); snprintf(str, sizeof(str), "I want to access index of the following id %d", id); FTI_Print(str, FTI_INFO); FTI_Print("+++++++++++++++ Data Hash INFO +++++++++++++++", FTI_INFO); - snprintf(str, sizeof(str), "Num Hashes are %d", dataHash->nbHashes); + snprintf(str, sizeof(str), "Num Hashes are %ld", dataHash->nbHashes); FTI_Print(str, FTI_INFO); snprintf(str, sizeof(str), "Type of hash is %d", dataHash->creationType); FTI_Print(str, FTI_INFO); @@ -658,7 +658,7 @@ void PrintDataHashInfo(FTIT_DataDiffHash* dataHash, int32_t chunkSize, int id) { dataHash->blockSize[dataHash->nbHashes-1]); FTI_Print(str, FTI_INFO); snprintf(str, sizeof(str), - "Total Block size is %d, Computed Block Size is %d", chunkSize, + "Total Block size is %d, Computed Block Size is %lu", chunkSize, (dataHash->nbHashes-1)*FTI_GetDiffBlockSize() + dataHash->blockSize[dataHash->nbHashes-1]); FTI_Print(str, FTI_INFO); @@ -681,7 +681,7 @@ void PrintDataHashInfo(FTIT_DataDiffHash* dataHash, int32_t chunkSize, int id) { It returns -1 if hashIdx is out of range. **/ /*-------------------------------------------------------------------------*/ -int FTI_HashCmp(int32_t hashIdx, FTIFF_dbvar* dbvar, unsigned char *ptr) { +int FTI_HashCmp(uint64_t hashIdx, FTIFF_dbvar* dbvar, unsigned char *ptr) { bool clean = true; uint32_t bit32hashNow = 0; unsigned char *prevHash = NULL; @@ -810,11 +810,11 @@ int FTI_UpdateDcpChanges(FTIT_execution* FTI_Exec) { dirty regions are found in which case 0 is returned. **/ /*-------------------------------------------------------------------------*/ -int FTI_ReceiveDataChunk(unsigned char** buffer_addr, size_t* buffer_size, - FTIFF_dbvar* dbvar, unsigned char *startAddr, size_t *totalBytes) { +int FTI_ReceiveDataChunk(unsigned char** buffer_addr, uint64_t* buffer_size, + FTIFF_dbvar* dbvar, unsigned char *startAddr, uint64_t *totalBytes) { static bool init = true; static bool reset; - static int32_t hashIdx; + static uint64_t hashIdx; unsigned char *ptr = startAddr; static int called = 0; diff --git a/src/IO/ftiff-dcp.h b/src/IO/ftiff-dcp.h index 71e1857c2..bed59cc08 100644 --- a/src/IO/ftiff-dcp.h +++ b/src/IO/ftiff-dcp.h @@ -16,14 +16,14 @@ int FTI_InitNextHashData(FTIT_DataDiffHash *hashes); int FTI_FreeDataDiff(FTIT_DataDiffHash *dhash); dcpBLK_t FTI_GetDiffBlockSize(); int FTI_GetDcpMode(); -int FTI_ReallocateDataDiff(FTIT_DataDiffHash *dhash, int32_t nbHashes); +int FTI_ReallocateDataDiff(FTIT_DataDiffHash *dhash, uint64_t nbHashes); int FTI_InitBlockHashArray(FTIFF_dbvar* dbvar); -int FTI_CollapseBlockHashArray(FTIT_DataDiffHash* hashes, int32_t chunkSize); -int FTI_ExpandBlockHashArray(FTIT_DataDiffHash* dataHash, int32_t chunkSize); -int32_t FTI_CalcNumHashes(int32_t chunkSize); -int FTI_HashCmp(int32_t hashIdx, FTIFF_dbvar* dbvar, unsigned char *ptr); +int FTI_CollapseBlockHashArray(FTIT_DataDiffHash* hashes, uint64_t chunkSize); +int FTI_ExpandBlockHashArray(FTIT_DataDiffHash* dataHash, uint64_t chunkSize); +uint64_t FTI_CalcNumHashes(uint32_t chunkSize); +int FTI_HashCmp(uint64_t hashIdx, FTIFF_dbvar* dbvar, unsigned char *ptr); int FTI_UpdateDcpChanges(FTIT_execution* FTI_Exec); -int FTI_ReceiveDataChunk(unsigned char** buffer_addr, size_t* buffer_size, - FTIFF_dbvar* dbvar, unsigned char *startAddr, size_t *totalBytes); +int FTI_ReceiveDataChunk(unsigned char** buffer_addr, uint64_t* buffer_size, + FTIFF_dbvar* dbvar, unsigned char *startAddr, uint64_t *totalBytes); #endif // FTI_SRC_IO_FTIFF_DCP_H_ diff --git a/src/IO/ftiff.c b/src/IO/ftiff.c index 7dc93c631..f1b27717e 100644 --- a/src/IO/ftiff.c +++ b/src/IO/ftiff.c @@ -107,7 +107,7 @@ int FTIFF_ReadDbFTIFF(FTIT_configuration *FTI_Conf, FTIT_execution *FTI_Exec, return FTI_NSCS; } - uint32_t fs = st.st_size; + uint64_t fs = st.st_size; // open checkpoint file for read only int fd = open(fn, O_RDONLY, 0); @@ -189,8 +189,8 @@ int FTIFF_ReadDbFTIFF(FTIT_configuration *FTI_Conf, FTIT_execution *FTI_Exec, // advance meta data offset seek_ptr += (FTI_ADDRVAL) FTI_dbstructsize; - snprintf(str, FTI_BUFS, "FTI-FF: Updatedb - dataBlock:%i, dbsize:" - " %d, numvars: %i.", dbcounter, currentdb->dbsize, + snprintf(str, FTI_BUFS, "FTI-FF: Updatedb - dataBlock:%d, dbsize:" + " %lu, numvars: %d.", dbcounter, currentdb->dbsize, currentdb->numvars); FTI_Print(str, FTI_DBUG); @@ -269,8 +269,8 @@ int FTIFF_ReadDbFTIFF(FTIT_configuration *FTI_Conf, FTIT_execution *FTI_Exec, // debug information snprintf(str, FTI_BUFS, "FTI-FF: Updatedb - dataBlock:%i/" - "dataBlockVar%i id: %i, destptr: %ld, fptr: %ld, " - "chunksize: %d.", dbcounter, dbvar_idx, + "dataBlockVar%i id: %i, destptr: %lu, fptr: %lu, " + "chunksize: %lu.", dbcounter, dbvar_idx, currentdbvar->id, currentdbvar->dptr, currentdbvar->fptr, currentdbvar->chunksize); FTI_Print(str, FTI_DBUG); @@ -517,7 +517,7 @@ int FTIFF_UpdateDatastructVarFTIFF(FTIT_execution* FTI_Exec, int editflags = 0; bool idFound = false; int isnextdb; - int32_t offset = 0; + uint64_t offset = 0; /* * - check if protected variable is in file info @@ -527,11 +527,11 @@ int FTIFF_UpdateDatastructVarFTIFF(FTIT_execution* FTI_Exec, FTI_Exec->lastdb = FTI_Exec->firstdb; int nbContainers = 0; - int32_t containerSizesAccu = 0; + uint64_t containerSizesAccu = 0; // init overflow with the datasizes and validBlock with true. bool validBlock = true; - int32_t overflow = data->size; + uint64_t overflow = data->size; // iterate though datablock list. Current datablock is 'lastdb'. // At the beginning of the loop 'lastdb = firstdb' @@ -568,7 +568,7 @@ int FTIFF_UpdateDatastructVarFTIFF(FTIT_execution* FTI_Exec, // set chunksize to containersize and ensure that // 'hascontent = true'. if (overflow > dbvar->containersize) { - int32_t chunksizeOld = dbvar->chunksize; + uint64_t chunksizeOld = dbvar->chunksize; dbvar->chunksize = dbvar->containersize; dbvar->cptr = data->ptr + dbvar->dptr; if (!dbvar->hascontent) { @@ -598,7 +598,7 @@ int FTIFF_UpdateDatastructVarFTIFF(FTIT_execution* FTI_Exec, // afterwards overflow to 0 and // ensure that 'hascontent = true'. if (overflow <= dbvar->containersize) { - int32_t chunksizeOld = dbvar->chunksize; + uint64_t chunksizeOld = dbvar->chunksize; dbvar->chunksize = overflow; dbvar->cptr = data->ptr + dbvar->dptr; if (!dbvar->hascontent) { @@ -674,7 +674,7 @@ int FTIFF_UpdateDatastructVarFTIFF(FTIT_execution* FTI_Exec, } int evar_idx = dblock->numvars; - int32_t dbsize = dblock->dbsize; + uint64_t dbsize = dblock->dbsize; switch (editflags) { case 1: // add new protected variable in next datablock @@ -760,20 +760,20 @@ int FTIFF_UpdateDatastructVarFTIFF(FTIT_execution* FTI_Exec, **/ /*-------------------------------------------------------------------------*/ int FTI_WriteMemFTIFFChunk(FTIT_execution *FTI_Exec, FTIFF_dbvar *currentdbvar, - unsigned char *dptr, size_t currentOffset, size_t fetchedBytes, - int32_t *dcpSize, WriteFTIFFInfo_t *fd) { + unsigned char *dptr, uint64_t currentOffset, uint64_t fetchedBytes, + uint64_t *dcpSize, WriteFTIFFInfo_t *fd) { unsigned char *chunk_addr = NULL; - size_t chunk_size, chunk_offset; - size_t remainingBytes = fetchedBytes; + uint64_t chunk_size, chunk_offset; + uint64_t remainingBytes = fetchedBytes; chunk_size = 0; chunk_offset = 0; - int32_t membs = 1024*1024*16; // 16 MB - int32_t cpybuf, cpynow, cpycnt; //, fptr; + uint64_t membs = 1024*1024*16; // 16 MB + uint64_t cpybuf, cpynow, cpycnt; //, fptr; uintptr_t fptr = currentdbvar-> fptr + currentOffset; uintptr_t fptrTemp = fptr; - size_t prevRemBytes = remainingBytes; + uint64_t prevRemBytes = remainingBytes; while (FTI_ReceiveDataChunk(&chunk_addr, &chunk_size, currentdbvar, dptr, &remainingBytes)) { @@ -820,13 +820,13 @@ int FTI_WriteMemFTIFFChunk(FTIT_execution *FTI_Exec, FTIFF_dbvar *currentdbvar, /*-------------------------------------------------------------------------*/ int FTI_ProcessDBVar(FTIT_execution *FTI_Exec, FTIT_configuration *FTI_Conf, FTIFF_dbvar *currentdbvar, FTIT_dataset *data, unsigned char *hashchk, - WriteFTIFFInfo_t *fd, int32_t *dcpSize, unsigned char **dptr) { + WriteFTIFFInfo_t *fd, uint64_t *dcpSize, unsigned char **dptr) { bool hascontent = currentdbvar->hascontent; unsigned char *cbasePtr = NULL; errno = 0; - size_t totalBytes; + uint64_t totalBytes; MD5_CTX dbContext; if (hascontent) { @@ -834,7 +834,7 @@ int FTI_ProcessDBVar(FTIT_execution *FTI_Exec, FTIT_configuration *FTI_Conf, FTI_InitNextHashData(currentdbvar->dataDiffHash); - size_t offset = 0; + uint64_t offset = 0; totalBytes = 0; FTIT_data_prefetch prefetcher; MD5_Init(&dbContext); @@ -843,7 +843,7 @@ int FTI_ProcessDBVar(FTIT_execution *FTI_Exec, FTIT_configuration *FTI_Conf, // I nitialize prefetcher to get data from device #ifdef GPUSUPPORT - size_t DCP_BLOCK_SIZE = FTI_GetDiffBlockSize(); + uint64_t DCP_BLOCK_SIZE = FTI_GetDiffBlockSize(); prefetcher.fetchSize = ((FTI_Conf->cHostBufSize) / DCP_BLOCK_SIZE) * DCP_BLOCK_SIZE; #else @@ -1024,9 +1024,9 @@ int FTI_WriteFtiffData(FTIT_dataset* data, void *fd) { unsigned char *dptr; int dbvar_idx, dbcounter = 0; int isnextdb; - int32_t dcpSize = 0; - int32_t dataSize = 0; - int32_t pureDataSize = 0; + uint64_t dcpSize = 0; + uint64_t dataSize = 0; + uint64_t pureDataSize = 0; FTIFF_UpdateDatastructVarFTIFF(write_info->FTI_Exec, data, write_info->FTI_Conf); @@ -1133,7 +1133,7 @@ int FTI_FinalizeFtiff(void *fd) { @return integer 0. **/ /*-------------------------------------------------------------------------*/ -size_t FTI_DummyFilePos(void *ignore) { +uint64_t FTI_DummyFilePos(void *ignore) { return 0; } @@ -1201,7 +1201,11 @@ int FTIFF_finalizeDatastructFTIFF(FTIT_execution* FTI_Exec) { return FTI_NSCS; } - uint32_t metaSize = FTI_filemetastructsize; + uint64_t metaSize = FTI_filemetastructsize; + + DBG_MSG("buffersize (FTI_filemetastructsize): %lu", 0, FTI_filemetastructsize); + DBG_MSG("buffersize (FTI_dbstructsize): %lu", 0, FTI_dbstructsize); + DBG_MSG("buffersize (FTI_dbvarstructsize): %lu", 0, db->numvars * FTI_dbvarstructsize); do { db->finalized = true; @@ -1241,6 +1245,8 @@ int FTIFF_writeMetaDataFTIFF(FTIT_execution* FTI_Exec, WriteFTIFFInfo_t *fd) { } FTI_ADDRVAL mbuf_pos = (FTI_ADDRVAL) mbuf; + + DBG_MSG("FTI_Exec->FTIFFMeta.metaSize: %lu",-1,FTI_Exec->FTIFFMeta.metaSize); MD5_CTX ctx; MD5_Init(&ctx); @@ -1308,14 +1314,14 @@ int FTIFF_CreateMetadata(FTIT_execution* FTI_Exec, FTIT_topology* FTI_Topo, FTI_Exec->ckptSize = FTI_Exec->FTIFFMeta.metaSize + FTI_Exec->FTIFFMeta.dataSize; - int32_t fs = FTI_Exec->ckptSize; + uint64_t fs = FTI_Exec->ckptSize; FTI_Exec->FTIFFMeta.ckptSize = fs; FTI_Exec->FTIFFMeta.fs = fs; // allgather not needed for L1 checkpoint if ((FTI_Exec->ckptMeta.level == 2) || (FTI_Exec->ckptMeta.level == 3)) { - int32_t fileSizes[FTI_BUFS], mfs = 0; - MPI_Allgather(&fs, 1, MPI_INT32_T, fileSizes, 1, MPI_INT32_T, + uint64_t fileSizes[FTI_BUFS], mfs = 0; + MPI_Allgather(&fs, 1, MPI_UINT64_T, fileSizes, 1, MPI_UINT64_T, FTI_Exec->groupComm); int ptnerGroupRank, i; switch (FTI_Exec->ckptMeta.level) { @@ -1415,8 +1421,8 @@ int FTIFF_Recover(FTIT_execution *FTI_Exec, FTIT_keymap *FTI_Data, int i = 0; for (; i < FTI_Exec->nbVar; i++) { if (data[i].size != data[i].sizeStored) { - snprintf(str, FTI_BUFS, "Cannot recover %d bytes to protected" - " variable (ID %d) size: %d", + snprintf(str, FTI_BUFS, "Cannot recover %lu bytes to protected" + " variable (ID %d) size: %lu", data[i].sizeStored, data[i].id, data[i].size); FTI_Print(str, FTI_WARN); @@ -1452,8 +1458,8 @@ int FTIFF_Recover(FTIT_execution *FTI_Exec, FTIT_keymap *FTI_Data, } // block size for memcpy of pointer. - int32_t membs = 1024*1024*16; // 16 MB - int32_t cpybuf, cpynow, cpycnt; + uint64_t membs = 1024*1024*16; // 16 MB + uint64_t cpybuf, cpynow, cpycnt; // open checkpoint file for read only int fd = open(fn, O_RDONLY, 0); @@ -1549,7 +1555,7 @@ int FTIFF_Recover(FTIT_execution *FTI_Exec, FTIT_keymap *FTI_Data, // debug information snprintf(str, FTI_BUFS, "FTI-FF: FTIFF_Recover - " "dataBlock:%i/dataBlockVar%i id: %i" - ", destptr: %ld, fptr: %ld, chunksize: %d, " + ", destptr: %lu, fptr: %lu, chunksize: %lu, " "base_ptr: 0x%" PRIxPTR " ptr_pos: 0x%" PRIxPTR ".", dbcounter, dbvar_idx, currentdbvar->id, currentdbvar->dptr, @@ -1667,8 +1673,8 @@ int FTIFF_RecoverVar(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, int dbvar_idx, dbcounter = 0; // block size for memcpy of pointer. - int32_t membs = 1024*1024*16; // 16 MB - int32_t cpybuf, cpynow, cpycnt; + uint64_t membs = 1024*1024*16; // 16 MB + uint64_t cpybuf, cpynow, cpycnt; // MD5 context for checksum of data chunks MD5_CTX mdContext; @@ -1698,8 +1704,8 @@ int FTIFF_RecoverVar(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, return FTI_NSCS; } if (data->size != data->sizeStored) { - snprintf(str, sizeof(str), "Cannot recover %d bytes to " - "protected variable (ID %d) size: %d", + snprintf(str, sizeof(str), "Cannot recover %lu bytes to " + "protected variable (ID %d) size: %lu", data->sizeStored, data->id, data->size); FTI_Print(str, FTI_WARN); return FTI_NREC; @@ -1723,7 +1729,7 @@ int FTIFF_RecoverVar(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, // debug information snprintf(str, FTI_BUFS, "FTIFF: FTIFF_RecoverVar -" " dataBlock:%i/dataBlockVar%i id: %i" - ", destptr: %ld, fptr: %ld, chunksize: %d, " + ", destptr: %lu, fptr: %lu, chunksize: %lu, " "base_ptr: 0x%" PRIxPTR " ptr_pos: 0x%" PRIxPTR ".", dbcounter, dbvar_idx, currentdbvar->id, currentdbvar->dptr, @@ -1965,7 +1971,7 @@ int FTIFF_LoadMetaPostprocessing(FTIT_execution* FTI_Exec, /*-------------------------------------------------------------------------*/ int FTIFF_GetEncodedFileChecksum(FTIFF_metaInfo *FTIFFMeta, int fd, char *checksum) { - int32_t rcount = 0, toRead, diff; + uint64_t rcount = 0, toRead, diff; int rbuffer; char buffer[CHUNK_SIZE], strerr[FTI_BUFS]; MD5_CTX mdContext; @@ -1983,7 +1989,7 @@ int FTIFF_GetEncodedFileChecksum(FTIFF_metaInfo *FTIFFMeta, int fd, rbuffer = read(fd, buffer, toRead); if (rbuffer == -1) { snprintf(strerr, FTI_BUFS, "FTI-FF: L3RecoveryInit - Failed to" - " read %d bytes from file", toRead); + " read %lu bytes from file", toRead); FTI_Print(strerr, FTI_EROR); errno = 0; return FTI_NSCS; @@ -2236,7 +2242,7 @@ int FTIFF_CheckL2RecoverInit(FTIT_execution* FTI_Exec, FTIT_topology* FTI_Topo, } } snprintf(dbgstr, FTI_BUFS, "FTI-FF: L2-Recovery - rank: %i, left: %i," - " right: %i, fs: %d, pfs: %d, ckptId: %i", + " right: %i, fs: %lu, pfs: %lu, ckptId: %i", FTI_Topo->myRank, leftIdx, rightIdx, FTI_Exec->ckptMeta.fs, FTI_Exec->ckptMeta.pfs, FTI_Exec->ckptId); FTI_Print(dbgstr, FTI_DBUG); @@ -2290,7 +2296,7 @@ int FTIFF_CheckL3RecoverInit(FTIT_execution* FTI_Exec, FTIT_topology* FTI_Topo, // check if recovery possible int i, saneCkptID = 0, saneMaxFs = 0, erasures = 0; - int32_t maxFs = 0; + uint64_t maxFs = 0; ckptId = 0; for (i = 0; i < FTI_Topo->groupSize; i++) { erased[i]=!groupInfo[i].FileExists; @@ -2313,7 +2319,7 @@ int FTIFF_CheckL3RecoverInit(FTIT_execution* FTI_Exec, FTIT_topology* FTI_Topo, } // for the case that all (and only) the encoded files are deleted if (saneMaxFs == 0 && !(erasures > FTI_Topo->groupSize)) { - MPI_Allreduce(&(info.maxFs), &FTI_Exec->ckptMeta.maxFs, 1, MPI_INT32_T, + MPI_Allreduce(&(info.maxFs), &FTI_Exec->ckptMeta.maxFs, 1, MPI_UINT64_T, MPI_SUM, FTI_Exec->groupComm); FTI_Exec->ckptMeta.maxFs /= FTI_Topo->groupSize; } @@ -2390,7 +2396,7 @@ void FTIFF_SetHashChunk(FTIFF_dbvar *dbvar, FTIT_keymap* FTI_Data) { } void * ptr = data->ptr + dbvar->dptr; - uint32_t size = dbvar->chunksize; + uint64_t size = dbvar->chunksize; MD5(ptr, size, dbvar->hash); } } @@ -2406,13 +2412,13 @@ void FTIFF_GetHashMetaInfo(unsigned char *hash, FTIFF_metaInfo *FTIFFMeta) { MD5_CTX md5Ctx; MD5_Init(&md5Ctx); MD5_Update(&md5Ctx, FTIFFMeta->checksum, MD5_DIGEST_STRING_LENGTH); - MD5_Update(&md5Ctx, &(FTIFFMeta->timestamp), sizeof(int32_t)); - MD5_Update(&md5Ctx, &(FTIFFMeta->ckptSize), sizeof(int32_t)); - MD5_Update(&md5Ctx, &(FTIFFMeta->metaSize), sizeof(int32_t)); - MD5_Update(&md5Ctx, &(FTIFFMeta->dataSize), sizeof(int32_t)); - MD5_Update(&md5Ctx, &(FTIFFMeta->fs), sizeof(int32_t)); - MD5_Update(&md5Ctx, &(FTIFFMeta->ptFs), sizeof(int32_t)); - MD5_Update(&md5Ctx, &(FTIFFMeta->maxFs), sizeof(int32_t)); + MD5_Update(&md5Ctx, &(FTIFFMeta->timestamp), sizeof(uint32_t)); + MD5_Update(&md5Ctx, &(FTIFFMeta->ckptSize), sizeof(uint64_t)); + MD5_Update(&md5Ctx, &(FTIFFMeta->metaSize), sizeof(uint64_t)); + MD5_Update(&md5Ctx, &(FTIFFMeta->dataSize), sizeof(uint64_t)); + MD5_Update(&md5Ctx, &(FTIFFMeta->fs), sizeof(uint64_t)); + MD5_Update(&md5Ctx, &(FTIFFMeta->ptFs), sizeof(uint64_t)); + MD5_Update(&md5Ctx, &(FTIFFMeta->maxFs), sizeof(uint64_t)); MD5_Final(hash, &md5Ctx); } @@ -2427,7 +2433,7 @@ void FTIFF_GetHashdb(unsigned char *hash, FTIFF_db *db) { MD5_CTX md5Ctx; MD5_Init(&md5Ctx); MD5_Update(&md5Ctx, &(db->numvars), sizeof(int)); - MD5_Update(&md5Ctx, &(db->dbsize), sizeof(int32_t)); + MD5_Update(&md5Ctx, &(db->dbsize), sizeof(uint64_t)); MD5_Final(hash, &md5Ctx); } @@ -2447,8 +2453,8 @@ void FTIFF_GetHashdbvar(unsigned char *hash, FTIFF_dbvar *dbvar) { MD5_Update(&md5Ctx, &(dbvar->hasCkpt), sizeof(bool)); MD5_Update(&md5Ctx, &(dbvar->dptr), sizeof(uintptr_t)); MD5_Update(&md5Ctx, &(dbvar->fptr), sizeof(uintptr_t)); - MD5_Update(&md5Ctx, &(dbvar->chunksize), sizeof(int32_t)); - MD5_Update(&md5Ctx, &(dbvar->containersize), sizeof(int32_t)); + MD5_Update(&md5Ctx, &(dbvar->chunksize), sizeof(uint64_t)); + MD5_Update(&md5Ctx, &(dbvar->containersize), sizeof(uint64_t)); MD5_Update(&md5Ctx, dbvar->hash, MD5_DIGEST_LENGTH); MD5_Final(hash, &md5Ctx); } @@ -2468,7 +2474,7 @@ void FTIFF_InitMpiTypes() { MBR_CNT(headInfo) = 7; MBR_BLK_LEN(headInfo) = { 1, 1, FTI_BUFS, 1, 1, 1, 1 }; MBR_TYPES(headInfo) = { MPI_INT, MPI_INT, MPI_CHAR, - MPI_INT32_T, MPI_INT32_T, MPI_INT32_T, MPI_INT }; + MPI_UINT64_T, MPI_UINT64_T, MPI_UINT64_T, MPI_INT }; MBR_DISP(headInfo) = { offsetof(FTIFF_headInfo, exists), offsetof(FTIFF_headInfo, nbVar), @@ -2487,7 +2493,7 @@ void FTIFF_InitMpiTypes() { MBR_CNT(RecoInfo) = 6; MBR_BLK_LEN(RecoInfo) = { 1, 1, 1, 1, 1, 1 }; MBR_TYPES(RecoInfo) = { MPI_INT, MPI_INT, MPI_INT, - MPI_INT, MPI_INT32_T, MPI_INT32_T }; + MPI_INT, MPI_INT64_T, MPI_INT64_T }; MBR_DISP(RecoInfo) = { offsetof(FTIFF_RecoveryInfo, FileExists), offsetof(FTIFF_RecoveryInfo, BackupExists), @@ -2543,19 +2549,19 @@ int FTIFF_DeserializeFileMeta(FTIFF_metaInfo* meta, char* buffer_ser) { pos += MD5_DIGEST_LENGTH; memcpy(&(meta->ckptId) , buffer_ser + pos, sizeof(int)); pos += sizeof(int); - memcpy(&(meta->ckptSize) , buffer_ser + pos, sizeof(int32_t)); - pos += sizeof(int32_t); - memcpy(&(meta->metaSize) , buffer_ser + pos, sizeof(int32_t)); - pos += sizeof(int32_t); - memcpy(&(meta->dataSize) , buffer_ser + pos, sizeof(int32_t)); - pos += sizeof(int32_t); - memcpy(&(meta->fs) , buffer_ser + pos, sizeof(int32_t)); - pos += sizeof(int32_t); - memcpy(&(meta->maxFs) , buffer_ser + pos, sizeof(int32_t)); - pos += sizeof(int32_t); - memcpy(&(meta->ptFs) , buffer_ser + pos, sizeof(int32_t)); - pos += sizeof(int32_t); - memcpy(&(meta->timestamp) , buffer_ser + pos, sizeof(int32_t)); + memcpy(&(meta->ckptSize) , buffer_ser + pos, sizeof(uint64_t)); + pos += sizeof(uint64_t); + memcpy(&(meta->metaSize) , buffer_ser + pos, sizeof(uint64_t)); + pos += sizeof(uint64_t); + memcpy(&(meta->dataSize) , buffer_ser + pos, sizeof(uint64_t)); + pos += sizeof(uint64_t); + memcpy(&(meta->fs) , buffer_ser + pos, sizeof(uint64_t)); + pos += sizeof(uint64_t); + memcpy(&(meta->maxFs) , buffer_ser + pos, sizeof(uint64_t)); + pos += sizeof(uint64_t); + memcpy(&(meta->ptFs) , buffer_ser + pos, sizeof(uint64_t)); + pos += sizeof(uint64_t); + memcpy(&(meta->timestamp) , buffer_ser + pos, sizeof(uint32_t)); return FTI_SCES; } @@ -2576,7 +2582,7 @@ int FTIFF_DeserializeDbMeta(FTIFF_db* db, char* buffer_ser) { int pos = 0; memcpy(&(db->numvars) , buffer_ser + pos, sizeof(int)); pos += sizeof(int); - memcpy(&(db->dbsize) , buffer_ser + pos, sizeof(int32_t)); + memcpy(&(db->dbsize) , buffer_ser + pos, sizeof(uint64_t)); return FTI_SCES; } @@ -2607,10 +2613,10 @@ int FTIFF_DeserializeDbVarMeta(FTIFF_dbvar* dbvar, char* buffer_ser) { pos += sizeof(uintptr_t); memcpy(&(dbvar->fptr) , buffer_ser + pos, sizeof(uintptr_t)); pos += sizeof(uintptr_t); - memcpy(&(dbvar->chunksize) , buffer_ser + pos, sizeof(int32_t)); - pos += sizeof(int32_t); - memcpy(&(dbvar->containersize) , buffer_ser + pos, sizeof(int32_t)); - pos += sizeof(int32_t); + memcpy(&(dbvar->chunksize) , buffer_ser + pos, sizeof(uint64_t)); + pos += sizeof(uint64_t); + memcpy(&(dbvar->containersize) , buffer_ser + pos, sizeof(uint64_t)); + pos += sizeof(uint64_t); memcpy(dbvar->hash , buffer_ser + pos, MD5_DIGEST_LENGTH); return FTI_SCES; @@ -2636,19 +2642,21 @@ int FTIFF_SerializeFileMeta(FTIFF_metaInfo* meta, char* buffer_ser) { pos += MD5_DIGEST_LENGTH; memcpy(buffer_ser + pos, &(meta->ckptId) , sizeof(int)); pos += sizeof(int); - memcpy(buffer_ser + pos, &(meta->ckptSize) , sizeof(int32_t)); - pos += sizeof(int32_t); - memcpy(buffer_ser + pos, &(meta->metaSize) , sizeof(int32_t)); - pos += sizeof(int32_t); - memcpy(buffer_ser + pos, &(meta->dataSize) , sizeof(int32_t)); - pos += sizeof(int32_t); - memcpy(buffer_ser + pos, &(meta->fs) , sizeof(int32_t)); - pos += sizeof(int32_t); - memcpy(buffer_ser + pos, &(meta->maxFs) , sizeof(int32_t)); - pos += sizeof(int32_t); - memcpy(buffer_ser + pos, &(meta->ptFs) , sizeof(int32_t)); - pos += sizeof(int32_t); - memcpy(buffer_ser + pos, &(meta->timestamp) , sizeof(int32_t)); + memcpy(buffer_ser + pos, &(meta->ckptSize) , sizeof(uint64_t)); + pos += sizeof(uint64_t); + memcpy(buffer_ser + pos, &(meta->metaSize) , sizeof(uint64_t)); + pos += sizeof(uint64_t); + memcpy(buffer_ser + pos, &(meta->dataSize) , sizeof(uint64_t)); + pos += sizeof(uint64_t); + memcpy(buffer_ser + pos, &(meta->fs) , sizeof(uint64_t)); + pos += sizeof(uint64_t); + memcpy(buffer_ser + pos, &(meta->maxFs) , sizeof(uint64_t)); + pos += sizeof(uint64_t); + memcpy(buffer_ser + pos, &(meta->ptFs) , sizeof(uint64_t)); + pos += sizeof(uint64_t); + memcpy(buffer_ser + pos, &(meta->timestamp) , sizeof(uint32_t)); + + DBG_MSG("metasize (FTIFF_SerializeFileMeta): %d", 0, pos + sizeof(uint32_t)); return FTI_SCES; } @@ -2669,7 +2677,9 @@ int FTIFF_SerializeDbMeta(FTIFF_db* db, char* buffer_ser) { int pos = 0; memcpy(buffer_ser + pos, &(db->numvars) , sizeof(int)); pos += sizeof(int); - memcpy(buffer_ser + pos, &(db->dbsize) , sizeof(int32_t)); + memcpy(buffer_ser + pos, &(db->dbsize) , sizeof(uint64_t)); + + DBG_MSG("metasize (FTIFF_SerializeDbMeta): %d", 0, pos + sizeof(uint64_t)); return FTI_SCES; } @@ -2700,11 +2710,13 @@ int FTIFF_SerializeDbVarMeta(FTIFF_dbvar* dbvar, char* buffer_ser) { pos += sizeof(uintptr_t); memcpy(buffer_ser + pos, &(dbvar->fptr) , sizeof(uintptr_t)); pos += sizeof(uintptr_t); - memcpy(buffer_ser + pos, &(dbvar->chunksize) , sizeof(int32_t)); - pos += sizeof(int32_t); - memcpy(buffer_ser + pos, &(dbvar->containersize) , sizeof(int32_t)); - pos += sizeof(int32_t); + memcpy(buffer_ser + pos, &(dbvar->chunksize) , sizeof(uint64_t)); + pos += sizeof(uint64_t); + memcpy(buffer_ser + pos, &(dbvar->containersize) , sizeof(uint64_t)); + pos += sizeof(uint64_t); memcpy(buffer_ser + pos, dbvar->hash , MD5_DIGEST_LENGTH); + + DBG_MSG("metasize (FTIFF_SerializeDbVarMeta): %d", 0, pos + MD5_DIGEST_LENGTH); return FTI_SCES; } @@ -2740,8 +2752,8 @@ void FTIFF_PrintDataStructure(int rank, FTIT_execution* FTI_Exec) { " [%d]----------------\n\n", rank); do { printf(" DataBase-id: %d\n", dbcnt); - printf(" dbsize: %d\n", dbgdb->dbsize); - printf(" metasize (offset: %d): %d\n\n", + printf(" dbsize: %lu\n", dbgdb->dbsize); + printf(" metasize (offset: %u): %u\n\n", FTI_filemetastructsize, FTI_dbstructsize+dbgdb->numvars*FTI_dbvarstructsize); dbcnt++; @@ -2754,8 +2766,8 @@ void FTIFF_PrintDataStructure(int rank, FTIT_execution* FTI_Exec) { " hasCkpt: %s\n" " dptr: %lu\n" " fptr: %lu\n" - " chunksize: %u\n" - " containersize: %u\n\n", + " chunksize: %lu\n" + " containersize: %lu\n\n", /* " nbHashes: %lu\n" " diffBlockSize: %d\n" diff --git a/src/IO/ftiff.h b/src/IO/ftiff.h index 74d7e150a..9fea2849d 100644 --- a/src/IO/ftiff.h +++ b/src/IO/ftiff.h @@ -96,9 +96,9 @@ typedef struct FTIFF_headInfo { int exists; int nbVar; char ckptFile[FTI_BUFS]; - int32_t maxFs; - int32_t fs; - int32_t pfs; + uint64_t maxFs; + uint64_t fs; + uint64_t pfs; int isDcp; } FTIFF_headInfo; @@ -113,9 +113,9 @@ typedef struct FTIFF_RecoveryInfo { int BackupExists; int ckptId; int rightIdx; - int32_t maxFs; - int32_t fs; - int32_t bfs; + uint64_t maxFs; + uint64_t fs; + uint64_t bfs; } FTIFF_RecoveryInfo; /** @@ -206,7 +206,7 @@ void FTIFF_SetHashChunk(FTIFF_dbvar *dbvar, FTIT_keymap* FTI_Data); void FTIFF_PrintDataStructure(int rank, FTIT_execution* FTI_Exec); int FTI_ProcessDBVar(FTIT_execution *FTI_Exec, FTIT_configuration *FTI_Conf, FTIFF_dbvar *currentdbvar, FTIT_dataset *data, unsigned char *hashchk, - WriteFTIFFInfo_t *fd, int32_t *dcpSize, unsigned char **dptr); + WriteFTIFFInfo_t *fd, uint64_t *dcpSize, unsigned char **dptr); int FTIFF_RecoverVarInit(char* fn); int FTIFF_RecoverVarFinalize(); #endif // FTI_SRC_IO_FTIFF_H_ diff --git a/src/IO/hdf5-fti.c b/src/IO/hdf5-fti.c index 250c36662..5d55a16cc 100644 --- a/src/IO/hdf5-fti.c +++ b/src/IO/hdf5-fti.c @@ -317,16 +317,16 @@ int FTI_ScanGroup(hid_t gid, char* fn) { char errstr[FTI_BUFS]; hsize_t nobj; if (H5Gget_num_objs(gid, &nobj) >= 0) { - int i; + hsize_t i; for (i = 0; i < nobj; i++) { int objtype; char dname[FTI_BUFS]; char gname[FTI_BUFS]; // determine if element is group or dataset - objtype = H5Gget_objtype_by_idx(gid, (size_t)i); + objtype = H5Gget_objtype_by_idx(gid, i); if (objtype == H5G_DATASET) { - H5Gget_objname_by_idx(gid, (hsize_t)i, dname, - (size_t)FTI_BUFS); + H5Gget_objname_by_idx(gid, i, dname, + (hsize_t)FTI_BUFS); // open dataset hid_t did = H5Dopen1(gid, dname); if (did > 0) { @@ -366,8 +366,8 @@ int FTI_ScanGroup(hid_t gid, char* fn) { } // step down other group if (objtype == H5G_GROUP) { - H5Gget_objname_by_idx(gid, (hsize_t)i, gname, - (size_t) FTI_BUFS); + H5Gget_objname_by_idx(gid, i, gname, + (hsize_t) FTI_BUFS); hid_t sgid = H5Gopen1(gid, gname); if (sgid > 0) { res += FTI_ScanGroup(sgid, fn); @@ -396,7 +396,7 @@ int FTI_ScanGroup(hid_t gid, char* fn) { @return integer The position in the file. **/ /*-------------------------------------------------------------------------*/ -size_t FTI_GetHDF5FilePos(void *fileDesc) { +uint64_t FTI_GetHDF5FilePos(void *fileDesc) { return 0; } @@ -492,7 +492,7 @@ int FTI_CommitDataType(FTIT_execution *FTI_Exec, FTIT_dataset *data) { **/ /*-------------------------------------------------------------------------*/ -int FTI_CheckHDF5File(char* fn, int32_t fs, char* checksum) { +int FTI_CheckHDF5File(char* fn, uint64_t fs, char* checksum) { char str[FTI_BUFS]; if (access(fn, F_OK) == 0) { struct stat fileStatus; @@ -544,7 +544,7 @@ hsize_t FTI_calculateCountDim(size_t sizeOfElement, hsize_t maxBytes, hsize_t *count, int numOfDimensions, hsize_t *dimensions, hsize_t *sep) { int i; memset(count, 0, sizeof(hsize_t)*numOfDimensions); - size_t maxElements = maxBytes/sizeOfElement; + uint64_t maxElements = maxBytes/sizeOfElement; hsize_t bytesToFetch; if (maxElements == 0) @@ -842,7 +842,7 @@ int FTI_WriteHDF5Var(FTIT_dataset *data, FTIT_execution* FTI_Exec) { prefetcher.totalBytesToFetch = data->size; prefetcher.isDevice = data->isDevicePtr; prefetcher.dptr = data->devicePtr; - size_t bytesToWrite; + uint64_t bytesToWrite; FTI_InitPrefetcher(&prefetcher); unsigned char *basePtr = NULL; @@ -969,8 +969,8 @@ int FTI_ReadHDF5Var(FTIT_dataset *data) { hsize_t seperator; - size_t fetchBytes; - size_t hostBufSize = FTI_getHostBuffSize(); + uint64_t fetchBytes; + uint64_t hostBufSize = FTI_getHostBuffSize(); // Calculate How many dimension I can compute each time // and how bug should the HOST-GPU communication buffer should be @@ -1506,8 +1506,8 @@ int FTI_RecoverVarHDF5(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, FTI_Conf->h5SingleFilePrefix, FTI_Exec->ckptId); } else { if (data->size != data->sizeStored) { - snprintf(str, sizeof(str), "Cannot recover %d bytes to " - "protected variable (ID %d) size: %d", + snprintf(str, sizeof(str), "Cannot recover %lu bytes to " + "protected variable (ID %d) size: %lu", data->sizeStored, data->id, data->size); FTI_Print(str, FTI_WARN); diff --git a/src/IO/hdf5-fti.h b/src/IO/hdf5-fti.h index 1f38ec5d0..abd2d69df 100644 --- a/src/IO/hdf5-fti.h +++ b/src/IO/hdf5-fti.h @@ -29,7 +29,7 @@ int FTI_ReadHDF5Var(FTIT_dataset *data); int FTI_GetDatasetRankReco(hid_t did); int FTI_GetDatasetSpanReco(hid_t did, hsize_t * span); int FTI_WriteHDF5Var(FTIT_dataset *data, FTIT_execution* FTI_Exec); -int FTI_CheckHDF5File(char* fn, int32_t fs, char* checksum); +int FTI_CheckHDF5File(char* fn, uint64_t fs, char* checksum); int FTI_OpenGlobalDatasets(FTIT_execution* FTI_Exec); herr_t FTI_ReadSharedFileData(FTIT_dataset FTI_Data); int FTI_H5CheckSingleFile(FTIT_configuration* FTI_Conf, int * ckptID); diff --git a/src/IO/posix-dcp.c b/src/IO/posix-dcp.c index ba4200813..03b670e88 100644 --- a/src/IO/posix-dcp.c +++ b/src/IO/posix-dcp.c @@ -183,16 +183,16 @@ int FTI_WritePosixDCPData(FTIT_dataset *data, void *fd) { unsigned char * block = (unsigned char*)malloc (FTI_Conf->dcpInfoPosix.BlockSize); size_t bytes; - int32_t varId = data->id; + int varId = data->id; FTI_Exec->dcpInfoPosix.dataSize += data->size; - uint32_t dataSize = data->size; - // uint32_t nbHashes = dataSize/FTI_Conf->dcpInfoPosix.BlockSize + + uint64_t dataSize = data->size; + // uint64_t nbHashes = dataSize/FTI_Conf->dcpInfoPosix.BlockSize + // (bool)(dataSize%FTI_Conf->dcpInfoPosix.BlockSize); if (dataSize > (MAX_BLOCK_IDX*FTI_Conf->dcpInfoPosix.BlockSize)) { snprintf(errstr, FTI_BUFS, "overflow in size of dataset with id:" - " %d (datasize: %u > MAX_DATA_SIZE: %u)", data->id, dataSize, + " %d (datasize: %lu > MAX_DATA_SIZE: %u)", data->id, dataSize, ((uint32_t)MAX_BLOCK_IDX)* ((uint32_t)FTI_Conf->dcpInfoPosix.BlockSize)); FTI_Print(errstr, FTI_EROR); @@ -216,13 +216,13 @@ int FTI_WritePosixDCPData(FTIT_dataset *data, void *fd) { if (dcpLayer == 0) { DFTI_EH_FWRITE(FTI_NSCS, bytes, &data->id, sizeof(int), 1, write_info->f, "p", block); - DFTI_EH_FWRITE(FTI_NSCS, bytes, &dataSize, sizeof(uint32_t), 1, + DFTI_EH_FWRITE(FTI_NSCS, bytes, &dataSize, sizeof(uint64_t), 1, write_info->f, "p", block); FTI_Exec->dcpInfoPosix.FileSize += (sizeof(int) + - sizeof(uint32_t)); - write_DCPinfo->layerSize += sizeof(int) + sizeof(uint32_t); + sizeof(uint64_t)); + write_DCPinfo->layerSize += sizeof(int) + sizeof(uint64_t); } - uint32_t pos = 0; + uint64_t pos = 0; FTIT_data_prefetch prefetcher; size_t totalBytes = 0; @@ -474,14 +474,14 @@ int FTI_RecoverDcpPosix(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, } for (i = 0; i < nbVarLayer; i++) { unsigned int varId; - uint32_t locDataSize; + uint64_t locDataSize; fread(&varId, sizeof(int), 1, fd); if (ferror(fd)) { snprintf(errstr, FTI_BUFS, "unable to read in file %s", fn); FTI_Print(errstr, FTI_EROR); return FTI_NSCS; } - fread(&locDataSize, sizeof(uint32_t), 1, fd); + fread(&locDataSize, sizeof(uint64_t), 1, fd); if (ferror(fd)) { snprintf(errstr, FTI_BUFS, "unable to read in file %s", fn); FTI_Print(errstr, FTI_EROR); @@ -527,7 +527,7 @@ int FTI_RecoverDcpPosix(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, } - uint32_t offset; + uint64_t offset; blockMetaInfo_t blockMeta; unsigned char *block = (unsigned char*) malloc(blockSize); if (!block) { @@ -539,7 +539,7 @@ int FTI_RecoverDcpPosix(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, for (i = 1; i < nbLayer; i++) { - uint32_t pos = 0; + uint64_t pos = 0; pos += fread(&ckptId, 1, sizeof(int), fd); if (ferror(fd)) { snprintf(errstr, FTI_BUFS, "unable to read in file %s", fn); @@ -635,7 +635,7 @@ int FTI_RecoverDcpPosix(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, return FTI_NSCS; } - uint32_t nbBlocks = (data[i].size % blockSize) ? + uint64_t nbBlocks = (data[i].size % blockSize) ? data[i].size/blockSize + 1 : data[i].size/blockSize; data[i].dcpInfoPosix.hashDataSize = data[i].size; int j = 0; @@ -645,7 +645,7 @@ int FTI_RecoverDcpPosix(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, totalBytes/blockSize + 1 : totalBytes/blockSize; int k; for (k = 0 ; k < currentBlocks && j < nbBlocks-1; k++) { - uint32_t hashIdx = j*MD5_DIGEST_LENGTH; + uint64_t hashIdx = j*MD5_DIGEST_LENGTH; FTI_Conf->dcpInfoPosix.hashFunc(ptr, blockSize, &data[i].dcpInfoPosix.oldHashArray[hashIdx]); ptr = ptr+blockSize; @@ -664,8 +664,8 @@ int FTI_RecoverDcpPosix(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, FTI_Print("unable to allocate memory!", FTI_EROR); return FTI_NSCS; } - uint32_t dataOffset = blockSize * (nbBlocks - 1); - uint32_t dataSize = data[i].size - dataOffset; + uint64_t dataOffset = blockSize * (nbBlocks - 1); + uint64_t dataSize = data[i].size - dataOffset; memcpy(buffer, ptr, dataSize); FTI_Conf->dcpInfoPosix.hashFunc(buffer, blockSize, &data[i].dcpInfoPosix.oldHashArray[(nbBlocks-1)*MD5_DIGEST_LENGTH]); @@ -766,14 +766,14 @@ int FTI_RecoverVarDcpPosix(FTIT_configuration* FTI_Conf, } for (i = 0; i < nbVarLayer; i++) { unsigned int varId; - uint32_t locDataSize; + uint64_t locDataSize; fread(&varId, sizeof(int), 1, fd); if (ferror(fd)) { snprintf(errstr, FTI_BUFS, "unable to read in file %s", fn); FTI_Print(errstr, FTI_EROR); return FTI_NSCS; } - fread(&locDataSize, sizeof(uint32_t), 1, fd); + fread(&locDataSize, sizeof(uint64_t), 1, fd); if (ferror(fd)) { snprintf(errstr, FTI_BUFS, "unable to read in file %s", fn); FTI_Print(errstr, FTI_EROR); @@ -806,7 +806,7 @@ int FTI_RecoverVarDcpPosix(FTIT_configuration* FTI_Conf, } } } else { - uint32_t skip = (locDataSize%blockSize == 0) ? + uint64_t skip = (locDataSize%blockSize == 0) ? locDataSize : (locDataSize/blockSize + 1)*blockSize; if (fseek(fd, skip, SEEK_CUR) == -1) { snprintf(errstr, FTI_BUFS, "unable to seek in file %s", fn); @@ -817,7 +817,7 @@ int FTI_RecoverVarDcpPosix(FTIT_configuration* FTI_Conf, } - uint32_t offset; + uint64_t offset; blockMetaInfo_t blockMeta; unsigned char *block = (unsigned char*) malloc(blockSize); @@ -828,7 +828,7 @@ int FTI_RecoverVarDcpPosix(FTIT_configuration* FTI_Conf, int nbLayer = FTI_Exec->dcpInfoPosix.nbLayerReco; for (i = 1; i < nbLayer; i++) { - uint32_t pos = 0; + uint64_t pos = 0; pos += fread(&ckptId, 1, sizeof(int), fd); if (ferror(fd)) { snprintf(errstr, FTI_BUFS, "unable to read in file %s", fn); @@ -921,7 +921,7 @@ int FTI_RecoverVarDcpPosix(FTIT_configuration* FTI_Conf, return FTI_NSCS; } - uint32_t nbBlocks = (data->size % blockSize) ? + uint64_t nbBlocks = (data->size % blockSize) ? data->size/blockSize + 1 : data->size/blockSize; data->dcpInfoPosix.hashDataSize = data->size; int j = 0; @@ -931,7 +931,7 @@ int FTI_RecoverVarDcpPosix(FTIT_configuration* FTI_Conf, totalBytes/blockSize + 1 : totalBytes/blockSize; int k; for (k = 0 ; k < currentBlocks && j < nbBlocks-1; k++) { - uint32_t hashIdx = j*MD5_DIGEST_LENGTH; + uint64_t hashIdx = j*MD5_DIGEST_LENGTH; FTI_Conf->dcpInfoPosix.hashFunc(ptr, blockSize, &data->dcpInfoPosix.oldHashArray[hashIdx]); ptr = ptr+blockSize; @@ -949,8 +949,8 @@ int FTI_RecoverVarDcpPosix(FTIT_configuration* FTI_Conf, FTI_Print("unable to allocate memory!", FTI_EROR); return FTI_NSCS; } - uint32_t dataOffset = blockSize * (nbBlocks - 1); - uint32_t dataSize = data->size - dataOffset; + uint64_t dataOffset = blockSize * (nbBlocks - 1); + uint64_t dataSize = data->size - dataOffset; memcpy(buffer, ptr, dataSize); FTI_Conf->dcpInfoPosix.hashFunc(buffer, blockSize, &data->dcpInfoPosix.oldHashArray[(nbBlocks-1)*MD5_DIGEST_LENGTH]); @@ -959,13 +959,13 @@ int FTI_RecoverVarDcpPosix(FTIT_configuration* FTI_Conf, /* // create hasharray for id i = FTI_DataGetIdx(id, FTI_Exec, FTI_Data); - uint32_t nbBlocks = (FTI_Data[i].size % blockSize) ? FTI_Data[i].size/blockSize + 1 : FTI_Data[i].size/blockSize; + uint64_t nbBlocks = (FTI_Data[i].size % blockSize) ? FTI_Data[i].size/blockSize + 1 : FTI_Data[i].size/blockSize; FTI_Data[i].dcpInfoPosix.hashDataSize = FTI_Data[i].size; int j; for(j=0; jsize != data->sizeStored) { - snprintf(str, sizeof(str), "Cannot recover %d bytes to protected " - "variable (ID %d) size: %d", data->sizeStored, data->id, data->size); + snprintf(str, sizeof(str), "Cannot recover %lu bytes to protected " + "variable (ID %d) size: %lu", data->sizeStored, data->id, data->size); FTI_Print(str, FTI_WARN); return FTI_NREC; } - int32_t filePos = data->filePos; + uint64_t filePos = data->filePos; if (fseek(fileposix, filePos, SEEK_SET) == 0) { fread(data->ptr, 1, data->size, fileposix); if (ferror(fileposix)) { diff --git a/src/api.c b/src/api.c index 48947ebf0..c8958ec81 100644 --- a/src/api.c +++ b/src/api.c @@ -249,7 +249,7 @@ int FTI_Status() { It is obscured from the user for API backwards compatibility. **/ /*-------------------------------------------------------------------------*/ -fti_id_t FTI_InitType_opaque(size_t size) { +fti_id_t FTI_InitType_opaque(uint64_t size) { FTIT_Datatype *type; fti_id_t new_id = FTI_Exec.datatypes.ntypes; @@ -324,7 +324,7 @@ FTIT_Datatype* FTI_GetType(fti_id_t id) { The fields can be added using FTI_AddScalarField and FTI_AddVectorField. **/ /*-------------------------------------------------------------------------*/ -fti_id_t FTI_InitCompositeType(const char* name, size_t size, +fti_id_t FTI_InitCompositeType(const char* name, uint64_t size, FTIT_H5Group* h5g) { FTIT_Datatype *type; FTIT_complexType *structure; @@ -369,7 +369,7 @@ fti_id_t FTI_InitCompositeType(const char* name, size_t size, **/ /*-------------------------------------------------------------------------*/ int FTI_AddScalarField(fti_id_t id, const char* name, fti_id_t fid, - size_t offset) { + uint64_t offset) { FTIT_Datatype *struct_ref, *field_type; int field_id; FTIT_typeField *field; @@ -428,7 +428,7 @@ int FTI_AddScalarField(fti_id_t id, const char* name, fti_id_t fid, **/ /*-------------------------------------------------------------------------*/ int FTI_AddVectorField(fti_id_t id, const char* name, - fti_id_t tid, size_t offset, int ndims, int* dim_sizes) { + fti_id_t tid, uint64_t offset, int ndims, int* dim_sizes) { FTIT_complexType *type; FTIT_typeField *field; int i; @@ -804,7 +804,7 @@ int FTI_RenameGroup(FTIT_H5Group* h5group, const char* name) { **/ /*-------------------------------------------------------------------------*/ -int FTI_Protect(int id, void* ptr, int32_t count, fti_id_t tid) { +int FTI_Protect(int id, void* ptr, uint64_t count, fti_id_t tid) { if (FTI_Exec.initSCES == 0) { FTI_Print("FTI is not initialized.", FTI_WARN); return FTI_NSCS; @@ -836,7 +836,7 @@ int FTI_Protect(int id, void* ptr, int32_t count, fti_id_t tid) { } if (data != NULL) { // Search for dataset with given id - int32_t prevSize = data->size; + uint64_t prevSize = data->size; #ifdef GPUSUPPORT if (ptrInfo.type == FTIT_PTRTYPE_CPU) { // strcpy(memLocation, "CPU"); @@ -895,7 +895,7 @@ int FTI_Protect(int id, void* ptr, int32_t count, fti_id_t tid) { FTI_Print(str, FTI_DBUG); if (prevSize != data->size && FTI_Conf.dcpPosix) { if (!(data->isDevicePtr)) { - uint32_t nbHashes = data->size / + uint64_t nbHashes = data->size / FTI_Conf.dcpInfoPosix.BlockSize + (bool)(data->size %FTI_Conf.dcpInfoPosix.BlockSize); data->dcpInfoPosix.currentHashArray = (unsigned char*) @@ -910,10 +910,10 @@ int FTI_Protect(int id, void* ptr, int32_t count, fti_id_t tid) { #ifdef GPUSUPPORT else { unsigned char *x; - uint32_t nbNewHashes = data->size / + uint64_t nbNewHashes = data->size / FTI_Conf.dcpInfoPosix.BlockSize + (bool)(data->size %FTI_Conf.dcpInfoPosix.BlockSize); - uint32_t nbOldHashes = prevSize / + uint64_t nbOldHashes = prevSize / FTI_Conf.dcpInfoPosix.BlockSize + (bool)(data->size %FTI_Conf.dcpInfoPosix.BlockSize); CUDA_ERROR_CHECK(cudaMallocManaged((void**) &x, @@ -1018,7 +1018,7 @@ int FTI_Protect(int id, void* ptr, int32_t count, fti_id_t tid) { if (FTI_Conf.dcpPosix) { if (!(data->isDevicePtr)) { - uint32_t nbHashes = data->size / + uint64_t nbHashes = data->size / FTI_Conf.dcpInfoPosix.BlockSize + (bool)(data->size %FTI_Conf.dcpInfoPosix.BlockSize); data->dcpInfoPosix.hashDataSize = 0; @@ -1032,7 +1032,7 @@ int FTI_Protect(int id, void* ptr, int32_t count, fti_id_t tid) { #ifdef GPUSUPPORT else { unsigned char *x; - uint32_t nbNewHashes = data->size / + uint64_t nbNewHashes = data->size / FTI_Conf.dcpInfoPosix.BlockSize + (bool)(data->size %FTI_Conf.dcpInfoPosix.BlockSize); CUDA_ERROR_CHECK(cudaMallocManaged((void**)&x, @@ -1616,7 +1616,7 @@ int FTI_DefineDataset(int id, int rank, int* dimLength, const char* name, } // check if size is correct - int expectedSize = 1; + uint64_t expectedSize = 1; int j; for (j = 0; j < rank; j++) { expectedSize *= dimLength[j]; // compute the number of elements @@ -1627,7 +1627,7 @@ int FTI_DefineDataset(int id, int rank, int* dimLength, const char* name, // sprintf(str, "Trying to define datasize: number of elements %d, // but the dataset count is %ld.", expectedSize, data->count); snprintf(str, sizeof(str), "Trying to define datasize: number of" - " elements %d, but the dataset count is %d.", + " elements %lu, but the dataset count is %lu.", expectedSize, data->count); FTI_Print(str, FTI_WARN); return FTI_NSCS; @@ -1653,7 +1653,7 @@ int FTI_DefineDataset(int id, int rank, int* dimLength, const char* name, /** @brief Returns size saved in metadata of variable @param id Variable ID. - @return int32_t Returns size of variable or 0 if size not saved. + @return uint64_t Returns size of variable or 0 if size not saved. This function returns size of variable of given ID that is saved in metadata. This may be different from size of variable that is in the program. If this @@ -1662,7 +1662,7 @@ int FTI_DefineDataset(int id, int rank, int* dimLength, const char* name, is no size saved in metadata it returns 0. **/ /*-------------------------------------------------------------------------*/ -int32_t FTI_GetStoredSize(int id) { +uint64_t FTI_GetStoredSize(int id) { if (FTI_Exec.initSCES == 0) { FTI_Print("FTI is not initialized.", FTI_WARN); return 0; @@ -1734,7 +1734,7 @@ void* FTI_Realloc(int id, void* ptr) { ptr = tmp; // sprintf(str, "Reallocated size: %ld", data->sizeStored); - snprintf(str, sizeof(str), "Reallocated size: %d", data->sizeStored); + snprintf(str, sizeof(str), "Reallocated size: %lu", data->sizeStored); FTI_Print(str, FTI_INFO); FTI_Exec.ckptSize += data->sizeStored - data->size; @@ -2295,13 +2295,13 @@ int FTI_FinalizeICP() { if ((FTI_Conf.dcpFtiff || FTI_Conf.dcpPosix) && FTI_Ckpt[4].isDcp) { // After dCP update store total data and dCP sizes in application rank0 - uint32_t *dataSize = (FTI_Conf.dcpFtiff)?(uint32_t*)& + uint64_t *dataSize = (FTI_Conf.dcpFtiff)?(uint64_t*)& FTI_Exec.FTIFFMeta.pureDataSize:&FTI_Exec.dcpInfoPosix.dataSize; - uint32_t *dcpSize = (FTI_Conf.dcpFtiff)?(uint32_t*)& + uint64_t *dcpSize = (FTI_Conf.dcpFtiff)?(uint64_t*)& FTI_Exec.FTIFFMeta.dcpSize:&FTI_Exec.dcpInfoPosix.dcpSize; - uint32_t dcpStats[2]; // 0:totalDcpSize, 1:totalDataSize - uint32_t sendBuf[] = { *dcpSize, *dataSize }; - MPI_Reduce(sendBuf, dcpStats, 2, MPI_UINT32_T, MPI_SUM, 0, + uint64_t dcpStats[2]; // 0:totalDcpSize, 1:totalDataSize + uint64_t sendBuf[] = { *dcpSize, *dataSize }; + MPI_Reduce(sendBuf, dcpStats, 2, MPI_UINT64_T, MPI_SUM, 0, FTI_COMM_WORLD); if (FTI_Topo.splitRank == 0) { *dcpSize = dcpStats[0]; @@ -2477,8 +2477,8 @@ int FTI_Recover() { protected variable (ID %d) size: %ld", data[i].sizeStored, data[i].id, data[i].size);*/ - snprintf(str, sizeof(str), "Cannot recover %d bytes to " - "protected variable (ID %d) size: %d", + snprintf(str, sizeof(str), "Cannot recover %lu bytes to " + "protected variable (ID %d) size: %lu", data[i].sizeStored, data[i].id, data[i].size); FTI_Print(str, FTI_WARN); @@ -2513,8 +2513,8 @@ int FTI_Recover() { FTI_Exec.dcpInfoPosix.datasetInfo[lidx][i].varSize, FTI_Exec.dcpInfoPosix.datasetInfo[lidx][i].varID, data->sizeStored);*/ - snprintf(str, sizeof(str), "Cannot recover %d bytes to " - "protected variable (ID %d) size: %d", + snprintf(str, sizeof(str), "Cannot recover %lu bytes to " + "protected variable (ID %d) size: %lu", FTI_Exec.dcpInfoPosix.datasetInfo[lidx][i].varSize, FTI_Exec.dcpInfoPosix.datasetInfo[lidx][i].varID, data->sizeStored); @@ -2573,7 +2573,7 @@ int FTI_Recover() { #ifdef GPUSUPPORT for (i = 0; i < FTI_Exec.nbVarStored; i++) { - size_t filePos = data[i].filePosStored; + uint64_t filePos = data[i].filePosStored; // strncpy(data[i].idChar, data[i].idChar, FTI_BUFS); fseek(fd, filePos, SEEK_SET); if (data[i].isDevicePtr) @@ -2591,7 +2591,7 @@ int FTI_Recover() { #else for (i = 0; i < FTI_Exec.nbVarStored; i++) { - size_t filePos = data[i].filePos; + uint64_t filePos = data[i].filePos; // strncpy(data[i].idChar, data[i].idChar, FTI_BUFS); fseek(fd, filePos, SEEK_SET); fread(data[i].ptr, 1, data[i].sizeStored, fd); diff --git a/src/checkpoint.c b/src/checkpoint.c index 631abc98b..c31e6ba61 100644 --- a/src/checkpoint.c +++ b/src/checkpoint.c @@ -183,15 +183,15 @@ int FTI_WriteCkpt(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, if ((FTI_Conf->dcpFtiff || FTI_Conf->dcpPosix) && FTI_Ckpt[4].isDcp) { // After dCP update store total data and dCP // sizes in application rank 0 - uint32_t *dataSize = (FTI_Conf->dcpFtiff)? - (uint32_t*)&FTI_Exec->FTIFFMeta.pureDataSize: + uint64_t *dataSize = (FTI_Conf->dcpFtiff)? + (uint64_t*)&FTI_Exec->FTIFFMeta.pureDataSize: &FTI_Exec->dcpInfoPosix.dataSize; - uint32_t *dcpSize = (FTI_Conf->dcpFtiff)? - (uint32_t*)&FTI_Exec->FTIFFMeta.dcpSize: + uint64_t *dcpSize = (FTI_Conf->dcpFtiff)? + (uint64_t*)&FTI_Exec->FTIFFMeta.dcpSize: &FTI_Exec->dcpInfoPosix.dcpSize; - uint32_t dcpStats[2]; // 0:totalDcpSize, 1:totalDataSize - uint32_t sendBuf[] = { *dcpSize, *dataSize }; - MPI_Reduce(sendBuf, dcpStats, 2, MPI_UINT32_T, MPI_SUM, 0, + uint64_t dcpStats[2]; // 0:totalDcpSize, 1:totalDataSize + uint64_t sendBuf[] = { *dcpSize, *dataSize }; + MPI_Reduce(sendBuf, dcpStats, 2, MPI_UINT64_T, MPI_SUM, 0, FTI_COMM_WORLD); if (FTI_Topo->splitRank == 0) { *dcpSize = dcpStats[0]; diff --git a/src/meta.c b/src/meta.c index 07fdd56b3..b3b7f2d72 100644 --- a/src/meta.c +++ b/src/meta.c @@ -362,7 +362,7 @@ int FTI_LoadMetaDcp(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, int k; for (k = 0; k < MAX_STACK_SIZE; k++) { snprintf(str, FTI_BUFS, "%d:dcp_layer%d_size", FTI_Topo->groupRank, k); - uint32_t LayerSize = ini.getLong(&ini, str); + uint64_t LayerSize = ini.getLong(&ini, str); if (LayerSize == -1) { // No more variables break; @@ -383,12 +383,12 @@ int FTI_LoadMetaDcp(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, FTI_Exec->dcpInfoPosix.datasetInfo[k][j].varID = varID; snprintf(str, FTI_BUFS, "%d:dcp_layer%d_var%d_size", FTI_Topo->groupRank, k, j); - int32_t varSize = ini.getLong(&ini, str); + int64_t varSize = ini.getLong(&ini, str); if (varID < 0) { break; } FTI_Exec->dcpInfoPosix.datasetInfo[k][j].varSize = - (uint32_t) varSize; + (uint64_t) varSize; } } @@ -677,12 +677,12 @@ int FTI_WriteCkptMetaData(FTIT_configuration* FTI_Conf, **/ /*-------------------------------------------------------------------------*/ int FTI_WriteMetadata(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, - FTIT_topology* FTI_Topo, FTIT_checkpoint* FTI_Ckpt, int32_t* fs, - int32_t mfs, char* fnl, char* checksums, int* allVarIDs, + FTIT_topology* FTI_Topo, FTIT_checkpoint* FTI_Ckpt, uint64_t* fs, + uint64_t mfs, char* fnl, char* checksums, int* allVarIDs, int* allRanks, uint64_t* allCounts, int* allVarTypeIDs, int* allVarTypeSizes, - int32_t* allVarSizes, uint32_t* allLayerSizes, char* allLayerHashes, - int32_t *allVarPositions, char *allNames, char *allCharIds) { + uint64_t* allVarSizes, uint64_t* allLayerSizes, char* allLayerHashes, + uint64_t *allVarPositions, char *allNames, char *allCharIds) { // no metadata files for FTI-FF if (FTI_Conf->ioMode == FTI_IO_FTIFF) { return FTI_SCES; } @@ -722,10 +722,10 @@ int FTI_WriteMetadata(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, snprintf(key, FTI_BUFS, "%d:Ckpt_file_name", i); ini.set(&ini, key, val); snprintf(key, FTI_BUFS, "%d:Ckpt_file_size", i); - snprintf(val, FTI_BUFS, "%u", fs[i]); + snprintf(val, FTI_BUFS, "%lu", fs[i]); ini.set(&ini, key, val); snprintf(key, FTI_BUFS, "%d:Ckpt_file_maxs", i); - snprintf(val, FTI_BUFS, "%u", mfs); + snprintf(val, FTI_BUFS, "%lu", mfs); ini.set(&ini, key, val); strncpy(val, checksums + (i * MD5_DIGEST_STRING_LENGTH), MD5_DIGEST_STRING_LENGTH); @@ -752,12 +752,12 @@ int FTI_WriteMetadata(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, // Save size of variable snprintf(key, FTI_BUFS, "%d:Var%d_size", i, j); - snprintf(val, FTI_BUFS, "%d", + snprintf(val, FTI_BUFS, "%lu", allVarSizes[i * FTI_Exec->nbVar + j]); ini.set(&ini, key, val); snprintf(key, FTI_BUFS, "%d:Var%d_pos", i, j); - snprintf(val, FTI_BUFS, "%d", + snprintf(val, FTI_BUFS, "%lu", allVarPositions[i * FTI_Exec->nbVar + j]); ini.set(&ini, key, val); @@ -790,7 +790,7 @@ int FTI_WriteMetadata(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, FTI_Conf->dcpInfoPosix.StackSize) + 1; for (j=0; j < nbLayer; j++) { snprintf(key, FTI_BUFS, "%d:dcp_layer%d_size", i, j); - snprintf(val, FTI_BUFS, "%u", allLayerSizes[i * nbLayer + j]); + snprintf(val, FTI_BUFS, "%lu", allLayerSizes[i * nbLayer + j]); ini.set(&ini, key, val); snprintf(key, FTI_BUFS, "%d:dcp_layer%d_hash", i, j); @@ -808,7 +808,7 @@ int FTI_WriteMetadata(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, // Save size of variable snprintf(key, FTI_BUFS, "%d:dcp_layer%d_var%d_size", i, j, k); - snprintf(val, FTI_BUFS, "%d", + snprintf(val, FTI_BUFS, "%lu", allVarSizes[i * FTI_Exec->nbVar + k]); ini.set(&ini, key, val); } @@ -881,9 +881,9 @@ int FTI_CreateMetadata(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, } #endif - int32_t fileSizes[FTI_BUFS]; - MPI_Allgather(&FTI_Exec->ckptMeta.fs, 1, MPI_INT32_T, - fileSizes, 1, MPI_INT32_T, FTI_Exec->groupComm); + uint64_t fileSizes[FTI_BUFS]; + MPI_Allgather(&FTI_Exec->ckptMeta.fs, 1, MPI_INT64_T, + fileSizes, 1, MPI_INT64_T, FTI_Exec->groupComm); // update partner file size: if (FTI_Exec->ckptMeta.level == 2) { @@ -892,7 +892,7 @@ int FTI_CreateMetadata(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, FTI_Exec->ckptMeta.pfs = fileSizes[ptnerGroupRank]; } - int32_t mfs = 0; // Max file size in group + uint64_t mfs = 0; // Max file size in group int i; for (i = 0; i < FTI_Topo->groupSize; i++) { if (fileSizes[i] > mfs) { @@ -901,7 +901,7 @@ int FTI_CreateMetadata(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, } FTI_Exec->ckptMeta.maxFs = mfs; char str[FTI_BUFS]; // For console output - snprintf(str, FTI_BUFS, "Max. file size in group %u.", mfs); + snprintf(str, FTI_BUFS, "Max. file size in group %lu.", mfs); FTI_Print(str, FTI_DBUG); char* ckptFileNames = NULL; @@ -936,11 +936,11 @@ int FTI_CreateMetadata(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, int* allVarIDs = NULL; int* allVarTypeIDs = NULL; int* allVarTypeSizes = NULL; - int32_t* allVarSizes = NULL; - int32_t *allVarPositions = NULL; + uint64_t* allVarSizes = NULL; + uint64_t *allVarPositions = NULL; // for posix dcp - uint32_t* allLayerSizes = NULL; + uint64_t* allLayerSizes = NULL; char* allLayerHashes = NULL; char* allCharIds = NULL; char* allNames = NULL; @@ -957,15 +957,15 @@ int FTI_CreateMetadata(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, allVarIDs = talloc(int, FTI_Topo->groupSize * FTI_Exec->nbVar); allVarTypeIDs = talloc(int, FTI_Topo->groupSize * FTI_Exec->nbVar); allVarTypeSizes = talloc(int, FTI_Topo->groupSize * FTI_Exec->nbVar); - allVarSizes = talloc(int32_t, FTI_Topo->groupSize * FTI_Exec->nbVar); - allVarPositions = talloc(int32_t, + allVarSizes = talloc(uint64_t, FTI_Topo->groupSize * FTI_Exec->nbVar); + allVarPositions = talloc(uint64_t, FTI_Topo->groupSize * FTI_Exec->nbVar); allCharIds = (char *)malloc(sizeof(char)*FTI_BUFS* FTI_Exec->nbVar*FTI_Topo->groupSize); allNames = (char *)malloc(sizeof(char)*FTI_BUFS* FTI_Exec->nbVar*FTI_Topo->groupSize); if (FTI_Ckpt[FTI_Exec->ckptMeta.level].isDcp) { - allLayerSizes = talloc(uint32_t, + allLayerSizes = talloc(uint64_t, FTI_Topo->groupSize * nbLayer); allLayerHashes = talloc(char, FTI_Topo->groupSize * nbLayer * MD5_DIGEST_STRING_LENGTH); @@ -977,8 +977,8 @@ int FTI_CreateMetadata(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, uint64_t* myCounts = talloc(uint64_t, 32 * FTI_Exec->nbVar); int* myVarTypeIDs = talloc(int, FTI_Exec->nbVar); int* myVarTypeSizes = talloc(int, FTI_Exec->nbVar); - int32_t* myVarSizes = talloc(int32_t, FTI_Exec->nbVar); - int32_t* myVarPositions = talloc(int32_t, FTI_Exec->nbVar); + uint64_t* myVarSizes = talloc(uint64_t, FTI_Exec->nbVar); + uint64_t* myVarPositions = talloc(uint64_t, FTI_Exec->nbVar); char *ArrayOfIdChars = (char *)malloc(FTI_Exec->nbVar * sizeof(char*) *FTI_BUFS); char *ArrayOfNames = (char *)malloc(FTI_Exec->nbVar * @@ -1019,11 +1019,11 @@ int FTI_CreateMetadata(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, MPI_Gather(myCounts, 32*FTI_Exec->nbVar, MPI_INT64_T, allCounts, 32*FTI_Exec->nbVar, MPI_INT64_T, 0, FTI_Exec->groupComm); // Gather variables sizes - MPI_Gather(myVarSizes, FTI_Exec->nbVar, MPI_INT32_T, allVarSizes, - FTI_Exec->nbVar, MPI_INT32_T, 0, FTI_Exec->groupComm); + MPI_Gather(myVarSizes, FTI_Exec->nbVar, MPI_INT64_T, allVarSizes, + FTI_Exec->nbVar, MPI_INT64_T, 0, FTI_Exec->groupComm); // Gather variables file positions - MPI_Gather(myVarPositions, FTI_Exec->nbVar, MPI_INT32_T, allVarPositions, - FTI_Exec->nbVar, MPI_INT32_T, 0, FTI_Exec->groupComm); + MPI_Gather(myVarPositions, FTI_Exec->nbVar, MPI_INT64_T, allVarPositions, + FTI_Exec->nbVar, MPI_INT64_T, 0, FTI_Exec->groupComm); // Gather all variable idchars MPI_Gather(ArrayOfIdChars, FTI_Exec->nbVar*FTI_BUFS, MPI_CHAR, allCharIds, FTI_Exec->nbVar*FTI_BUFS, MPI_CHAR, 0, FTI_Exec->groupComm); @@ -1034,8 +1034,8 @@ int FTI_CreateMetadata(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, if (FTI_Ckpt[FTI_Exec->ckptMeta.level].isDcp) { // Gather dcp layer sizes - MPI_Gather(FTI_Exec->dcpInfoPosix.LayerSize, nbLayer, MPI_UINT32_T, - allLayerSizes, nbLayer, MPI_UINT32_T, 0, FTI_Exec->groupComm); + MPI_Gather(FTI_Exec->dcpInfoPosix.LayerSize, nbLayer, MPI_UINT64_T, + allLayerSizes, nbLayer, MPI_UINT64_T, 0, FTI_Exec->groupComm); // Gather dcp layer hashes MPI_Gather(FTI_Exec->dcpInfoPosix.LayerHash, nbLayer * MD5_DIGEST_STRING_LENGTH, MPI_CHAR, allLayerHashes, diff --git a/src/meta.h b/src/meta.h index a498b5523..25261f227 100644 --- a/src/meta.h +++ b/src/meta.h @@ -28,12 +28,12 @@ int FTI_LoadMetaDataset(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, FTIT_topology* FTI_Topo, FTIT_checkpoint* FTI_Ckpt, FTIT_keymap* FTI_Data); int FTI_WriteMetadata(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, - FTIT_topology* FTI_Topo, FTIT_checkpoint* FTI_Ckpt, int32_t* fs, - int32_t mfs, char* fnl, char* checksums, int* allVarIDs, + FTIT_topology* FTI_Topo, FTIT_checkpoint* FTI_Ckpt, uint64_t* fs, + uint64_t mfs, char* fnl, char* checksums, int* allVarIDs, int* allRanks, uint64_t* allCounts, int* allVarTypeIDs, int* allVarTypeSizes, - int32_t* allVarSizes, uint32_t* allLayerSizes, char* allLayerHashes, - int32_t *allVarPositions, char *allNames, char *allCharIds); + uint64_t* allVarSizes, uint64_t* allLayerSizes, char* allLayerHashes, + uint64_t *allVarPositions, char *allNames, char *allCharIds); int FTI_CreateMetadata(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, FTIT_topology* FTI_Topo, FTIT_checkpoint* FTI_Ckpt, FTIT_keymap* FTI_Data); diff --git a/src/postckpt.c b/src/postckpt.c index 2b720c58e..2a8a8204f 100644 --- a/src/postckpt.c +++ b/src/postckpt.c @@ -99,7 +99,7 @@ int FTI_SendCkpt(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, } char* buffer = talloc(char, FTI_Conf->blockSize); - int32_t toSend = FTI_Exec->ckptMeta.fs; // remaining data to send + uint64_t toSend = FTI_Exec->ckptMeta.fs; // remaining data to send while (toSend > 0) { int sendSize = (toSend > FTI_Conf->blockSize) ? FTI_Conf->blockSize : toSend; @@ -152,7 +152,7 @@ int FTI_RecvPtner(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, } char* buffer = talloc(char, FTI_Conf->blockSize); - uint32_t toRecv = FTI_Exec->ckptMeta.pfs; + uint64_t toRecv = FTI_Exec->ckptMeta.pfs; // remaining data to receive while (toRecv > 0) { int recvSize = (toRecv > FTI_Conf->blockSize) ? @@ -285,7 +285,7 @@ int FTI_RSenc(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, FTI_Print(str, FTI_DBUG); // all files in group must have the same size - int32_t maxFs = FTI_Exec->ckptMeta.maxFs; // max file size in group + uint64_t maxFs = FTI_Exec->ckptMeta.maxFs; // max file size in group // determine file size in order to write at the end of the elongated // file (i.e. write at the end of file after 'truncate(..., maxFs)'. @@ -298,6 +298,7 @@ int FTI_RSenc(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, FTI_Print("Error with truncate on checkpoint file", FTI_WARN); return FTI_NSCS; } + DBG_MSG("fs: %lu",-1,maxFs); // write file size at the end of elongated file to recover original // size during restart. The file size, thus, will be included in the @@ -322,6 +323,7 @@ int FTI_RSenc(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, return FTI_NSCS; } close(lftmp_); + } FILE* lfd = fopen(lfn, "rb"); @@ -357,7 +359,7 @@ int FTI_RSenc(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, int remBsize = bs; - int32_t ps = ((maxFs / bs)) * bs; + uint64_t ps = ((maxFs / bs)) * bs; if (ps < maxFs) { ps = ps + bs; } @@ -367,7 +369,7 @@ int FTI_RSenc(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, MD5_Init(&mdContext); // For each block - int32_t pos = 0; + uint64_t pos = 0; while (pos < ps) { if ((maxFs - pos) < bs) { remBsize = maxFs - pos; @@ -508,7 +510,7 @@ int FTI_RSenc(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, efd, "ppppf", data, matrix, coding, myData, lfd); free(buffer_ser); } - + free(data); free(matrix); free(coding); @@ -516,12 +518,15 @@ int FTI_RSenc(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, fclose(lfd); fclose(efd); - int32_t fs = FTI_Exec->ckptMeta.fs; // ckpt file size - + uint64_t fs = FTI_Exec->ckptMeta.fs; // ckpt file size + if (truncate(lfn, fs) == -1) { FTI_Print("Error with re-truncate on checkpoint file", FTI_WARN); return FTI_NSCS; } + + stat(lfn, &st_); + DBG_MSG("stat.st: %lu, fs: %lu", -1, st_.st_size, fs); int res = FTI_WriteRSedChecksum(FTI_Conf, FTI_Exec, FTI_Topo, FTI_Ckpt, rank, checksum); @@ -787,11 +792,11 @@ int FTI_FlushPosix(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, } char *readData = talloc(char, FTI_Conf->transferSize); - int32_t bSize = FTI_Conf->transferSize; - int32_t fs = FTI_Exec->ckptMeta.fs; - snprintf(str, FTI_BUFS, "Local file size for proc %d: %d", proc, fs); + int bSize = FTI_Conf->transferSize; + uint64_t fs = FTI_Exec->ckptMeta.fs; + snprintf(str, FTI_BUFS, "Local file size for proc %d: %lu", proc, fs); FTI_Print(str, FTI_DBUG); - int32_t pos = 0; + uint64_t pos = 0; // Checkpoint files exchange while (pos < fs) { if ((fs - pos) < FTI_Conf->transferSize) @@ -912,10 +917,10 @@ int FTI_FlushMPI(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, } char* readData = talloc(char, FTI_Conf->transferSize); - int32_t bSize = FTI_Conf->transferSize; - int32_t fs = FTI_Exec->ckptMeta.fs; + int bSize = FTI_Conf->transferSize; + uint64_t fs = FTI_Exec->ckptMeta.fs; - int32_t pos = 0; + uint64_t pos = 0; // Checkpoint files exchange while (pos < fs) { if ((fs - pos) < FTI_Conf->transferSize) { @@ -970,7 +975,7 @@ int FTI_FlushSionlib(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, } int nbProc = endProc - startProc; - int32_t* localFileSizes = talloc(int32_t, nbProc); + uint64_t* localFileSizes = talloc(uint64_t, nbProc); char* localFileNames = talloc(char, FTI_BUFS * nbProc); int* splitRanks = talloc(int, nbProc); // rank of process in FTI_COMM_WORLD for (proc = startProc; proc < endProc; proc++) { @@ -1076,10 +1081,10 @@ int FTI_FlushSionlib(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, } char *readData = talloc(char, FTI_Conf->transferSize); - int32_t bSize = FTI_Conf->transferSize; - int32_t fs = FTI_Exec->ckptMeta.fs; + int bSize = FTI_Conf->transferSize; + uint64_t fs = FTI_Exec->ckptMeta.fs; - int32_t pos = 0; + uint64_t pos = 0; // Checkpoint files exchange while (pos < fs) { if ((fs - pos) < FTI_Conf->transferSize) @@ -1091,7 +1096,7 @@ int FTI_FlushSionlib(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, ranks, rank_map, chunkSizes); - int32_t data_written = sion_fwrite(readData, sizeof(char), + uint64_t data_written = sion_fwrite(readData, sizeof(char), bytes, sid); if (data_written < 0) { diff --git a/src/postreco.c b/src/postreco.c index 27435527b..a3984acab 100644 --- a/src/postreco.c +++ b/src/postreco.c @@ -68,7 +68,7 @@ int FTI_Decode(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, int k = FTI_Topo->groupSize; int m = k; - int32_t fs = FTI_Exec->ckptMeta.fs; + uint64_t fs = FTI_Exec->ckptMeta.fs; char** data = talloc(char*, k); char** coding = talloc(char*, m); @@ -130,8 +130,8 @@ int FTI_Decode(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, } FILE *fd, *efd; - int32_t maxFs = FTI_Exec->ckptMeta.maxFs; - int32_t ps = ((maxFs / FTI_Conf->blockSize)) * FTI_Conf->blockSize; + uint64_t maxFs = FTI_Exec->ckptMeta.maxFs; + uint64_t ps = ((maxFs / FTI_Conf->blockSize)) * FTI_Conf->blockSize; if (ps < maxFs) { ps = ps + FTI_Conf->blockSize; // Calculating padding size } @@ -237,7 +237,7 @@ int FTI_Decode(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, } // Main loop, block by block - int32_t pos = 0; + uint64_t pos = 0; int remBsize = bs; MD5_CTX md5ctxRS; @@ -381,8 +381,10 @@ int FTI_Decode(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, close(ifd); return FTI_NSCS; } + + DBG_MSG("fs: %lu",-1,fs_); - fs = (int32_t) fs_; + fs = (uint64_t) fs_; FTI_Exec->ckptMeta.fs = fs; close(ifd); @@ -441,7 +443,7 @@ int FTI_Decode(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, close(ifd); free(buffer_ser); } - + if (truncate(fn, fs) == -1) { FTI_Print("R3 cannot re-truncate checkpoint file.", FTI_WARN); diff --git a/src/recover.c b/src/recover.c index 2f454b929..f2125ec44 100644 --- a/src/recover.c +++ b/src/recover.c @@ -51,7 +51,7 @@ **/ /*-------------------------------------------------------------------------*/ -int FTI_CheckFile(char* fn, int32_t fs, char* checksum) { +int FTI_CheckFile(char* fn, uint64_t fs, char* checksum) { char str[FTI_BUFS]; if (access(fn, F_OK) == 0) { struct stat fileStatus; @@ -105,9 +105,9 @@ int FTI_CheckErasures(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, FTIT_topology* FTI_Topo, FTIT_checkpoint* FTI_Ckpt, int *erased) { int level = FTI_Exec->ckptMeta.level; - int32_t fs = FTI_Exec->ckptMeta.fs; - int32_t pfs = FTI_Exec->ckptMeta.pfs; - int32_t maxFs = FTI_Exec->ckptMeta.maxFs; + uint64_t fs = FTI_Exec->ckptMeta.fs; + uint64_t pfs = FTI_Exec->ckptMeta.pfs; + uint64_t maxFs = FTI_Exec->ckptMeta.maxFs; char ckptFile[FTI_BUFS]; strncpy(ckptFile, FTI_Exec->ckptMeta.ckptFile, FTI_BUFS); @@ -123,7 +123,7 @@ int FTI_CheckErasures(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, char fn[FTI_BUFS]; // Path to the checkpoint/partner file name int buf; int ckptId, rank; // Variables for proper partner file name - int (*consistency)(char *, int32_t , char*); + int (*consistency)(char *, uint64_t , char*); #ifdef ENABLE_HDF5 if (FTI_Conf->ioMode == FTI_IO_HDF5) { consistency = &FTI_CheckHDF5File; diff --git a/src/recover.h b/src/recover.h index 684ee80f4..b372a1272 100644 --- a/src/recover.h +++ b/src/recover.h @@ -10,7 +10,7 @@ #include "interface.h" -int FTI_CheckFile(char *fn, int32_t fs, char* checksum); +int FTI_CheckFile(char *fn, uint64_t fs, char* checksum); int FTI_CheckErasures(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, FTIT_topology* FTI_Topo, FTIT_checkpoint* FTI_Ckpt, int *erased); diff --git a/src/util/keymap.c b/src/util/keymap.c index 677e8ca3c..7956cfec9 100644 --- a/src/util/keymap.c +++ b/src/util/keymap.c @@ -47,7 +47,7 @@ /*-------------------------------------------------------------------------*/ static FTIT_keymap self; -int FTI_KeyMap(FTIT_keymap** instance, int32_t type_size, int32_t max_key, +int FTI_KeyMap(FTIT_keymap** instance, size_t type_size, int max_key, bool reset) { if (type_size == 0) { FTI_Print("Call to FTI_KeyMap with typesize '0' is invalid", FTI_EROR); @@ -109,8 +109,8 @@ int FTI_KeyMapPushBack(void* new_item, int key) { return FTI_NSCS; } - int32_t new_size = self._size; - int32_t new_used = self._used + 1; + uint64_t new_size = self._size; + uint64_t new_used = self._used + 1; if (new_used > self._size) { // double container size each time limit is reached except @@ -175,7 +175,7 @@ int FTI_KeyMapGet(FTIT_dataset** data, int key) { return FTI_NSCS; } - int32_t check_pos = self._key[key]; + int64_t check_pos = self._key[key]; if (check_pos > (self._used - 1)) { FTI_Print("data location out of bounds", FTI_EROR); diff --git a/src/util/keymap.h b/src/util/keymap.h index 2ce8019b5..62d85cfb4 100644 --- a/src/util/keymap.h +++ b/src/util/keymap.h @@ -104,7 +104,7 @@ extern "C" { --------------------------------------------------------------------------**/ - int FTI_KeyMap(FTIT_keymap**, int32_t, int32_t, bool); + int FTI_KeyMap(FTIT_keymap**, size_t, int, bool); /**-------------------------------------------------------------------------- diff --git a/src/util/macros.h b/src/util/macros.h index 7916c3fbe..e639b8dcb 100644 --- a/src/util/macros.h +++ b/src/util/macros.h @@ -73,7 +73,7 @@ void cleanup(char* pattern, ...); bytes = fread(buff, size, number, fd); \ if (ferror(fd)) { \ char ErrorString[400]; \ - snprintf(ErrorString, sizeof(ErrorString), "FILE %s FUNC %s:%d Error Reading File Bytes Read : %d", __FILE__, __FUNCTION__, __LINE__, (int32_t)bytes); \ + snprintf(ErrorString, sizeof(ErrorString), "FILE %s FUNC %s:%d Error Reading File Bytes Read : %lu", __FILE__, __FUNCTION__, __LINE__, (uint64_t)bytes); \ FTI_Print(ErrorString, FTI_EROR); \ cleanup(format, __VA_ARGS__, NULL); \ fclose(fd); \ @@ -86,7 +86,7 @@ void cleanup(char* pattern, ...); bytes = fwrite(buff, size, number, fd); \ if (ferror(fd)) { \ char ErrorString[400]; \ - snprintf(ErrorString, sizeof(ErrorString), "FILE %s FUNC %s:%d Error Writing File Bytes Written : %d", __FILE__, __FUNCTION__, __LINE__, (int32_t)bytes); \ + snprintf(ErrorString, sizeof(ErrorString), "FILE %s FUNC %s:%d Error Writing File Bytes Written : %lu", __FILE__, __FUNCTION__, __LINE__, (uint64_t)bytes); \ FTI_Print(ErrorString, FTI_EROR); \ cleanup(format, __VA_ARGS__, NULL); \ fclose(fd); \ diff --git a/src/util/tools.c b/src/util/tools.c index fc6b20f03..c6c448cb1 100644 --- a/src/util/tools.c +++ b/src/util/tools.c @@ -95,8 +95,9 @@ int FTI_InitExecVars(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, FTI_filemetastructsize = MD5_DIGEST_STRING_LENGTH + MD5_DIGEST_LENGTH - + 7*sizeof(int32_t) - + sizeof(int); + + sizeof(int) + + 6*sizeof(uint64_t) + + sizeof(uint32_t); // TODO(leobago) RS L3 only works for even file sizes. // This accounts for many but clearly not all cases. @@ -105,13 +106,13 @@ int FTI_InitExecVars(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, FTI_dbstructsize = sizeof(int) /* numvars */ - + sizeof(int32_t); /* dbsize */ + + sizeof(uint64_t); /* dbsize */ FTI_dbvarstructsize = 2*sizeof(int) /* numvars */ + 2*sizeof(bool) + 2*sizeof(uintptr_t) - + 2*sizeof(int32_t) + + 2*sizeof(uint64_t) + MD5_DIGEST_LENGTH; // diff --git a/testing/suites/core/ckptDiffSizes/diffSizes.c b/testing/suites/core/ckptDiffSizes/diffSizes.c index 560baa100..d3a84b519 100644 --- a/testing/suites/core/ckptDiffSizes/diffSizes.c +++ b/testing/suites/core/ckptDiffSizes/diffSizes.c @@ -196,7 +196,8 @@ int do_work(int world_rank, int world_size, int checkpoint_level, int fail) { } } - int rtn = verify(buf, world_rank); + // UNUSED!! + //int rtn = verify(buf, world_rank); free(buf); @@ -351,7 +352,7 @@ int main(int argc, char** argv) { int res; if (checkpoint_level != 1) { int isInline = -1; - int heads = (int)iniparser_getint(ini, "Basic:head", -1); + //int heads = (int)iniparser_getint(ini, "Basic:head", -1); switch (checkpoint_level) { case 2: isInline = (int)iniparser_getint(ini, "Basic:inline_l2", 1); diff --git a/testing/suites/core/multiLevelCkpt/check.c b/testing/suites/core/multiLevelCkpt/check.c index ef2680e4c..95c753703 100644 --- a/testing/suites/core/multiLevelCkpt/check.c +++ b/testing/suites/core/multiLevelCkpt/check.c @@ -297,7 +297,6 @@ int main(int argc, char* argv[]) { void init_arrays(double* A, double* B, size_t asize) { int i; - double r; for (i = 0; i < asize; i++) { A[i] = 1.0; B[i] = ((double)rand() / RAND_MAX) * 5.0; diff --git a/testing/suites/features/cornerCases/ckptHierarchy.c b/testing/suites/features/cornerCases/ckptHierarchy.c index 374b046d9..518bd632e 100644 --- a/testing/suites/features/cornerCases/ckptHierarchy.c +++ b/testing/suites/features/cornerCases/ckptHierarchy.c @@ -40,7 +40,7 @@ void simulateCrash() { int res; if (checkpoint_level[3] != 1) { int isInline = -1; - int heads = (int)iniparser_getint(ini, "Basic:head", -1); + //int heads = (int)iniparser_getint(ini, "Basic:head", -1); switch (checkpoint_level[3]) { case 2: isInline = (int)iniparser_getint(ini, "Basic:inline_l2", 1); diff --git a/testing/suites/features/cornerCases/consistency.c b/testing/suites/features/cornerCases/consistency.c index 3a5450778..a77eced3b 100644 --- a/testing/suites/features/cornerCases/consistency.c +++ b/testing/suites/features/cornerCases/consistency.c @@ -33,7 +33,7 @@ void simulateCrash() { int res; if (checkpoint_level != 1) { int isInline = -1; - int heads = (int)iniparser_getint(ini, "Basic:head", -1); + //int heads = (int)iniparser_getint(ini, "Basic:head", -1); switch (checkpoint_level) { case 2: isInline = (int)iniparser_getint(ini, "Basic:inline_l2", 1); @@ -81,7 +81,7 @@ void simulateCrashWithoutCkpt() { dictionary* ini = iniparser_load(configfile); int heads = (int)iniparser_getint(ini, "Basic:head", -1); int nodeSize = (int)iniparser_getint(ini, "Basic:node_size", -1); - int general_tag = (int)iniparser_getint(ini, "Advanced:general_tag", 2612); + //int general_tag = (int)iniparser_getint(ini, "Advanced:general_tag", 2612); int final_tag = (int)iniparser_getint(ini, "Advanced:final_tag", 3107); int res; iniparser_freedict(ini); @@ -183,6 +183,8 @@ int afterSuccessfulRecovery() { if (world_rank == 0) printf("Recover done.\n"); return 0; } + // no case from above + return -1; } void initKeep_last_ckpt() { @@ -219,6 +221,8 @@ int keep_last_ckpt() { return 1; } } + // no case from above + return -1; } int initReInit() { @@ -229,7 +233,7 @@ int initReInit() { /* check for a correct restart without the crash of the application. */ int reInit() { - int initres = FTI_Init(configfile2, MPI_COMM_WORLD); + //int initres = FTI_Init(configfile2, MPI_COMM_WORLD); int* array2 = malloc(sizeof(int) * ARRAY_SIZE); initArray(array2); diff --git a/testing/suites/features/differentialCkpt/checkDCPPosix.c b/testing/suites/features/differentialCkpt/checkDCPPosix.c index bf8299414..22a072194 100644 --- a/testing/suites/features/differentialCkpt/checkDCPPosix.c +++ b/testing/suites/features/differentialCkpt/checkDCPPosix.c @@ -144,7 +144,7 @@ int read_data(double* B_chk, size_t* asize_chk, int rank, size_t asize, */ int main(int argc, char* argv[]) { - unsigned char parity, crash, state, diff_sizes, enable_icp = -1; + unsigned char parity, crash, state, diff_sizes; int FTI_APP_RANK, result, tmp, success = 1; double *A, *B, *B_chk; size_t i; @@ -342,7 +342,6 @@ int main(int argc, char* argv[]) { void init_arrays(double* A, double* B, size_t asize) { int i; - double r; for (i = 0; i < asize; i++) { A[i] = 1.0; B[i] = ((double)rand() / RAND_MAX) * 5.0; diff --git a/testing/suites/features/differentialCkpt/diff_test.c b/testing/suites/features/differentialCkpt/diff_test.c index 0f3eb7f87..8e107c562 100644 --- a/testing/suites/features/differentialCkpt/diff_test.c +++ b/testing/suites/features/differentialCkpt/diff_test.c @@ -32,8 +32,9 @@ int main(int argc, char* argv[]) { checkpoint(&info, 1, FTI_L4_DCP); int i; for (i = 0; i < NUM_DCKPT-1; ++i) { - uint32_t allocated = reallocate_buffers(&info, - ALLOC_SIZE, ALLOC_RANDOM); + // UNUSED!! + //uint32_t allocated = reallocate_buffers(&info, + // ALLOC_SIZE, ALLOC_RANDOM); protect_buffers(&info); xor_data(i, &info); // FTI_Checkpoint(i+2, FTI_L4_DCP); diff --git a/testing/suites/features/differentialCkpt/diff_test.h b/testing/suites/features/differentialCkpt/diff_test.h index d3b139ccf..42b6c575e 100644 --- a/testing/suites/features/differentialCkpt/diff_test.h +++ b/testing/suites/features/differentialCkpt/diff_test.h @@ -89,9 +89,10 @@ extern int numHeads; extern int finalTag; extern int headRank; -static fti_id_t FTI_UI, FTI_XOR_INFO; -static uint32_t pat; -static double share_ratio; +fti_id_t FTI_UI, FTI_XOR_INFO; +// UNUSED!! +uint32_t pat; +//static double share_ratio; extern int A[1]; extern int B[2]; diff --git a/testing/suites/features/differentialCkpt/diff_test_func.c b/testing/suites/features/differentialCkpt/diff_test_func.c index b367409fc..66bf2f361 100644 --- a/testing/suites/features/differentialCkpt/diff_test_func.c +++ b/testing/suites/features/differentialCkpt/diff_test_func.c @@ -303,8 +303,9 @@ void allocate_buffers(dcp_info_t * info, uint32_t alloc_size) { idx-1, info->size[idx-1]); } } - uint32_t ckptsize = allocated + sizeof(int) + - NUM_DCKPT*sizeof(xor_info_t) + sizeof(unsigned int); + // UNUSED!! + //uint32_t ckptsize = allocated + sizeof(int) + + // NUM_DCKPT*sizeof(xor_info_t) + sizeof(unsigned int); // DBG_MSG_APP("allocated (total): %lu, [ckptsize: %lu]", // -1, allocated, ckptsize); assert((alloc_size == allocated)); @@ -336,8 +337,9 @@ uint32_t reallocate_buffers(dcp_info_t * info, uint32_t _alloc_size, allocated += rest; info->size[idx-1] += rest; } - uint32_t ckptsize = allocated + sizeof(int) + - NUM_DCKPT*sizeof(xor_info_t) + sizeof(unsigned int); + // UNUSED!! + //uint32_t ckptsize = allocated + sizeof(int) + + // NUM_DCKPT*sizeof(xor_info_t) + sizeof(unsigned int); // DBG_MSG_APP("re-allocated (total): %lu, [ckptsize: %lu]", // -1, allocated, ckptsize); assert((alloc_size == allocated)); diff --git a/testing/suites/features/getConfig/getConfig.c b/testing/suites/features/getConfig/getConfig.c index 873263fc2..df582406b 100644 --- a/testing/suites/features/getConfig/getConfig.c +++ b/testing/suites/features/getConfig/getConfig.c @@ -13,7 +13,7 @@ #define CONFIG_EROR 30 int main(int argc, char *argv[]){ - int nrank, nbProcs, i; + int nrank, nbProcs; char *path = argv[1]; MPI_Init(&argc, &argv); diff --git a/testing/suites/features/hdf5/hdf5Test.c b/testing/suites/features/hdf5/hdf5Test.c index 750a726d1..ca8e577c7 100644 --- a/testing/suites/features/hdf5/hdf5Test.c +++ b/testing/suites/features/hdf5/hdf5Test.c @@ -489,7 +489,6 @@ int main(int argc, char** argv) { int res; if (checkpoint_level != 1) { int isInline = -1; - int heads = (int)iniparser_getint(ini, "Basic:head", -1); switch (checkpoint_level) { case 2: isInline = (int)iniparser_getint(ini, "Basic:inline_l2", 1); diff --git a/testing/suites/features/recoverName/checkRecoverName.c b/testing/suites/features/recoverName/checkRecoverName.c index ab20da105..b12dd7805 100644 --- a/testing/suites/features/recoverName/checkRecoverName.c +++ b/testing/suites/features/recoverName/checkRecoverName.c @@ -101,8 +101,8 @@ int checkCorrectness(int *array, int size, int val) { } int main(int argc, char *argv[]) { - unsigned char parity, crash, level, state, diff_sizes, enable_icp = -1; - int FTI_APP_RANK, result, tmp, success = 1; + unsigned char crash, level, state; + int FTI_APP_RANK, result; int *array[10]; int sizes[10] = {42, 85, 8, 19, 95, 26, 66, 33, 65, 83}; int order[10] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; @@ -110,7 +110,6 @@ int main(int argc, char *argv[]) { "five", "six", "seven", "eight", "nine"}; srand(time(NULL)); - size_t asize, asize_chk; allocArray(array, sizes, 10); MPI_Init(&argc, &argv); diff --git a/testing/suites/features/recoverVar/checkRecoverVar.c b/testing/suites/features/recoverVar/checkRecoverVar.c index 1a2948cb4..fd0d89b7b 100644 --- a/testing/suites/features/recoverVar/checkRecoverVar.c +++ b/testing/suites/features/recoverVar/checkRecoverVar.c @@ -101,14 +101,13 @@ int checkCorrectness(int *array, int size, int val) { } int main(int argc, char *argv[]) { - unsigned char parity, crash, level, state, diff_sizes, enable_icp = -1; - int FTI_APP_RANK, result, tmp, success = 1; + unsigned char crash, level, state; + int FTI_APP_RANK, result; int *array[10]; int sizes[10] = {42, 85, 8, 19, 95, 26, 66, 33, 65, 83}; int order[10] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; srand(time(NULL)); - size_t asize, asize_chk; allocArray(array, sizes, 10); MPI_Init(&argc, &argv); diff --git a/testing/suites/features/syncIntv/syncIntv.c b/testing/suites/features/syncIntv/syncIntv.c index 214e6d450..bdd89b840 100644 --- a/testing/suites/features/syncIntv/syncIntv.c +++ b/testing/suites/features/syncIntv/syncIntv.c @@ -149,10 +149,8 @@ int main(int argc, char **argv) { FTI_Protect(0, &i, 1, FTI_INTG); FTI_Protect(1, h, M * nbLines, FTI_DBLE); FTI_Protect(2, g, M * nbLines, FTI_DBLE); - int iTmp = 0; wtime = MPI_Wtime(); for (i = 0; i < ITER_TIMES; i++) { - iTmp = i; int checkpointed = FTI_Snapshot(); if (!(checkpointed != FTI_SCES || checkpointed != FTI_DONE)) { printf("%d: Snapshot failed! Returned %d.\n", rank, checkpointed); diff --git a/testing/suites/features/variateProcessorRestart/test.c b/testing/suites/features/variateProcessorRestart/test.c index ac1e24180..59a3276ed 100644 --- a/testing/suites/features/variateProcessorRestart/test.c +++ b/testing/suites/features/variateProcessorRestart/test.c @@ -95,7 +95,6 @@ int main(int argc, char **argv) { RECOVERVAR = atoi(argv[3]); int rank, grank; int size; - int size_bac; int i, j; int *ids; @@ -139,8 +138,6 @@ int main(int argc, char **argv) { MPI_Abort(MPI_COMM_WORLD, -1); } - size_bac = size; - int ldim0 = fdimX / ((int)sqrt(size)); int ldim1 = fdimY1 / ((int)sqrt(size)); @@ -295,7 +292,6 @@ int main(int argc, char **argv) { // int base = offset[0] * fdim[1] + offset[1] + 1; - int stride = 0; // init array elements with increasing integers for (i = 0; i < ldim0; ++i) { for (j = 0; j < ldim1; ++j) { @@ -349,6 +345,7 @@ int checkpoint(int id, int level, int *ids, int nids) { } else { FTI_Checkpoint(id, level); } + return 0; } int recover(int *ids, int nids) { @@ -367,6 +364,7 @@ int recover(int *ids, int nids) { } else { FTI_Recover(); } + return 0; } void shuffle(int *array, size_t n) { From cffa548d97ec43997e267102a6683b421a6160ab Mon Sep 17 00:00:00 2001 From: kellekai Date: Wed, 21 Jul 2021 13:29:31 +0200 Subject: [PATCH 04/11] works now for l3 and ftiff --- src/IO/ftiff.c | 41 +++++++++++++++++++++++++++++++++++++++++ src/IO/ftiff.h | 6 +++--- src/postckpt.c | 46 +++++++++++++++++++++------------------------- src/postreco.c | 1 + 4 files changed, 66 insertions(+), 28 deletions(-) diff --git a/src/IO/ftiff.c b/src/IO/ftiff.c index f1b27717e..eb63f6c9b 100644 --- a/src/IO/ftiff.c +++ b/src/IO/ftiff.c @@ -1348,6 +1348,8 @@ int FTIFF_CreateMetadata(FTIT_execution* FTI_Exec, FTIT_topology* FTI_Topo, // [Important for FTI_RSenc after file truncation to maxFs] mfs += sizeof(off_t); + DBG_MSG("mfs: %lu", 0, mfs); + FTI_Exec->FTIFFMeta.maxFs = mfs; FTI_Exec->FTIFFMeta.ptFs = -1; } @@ -2328,6 +2330,45 @@ int FTIFF_CheckL3RecoverInit(FTIT_execution* FTI_Exec, FTIT_topology* FTI_Topo, snprintf(FTI_Exec->ckptMeta.ckptFile, FTI_BUFS, "Ckpt%d-Rank%d.fti", FTI_Exec->ckptId, FTI_Topo->myRank); +#warning REMOVE + //char lfn[FTI_BUFS]; + + //snprintf(lfn, FTI_BUFS, "%s/%s", FTI_Ckpt[3].dir, + // FTI_Exec->ckptMeta.ckptFile); + //char str[FTI_BUFS]; + + //int ifd = open(lfn, O_RDONLY); + //if (ifd == -1) { + // snprintf(str, FTI_BUFS, + // "failed to read FTI-FF file meta data from file '%s'", lfn); + // FTI_Print(str, FTI_EROR); + // errno = 0; + // return FTI_NSCS; + //} + + //if (lseek(ifd, -sizeof(off_t), SEEK_END) == -1) { + // snprintf(str, FTI_BUFS, + // "failed to read FTI-FF file meta data from file '%s'", lfn); + // FTI_Print(str, FTI_EROR); + // errno = 0; + // close(ifd); + // return FTI_NSCS; + //} + + //off_t fs_; + //if (read(ifd, &fs_, sizeof(off_t)) == -1) { + // snprintf(str, FTI_BUFS, + // "failed to read FTI-FF file meta data from file '%s'", lfn); + // FTI_Print(str, FTI_EROR); + // errno = 0; + // close(ifd); + // return FTI_NSCS; + //} + // + //DBG_MSG("fs: %lu",-1,fs_); + + //close(ifd); +#warning REMOVE free(groupInfo); diff --git a/src/IO/ftiff.h b/src/IO/ftiff.h index 9fea2849d..7b0b48c92 100644 --- a/src/IO/ftiff.h +++ b/src/IO/ftiff.h @@ -113,9 +113,9 @@ typedef struct FTIFF_RecoveryInfo { int BackupExists; int ckptId; int rightIdx; - uint64_t maxFs; - uint64_t fs; - uint64_t bfs; + int64_t maxFs; + int64_t fs; + int64_t bfs; } FTIFF_RecoveryInfo; /** diff --git a/src/postckpt.c b/src/postckpt.c index 2a8a8204f..bef191f45 100644 --- a/src/postckpt.c +++ b/src/postckpt.c @@ -99,7 +99,7 @@ int FTI_SendCkpt(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, } char* buffer = talloc(char, FTI_Conf->blockSize); - uint64_t toSend = FTI_Exec->ckptMeta.fs; // remaining data to send + int32_t toSend = FTI_Exec->ckptMeta.fs; // remaining data to send while (toSend > 0) { int sendSize = (toSend > FTI_Conf->blockSize) ? FTI_Conf->blockSize : toSend; @@ -152,7 +152,7 @@ int FTI_RecvPtner(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, } char* buffer = talloc(char, FTI_Conf->blockSize); - uint64_t toRecv = FTI_Exec->ckptMeta.pfs; + uint32_t toRecv = FTI_Exec->ckptMeta.pfs; // remaining data to receive while (toRecv > 0) { int recvSize = (toRecv > FTI_Conf->blockSize) ? @@ -285,7 +285,7 @@ int FTI_RSenc(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, FTI_Print(str, FTI_DBUG); // all files in group must have the same size - uint64_t maxFs = FTI_Exec->ckptMeta.maxFs; // max file size in group + int32_t maxFs = FTI_Exec->ckptMeta.maxFs; // max file size in group // determine file size in order to write at the end of the elongated // file (i.e. write at the end of file after 'truncate(..., maxFs)'. @@ -298,7 +298,6 @@ int FTI_RSenc(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, FTI_Print("Error with truncate on checkpoint file", FTI_WARN); return FTI_NSCS; } - DBG_MSG("fs: %lu",-1,maxFs); // write file size at the end of elongated file to recover original // size during restart. The file size, thus, will be included in the @@ -323,7 +322,6 @@ int FTI_RSenc(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, return FTI_NSCS; } close(lftmp_); - } FILE* lfd = fopen(lfn, "rb"); @@ -359,17 +357,18 @@ int FTI_RSenc(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, int remBsize = bs; - uint64_t ps = ((maxFs / bs)) * bs; + int32_t ps = ((maxFs / bs)) * bs; if (ps < maxFs) { ps = ps + bs; } + // for MD5 checksum MD5_CTX mdContext; MD5_Init(&mdContext); // For each block - uint64_t pos = 0; + int32_t pos = 0; while (pos < ps) { if ((maxFs - pos) < bs) { remBsize = maxFs - pos; @@ -510,7 +509,7 @@ int FTI_RSenc(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, efd, "ppppf", data, matrix, coding, myData, lfd); free(buffer_ser); } - + free(data); free(matrix); free(coding); @@ -518,15 +517,12 @@ int FTI_RSenc(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, fclose(lfd); fclose(efd); - uint64_t fs = FTI_Exec->ckptMeta.fs; // ckpt file size - + int32_t fs = FTI_Exec->ckptMeta.fs; // ckpt file size + if (truncate(lfn, fs) == -1) { FTI_Print("Error with re-truncate on checkpoint file", FTI_WARN); return FTI_NSCS; } - - stat(lfn, &st_); - DBG_MSG("stat.st: %lu, fs: %lu", -1, st_.st_size, fs); int res = FTI_WriteRSedChecksum(FTI_Conf, FTI_Exec, FTI_Topo, FTI_Ckpt, rank, checksum); @@ -792,11 +788,11 @@ int FTI_FlushPosix(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, } char *readData = talloc(char, FTI_Conf->transferSize); - int bSize = FTI_Conf->transferSize; - uint64_t fs = FTI_Exec->ckptMeta.fs; - snprintf(str, FTI_BUFS, "Local file size for proc %d: %lu", proc, fs); + int32_t bSize = FTI_Conf->transferSize; + int32_t fs = FTI_Exec->ckptMeta.fs; + snprintf(str, FTI_BUFS, "Local file size for proc %d: %d", proc, fs); FTI_Print(str, FTI_DBUG); - uint64_t pos = 0; + int32_t pos = 0; // Checkpoint files exchange while (pos < fs) { if ((fs - pos) < FTI_Conf->transferSize) @@ -917,10 +913,10 @@ int FTI_FlushMPI(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, } char* readData = talloc(char, FTI_Conf->transferSize); - int bSize = FTI_Conf->transferSize; - uint64_t fs = FTI_Exec->ckptMeta.fs; + int32_t bSize = FTI_Conf->transferSize; + int32_t fs = FTI_Exec->ckptMeta.fs; - uint64_t pos = 0; + int32_t pos = 0; // Checkpoint files exchange while (pos < fs) { if ((fs - pos) < FTI_Conf->transferSize) { @@ -975,7 +971,7 @@ int FTI_FlushSionlib(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, } int nbProc = endProc - startProc; - uint64_t* localFileSizes = talloc(uint64_t, nbProc); + int32_t* localFileSizes = talloc(int32_t, nbProc); char* localFileNames = talloc(char, FTI_BUFS * nbProc); int* splitRanks = talloc(int, nbProc); // rank of process in FTI_COMM_WORLD for (proc = startProc; proc < endProc; proc++) { @@ -1081,10 +1077,10 @@ int FTI_FlushSionlib(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, } char *readData = talloc(char, FTI_Conf->transferSize); - int bSize = FTI_Conf->transferSize; - uint64_t fs = FTI_Exec->ckptMeta.fs; + int32_t bSize = FTI_Conf->transferSize; + int32_t fs = FTI_Exec->ckptMeta.fs; - uint64_t pos = 0; + int32_t pos = 0; // Checkpoint files exchange while (pos < fs) { if ((fs - pos) < FTI_Conf->transferSize) @@ -1096,7 +1092,7 @@ int FTI_FlushSionlib(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, ranks, rank_map, chunkSizes); - uint64_t data_written = sion_fwrite(readData, sizeof(char), + int32_t data_written = sion_fwrite(readData, sizeof(char), bytes, sid); if (data_written < 0) { diff --git a/src/postreco.c b/src/postreco.c index a3984acab..6a051e05e 100644 --- a/src/postreco.c +++ b/src/postreco.c @@ -134,6 +134,7 @@ int FTI_Decode(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, uint64_t ps = ((maxFs / FTI_Conf->blockSize)) * FTI_Conf->blockSize; if (ps < maxFs) { ps = ps + FTI_Conf->blockSize; // Calculating padding size + DBG_MSG("maxFs: %lu, ps: %lu", 0, maxFs, ps); } if (erased[FTI_Topo->groupRank] == 0) { // Resize and open files // determine file size in order to write at the end of the From 5d4638b48396d715844fa96dbc93295a1531aa39 Mon Sep 17 00:00:00 2001 From: kellekai Date: Wed, 21 Jul 2021 18:28:32 +0200 Subject: [PATCH 05/11] changed most of the data types to int64_t --- include/fti-intern.h | 54 ++++---- include/fti.h | 8 +- src/IO/cuda-md5/md5Opt.c | 4 +- src/IO/ftiff-dcp.c | 26 ++-- src/IO/ftiff-dcp.h | 14 +- src/IO/ftiff.c | 255 ++++++++++++++--------------------- src/IO/ftiff.h | 10 +- src/IO/hdf5-fti.c | 12 +- src/IO/hdf5-fti.h | 2 +- src/IO/ime.c | 2 - src/IO/mpio.c | 10 +- src/IO/posix-dcp.c | 90 ++++++------- src/IO/posix-dcp.h | 4 +- src/IO/posix.c | 14 +- src/IO/posix.h | 8 +- src/api-cuda.c | 18 +-- src/api-cuda.h | 16 +-- src/api.c | 40 +++--- src/checkpoint.c | 12 +- src/icp.c | 2 +- src/meta.c | 42 +++--- src/meta.h | 10 +- src/postckpt.c | 4 +- src/postreco.c | 13 +- src/recover.c | 10 +- src/recover.h | 2 +- src/util/failure-injection.h | 8 +- src/util/keymap.c | 4 +- src/util/macros.h | 4 +- src/util/tools.c | 6 +- src/util/utility.h | 26 ++-- 31 files changed, 336 insertions(+), 394 deletions(-) diff --git a/include/fti-intern.h b/include/fti-intern.h index 8b16e4006..f39012fc4 100644 --- a/include/fti-intern.h +++ b/include/fti-intern.h @@ -30,7 +30,7 @@ #endif /** Malloc macro. */ -#define talloc(type, num) (type *)malloc(sizeof(type) * (num)) +#define talloc(type, num) (type *)malloc(sizeof(type) * ((size_t)num)) #define LOCAL 0 #define GLOBAL 1 @@ -127,12 +127,12 @@ extern "C" { #ifdef ENABLE_HDF5 // --> If HDF5 is installed typedef hsize_t FTIT_hsize_t; #else - typedef uint64_t FTIT_hsize_t; + typedef int64_t FTIT_hsize_t; #endif typedef struct FTIT_dimension { int ndims; - uint64_t count[32]; + int64_t count[32]; } FTIT_dimension; typedef uintptr_t FTI_ADDRVAL; /**< for ptr manipulation */ @@ -140,13 +140,13 @@ extern "C" { typedef struct FTIT_datasetInfo { int varID; - uint64_t varSize; + int64_t varSize; } FTIT_datasetInfo; typedef struct FTIT_dcpConfigurationPosix { unsigned int digestWidth; unsigned char* (*hashFunc)(const unsigned char *data, - uint64_t nBytes, unsigned char *hash); + unsigned long nBytes, unsigned char *hash); unsigned int StackSize; uint32_t BlockSize; unsigned int cachedCkpt; @@ -156,23 +156,23 @@ extern "C" { int nbLayerReco; int nbVarReco; unsigned int Counter; - uint64_t FileSize; - uint64_t dataSize; - uint64_t dcpSize; - uint64_t LayerSize[MAX_STACK_SIZE]; + int64_t FileSize; + int64_t dataSize; + int64_t dcpSize; + int64_t LayerSize[MAX_STACK_SIZE]; FTIT_datasetInfo datasetInfo[MAX_STACK_SIZE][FTI_BUFS]; char LayerHash[MAX_STACK_SIZE*MD5_DIGEST_STRING_LENGTH]; } FTIT_dcpExecutionPosix; typedef struct FTIT_dcpDatasetPosix { - uint64_t hashDataSize; + int64_t hashDataSize; unsigned char* currentHashArray; unsigned char* oldHashArray; } FTIT_dcpDatasetPosix; typedef struct blockMetaInfo_t { - uint64_t varId : 18; - uint64_t blockId : 30; + int64_t varId : 18; + int64_t blockId : 30; } blockMetaInfo_t; /*----------------------------------------------------------------------- @@ -229,15 +229,15 @@ extern "C" { char checksum[MD5_DIGEST_STRING_LENGTH]; /**< file hash without meta */ unsigned char myHash[MD5_DIGEST_LENGTH]; /**< hash of this struct */ int ckptId; /**< Checkpoint ID */ - uint64_t metaSize; /**< size of ckpt data */ - uint64_t ckptSize; /**< also file size TODO remove */ - uint64_t dataSize; /**< size of protected data without meta data */ - uint64_t pureDataSize; /**< total data size (for printouts) */ - uint64_t fs; /**< file size */ - uint64_t maxFs; /**< maximum file size in group */ - uint64_t ptFs; /**< partner copy file size */ + int64_t metaSize; /**< size of ckpt data */ + int64_t ckptSize; /**< also file size TODO remove */ + int64_t dataSize; /**< size of protected data without meta data */ + int64_t pureDataSize; /**< total data size (for printouts) */ + int64_t fs; /**< file size */ + int64_t maxFs; /**< maximum file size in group */ + int64_t ptFs; /**< partner copy file size */ uint32_t timestamp; /**< time (ns) cp was created (CLOCK_REALTIME) */ - uint64_t dcpSize; /**< how much actually written by rank */ + int64_t dcpSize; /**< how much actually written by rank */ } FTIFF_metaInfo; /** @typedef FTIT_DataDiffHash @@ -253,7 +253,7 @@ extern "C" { uint32_t* bit32hash[2]; /**< CRC32 digest */ uint16_t* blockSize; /**< data block size */ bool* isValid; /**< indicates if data block is valid */ - uint64_t nbHashes; /**< number of hashes for the data chunk */ + int64_t nbHashes; /**< number of hashes for the data chunk */ int currentId; int creationType; int lifetime; @@ -293,7 +293,7 @@ extern "C" { */ typedef struct FTIFF_db { int numvars; /**< number of protected variables in datablock */ - uint64_t dbsize; /**< size of metadata + data for block in bytes */ + int64_t dbsize; /**< size of metadata + data for block in bytes */ unsigned char myhash[MD5_DIGEST_LENGTH]; /**< hash of variable chunk */ bool update; /**< TRUE if struct needs to be updated in ckpt file */ bool finalized; /**< TRUE if block is stored in cp file */ @@ -366,7 +366,7 @@ extern "C" { */ typedef struct FTIT_Datatype { int id; /**< ID of the data type. */ - uint64_t size; /**< Size of the data type. */ + int64_t size; /**< Size of the data type. */ FTIT_complexType* structure; /**< Logical structure for HDF5. */ FTIT_H5Group* h5group; /**< Group of this datatype. */ #ifdef ENABLE_HDF5 @@ -409,7 +409,7 @@ extern "C" { typedef struct FTIT_typeField { FTIT_Datatype *type; /**< FTI type ID of the field. */ int id; /**< Order of the field in the structure */ - uint64_t offset; /**< Offset of the field in structure. */ + int64_t offset; /**< Offset of the field in structure. */ int rank; /**< Field rank (max. 32) */ int dimLength[32]; /**< Lenght of each dimention */ char name[FTI_BUFS]; /**< Name of the field */ @@ -452,7 +452,7 @@ extern "C" { int64_t count; /**< nb of elements in dataset */ int64_t size; /**< size of the data */ int64_t sizeStored; /**< size of the data in last CP */ - uint64_t filePos; /**< offset of buffer in CP file */ + int64_t filePos; /**< offset of buffer in CP file */ FTIT_attribute attribute; FTIT_sharedData sharedData; /**< Info if dataset is subset */ FTIT_dcpDatasetPosix dcpInfoPosix; /**< dCP info for posix I/O */ @@ -522,7 +522,7 @@ extern "C" { char lTmpDir[FTI_BUFS]; /**< Local temporary directory. */ char gTmpDir[FTI_BUFS]; /**< Global temporary directory. */ char mTmpDir[FTI_BUFS]; /**< Metadata temporary directory. */ - uint64_t cHostBufSize; /**< Host buffer size for GPU data.*/ + int64_t cHostBufSize; /**< Host buffer size for GPU data.*/ char suffix[4]; /** Suffix of the checkpoint files */ FTIT_dcpConfigurationPosix dcpInfoPosix; /**< dCP info for posix I/O */ // int fastForward; /**< Fast forward rate for ckpt intervals */ @@ -618,7 +618,7 @@ extern "C" { int(*WriteData) (FTIT_dataset * , void *write_info); int(*finCKPT) (void *fileDesc); - uint64_t(*getPos) (void *fileDesc); + int64_t(*getPos) (void *fileDesc); void(*finIntegrity) (unsigned char *, void*); }FTIT_IO; diff --git a/include/fti.h b/include/fti.h index 5fb58fac3..bfb409f45 100644 --- a/include/fti.h +++ b/include/fti.h @@ -87,7 +87,7 @@ extern "C" { int FTI_InitGroup(FTIT_H5Group* h5group, const char* name, FTIT_H5Group* parent); int FTI_RenameGroup(FTIT_H5Group* h5group, const char* name); - int FTI_Protect(int id, void* ptr, uint64_t count, fti_id_t tid); + int FTI_Protect(int id, void* ptr, int64_t count, fti_id_t tid); int FTI_SetAttribute(int id, FTIT_attribute attribute, FTIT_attributeFlag flag); int FTI_DefineDataset(int id, int rank, int* dimLength, const char* name, @@ -102,7 +102,7 @@ extern "C" { int FTI_UpdateGlobalDataset(int id, int rank, FTIT_hsize_t* dimLength); int FTI_UpdateSubset(int id, int rank, FTIT_hsize_t* offset, FTIT_hsize_t* count, int did); - uint64_t FTI_GetStoredSize(int id); + int64_t FTI_GetStoredSize(int id); void* FTI_Realloc(int id, void* ptr); int FTI_BitFlip(int datasetID); int FTI_Checkpoint(int id, int level); @@ -130,9 +130,9 @@ extern "C" { fti_id_t FTI_InitCompositeType(const char* name, size_t size, FTIT_H5Group* h5g); int FTI_AddScalarField(fti_id_t id, const char* name, fti_id_t fid, - size_t offset); + int64_t offset); int FTI_AddVectorField(fti_id_t id, const char* name, fti_id_t tid, - size_t offset, int ndims, int* dim_sizes); + int64_t offset, int ndims, int* dim_sizes); #ifdef __cplusplus } diff --git a/src/IO/cuda-md5/md5Opt.c b/src/IO/cuda-md5/md5Opt.c index 0deff8b3e..a5e7718bd 100644 --- a/src/IO/cuda-md5/md5Opt.c +++ b/src/IO/cuda-md5/md5Opt.c @@ -59,7 +59,7 @@ pthread_mutex_t application; int32_t totalWork = 0; int32_t worker_exit = 0; int deviceId; -unsigned char* (*cpuHash)(const unsigned char *data, uint64_t nBytes, +unsigned char* (*cpuHash)(const unsigned char *data, unsigned long nBytes, unsigned char *hash); int32_t tempBufferSize; int32_t md5ChunkSize; @@ -98,7 +98,7 @@ int FTI_initMD5(int32_t cSize, int32_t tempSize, FTIT_configuration *FTI_Conf) { **/ /*-------------------------------------------------------------------------*/ int MD5CPU(FTIT_dataset *data) { - uint64_t dataSize = data->size; + int64_t dataSize = data->size; unsigned char block[md5ChunkSize]; size_t i; unsigned char *ptr = (unsigned char *) data->ptr; diff --git a/src/IO/ftiff-dcp.c b/src/IO/ftiff-dcp.c index a64c5e96c..4c22106e9 100644 --- a/src/IO/ftiff-dcp.c +++ b/src/IO/ftiff-dcp.c @@ -342,13 +342,13 @@ int FTI_GetDcpMode() { /** @brief Reallocate meta data related to dCP @param FTIT_DataDiffHash metadata to be reallocated. - @param uint64_t number of hashes that i need to reallocate + @param int64_t number of hashes that i need to reallocate @return integer FTI_SCES if successful. This function reallocates all the the metadata related to the dCP (isValid & blockSize); **/ /*-------------------------------------------------------------------------*/ -int FTI_ReallocateDataDiff(FTIT_DataDiffHash *dhash, uint64_t nbHashes) { +int FTI_ReallocateDataDiff(FTIT_DataDiffHash *dhash, int64_t nbHashes) { if (!dcpEnabled) return FTI_SCES; @@ -489,7 +489,7 @@ When the checkpoint will terminate the current hash will be freed, whereas the n will be used as current. Keep in mind that the next has the correct size **/ /*-------------------------------------------------------------------------*/ -int FTI_CollapseBlockHashArray(FTIT_DataDiffHash* hashes, uint64_t chunkSize) { +int FTI_CollapseBlockHashArray(FTIT_DataDiffHash* hashes, int64_t chunkSize) { if (!dcpEnabled) return FTI_SCES; @@ -498,8 +498,8 @@ int FTI_CollapseBlockHashArray(FTIT_DataDiffHash* hashes, uint64_t chunkSize) { bool changeSize = true; - uint64_t nbHashesOld = hashes->nbHashes; - uint64_t newNumber = FTI_CalcNumHashes(chunkSize); + int64_t nbHashesOld = hashes->nbHashes; + int64_t newNumber = FTI_CalcNumHashes(chunkSize); // update to new number of hashes (which might be actually unchanged) hashes->nbHashes = newNumber; @@ -554,7 +554,7 @@ When the checkpoint will terminate the current hash will be freed, whereas the n will be used as current. Keep in mind that the next has the correct size **/ /*-------------------------------------------------------------------------*/ -int FTI_ExpandBlockHashArray(FTIT_DataDiffHash* dataHash, uint64_t chunkSize) { +int FTI_ExpandBlockHashArray(FTIT_DataDiffHash* dataHash, int64_t chunkSize) { if (!dcpEnabled) return FTI_SCES; @@ -562,10 +562,10 @@ int FTI_ExpandBlockHashArray(FTIT_DataDiffHash* dataHash, uint64_t chunkSize) { return FTI_SCES; bool changeSize = true; - uint64_t nbHashesOld = dataHash->nbHashes; + int64_t nbHashesOld = dataHash->nbHashes; // - uint64_t newNumber = FTI_CalcNumHashes(chunkSize); + int64_t newNumber = FTI_CalcNumHashes(chunkSize); assert(nbHashesOld <= newNumber); if (newNumber == nbHashesOld) { @@ -615,7 +615,7 @@ int FTI_ExpandBlockHashArray(FTIT_DataDiffHash* dataHash, uint64_t chunkSize) { block size corresponding to chunkSize. **/ /*-------------------------------------------------------------------------*/ -uint64_t FTI_CalcNumHashes(uint32_t chunkSize) { +int64_t FTI_CalcNumHashes(uint32_t chunkSize) { if ((chunkSize%((uint32_t)DCP_BLOCK_SIZE)) == 0) { return chunkSize/DCP_BLOCK_SIZE; } else { @@ -681,7 +681,7 @@ void PrintDataHashInfo(FTIT_DataDiffHash* dataHash, uint32_t chunkSize, int id) It returns -1 if hashIdx is out of range. **/ /*-------------------------------------------------------------------------*/ -int FTI_HashCmp(uint64_t hashIdx, FTIFF_dbvar* dbvar, unsigned char *ptr) { +int FTI_HashCmp(int64_t hashIdx, FTIFF_dbvar* dbvar, unsigned char *ptr) { bool clean = true; uint32_t bit32hashNow = 0; unsigned char *prevHash = NULL; @@ -810,11 +810,11 @@ int FTI_UpdateDcpChanges(FTIT_execution* FTI_Exec) { dirty regions are found in which case 0 is returned. **/ /*-------------------------------------------------------------------------*/ -int FTI_ReceiveDataChunk(unsigned char** buffer_addr, uint64_t* buffer_size, - FTIFF_dbvar* dbvar, unsigned char *startAddr, uint64_t *totalBytes) { +int FTI_ReceiveDataChunk(unsigned char** buffer_addr, int64_t* buffer_size, + FTIFF_dbvar* dbvar, unsigned char *startAddr, int64_t *totalBytes) { static bool init = true; static bool reset; - static uint64_t hashIdx; + static int64_t hashIdx; unsigned char *ptr = startAddr; static int called = 0; diff --git a/src/IO/ftiff-dcp.h b/src/IO/ftiff-dcp.h index bed59cc08..658701496 100644 --- a/src/IO/ftiff-dcp.h +++ b/src/IO/ftiff-dcp.h @@ -16,14 +16,14 @@ int FTI_InitNextHashData(FTIT_DataDiffHash *hashes); int FTI_FreeDataDiff(FTIT_DataDiffHash *dhash); dcpBLK_t FTI_GetDiffBlockSize(); int FTI_GetDcpMode(); -int FTI_ReallocateDataDiff(FTIT_DataDiffHash *dhash, uint64_t nbHashes); +int FTI_ReallocateDataDiff(FTIT_DataDiffHash *dhash, int64_t nbHashes); int FTI_InitBlockHashArray(FTIFF_dbvar* dbvar); -int FTI_CollapseBlockHashArray(FTIT_DataDiffHash* hashes, uint64_t chunkSize); -int FTI_ExpandBlockHashArray(FTIT_DataDiffHash* dataHash, uint64_t chunkSize); -uint64_t FTI_CalcNumHashes(uint32_t chunkSize); -int FTI_HashCmp(uint64_t hashIdx, FTIFF_dbvar* dbvar, unsigned char *ptr); +int FTI_CollapseBlockHashArray(FTIT_DataDiffHash* hashes, int64_t chunkSize); +int FTI_ExpandBlockHashArray(FTIT_DataDiffHash* dataHash, int64_t chunkSize); +int64_t FTI_CalcNumHashes(uint32_t chunkSize); +int FTI_HashCmp(int64_t hashIdx, FTIFF_dbvar* dbvar, unsigned char *ptr); int FTI_UpdateDcpChanges(FTIT_execution* FTI_Exec); -int FTI_ReceiveDataChunk(unsigned char** buffer_addr, uint64_t* buffer_size, - FTIFF_dbvar* dbvar, unsigned char *startAddr, uint64_t *totalBytes); +int FTI_ReceiveDataChunk(unsigned char** buffer_addr, int64_t* buffer_size, + FTIFF_dbvar* dbvar, unsigned char *startAddr, int64_t *totalBytes); #endif // FTI_SRC_IO_FTIFF_DCP_H_ diff --git a/src/IO/ftiff.c b/src/IO/ftiff.c index eb63f6c9b..f7d98c921 100644 --- a/src/IO/ftiff.c +++ b/src/IO/ftiff.c @@ -86,7 +86,7 @@ int FTIFF_ReadDbFTIFF(FTIT_configuration *FTI_Conf, FTIT_execution *FTI_Exec, char strerr[FTI_BUFS]; int *varsFound = NULL; - int varCnt = 0; + unsigned int varCnt = 0; // Recovering from local for L4 case in FTI_Recover if (FTI_Exec->ckptLvel == 4) { @@ -107,7 +107,7 @@ int FTIFF_ReadDbFTIFF(FTIT_configuration *FTI_Conf, FTIT_execution *FTI_Exec, return FTI_NSCS; } - uint64_t fs = st.st_size; + int64_t fs = st.st_size; // open checkpoint file for read only int fd = open(fn, O_RDONLY, 0); @@ -120,7 +120,7 @@ int FTIFF_ReadDbFTIFF(FTIT_configuration *FTI_Conf, FTIT_execution *FTI_Exec, } // map file into memory - unsigned char* fmmap = (unsigned char*) mmap(0, fs, PROT_READ, + unsigned char* fmmap = (unsigned char*) mmap(0, (size_t)fs, PROT_READ, MAP_SHARED, fd, 0); if (fmmap == MAP_FAILED) { snprintf(strerr, FTI_BUFS, "FTI-FF: ReadDbFTIFF - could not map" @@ -144,7 +144,7 @@ int FTIFF_ReadDbFTIFF(FTIT_configuration *FTI_Conf, FTIT_execution *FTI_Exec, seek_ptr) != FTI_SCES) { FTI_Print("FTI-FF: ReadDbFTIFF - failed to deserialize" " 'FTI_Exec->FTIFFMeta'", FTI_EROR); - munmap(fmmap, fs); + munmap(fmmap, (size_t)fs); errno = 0; return FTI_NSCS; } @@ -161,7 +161,7 @@ int FTIFF_ReadDbFTIFF(FTIT_configuration *FTI_Conf, FTIT_execution *FTI_Exec, snprintf(strerr, FTI_BUFS, "FTI-FF: ReadDbFTIFF - failed to allocate" " %ld bytes for 'currentdb'", sizeof(FTIFF_db)); FTI_Print(strerr, FTI_EROR); - munmap(fmmap, fs); + munmap(fmmap,(size_t)fs); errno = 0; return FTI_NSCS; } @@ -176,7 +176,7 @@ int FTIFF_ReadDbFTIFF(FTIT_configuration *FTI_Conf, FTIT_execution *FTI_Exec, if (FTIFF_DeserializeDbMeta(currentdb, seek_ptr) != FTI_SCES) { FTI_Print("FTI-FF: ReadDbFTIFF - failed to deserialize " "'currentdb'", FTI_EROR); - munmap(fmmap, fs); + munmap(fmmap, (size_t)fs); errno = 0; return FTI_NSCS; } @@ -198,9 +198,9 @@ int FTIFF_ReadDbFTIFF(FTIT_configuration *FTI_Conf, FTIT_execution *FTI_Exec, if (currentdb->dbvars == NULL) { snprintf(strerr, FTI_BUFS, "FTI-FF: Updatedb - failed to allocate" " %ld bytes for 'currentdb->dbvars'", - sizeof(FTIFF_dbvar) * currentdb->numvars); + sizeof(FTIFF_dbvar) * (size_t)currentdb->numvars); FTI_Print(strerr, FTI_EROR); - munmap(fmmap, fs); + munmap(fmmap, (size_t)fs); errno = 0; return FTI_NSCS; } @@ -214,7 +214,7 @@ int FTIFF_ReadDbFTIFF(FTIT_configuration *FTI_Conf, FTIT_execution *FTI_Exec, seek_ptr) != FTI_SCES) { FTI_Print("FTI-FF: ReadDbFTIFF - failed to deserialize" " 'dbvar'", FTI_EROR); - munmap(fmmap, fs); + munmap(fmmap, (size_t)fs); errno = 0; return FTI_NSCS; } @@ -252,7 +252,7 @@ int FTIFF_ReadDbFTIFF(FTIT_configuration *FTI_Conf, FTIT_execution *FTI_Exec, } if (varCnt == 0) { - varsFound = realloc(varsFound, sizeof(int) * (varCnt+1)); + varsFound = realloc(varsFound, sizeof(int) * (size_t)(varCnt+1)); varsFound[varCnt++] = currentdbvar->id; } else { int i; @@ -262,7 +262,7 @@ int FTIFF_ReadDbFTIFF(FTIT_configuration *FTI_Conf, FTIT_execution *FTI_Exec, } } if (i == varCnt) { - varsFound = realloc(varsFound, sizeof(int) * (varCnt+1)); + varsFound = realloc(varsFound, sizeof(int) * (size_t)(varCnt+1)); varsFound[varCnt++] = currentdbvar->id; } } @@ -282,7 +282,7 @@ int FTIFF_ReadDbFTIFF(FTIT_configuration *FTI_Conf, FTIT_execution *FTI_Exec, snprintf(strerr, FTI_BUFS, "FTI-FF: ReadDbFTIFF - failed to " "allocate %ld bytes for 'nextdb'", sizeof(FTIFF_db)); FTI_Print(strerr, FTI_EROR); - munmap(fmmap, fs); + munmap(fmmap, (size_t)fs); errno = 0; return FTI_NSCS; } @@ -303,7 +303,7 @@ int FTIFF_ReadDbFTIFF(FTIT_configuration *FTI_Conf, FTIT_execution *FTI_Exec, FTI_Exec->lastdb->next = NULL; // unmap memory. - if (munmap(fmmap, fs) == -1) { + if (munmap(fmmap, (size_t)fs) == -1) { FTI_Print("FTI-FF: ReadDbFTIFF - unable to unmap memory", FTI_EROR); errno = 0; return FTI_NSCS; @@ -333,7 +333,7 @@ int FTIFF_GetFileChecksum(FTIFF_metaInfo *FTIFFMeta, int fd, char *checksum) { MD5_Init(&ctx); // map file into memory - unsigned char* fmmap = (unsigned char*) mmap(0, FTIFFMeta->ckptSize, + unsigned char* fmmap = (unsigned char*) mmap(0, (size_t)FTIFFMeta->ckptSize, PROT_READ, MAP_SHARED, fd, 0); if (fmmap == MAP_FAILED) { snprintf(strerr, FTI_BUFS, "FTI-FF: GetFileChecksum - could not " @@ -355,7 +355,7 @@ int FTIFF_GetFileChecksum(FTIFF_metaInfo *FTIFFMeta, int fd, char *checksum) { snprintf(strerr, FTI_BUFS, "FTI-FF: ReadDbFTIFF - failed to allocate " "%ld bytes for 'db'", sizeof(FTIFF_db)); FTI_Print(strerr, FTI_EROR); - munmap(fmmap, FTIFFMeta->ckptSize); + munmap(fmmap, (size_t)FTIFFMeta->ckptSize); errno = 0; return FTI_NSCS; } @@ -364,7 +364,7 @@ int FTIFF_GetFileChecksum(FTIFF_metaInfo *FTIFFMeta, int fd, char *checksum) { if (FTIFF_DeserializeDbMeta(db, seek_ptr) != FTI_SCES) { FTI_Print("FTI-FF: ReadDbFTIFF - failed to deserialize 'db'", FTI_EROR); - munmap(fmmap, FTIFFMeta->ckptSize); + munmap(fmmap, (size_t)FTIFFMeta->ckptSize); errno = 0; return FTI_NSCS; } @@ -374,9 +374,9 @@ int FTIFF_GetFileChecksum(FTIFF_metaInfo *FTIFFMeta, int fd, char *checksum) { FTIFF_dbvar *dbvars = talloc(FTIFF_dbvar, db->numvars); if (dbvars == NULL) { snprintf(strerr, FTI_BUFS, "FTI-FF: Updatedb - failed to allocate" - " %ld bytes for 'dbvars'", sizeof(FTIFF_dbvar) * db->numvars); + " %lu bytes for 'dbvars'", sizeof(FTIFF_dbvar) * (size_t)db->numvars); FTI_Print(strerr, FTI_EROR); - munmap(fmmap, FTIFFMeta->ckptSize); + munmap(fmmap, (size_t)FTIFFMeta->ckptSize); errno = 0; return FTI_NSCS; } @@ -389,7 +389,7 @@ int FTIFF_GetFileChecksum(FTIFF_metaInfo *FTIFFMeta, int fd, char *checksum) { if (FTIFF_DeserializeDbVarMeta(dbvar, seek_ptr) != FTI_SCES) { FTI_Print("FTI-FF: ReadDbFTIFF - failed to deserialize " "'dbvar'", FTI_EROR); - munmap(fmmap, FTIFFMeta->ckptSize); + munmap(fmmap, (size_t)FTIFFMeta->ckptSize); errno = 0; return FTI_NSCS; } @@ -401,7 +401,7 @@ int FTIFF_GetFileChecksum(FTIFF_metaInfo *FTIFFMeta, int fd, char *checksum) { // (Note: we create the file hash from the chunk hashes due to ICP) if (dbvar->hascontent) { unsigned char chash[MD5_DIGEST_LENGTH]; - MD5(fmmap + dbvar->fptr, dbvar->chunksize, chash); + MD5(fmmap + dbvar->fptr, (unsigned long)dbvar->chunksize, chash); MD5_Update(&ctx, chash, MD5_DIGEST_LENGTH); } } @@ -419,7 +419,7 @@ int FTIFF_GetFileChecksum(FTIFF_metaInfo *FTIFFMeta, int fd, char *checksum) { } // unmap memory. - if (munmap(fmmap, FTIFFMeta->ckptSize) == -1) { + if (munmap(fmmap, (size_t)FTIFFMeta->ckptSize) == -1) { FTI_Print("FTI-FF: ReadDbFTIFF - unable to unmap memory", FTI_EROR); errno = 0; return FTI_NSCS; @@ -531,7 +531,7 @@ int FTIFF_UpdateDatastructVarFTIFF(FTIT_execution* FTI_Exec, // init overflow with the datasizes and validBlock with true. bool validBlock = true; - uint64_t overflow = data->size; + int64_t overflow = data->size; // iterate though datablock list. Current datablock is 'lastdb'. // At the beginning of the loop 'lastdb = firstdb' @@ -568,7 +568,7 @@ int FTIFF_UpdateDatastructVarFTIFF(FTIT_execution* FTI_Exec, // set chunksize to containersize and ensure that // 'hascontent = true'. if (overflow > dbvar->containersize) { - uint64_t chunksizeOld = dbvar->chunksize; + int64_t chunksizeOld = dbvar->chunksize; dbvar->chunksize = dbvar->containersize; dbvar->cptr = data->ptr + dbvar->dptr; if (!dbvar->hascontent) { @@ -598,7 +598,7 @@ int FTIFF_UpdateDatastructVarFTIFF(FTIT_execution* FTI_Exec, // afterwards overflow to 0 and // ensure that 'hascontent = true'. if (overflow <= dbvar->containersize) { - uint64_t chunksizeOld = dbvar->chunksize; + int64_t chunksizeOld = dbvar->chunksize; dbvar->chunksize = overflow; dbvar->cptr = data->ptr + dbvar->dptr; if (!dbvar->hascontent) { @@ -629,7 +629,7 @@ int FTIFF_UpdateDatastructVarFTIFF(FTIT_execution* FTI_Exec, } } } - offset += FTI_Exec->lastdb->dbsize; + offset += (uint64_t)FTI_Exec->lastdb->dbsize; if (FTI_Exec->lastdb->next) { FTI_Exec->lastdb = FTI_Exec->lastdb->next; isnextdb = 1; @@ -674,12 +674,12 @@ int FTIFF_UpdateDatastructVarFTIFF(FTIT_execution* FTI_Exec, } int evar_idx = dblock->numvars; - uint64_t dbsize = dblock->dbsize; + int64_t dbsize = dblock->dbsize; switch (editflags) { case 1: // add new protected variable in next datablock dbvars = (FTIFF_dbvar*) realloc(dblock->dbvars, - (evar_idx+1) * sizeof(FTIFF_dbvar)); + (size_t)(evar_idx+1) * sizeof(FTIFF_dbvar)); dbvars[evar_idx].fptr = offset; dbvars[evar_idx].dptr = 0; dbvars[evar_idx].id = data->id; @@ -705,7 +705,7 @@ int FTIFF_UpdateDatastructVarFTIFF(FTIT_execution* FTI_Exec, // create data chunk info dbvars = (FTIFF_dbvar*) realloc(dblock->dbvars, - (evar_idx+1) * sizeof(FTIFF_dbvar)); + (size_t)(evar_idx+1) * sizeof(FTIFF_dbvar)); dbvars[evar_idx].fptr = offset; dbvars[evar_idx].dptr = containerSizesAccu; dbvars[evar_idx].id = data->id; @@ -760,20 +760,20 @@ int FTIFF_UpdateDatastructVarFTIFF(FTIT_execution* FTI_Exec, **/ /*-------------------------------------------------------------------------*/ int FTI_WriteMemFTIFFChunk(FTIT_execution *FTI_Exec, FTIFF_dbvar *currentdbvar, - unsigned char *dptr, uint64_t currentOffset, uint64_t fetchedBytes, - uint64_t *dcpSize, WriteFTIFFInfo_t *fd) { + unsigned char *dptr, int64_t currentOffset, int64_t fetchedBytes, + int64_t *dcpSize, WriteFTIFFInfo_t *fd) { unsigned char *chunk_addr = NULL; - uint64_t chunk_size, chunk_offset; - uint64_t remainingBytes = fetchedBytes; + int64_t chunk_size, chunk_offset; + int64_t remainingBytes = fetchedBytes; chunk_size = 0; chunk_offset = 0; - uint64_t membs = 1024*1024*16; // 16 MB - uint64_t cpybuf, cpynow, cpycnt; //, fptr; + int64_t membs = 1024*1024*16; // 16 MB + int64_t cpybuf, cpynow, cpycnt; //, fptr; uintptr_t fptr = currentdbvar-> fptr + currentOffset; uintptr_t fptrTemp = fptr; - uint64_t prevRemBytes = remainingBytes; + int64_t prevRemBytes = remainingBytes; while (FTI_ReceiveDataChunk(&chunk_addr, &chunk_size, currentdbvar, dptr, &remainingBytes)) { @@ -820,13 +820,13 @@ int FTI_WriteMemFTIFFChunk(FTIT_execution *FTI_Exec, FTIFF_dbvar *currentdbvar, /*-------------------------------------------------------------------------*/ int FTI_ProcessDBVar(FTIT_execution *FTI_Exec, FTIT_configuration *FTI_Conf, FTIFF_dbvar *currentdbvar, FTIT_dataset *data, unsigned char *hashchk, - WriteFTIFFInfo_t *fd, uint64_t *dcpSize, unsigned char **dptr) { + WriteFTIFFInfo_t *fd, int64_t *dcpSize, unsigned char **dptr) { bool hascontent = currentdbvar->hascontent; unsigned char *cbasePtr = NULL; errno = 0; - uint64_t totalBytes; + int64_t totalBytes; MD5_CTX dbContext; if (hascontent) { @@ -834,7 +834,7 @@ int FTI_ProcessDBVar(FTIT_execution *FTI_Exec, FTIT_configuration *FTI_Conf, FTI_InitNextHashData(currentdbvar->dataDiffHash); - uint64_t offset = 0; + int64_t offset = 0; totalBytes = 0; FTIT_data_prefetch prefetcher; MD5_Init(&dbContext); @@ -843,7 +843,7 @@ int FTI_ProcessDBVar(FTIT_execution *FTI_Exec, FTIT_configuration *FTI_Conf, // I nitialize prefetcher to get data from device #ifdef GPUSUPPORT - uint64_t DCP_BLOCK_SIZE = FTI_GetDiffBlockSize(); + int64_t DCP_BLOCK_SIZE = FTI_GetDiffBlockSize(); prefetcher.fetchSize = ((FTI_Conf->cHostBufSize) / DCP_BLOCK_SIZE) * DCP_BLOCK_SIZE; #else @@ -1024,9 +1024,9 @@ int FTI_WriteFtiffData(FTIT_dataset* data, void *fd) { unsigned char *dptr; int dbvar_idx, dbcounter = 0; int isnextdb; - uint64_t dcpSize = 0; - uint64_t dataSize = 0; - uint64_t pureDataSize = 0; + int64_t dcpSize = 0; + int64_t dataSize = 0; + int64_t pureDataSize = 0; FTIFF_UpdateDatastructVarFTIFF(write_info->FTI_Exec, data, write_info->FTI_Conf); @@ -1133,7 +1133,7 @@ int FTI_FinalizeFtiff(void *fd) { @return integer 0. **/ /*-------------------------------------------------------------------------*/ -uint64_t FTI_DummyFilePos(void *ignore) { +int64_t FTI_DummyFilePos(void *ignore) { return 0; } @@ -1201,12 +1201,8 @@ int FTIFF_finalizeDatastructFTIFF(FTIT_execution* FTI_Exec) { return FTI_NSCS; } - uint64_t metaSize = FTI_filemetastructsize; + int64_t metaSize = FTI_filemetastructsize; - DBG_MSG("buffersize (FTI_filemetastructsize): %lu", 0, FTI_filemetastructsize); - DBG_MSG("buffersize (FTI_dbstructsize): %lu", 0, FTI_dbstructsize); - DBG_MSG("buffersize (FTI_dbvarstructsize): %lu", 0, db->numvars * FTI_dbvarstructsize); - do { db->finalized = true; @@ -1246,8 +1242,6 @@ int FTIFF_writeMetaDataFTIFF(FTIT_execution* FTI_Exec, WriteFTIFFInfo_t *fd) { FTI_ADDRVAL mbuf_pos = (FTI_ADDRVAL) mbuf; - DBG_MSG("FTI_Exec->FTIFFMeta.metaSize: %lu",-1,FTI_Exec->FTIFFMeta.metaSize); - MD5_CTX ctx; MD5_Init(&ctx); @@ -1314,13 +1308,13 @@ int FTIFF_CreateMetadata(FTIT_execution* FTI_Exec, FTIT_topology* FTI_Topo, FTI_Exec->ckptSize = FTI_Exec->FTIFFMeta.metaSize + FTI_Exec->FTIFFMeta.dataSize; - uint64_t fs = FTI_Exec->ckptSize; + int64_t fs = FTI_Exec->ckptSize; FTI_Exec->FTIFFMeta.ckptSize = fs; FTI_Exec->FTIFFMeta.fs = fs; // allgather not needed for L1 checkpoint if ((FTI_Exec->ckptMeta.level == 2) || (FTI_Exec->ckptMeta.level == 3)) { - uint64_t fileSizes[FTI_BUFS], mfs = 0; + int64_t fileSizes[FTI_BUFS], mfs = 0; MPI_Allgather(&fs, 1, MPI_UINT64_T, fileSizes, 1, MPI_UINT64_T, FTI_Exec->groupComm); int ptnerGroupRank, i; @@ -1348,8 +1342,6 @@ int FTIFF_CreateMetadata(FTIT_execution* FTI_Exec, FTIT_topology* FTI_Topo, // [Important for FTI_RSenc after file truncation to maxFs] mfs += sizeof(off_t); - DBG_MSG("mfs: %lu", 0, mfs); - FTI_Exec->FTIFFMeta.maxFs = mfs; FTI_Exec->FTIFFMeta.ptFs = -1; } @@ -1460,8 +1452,8 @@ int FTIFF_Recover(FTIT_execution *FTI_Exec, FTIT_keymap *FTI_Data, } // block size for memcpy of pointer. - uint64_t membs = 1024*1024*16; // 16 MB - uint64_t cpybuf, cpynow, cpycnt; + int64_t membs = 1024*1024*16; // 16 MB + int64_t cpybuf, cpynow, cpycnt; // open checkpoint file for read only int fd = open(fn, O_RDONLY, 0); @@ -1591,7 +1583,7 @@ int FTIFF_Recover(FTIT_execution *FTI_Exec, FTIT_keymap *FTI_Data, " (%s!=%s).", currentdbvar->id, currentdbvar->containerid, checkSum, checkSum_struct); FTI_Print(strerr, FTI_WARN); - if (munmap(fmmap, st.st_size) == -1) { + if (munmap(fmmap, (size_t)st.st_size) == -1) { FTI_Print("FTIFF: FTIFF_Recover - unable to unmap memory", FTI_EROR); errno = 0; @@ -1610,7 +1602,7 @@ int FTIFF_Recover(FTIT_execution *FTI_Exec, FTIT_keymap *FTI_Data, FTI_device_sync(); // unmap memory - if (munmap(fmmap, st.st_size) == -1) { + if (munmap(fmmap, (size_t)st.st_size) == -1) { FTI_Print("FTIFF: FTIFF_Recover - unable to unmap memory", FTI_EROR); errno = 0; return FTI_NREC; @@ -1675,8 +1667,8 @@ int FTIFF_RecoverVar(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, int dbvar_idx, dbcounter = 0; // block size for memcpy of pointer. - uint64_t membs = 1024*1024*16; // 16 MB - uint64_t cpybuf, cpynow, cpycnt; + int64_t membs = 1024*1024*16; // 16 MB + int64_t cpybuf, cpynow, cpycnt; // MD5 context for checksum of data chunks MD5_CTX mdContext; @@ -1746,7 +1738,7 @@ int FTIFF_RecoverVar(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, "FTIFF: FTIFF_RecoverVar - dataset with id:%i has been " "corrupted! Discard recovery.", currentdbvar->id); FTI_Print(strerr, FTI_WARN); - if (munmap(filemmap, filestats.st_size) == -1) { + if (munmap(filemmap, (size_t)filestats.st_size) == -1) { FTI_Print("FTIFF: FTIFF_RecoverVar - unable to" " unmap memory", FTI_EROR); errno = 0; @@ -1775,7 +1767,7 @@ int FTIFF_RecoverVar(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, /*-------------------------------------------------------------------------*/ int FTIFF_RecoverVarFinalize() { int res = FTI_NREC; - if (munmap(filemmap, filestats.st_size) == -1) { + if (munmap(filemmap, (size_t)filestats.st_size) == -1) { FTI_Print("Could not close FTI checkpoint file.", FTI_EROR); } else { res = FTI_SCES; @@ -1973,7 +1965,7 @@ int FTIFF_LoadMetaPostprocessing(FTIT_execution* FTI_Exec, /*-------------------------------------------------------------------------*/ int FTIFF_GetEncodedFileChecksum(FTIFF_metaInfo *FTIFFMeta, int fd, char *checksum) { - uint64_t rcount = 0, toRead, diff; + int64_t rcount = 0, toRead, diff; int rbuffer; char buffer[CHUNK_SIZE], strerr[FTI_BUFS]; MD5_CTX mdContext; @@ -2298,7 +2290,7 @@ int FTIFF_CheckL3RecoverInit(FTIT_execution* FTI_Exec, FTIT_topology* FTI_Topo, // check if recovery possible int i, saneCkptID = 0, saneMaxFs = 0, erasures = 0; - uint64_t maxFs = 0; + int64_t maxFs = 0; ckptId = 0; for (i = 0; i < FTI_Topo->groupSize; i++) { erased[i]=!groupInfo[i].FileExists; @@ -2330,45 +2322,6 @@ int FTIFF_CheckL3RecoverInit(FTIT_execution* FTI_Exec, FTIT_topology* FTI_Topo, snprintf(FTI_Exec->ckptMeta.ckptFile, FTI_BUFS, "Ckpt%d-Rank%d.fti", FTI_Exec->ckptId, FTI_Topo->myRank); -#warning REMOVE - //char lfn[FTI_BUFS]; - - //snprintf(lfn, FTI_BUFS, "%s/%s", FTI_Ckpt[3].dir, - // FTI_Exec->ckptMeta.ckptFile); - //char str[FTI_BUFS]; - - //int ifd = open(lfn, O_RDONLY); - //if (ifd == -1) { - // snprintf(str, FTI_BUFS, - // "failed to read FTI-FF file meta data from file '%s'", lfn); - // FTI_Print(str, FTI_EROR); - // errno = 0; - // return FTI_NSCS; - //} - - //if (lseek(ifd, -sizeof(off_t), SEEK_END) == -1) { - // snprintf(str, FTI_BUFS, - // "failed to read FTI-FF file meta data from file '%s'", lfn); - // FTI_Print(str, FTI_EROR); - // errno = 0; - // close(ifd); - // return FTI_NSCS; - //} - - //off_t fs_; - //if (read(ifd, &fs_, sizeof(off_t)) == -1) { - // snprintf(str, FTI_BUFS, - // "failed to read FTI-FF file meta data from file '%s'", lfn); - // FTI_Print(str, FTI_EROR); - // errno = 0; - // close(ifd); - // return FTI_NSCS; - //} - // - //DBG_MSG("fs: %lu",-1,fs_); - - //close(ifd); -#warning REMOVE free(groupInfo); @@ -2437,7 +2390,7 @@ void FTIFF_SetHashChunk(FTIFF_dbvar *dbvar, FTIT_keymap* FTI_Data) { } void * ptr = data->ptr + dbvar->dptr; - uint64_t size = dbvar->chunksize; + int64_t size = dbvar->chunksize; MD5(ptr, size, dbvar->hash); } } @@ -2454,12 +2407,12 @@ void FTIFF_GetHashMetaInfo(unsigned char *hash, FTIFF_metaInfo *FTIFFMeta) { MD5_Init(&md5Ctx); MD5_Update(&md5Ctx, FTIFFMeta->checksum, MD5_DIGEST_STRING_LENGTH); MD5_Update(&md5Ctx, &(FTIFFMeta->timestamp), sizeof(uint32_t)); - MD5_Update(&md5Ctx, &(FTIFFMeta->ckptSize), sizeof(uint64_t)); - MD5_Update(&md5Ctx, &(FTIFFMeta->metaSize), sizeof(uint64_t)); - MD5_Update(&md5Ctx, &(FTIFFMeta->dataSize), sizeof(uint64_t)); - MD5_Update(&md5Ctx, &(FTIFFMeta->fs), sizeof(uint64_t)); - MD5_Update(&md5Ctx, &(FTIFFMeta->ptFs), sizeof(uint64_t)); - MD5_Update(&md5Ctx, &(FTIFFMeta->maxFs), sizeof(uint64_t)); + MD5_Update(&md5Ctx, &(FTIFFMeta->ckptSize), sizeof(int64_t)); + MD5_Update(&md5Ctx, &(FTIFFMeta->metaSize), sizeof(int64_t)); + MD5_Update(&md5Ctx, &(FTIFFMeta->dataSize), sizeof(int64_t)); + MD5_Update(&md5Ctx, &(FTIFFMeta->fs), sizeof(int64_t)); + MD5_Update(&md5Ctx, &(FTIFFMeta->ptFs), sizeof(int64_t)); + MD5_Update(&md5Ctx, &(FTIFFMeta->maxFs), sizeof(int64_t)); MD5_Final(hash, &md5Ctx); } @@ -2474,7 +2427,7 @@ void FTIFF_GetHashdb(unsigned char *hash, FTIFF_db *db) { MD5_CTX md5Ctx; MD5_Init(&md5Ctx); MD5_Update(&md5Ctx, &(db->numvars), sizeof(int)); - MD5_Update(&md5Ctx, &(db->dbsize), sizeof(uint64_t)); + MD5_Update(&md5Ctx, &(db->dbsize), sizeof(int64_t)); MD5_Final(hash, &md5Ctx); } @@ -2494,8 +2447,8 @@ void FTIFF_GetHashdbvar(unsigned char *hash, FTIFF_dbvar *dbvar) { MD5_Update(&md5Ctx, &(dbvar->hasCkpt), sizeof(bool)); MD5_Update(&md5Ctx, &(dbvar->dptr), sizeof(uintptr_t)); MD5_Update(&md5Ctx, &(dbvar->fptr), sizeof(uintptr_t)); - MD5_Update(&md5Ctx, &(dbvar->chunksize), sizeof(uint64_t)); - MD5_Update(&md5Ctx, &(dbvar->containersize), sizeof(uint64_t)); + MD5_Update(&md5Ctx, &(dbvar->chunksize), sizeof(int64_t)); + MD5_Update(&md5Ctx, &(dbvar->containersize), sizeof(int64_t)); MD5_Update(&md5Ctx, dbvar->hash, MD5_DIGEST_LENGTH); MD5_Final(hash, &md5Ctx); } @@ -2590,18 +2543,18 @@ int FTIFF_DeserializeFileMeta(FTIFF_metaInfo* meta, char* buffer_ser) { pos += MD5_DIGEST_LENGTH; memcpy(&(meta->ckptId) , buffer_ser + pos, sizeof(int)); pos += sizeof(int); - memcpy(&(meta->ckptSize) , buffer_ser + pos, sizeof(uint64_t)); - pos += sizeof(uint64_t); - memcpy(&(meta->metaSize) , buffer_ser + pos, sizeof(uint64_t)); - pos += sizeof(uint64_t); - memcpy(&(meta->dataSize) , buffer_ser + pos, sizeof(uint64_t)); - pos += sizeof(uint64_t); - memcpy(&(meta->fs) , buffer_ser + pos, sizeof(uint64_t)); - pos += sizeof(uint64_t); - memcpy(&(meta->maxFs) , buffer_ser + pos, sizeof(uint64_t)); - pos += sizeof(uint64_t); - memcpy(&(meta->ptFs) , buffer_ser + pos, sizeof(uint64_t)); - pos += sizeof(uint64_t); + memcpy(&(meta->ckptSize) , buffer_ser + pos, sizeof(int64_t)); + pos += sizeof(int64_t); + memcpy(&(meta->metaSize) , buffer_ser + pos, sizeof(int64_t)); + pos += sizeof(int64_t); + memcpy(&(meta->dataSize) , buffer_ser + pos, sizeof(int64_t)); + pos += sizeof(int64_t); + memcpy(&(meta->fs) , buffer_ser + pos, sizeof(int64_t)); + pos += sizeof(int64_t); + memcpy(&(meta->maxFs) , buffer_ser + pos, sizeof(int64_t)); + pos += sizeof(int64_t); + memcpy(&(meta->ptFs) , buffer_ser + pos, sizeof(int64_t)); + pos += sizeof(int64_t); memcpy(&(meta->timestamp) , buffer_ser + pos, sizeof(uint32_t)); return FTI_SCES; @@ -2623,7 +2576,7 @@ int FTIFF_DeserializeDbMeta(FTIFF_db* db, char* buffer_ser) { int pos = 0; memcpy(&(db->numvars) , buffer_ser + pos, sizeof(int)); pos += sizeof(int); - memcpy(&(db->dbsize) , buffer_ser + pos, sizeof(uint64_t)); + memcpy(&(db->dbsize) , buffer_ser + pos, sizeof(int64_t)); return FTI_SCES; } @@ -2654,10 +2607,10 @@ int FTIFF_DeserializeDbVarMeta(FTIFF_dbvar* dbvar, char* buffer_ser) { pos += sizeof(uintptr_t); memcpy(&(dbvar->fptr) , buffer_ser + pos, sizeof(uintptr_t)); pos += sizeof(uintptr_t); - memcpy(&(dbvar->chunksize) , buffer_ser + pos, sizeof(uint64_t)); - pos += sizeof(uint64_t); - memcpy(&(dbvar->containersize) , buffer_ser + pos, sizeof(uint64_t)); - pos += sizeof(uint64_t); + memcpy(&(dbvar->chunksize) , buffer_ser + pos, sizeof(int64_t)); + pos += sizeof(int64_t); + memcpy(&(dbvar->containersize) , buffer_ser + pos, sizeof(int64_t)); + pos += sizeof(int64_t); memcpy(dbvar->hash , buffer_ser + pos, MD5_DIGEST_LENGTH); return FTI_SCES; @@ -2683,22 +2636,20 @@ int FTIFF_SerializeFileMeta(FTIFF_metaInfo* meta, char* buffer_ser) { pos += MD5_DIGEST_LENGTH; memcpy(buffer_ser + pos, &(meta->ckptId) , sizeof(int)); pos += sizeof(int); - memcpy(buffer_ser + pos, &(meta->ckptSize) , sizeof(uint64_t)); - pos += sizeof(uint64_t); - memcpy(buffer_ser + pos, &(meta->metaSize) , sizeof(uint64_t)); - pos += sizeof(uint64_t); - memcpy(buffer_ser + pos, &(meta->dataSize) , sizeof(uint64_t)); - pos += sizeof(uint64_t); - memcpy(buffer_ser + pos, &(meta->fs) , sizeof(uint64_t)); - pos += sizeof(uint64_t); - memcpy(buffer_ser + pos, &(meta->maxFs) , sizeof(uint64_t)); - pos += sizeof(uint64_t); - memcpy(buffer_ser + pos, &(meta->ptFs) , sizeof(uint64_t)); - pos += sizeof(uint64_t); + memcpy(buffer_ser + pos, &(meta->ckptSize) , sizeof(int64_t)); + pos += sizeof(int64_t); + memcpy(buffer_ser + pos, &(meta->metaSize) , sizeof(int64_t)); + pos += sizeof(int64_t); + memcpy(buffer_ser + pos, &(meta->dataSize) , sizeof(int64_t)); + pos += sizeof(int64_t); + memcpy(buffer_ser + pos, &(meta->fs) , sizeof(int64_t)); + pos += sizeof(int64_t); + memcpy(buffer_ser + pos, &(meta->maxFs) , sizeof(int64_t)); + pos += sizeof(int64_t); + memcpy(buffer_ser + pos, &(meta->ptFs) , sizeof(int64_t)); + pos += sizeof(int64_t); memcpy(buffer_ser + pos, &(meta->timestamp) , sizeof(uint32_t)); - DBG_MSG("metasize (FTIFF_SerializeFileMeta): %d", 0, pos + sizeof(uint32_t)); - return FTI_SCES; } @@ -2718,10 +2669,8 @@ int FTIFF_SerializeDbMeta(FTIFF_db* db, char* buffer_ser) { int pos = 0; memcpy(buffer_ser + pos, &(db->numvars) , sizeof(int)); pos += sizeof(int); - memcpy(buffer_ser + pos, &(db->dbsize) , sizeof(uint64_t)); + memcpy(buffer_ser + pos, &(db->dbsize) , sizeof(int64_t)); - DBG_MSG("metasize (FTIFF_SerializeDbMeta): %d", 0, pos + sizeof(uint64_t)); - return FTI_SCES; } @@ -2738,7 +2687,7 @@ int FTIFF_SerializeDbVarMeta(FTIFF_dbvar* dbvar, char* buffer_ser) { return FTI_NSCS; } - int pos = 0; + size_t pos = 0; memcpy(buffer_ser + pos, &(dbvar->id) , sizeof(int)); pos += sizeof(int); memcpy(buffer_ser + pos, &(dbvar->containerid) , sizeof(int)); @@ -2751,14 +2700,12 @@ int FTIFF_SerializeDbVarMeta(FTIFF_dbvar* dbvar, char* buffer_ser) { pos += sizeof(uintptr_t); memcpy(buffer_ser + pos, &(dbvar->fptr) , sizeof(uintptr_t)); pos += sizeof(uintptr_t); - memcpy(buffer_ser + pos, &(dbvar->chunksize) , sizeof(uint64_t)); - pos += sizeof(uint64_t); - memcpy(buffer_ser + pos, &(dbvar->containersize) , sizeof(uint64_t)); - pos += sizeof(uint64_t); + memcpy(buffer_ser + pos, &(dbvar->chunksize) , sizeof(int64_t)); + pos += sizeof(int64_t); + memcpy(buffer_ser + pos, &(dbvar->containersize) , sizeof(int64_t)); + pos += sizeof(int64_t); memcpy(buffer_ser + pos, dbvar->hash , MD5_DIGEST_LENGTH); - DBG_MSG("metasize (FTIFF_SerializeDbVarMeta): %d", 0, pos + MD5_DIGEST_LENGTH); - return FTI_SCES; } diff --git a/src/IO/ftiff.h b/src/IO/ftiff.h index 7b0b48c92..89c154b27 100644 --- a/src/IO/ftiff.h +++ b/src/IO/ftiff.h @@ -96,9 +96,9 @@ typedef struct FTIFF_headInfo { int exists; int nbVar; char ckptFile[FTI_BUFS]; - uint64_t maxFs; - uint64_t fs; - uint64_t pfs; + int64_t maxFs; + int64_t fs; + int64_t pfs; int isDcp; } FTIFF_headInfo; @@ -152,7 +152,7 @@ typedef struct FTIFF_MPITypeInfo { **/ int FTI_FinalizeFtiff(void *fd); -size_t FTI_DummyFilePos(void *ignore); +int64_t FTI_DummyFilePos(void *ignore); int FTI_WriteFtiffData(FTIT_dataset* data, void *fd); void* FTI_InitFtiff(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, FTIT_topology* FTI_Topo, FTIT_checkpoint* FTI_Ckpt, @@ -206,7 +206,7 @@ void FTIFF_SetHashChunk(FTIFF_dbvar *dbvar, FTIT_keymap* FTI_Data); void FTIFF_PrintDataStructure(int rank, FTIT_execution* FTI_Exec); int FTI_ProcessDBVar(FTIT_execution *FTI_Exec, FTIT_configuration *FTI_Conf, FTIFF_dbvar *currentdbvar, FTIT_dataset *data, unsigned char *hashchk, - WriteFTIFFInfo_t *fd, uint64_t *dcpSize, unsigned char **dptr); + WriteFTIFFInfo_t *fd, int64_t *dcpSize, unsigned char **dptr); int FTIFF_RecoverVarInit(char* fn); int FTIFF_RecoverVarFinalize(); #endif // FTI_SRC_IO_FTIFF_H_ diff --git a/src/IO/hdf5-fti.c b/src/IO/hdf5-fti.c index 5d55a16cc..924a288d9 100644 --- a/src/IO/hdf5-fti.c +++ b/src/IO/hdf5-fti.c @@ -396,7 +396,7 @@ int FTI_ScanGroup(hid_t gid, char* fn) { @return integer The position in the file. **/ /*-------------------------------------------------------------------------*/ -uint64_t FTI_GetHDF5FilePos(void *fileDesc) { +int64_t FTI_GetHDF5FilePos(void *fileDesc) { return 0; } @@ -492,7 +492,7 @@ int FTI_CommitDataType(FTIT_execution *FTI_Exec, FTIT_dataset *data) { **/ /*-------------------------------------------------------------------------*/ -int FTI_CheckHDF5File(char* fn, uint64_t fs, char* checksum) { +int FTI_CheckHDF5File(char* fn, int64_t fs, char* checksum) { char str[FTI_BUFS]; if (access(fn, F_OK) == 0) { struct stat fileStatus; @@ -544,7 +544,7 @@ hsize_t FTI_calculateCountDim(size_t sizeOfElement, hsize_t maxBytes, hsize_t *count, int numOfDimensions, hsize_t *dimensions, hsize_t *sep) { int i; memset(count, 0, sizeof(hsize_t)*numOfDimensions); - uint64_t maxElements = maxBytes/sizeOfElement; + int64_t maxElements = maxBytes/sizeOfElement; hsize_t bytesToFetch; if (maxElements == 0) @@ -842,7 +842,7 @@ int FTI_WriteHDF5Var(FTIT_dataset *data, FTIT_execution* FTI_Exec) { prefetcher.totalBytesToFetch = data->size; prefetcher.isDevice = data->isDevicePtr; prefetcher.dptr = data->devicePtr; - uint64_t bytesToWrite; + int64_t bytesToWrite; FTI_InitPrefetcher(&prefetcher); unsigned char *basePtr = NULL; @@ -969,8 +969,8 @@ int FTI_ReadHDF5Var(FTIT_dataset *data) { hsize_t seperator; - uint64_t fetchBytes; - uint64_t hostBufSize = FTI_getHostBuffSize(); + int64_t fetchBytes; + int64_t hostBufSize = FTI_getHostBuffSize(); // Calculate How many dimension I can compute each time // and how bug should the HOST-GPU communication buffer should be diff --git a/src/IO/hdf5-fti.h b/src/IO/hdf5-fti.h index abd2d69df..495e4bc4f 100644 --- a/src/IO/hdf5-fti.h +++ b/src/IO/hdf5-fti.h @@ -29,7 +29,7 @@ int FTI_ReadHDF5Var(FTIT_dataset *data); int FTI_GetDatasetRankReco(hid_t did); int FTI_GetDatasetSpanReco(hid_t did, hsize_t * span); int FTI_WriteHDF5Var(FTIT_dataset *data, FTIT_execution* FTI_Exec); -int FTI_CheckHDF5File(char* fn, uint64_t fs, char* checksum); +int FTI_CheckHDF5File(char* fn, int64_t fs, char* checksum); int FTI_OpenGlobalDatasets(FTIT_execution* FTI_Exec); herr_t FTI_ReadSharedFileData(FTIT_dataset FTI_Data); int FTI_H5CheckSingleFile(FTIT_configuration* FTI_Conf, int * ckptID); diff --git a/src/IO/ime.c b/src/IO/ime.c index 7801f6c1d..88cb256a0 100644 --- a/src/IO/ime.c +++ b/src/IO/ime.c @@ -212,8 +212,6 @@ void* FTI_InitIME(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, FTI_Exec->ckptMeta.ckptFile); } - DBG_MSG("WRITING WITH IME: fn -> '%s'", -1, fn); - write_info->flag = O_WRONLY; write_info->offset = 0; FTI_IMEOpen(fn, write_info); diff --git a/src/IO/mpio.c b/src/IO/mpio.c index 744f9ab43..8376e6052 100644 --- a/src/IO/mpio.c +++ b/src/IO/mpio.c @@ -122,10 +122,10 @@ int FTI_MPIOClose(void *fileDesc) { **/ /*-------------------------------------------------------------------------*/ -int FTI_MPIOWrite(void *src, size_t size, void *fileDesc) { +int FTI_MPIOWrite(void *src, int64_t size, void *fileDesc) { WriteMPIInfo_t *fd = (WriteMPIInfo_t *)fileDesc; - size_t pos = 0; - size_t bSize = fd->FTI_Conf->transferSize; + int64_t pos = 0; + int64_t bSize = fd->FTI_Conf->transferSize; while (pos < size) { if ((size - pos) < fd->FTI_Conf->transferSize) { bSize = size - pos; @@ -165,7 +165,7 @@ int FTI_MPIOWrite(void *src, size_t size, void *fileDesc) { @return integer The position in virtual local file. **/ /*-------------------------------------------------------------------------*/ -size_t FTI_GetMPIOFilePos(void *fileDesc) { +int64_t FTI_GetMPIOFilePos(void *fileDesc) { WriteMPIInfo_t *fd = (WriteMPIInfo_t *)fileDesc; return fd->loffset; } @@ -181,7 +181,7 @@ size_t FTI_GetMPIOFilePos(void *fileDesc) { **/ /*-------------------------------------------------------------------------*/ -int FTI_MPIORead(void *dest, size_t size, void *fileDesc) { +int FTI_MPIORead(void *dest, int64_t size, void *fileDesc) { WriteMPIInfo_t *fd = (WriteMPIInfo_t *)fileDesc; return MPI_File_read_at(fd->pfh, fd->offset, dest, size, MPI_BYTE, MPI_STATUS_IGNORE); diff --git a/src/IO/posix-dcp.c b/src/IO/posix-dcp.c index 03b670e88..951311d19 100644 --- a/src/IO/posix-dcp.c +++ b/src/IO/posix-dcp.c @@ -50,7 +50,7 @@ @return integer The position in the file. **/ /*-------------------------------------------------------------------------*/ -size_t FTI_GetDCPPosixFilePos(void *fileDesc) { +int64_t FTI_GetDCPPosixFilePos(void *fileDesc) { WriteDCPPosixInfo_t *fd = (WriteDCPPosixInfo_t*) fileDesc; return ftell(fd->write_info.f); } @@ -75,7 +75,7 @@ void *FTI_InitDCPPosix(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, FTI_Print("I/O mode: Posix.", FTI_DBUG); char fn[FTI_BUFS]; - size_t bytes; + int64_t bytes; WriteDCPPosixInfo_t *write_DCPinfo = (WriteDCPPosixInfo_t*) malloc(sizeof(WriteDCPPosixInfo_t)); @@ -182,12 +182,12 @@ int FTI_WritePosixDCPData(FTIT_dataset *data, void *fd) { char errstr[FTI_BUFS]; unsigned char * block = (unsigned char*)malloc (FTI_Conf->dcpInfoPosix.BlockSize); - size_t bytes; + int64_t bytes; int varId = data->id; FTI_Exec->dcpInfoPosix.dataSize += data->size; - uint64_t dataSize = data->size; - // uint64_t nbHashes = dataSize/FTI_Conf->dcpInfoPosix.BlockSize + + int64_t dataSize = data->size; + // int64_t nbHashes = dataSize/FTI_Conf->dcpInfoPosix.BlockSize + // (bool)(dataSize%FTI_Conf->dcpInfoPosix.BlockSize); if (dataSize > (MAX_BLOCK_IDX*FTI_Conf->dcpInfoPosix.BlockSize)) { @@ -216,16 +216,16 @@ int FTI_WritePosixDCPData(FTIT_dataset *data, void *fd) { if (dcpLayer == 0) { DFTI_EH_FWRITE(FTI_NSCS, bytes, &data->id, sizeof(int), 1, write_info->f, "p", block); - DFTI_EH_FWRITE(FTI_NSCS, bytes, &dataSize, sizeof(uint64_t), 1, + DFTI_EH_FWRITE(FTI_NSCS, bytes, &dataSize, sizeof(int64_t), 1, write_info->f, "p", block); FTI_Exec->dcpInfoPosix.FileSize += (sizeof(int) + - sizeof(uint64_t)); - write_DCPinfo->layerSize += sizeof(int) + sizeof(uint64_t); + sizeof(int64_t)); + write_DCPinfo->layerSize += sizeof(int) + sizeof(int64_t); } - uint64_t pos = 0; + int64_t pos = 0; FTIT_data_prefetch prefetcher; - size_t totalBytes = 0; + int64_t totalBytes = 0; unsigned char * ptr; #ifdef GPUSUPPORT prefetcher.fetchSize = ((FTI_Conf->cHostBufSize) / @@ -250,7 +250,7 @@ int FTI_WritePosixDCPData(FTIT_dataset *data, void *fd) { " Fetching Next Memory block from memory") != FTI_SCES) { return FTI_NSCS; } - size_t offset = 0; + int64_t offset = 0; FTI_SyncMD5(); while (ptr) { pos = 0; @@ -474,14 +474,14 @@ int FTI_RecoverDcpPosix(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, } for (i = 0; i < nbVarLayer; i++) { unsigned int varId; - uint64_t locDataSize; + int64_t locDataSize; fread(&varId, sizeof(int), 1, fd); if (ferror(fd)) { snprintf(errstr, FTI_BUFS, "unable to read in file %s", fn); FTI_Print(errstr, FTI_EROR); return FTI_NSCS; } - fread(&locDataSize, sizeof(uint64_t), 1, fd); + fread(&locDataSize, sizeof(int64_t), 1, fd); if (ferror(fd)) { snprintf(errstr, FTI_BUFS, "unable to read in file %s", fn); FTI_Print(errstr, FTI_EROR); @@ -527,7 +527,7 @@ int FTI_RecoverDcpPosix(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, } - uint64_t offset; + int64_t offset; blockMetaInfo_t blockMeta; unsigned char *block = (unsigned char*) malloc(blockSize); if (!block) { @@ -539,7 +539,7 @@ int FTI_RecoverDcpPosix(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, for (i = 1; i < nbLayer; i++) { - uint64_t pos = 0; + int64_t pos = 0; pos += fread(&ckptId, 1, sizeof(int), fd); if (ferror(fd)) { snprintf(errstr, FTI_BUFS, "unable to read in file %s", fn); @@ -611,7 +611,7 @@ int FTI_RecoverDcpPosix(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, for (i = 0; i < FTI_Exec->nbVarStored; i++) { FTIT_data_prefetch prefetcher; - size_t totalBytes = 0; + int64_t totalBytes = 0; unsigned char * ptr = NULL, *startPtr = NULL; #ifdef GPUSUPPORT @@ -635,7 +635,7 @@ int FTI_RecoverDcpPosix(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, return FTI_NSCS; } - uint64_t nbBlocks = (data[i].size % blockSize) ? + int64_t nbBlocks = (data[i].size % blockSize) ? data[i].size/blockSize + 1 : data[i].size/blockSize; data[i].dcpInfoPosix.hashDataSize = data[i].size; int j = 0; @@ -645,7 +645,7 @@ int FTI_RecoverDcpPosix(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, totalBytes/blockSize + 1 : totalBytes/blockSize; int k; for (k = 0 ; k < currentBlocks && j < nbBlocks-1; k++) { - uint64_t hashIdx = j*MD5_DIGEST_LENGTH; + int64_t hashIdx = j*MD5_DIGEST_LENGTH; FTI_Conf->dcpInfoPosix.hashFunc(ptr, blockSize, &data[i].dcpInfoPosix.oldHashArray[hashIdx]); ptr = ptr+blockSize; @@ -664,8 +664,8 @@ int FTI_RecoverDcpPosix(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, FTI_Print("unable to allocate memory!", FTI_EROR); return FTI_NSCS; } - uint64_t dataOffset = blockSize * (nbBlocks - 1); - uint64_t dataSize = data[i].size - dataOffset; + int64_t dataOffset = blockSize * (nbBlocks - 1); + int64_t dataSize = data[i].size - dataOffset; memcpy(buffer, ptr, dataSize); FTI_Conf->dcpInfoPosix.hashFunc(buffer, blockSize, &data[i].dcpInfoPosix.oldHashArray[(nbBlocks-1)*MD5_DIGEST_LENGTH]); @@ -766,14 +766,14 @@ int FTI_RecoverVarDcpPosix(FTIT_configuration* FTI_Conf, } for (i = 0; i < nbVarLayer; i++) { unsigned int varId; - uint64_t locDataSize; + int64_t locDataSize; fread(&varId, sizeof(int), 1, fd); if (ferror(fd)) { snprintf(errstr, FTI_BUFS, "unable to read in file %s", fn); FTI_Print(errstr, FTI_EROR); return FTI_NSCS; } - fread(&locDataSize, sizeof(uint64_t), 1, fd); + fread(&locDataSize, sizeof(int64_t), 1, fd); if (ferror(fd)) { snprintf(errstr, FTI_BUFS, "unable to read in file %s", fn); FTI_Print(errstr, FTI_EROR); @@ -806,7 +806,7 @@ int FTI_RecoverVarDcpPosix(FTIT_configuration* FTI_Conf, } } } else { - uint64_t skip = (locDataSize%blockSize == 0) ? + int64_t skip = (locDataSize%blockSize == 0) ? locDataSize : (locDataSize/blockSize + 1)*blockSize; if (fseek(fd, skip, SEEK_CUR) == -1) { snprintf(errstr, FTI_BUFS, "unable to seek in file %s", fn); @@ -817,7 +817,7 @@ int FTI_RecoverVarDcpPosix(FTIT_configuration* FTI_Conf, } - uint64_t offset; + int64_t offset; blockMetaInfo_t blockMeta; unsigned char *block = (unsigned char*) malloc(blockSize); @@ -828,7 +828,7 @@ int FTI_RecoverVarDcpPosix(FTIT_configuration* FTI_Conf, int nbLayer = FTI_Exec->dcpInfoPosix.nbLayerReco; for (i = 1; i < nbLayer; i++) { - uint64_t pos = 0; + int64_t pos = 0; pos += fread(&ckptId, 1, sizeof(int), fd); if (ferror(fd)) { snprintf(errstr, FTI_BUFS, "unable to read in file %s", fn); @@ -897,7 +897,7 @@ int FTI_RecoverVarDcpPosix(FTIT_configuration* FTI_Conf, return FTI_NSCS; } FTIT_data_prefetch prefetcher; - size_t totalBytes = 0; + int64_t totalBytes = 0; unsigned char * ptr = NULL, *startPtr = NULL; #ifdef GPUSUPPORT @@ -921,7 +921,7 @@ int FTI_RecoverVarDcpPosix(FTIT_configuration* FTI_Conf, return FTI_NSCS; } - uint64_t nbBlocks = (data->size % blockSize) ? + int64_t nbBlocks = (data->size % blockSize) ? data->size/blockSize + 1 : data->size/blockSize; data->dcpInfoPosix.hashDataSize = data->size; int j = 0; @@ -931,7 +931,7 @@ int FTI_RecoverVarDcpPosix(FTIT_configuration* FTI_Conf, totalBytes/blockSize + 1 : totalBytes/blockSize; int k; for (k = 0 ; k < currentBlocks && j < nbBlocks-1; k++) { - uint64_t hashIdx = j*MD5_DIGEST_LENGTH; + int64_t hashIdx = j*MD5_DIGEST_LENGTH; FTI_Conf->dcpInfoPosix.hashFunc(ptr, blockSize, &data->dcpInfoPosix.oldHashArray[hashIdx]); ptr = ptr+blockSize; @@ -949,8 +949,8 @@ int FTI_RecoverVarDcpPosix(FTIT_configuration* FTI_Conf, FTI_Print("unable to allocate memory!", FTI_EROR); return FTI_NSCS; } - uint64_t dataOffset = blockSize * (nbBlocks - 1); - uint64_t dataSize = data->size - dataOffset; + int64_t dataOffset = blockSize * (nbBlocks - 1); + int64_t dataSize = data->size - dataOffset; memcpy(buffer, ptr, dataSize); FTI_Conf->dcpInfoPosix.hashFunc(buffer, blockSize, &data->dcpInfoPosix.oldHashArray[(nbBlocks-1)*MD5_DIGEST_LENGTH]); @@ -959,13 +959,13 @@ int FTI_RecoverVarDcpPosix(FTIT_configuration* FTI_Conf, /* // create hasharray for id i = FTI_DataGetIdx(id, FTI_Exec, FTI_Data); - uint64_t nbBlocks = (FTI_Data[i].size % blockSize) ? FTI_Data[i].size/blockSize + 1 : FTI_Data[i].size/blockSize; + int64_t nbBlocks = (FTI_Data[i].size % blockSize) ? FTI_Data[i].size/blockSize + 1 : FTI_Data[i].size/blockSize; FTI_Data[i].dcpInfoPosix.hashDataSize = FTI_Data[i].size; int j; for(j=0; jdcpInfoPosix.Counter = counter; bool readLayer = true; - size_t bytes; - size_t layerSize; + int64_t bytes; + int64_t layerSize; // now treat other layers for (; layer < stackSize && readLayer; layer++) { @@ -1324,7 +1324,7 @@ void* FTI_DcpPosixRecoverRuntimeInfo(int tag, void* exec_, void* conf_) { } // have the same for for MD5 and CRC32 -unsigned char* CRC32(const unsigned char *d, uint64_t nBytes, +unsigned char* CRC32(const unsigned char *d, unsigned long nBytes, unsigned char *hash) { static unsigned char hash_[CRC32_DIGEST_LENGTH]; if (hash == NULL) { diff --git a/src/IO/posix-dcp.h b/src/IO/posix-dcp.h index 08efd0bbd..defb2088d 100644 --- a/src/IO/posix-dcp.h +++ b/src/IO/posix-dcp.h @@ -22,7 +22,7 @@ #define DCP_POSIX_CONF_TAG 1 #define DCP_POSIX_INIT_TAG -1 -int FTI_CheckFileDcpPosix(char* fn, uint64_t fs, char* checksum); +int FTI_CheckFileDcpPosix(char* fn, int64_t fs, char* checksum); int FTI_VerifyChecksumDcpPosix(char* fileName); void* FTI_DcpPosixRecoverRuntimeInfo(int tag, void* exec_, void* conf_); int FTI_RecoverDcpPosix(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, @@ -33,7 +33,7 @@ int FTI_RecoverVarDcpPosix(FTIT_configuration* FTI_Conf, char* FTI_GetHashHexStr(unsigned char* hash, int digestWidth, char* hashHexStr); // wrapper for CRC32 hash algorithm -unsigned char* CRC32(const unsigned char *d, uint64_t nBytes, +unsigned char* CRC32(const unsigned char *d, unsigned long nBytes, unsigned char *hash); int FTI_RecoverVarDcpPosixInit(); diff --git a/src/IO/posix.c b/src/IO/posix.c index 7fea01042..951842729 100644 --- a/src/IO/posix.c +++ b/src/IO/posix.c @@ -119,9 +119,9 @@ int FTI_PosixClose(void *fileDesc) { **/ /*-------------------------------------------------------------------------*/ -int FTI_PosixWrite(void *src, size_t size, void *fileDesc) { +int FTI_PosixWrite(void *src, int64_t size, void *fileDesc) { WritePosixInfo_t *fd = (WritePosixInfo_t *)fileDesc; - size_t written = 0; + int64_t written = 0; int fwrite_errno = 0; char str[FTI_BUFS]; @@ -156,7 +156,7 @@ int FTI_PosixWrite(void *src, size_t size, void *fileDesc) { **/ /*-------------------------------------------------------------------------*/ -int FTI_PosixSeek(size_t pos, void *fileDesc) { +int FTI_PosixSeek(int64_t pos, void *fileDesc) { WritePosixInfo_t *fd = (WritePosixInfo_t *) fileDesc; if (fseek(fd->f, pos, SEEK_SET) == -1) { char error_msg[FTI_BUFS]; @@ -172,11 +172,11 @@ int FTI_PosixSeek(size_t pos, void *fileDesc) { /** @brief Return the current file postion @param fileDesc The fileDescriptor - @return size_t Position of the file descriptor + @return int64_t Position of the file descriptor **/ /*-------------------------------------------------------------------------*/ -size_t FTI_GetPosixFilePos(void *fileDesc) { +int64_t FTI_GetPosixFilePos(void *fileDesc) { WritePosixInfo_t *fd = (WritePosixInfo_t *) fileDesc; return ftell(fd->f); } @@ -191,7 +191,7 @@ size_t FTI_GetPosixFilePos(void *fileDesc) { **/ /*-------------------------------------------------------------------------*/ -int FTI_PosixRead(void *dest, size_t size, void *fileDesc) { +int FTI_PosixRead(void *dest, int64_t size, void *fileDesc) { return FTI_SCES; } @@ -359,7 +359,7 @@ int FTI_RecoverVarPOSIX(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, return FTI_NREC; } - uint64_t filePos = data->filePos; + int64_t filePos = data->filePos; if (fseek(fileposix, filePos, SEEK_SET) == 0) { fread(data->ptr, 1, data->size, fileposix); if (ferror(fileposix)) { diff --git a/src/IO/posix.h b/src/IO/posix.h index dae9b5033..4ed5fd3cb 100644 --- a/src/IO/posix.h +++ b/src/IO/posix.h @@ -21,10 +21,10 @@ int FTI_WritePosixData(FTIT_dataset * data, void *fd); void* FTI_InitPosix(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, FTIT_topology* FTI_Topo, FTIT_checkpoint *FTI_Ckpt, FTIT_keymap *FTI_Data); int FTI_PosixSync(void *fileDesc); -int FTI_PosixRead(void *dest, size_t size, void *fileDesc); -size_t FTI_GetPosixFilePos(void *fileDesc); -int FTI_PosixSeek(size_t pos, void *fileDesc); -int FTI_PosixWrite(void *src, size_t size, void *fileDesc); +int FTI_PosixRead(void *dest, int64_t size, void *fileDesc); +int64_t FTI_GetPosixFilePos(void *fileDesc); +int FTI_PosixSeek(int64_t pos, void *fileDesc); +int FTI_PosixWrite(void *src, int64_t size, void *fileDesc); int FTI_PosixClose(void *fileDesc); int FTI_PosixOpen(char *fn, void *fileDesc); int FTI_RecoverVarInitPOSIX(char* fn); diff --git a/src/api-cuda.c b/src/api-cuda.c index a2d72e579..6012446ce 100644 --- a/src/api-cuda.c +++ b/src/api-cuda.c @@ -46,7 +46,7 @@ cudaStream_t Gstream; void *hostBuffers[2]; -size_t bufferSize; +int64_t bufferSize; /*-------------------------------------------------------------------------*/ @@ -106,7 +106,7 @@ int FTI_get_pointer_info(const void *ptr, FTIT_ptrinfo *ptrInfo) { to the CPU memory area pointed to by dst. **/ /*-------------------------------------------------------------------------*/ -int FTI_copy_from_device(void *dst, const void *src, size_t count, +int FTI_copy_from_device(void *dst, const void *src, int64_t count, FTIT_execution *exec) { #ifdef GPUSUPPORT CUDA_ERROR_CHECK(cudaMemcpy(dst, src, count, cudaMemcpyDeviceToHost)); @@ -130,7 +130,7 @@ int FTI_copy_from_device(void *dst, const void *src, size_t count, to the GPU memory area pointed to by dst. **/ /*-------------------------------------------------------------------------*/ -int FTI_copy_to_device(void *dst, const void *src, size_t count, +int FTI_copy_to_device(void *dst, const void *src, int64_t count, FTIT_execution *exec) { #ifdef GPUSUPPORT CUDA_ERROR_CHECK(cudaMemcpy(dst, src, count, cudaMemcpyHostToDevice)); @@ -166,7 +166,7 @@ int FTI_TransferDeviceMemToFileAsync(FTIT_dataset *data, prefetcher.isDevice = data->isDevicePtr; prefetcher.dptr = data->devicePtr; FTI_InitPrefetcher(&prefetcher); - size_t bytesToWrite; + int64_t bytesToWrite; unsigned char *basePtr = NULL; if (FTI_Try(FTI_getPrefetchedData(&prefetcher, &bytesToWrite, &basePtr), @@ -212,7 +212,7 @@ int FTI_InitPrefetcher(FTIT_data_prefetch *dfls) { if (dfls->isDevice) { CUDA_ERROR_CHECK(cudaStreamCreate(&(dfls->streams[0]))); CUDA_ERROR_CHECK(cudaStreamCreate(&(dfls->streams[1]))); - size_t copy_size = MIN(dfls->fetchSize, dfls->totalBytesToFetch); + int64_t copy_size = MIN(dfls->fetchSize, dfls->totalBytesToFetch); if (copy_size > bufferSize) { FTI_Print("I am requesting more bytes than the ones reserved" @@ -247,7 +247,7 @@ int FTI_InitPrefetcher(FTIT_data_prefetch *dfls) { requests more data from the GPU side. **/ /*-------------------------------------------------------------------------*/ -int FTI_getPrefetchedData(FTIT_data_prefetch *dfls, size_t *size, +int FTI_getPrefetchedData(FTIT_data_prefetch *dfls, int64_t *size, unsigned char **fetchedData) { if (dfls->end) { *fetchedData = NULL; @@ -259,7 +259,7 @@ int FTI_getPrefetchedData(FTIT_data_prefetch *dfls, size_t *size, if (dfls->isDevice) { *size = dfls->requestedData; if (dfls->totalBytesToFetch > 0) { - size_t copy_size = MIN(dfls->fetchSize, dfls->totalBytesToFetch); + int64_t copy_size = MIN(dfls->fetchSize, dfls->totalBytesToFetch); CUDA_ERROR_CHECK(cudaMemcpyAsync(hostBuffers[dfls->Id], dfls->dptr, copy_size, cudaMemcpyDeviceToHost, dfls->streams[dfls->Id])); dfls->dptr += copy_size; @@ -399,7 +399,7 @@ int FTI_TransferFileToDeviceAsync(FILE *fd, void *dptr, int numBytes) { **/ /*-------------------------------------------------------------------------*/ -size_t FTI_getHostBuffSize() { +int64_t FTI_getHostBuffSize() { return bufferSize; } @@ -432,7 +432,7 @@ BYTE *FTI_getHostBuffer(int id) { **/ /*-------------------------------------------------------------------------*/ -int FTI_copy_to_device_async(void *dst, const void *src, size_t count) { +int FTI_copy_to_device_async(void *dst, const void *src, int64_t count) { #ifdef GPUSUPPORT CUDA_ERROR_CHECK(cudaMemcpyAsync(dst, src, count, cudaMemcpyHostToDevice, Gstream)); diff --git a/src/api-cuda.h b/src/api-cuda.h index a6787c531..2abcdcb36 100644 --- a/src/api-cuda.h +++ b/src/api-cuda.h @@ -59,9 +59,9 @@ typedef struct FTIT_data_prefetch { // Pointer pointing to data to be fetched; unsigned char *dptr; // total number of bytes to be streamed; - size_t totalBytesToFetch; + int64_t totalBytesToFetch; // Bytes I will fetch each time; - size_t fetchSize; + int64_t fetchSize; // Check Whether Bytes are stored in device int isDevice; // Points to the current stream @@ -77,25 +77,25 @@ typedef struct FTIT_data_prefetch { } FTIT_data_prefetch; -typedef int (*FTIT_fwritefunc)(void *src, size_t size, void *opaque); +typedef int (*FTIT_fwritefunc)(void *src, int64_t size, void *opaque); int FTI_get_pointer_info(const void *ptr, FTIT_ptrinfo *ptrInfo); -int FTI_copy_from_device(void *dst, const void *src, size_t count, +int FTI_copy_from_device(void *dst, const void *src, int64_t count, FTIT_execution *exec); -int FTI_copy_to_device(void *dst, const void *src, size_t count, +int FTI_copy_to_device(void *dst, const void *src, int64_t count, FTIT_execution *exec); int FTI_TransferDeviceMemToFileAsync(FTIT_dataset *data, FTIT_fwritefunc fwritefunc, void *opaque); int FTI_InitPrefetcher(FTIT_data_prefetch *dfls); -int FTI_getPrefetchedData(FTIT_data_prefetch *dfls, size_t *size, +int FTI_getPrefetchedData(FTIT_data_prefetch *dfls, int64_t *size, unsigned char **fetchedData); int FTI_destroyPrefetcher(FTIT_data_prefetch *dfls); int FTI_DestroyDevices(); int FTI_InitDevices(int HostBuffSize); int FTI_TransferFileToDeviceAsync(FILE *fd, void *dptr, int numBytes); BYTE *FTI_getHostBuffer(int id); -size_t FTI_getHostBuffSize(); -int FTI_copy_to_device_async(void *dst, const void *src, size_t count); +int64_t FTI_getHostBuffSize(); +int FTI_copy_to_device_async(void *dst, const void *src, int64_t count); int FTI_device_sync(); #endif // FTI_SRC_API_CUDA_H_ diff --git a/src/api.c b/src/api.c index c8958ec81..d14e41636 100644 --- a/src/api.c +++ b/src/api.c @@ -249,7 +249,7 @@ int FTI_Status() { It is obscured from the user for API backwards compatibility. **/ /*-------------------------------------------------------------------------*/ -fti_id_t FTI_InitType_opaque(uint64_t size) { +fti_id_t FTI_InitType_opaque(size_t size) { FTIT_Datatype *type; fti_id_t new_id = FTI_Exec.datatypes.ntypes; @@ -324,7 +324,7 @@ FTIT_Datatype* FTI_GetType(fti_id_t id) { The fields can be added using FTI_AddScalarField and FTI_AddVectorField. **/ /*-------------------------------------------------------------------------*/ -fti_id_t FTI_InitCompositeType(const char* name, uint64_t size, +fti_id_t FTI_InitCompositeType(const char* name, size_t size, FTIT_H5Group* h5g) { FTIT_Datatype *type; FTIT_complexType *structure; @@ -369,7 +369,7 @@ fti_id_t FTI_InitCompositeType(const char* name, uint64_t size, **/ /*-------------------------------------------------------------------------*/ int FTI_AddScalarField(fti_id_t id, const char* name, fti_id_t fid, - uint64_t offset) { + int64_t offset) { FTIT_Datatype *struct_ref, *field_type; int field_id; FTIT_typeField *field; @@ -428,7 +428,7 @@ int FTI_AddScalarField(fti_id_t id, const char* name, fti_id_t fid, **/ /*-------------------------------------------------------------------------*/ int FTI_AddVectorField(fti_id_t id, const char* name, - fti_id_t tid, uint64_t offset, int ndims, int* dim_sizes) { + fti_id_t tid, int64_t offset, int ndims, int* dim_sizes) { FTIT_complexType *type; FTIT_typeField *field; int i; @@ -804,7 +804,7 @@ int FTI_RenameGroup(FTIT_H5Group* h5group, const char* name) { **/ /*-------------------------------------------------------------------------*/ -int FTI_Protect(int id, void* ptr, uint64_t count, fti_id_t tid) { +int FTI_Protect(int id, void* ptr, int64_t count, fti_id_t tid) { if (FTI_Exec.initSCES == 0) { FTI_Print("FTI is not initialized.", FTI_WARN); return FTI_NSCS; @@ -836,7 +836,7 @@ int FTI_Protect(int id, void* ptr, uint64_t count, fti_id_t tid) { } if (data != NULL) { // Search for dataset with given id - uint64_t prevSize = data->size; + int64_t prevSize = data->size; #ifdef GPUSUPPORT if (ptrInfo.type == FTIT_PTRTYPE_CPU) { // strcpy(memLocation, "CPU"); @@ -895,7 +895,7 @@ int FTI_Protect(int id, void* ptr, uint64_t count, fti_id_t tid) { FTI_Print(str, FTI_DBUG); if (prevSize != data->size && FTI_Conf.dcpPosix) { if (!(data->isDevicePtr)) { - uint64_t nbHashes = data->size / + int64_t nbHashes = data->size / FTI_Conf.dcpInfoPosix.BlockSize + (bool)(data->size %FTI_Conf.dcpInfoPosix.BlockSize); data->dcpInfoPosix.currentHashArray = (unsigned char*) @@ -910,10 +910,10 @@ int FTI_Protect(int id, void* ptr, uint64_t count, fti_id_t tid) { #ifdef GPUSUPPORT else { unsigned char *x; - uint64_t nbNewHashes = data->size / + int64_t nbNewHashes = data->size / FTI_Conf.dcpInfoPosix.BlockSize + (bool)(data->size %FTI_Conf.dcpInfoPosix.BlockSize); - uint64_t nbOldHashes = prevSize / + int64_t nbOldHashes = prevSize / FTI_Conf.dcpInfoPosix.BlockSize + (bool)(data->size %FTI_Conf.dcpInfoPosix.BlockSize); CUDA_ERROR_CHECK(cudaMallocManaged((void**) &x, @@ -1018,7 +1018,7 @@ int FTI_Protect(int id, void* ptr, uint64_t count, fti_id_t tid) { if (FTI_Conf.dcpPosix) { if (!(data->isDevicePtr)) { - uint64_t nbHashes = data->size / + int64_t nbHashes = data->size / FTI_Conf.dcpInfoPosix.BlockSize + (bool)(data->size %FTI_Conf.dcpInfoPosix.BlockSize); data->dcpInfoPosix.hashDataSize = 0; @@ -1032,7 +1032,7 @@ int FTI_Protect(int id, void* ptr, uint64_t count, fti_id_t tid) { #ifdef GPUSUPPORT else { unsigned char *x; - uint64_t nbNewHashes = data->size / + int64_t nbNewHashes = data->size / FTI_Conf.dcpInfoPosix.BlockSize + (bool)(data->size %FTI_Conf.dcpInfoPosix.BlockSize); CUDA_ERROR_CHECK(cudaMallocManaged((void**)&x, @@ -1616,7 +1616,7 @@ int FTI_DefineDataset(int id, int rank, int* dimLength, const char* name, } // check if size is correct - uint64_t expectedSize = 1; + int64_t expectedSize = 1; int j; for (j = 0; j < rank; j++) { expectedSize *= dimLength[j]; // compute the number of elements @@ -1653,7 +1653,7 @@ int FTI_DefineDataset(int id, int rank, int* dimLength, const char* name, /** @brief Returns size saved in metadata of variable @param id Variable ID. - @return uint64_t Returns size of variable or 0 if size not saved. + @return int64_t Returns size of variable or 0 if size not saved. This function returns size of variable of given ID that is saved in metadata. This may be different from size of variable that is in the program. If this @@ -1662,7 +1662,7 @@ int FTI_DefineDataset(int id, int rank, int* dimLength, const char* name, is no size saved in metadata it returns 0. **/ /*-------------------------------------------------------------------------*/ -uint64_t FTI_GetStoredSize(int id) { +int64_t FTI_GetStoredSize(int id) { if (FTI_Exec.initSCES == 0) { FTI_Print("FTI is not initialized.", FTI_WARN); return 0; @@ -2295,12 +2295,12 @@ int FTI_FinalizeICP() { if ((FTI_Conf.dcpFtiff || FTI_Conf.dcpPosix) && FTI_Ckpt[4].isDcp) { // After dCP update store total data and dCP sizes in application rank0 - uint64_t *dataSize = (FTI_Conf.dcpFtiff)?(uint64_t*)& + int64_t *dataSize = (FTI_Conf.dcpFtiff)?(int64_t*)& FTI_Exec.FTIFFMeta.pureDataSize:&FTI_Exec.dcpInfoPosix.dataSize; - uint64_t *dcpSize = (FTI_Conf.dcpFtiff)?(uint64_t*)& + int64_t *dcpSize = (FTI_Conf.dcpFtiff)?(int64_t*)& FTI_Exec.FTIFFMeta.dcpSize:&FTI_Exec.dcpInfoPosix.dcpSize; - uint64_t dcpStats[2]; // 0:totalDcpSize, 1:totalDataSize - uint64_t sendBuf[] = { *dcpSize, *dataSize }; + int64_t dcpStats[2]; // 0:totalDcpSize, 1:totalDataSize + int64_t sendBuf[] = { *dcpSize, *dataSize }; MPI_Reduce(sendBuf, dcpStats, 2, MPI_UINT64_T, MPI_SUM, 0, FTI_COMM_WORLD); if (FTI_Topo.splitRank == 0) { @@ -2573,7 +2573,7 @@ int FTI_Recover() { #ifdef GPUSUPPORT for (i = 0; i < FTI_Exec.nbVarStored; i++) { - uint64_t filePos = data[i].filePosStored; + int64_t filePos = data[i].filePosStored; // strncpy(data[i].idChar, data[i].idChar, FTI_BUFS); fseek(fd, filePos, SEEK_SET); if (data[i].isDevicePtr) @@ -2591,7 +2591,7 @@ int FTI_Recover() { #else for (i = 0; i < FTI_Exec.nbVarStored; i++) { - uint64_t filePos = data[i].filePos; + size_t filePos = data[i].filePos; // strncpy(data[i].idChar, data[i].idChar, FTI_BUFS); fseek(fd, filePos, SEEK_SET); fread(data[i].ptr, 1, data[i].sizeStored, fd); diff --git a/src/checkpoint.c b/src/checkpoint.c index c31e6ba61..2944cef9a 100644 --- a/src/checkpoint.c +++ b/src/checkpoint.c @@ -183,14 +183,14 @@ int FTI_WriteCkpt(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, if ((FTI_Conf->dcpFtiff || FTI_Conf->dcpPosix) && FTI_Ckpt[4].isDcp) { // After dCP update store total data and dCP // sizes in application rank 0 - uint64_t *dataSize = (FTI_Conf->dcpFtiff)? - (uint64_t*)&FTI_Exec->FTIFFMeta.pureDataSize: + int64_t *dataSize = (FTI_Conf->dcpFtiff)? + (int64_t*)&FTI_Exec->FTIFFMeta.pureDataSize: &FTI_Exec->dcpInfoPosix.dataSize; - uint64_t *dcpSize = (FTI_Conf->dcpFtiff)? - (uint64_t*)&FTI_Exec->FTIFFMeta.dcpSize: + int64_t *dcpSize = (FTI_Conf->dcpFtiff)? + (int64_t*)&FTI_Exec->FTIFFMeta.dcpSize: &FTI_Exec->dcpInfoPosix.dcpSize; - uint64_t dcpStats[2]; // 0:totalDcpSize, 1:totalDataSize - uint64_t sendBuf[] = { *dcpSize, *dataSize }; + int64_t dcpStats[2]; // 0:totalDcpSize, 1:totalDataSize + int64_t sendBuf[] = { *dcpSize, *dataSize }; MPI_Reduce(sendBuf, dcpStats, 2, MPI_UINT64_T, MPI_SUM, 0, FTI_COMM_WORLD); if (FTI_Topo->splitRank == 0) { diff --git a/src/icp.c b/src/icp.c index 16c15b497..ab0c28aec 100644 --- a/src/icp.c +++ b/src/icp.c @@ -218,7 +218,7 @@ int FTI_WriteSionlibVar(int varID, FTIT_configuration* FTI_Conf, int sid; memcpy( &sid, FTI_Exec->iCPInfo.fh, sizeof(FTI_SL_FH) ); - uint64_t offset = 0; + int64_t offset = 0; // write datasets into file int i; for (i = 0; i < FTI_Exec->nbVar; i++) { diff --git a/src/meta.c b/src/meta.c index b3b7f2d72..ffcead2ba 100644 --- a/src/meta.c +++ b/src/meta.c @@ -362,7 +362,7 @@ int FTI_LoadMetaDcp(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, int k; for (k = 0; k < MAX_STACK_SIZE; k++) { snprintf(str, FTI_BUFS, "%d:dcp_layer%d_size", FTI_Topo->groupRank, k); - uint64_t LayerSize = ini.getLong(&ini, str); + int64_t LayerSize = ini.getLong(&ini, str); if (LayerSize == -1) { // No more variables break; @@ -388,7 +388,7 @@ int FTI_LoadMetaDcp(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, break; } FTI_Exec->dcpInfoPosix.datasetInfo[k][j].varSize = - (uint64_t) varSize; + (int64_t) varSize; } } @@ -677,12 +677,12 @@ int FTI_WriteCkptMetaData(FTIT_configuration* FTI_Conf, **/ /*-------------------------------------------------------------------------*/ int FTI_WriteMetadata(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, - FTIT_topology* FTI_Topo, FTIT_checkpoint* FTI_Ckpt, uint64_t* fs, - uint64_t mfs, char* fnl, char* checksums, int* allVarIDs, - int* allRanks, uint64_t* allCounts, + FTIT_topology* FTI_Topo, FTIT_checkpoint* FTI_Ckpt, int64_t* fs, + int64_t mfs, char* fnl, char* checksums, int* allVarIDs, + int* allRanks, int64_t* allCounts, int* allVarTypeIDs, int* allVarTypeSizes, - uint64_t* allVarSizes, uint64_t* allLayerSizes, char* allLayerHashes, - uint64_t *allVarPositions, char *allNames, char *allCharIds) { + int64_t* allVarSizes, int64_t* allLayerSizes, char* allLayerHashes, + int64_t *allVarPositions, char *allNames, char *allCharIds) { // no metadata files for FTI-FF if (FTI_Conf->ioMode == FTI_IO_FTIFF) { return FTI_SCES; } @@ -881,7 +881,7 @@ int FTI_CreateMetadata(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, } #endif - uint64_t fileSizes[FTI_BUFS]; + int64_t fileSizes[FTI_BUFS]; MPI_Allgather(&FTI_Exec->ckptMeta.fs, 1, MPI_INT64_T, fileSizes, 1, MPI_INT64_T, FTI_Exec->groupComm); @@ -892,7 +892,7 @@ int FTI_CreateMetadata(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, FTI_Exec->ckptMeta.pfs = fileSizes[ptnerGroupRank]; } - uint64_t mfs = 0; // Max file size in group + int64_t mfs = 0; // Max file size in group int i; for (i = 0; i < FTI_Topo->groupSize; i++) { if (fileSizes[i] > mfs) { @@ -936,15 +936,15 @@ int FTI_CreateMetadata(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, int* allVarIDs = NULL; int* allVarTypeIDs = NULL; int* allVarTypeSizes = NULL; - uint64_t* allVarSizes = NULL; - uint64_t *allVarPositions = NULL; + int64_t* allVarSizes = NULL; + int64_t *allVarPositions = NULL; // for posix dcp - uint64_t* allLayerSizes = NULL; + int64_t* allLayerSizes = NULL; char* allLayerHashes = NULL; char* allCharIds = NULL; char* allNames = NULL; - uint64_t* allCounts = NULL; + int64_t* allCounts = NULL; int* allRanks = NULL; int nbLayer = ((FTI_Exec->dcpInfoPosix.Counter-1) % @@ -952,20 +952,20 @@ int FTI_CreateMetadata(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, if (FTI_Topo->groupRank == 0) { allRanks = talloc(int, FTI_Topo->groupSize * FTI_Exec->nbVar); - allCounts = talloc(uint64_t, 32 * FTI_Topo->groupSize * + allCounts = talloc(int64_t, 32 * FTI_Topo->groupSize * FTI_Exec->nbVar); allVarIDs = talloc(int, FTI_Topo->groupSize * FTI_Exec->nbVar); allVarTypeIDs = talloc(int, FTI_Topo->groupSize * FTI_Exec->nbVar); allVarTypeSizes = talloc(int, FTI_Topo->groupSize * FTI_Exec->nbVar); - allVarSizes = talloc(uint64_t, FTI_Topo->groupSize * FTI_Exec->nbVar); - allVarPositions = talloc(uint64_t, + allVarSizes = talloc(int64_t, FTI_Topo->groupSize * FTI_Exec->nbVar); + allVarPositions = talloc(int64_t, FTI_Topo->groupSize * FTI_Exec->nbVar); allCharIds = (char *)malloc(sizeof(char)*FTI_BUFS* FTI_Exec->nbVar*FTI_Topo->groupSize); allNames = (char *)malloc(sizeof(char)*FTI_BUFS* FTI_Exec->nbVar*FTI_Topo->groupSize); if (FTI_Ckpt[FTI_Exec->ckptMeta.level].isDcp) { - allLayerSizes = talloc(uint64_t, + allLayerSizes = talloc(int64_t, FTI_Topo->groupSize * nbLayer); allLayerHashes = talloc(char, FTI_Topo->groupSize * nbLayer * MD5_DIGEST_STRING_LENGTH); @@ -974,11 +974,11 @@ int FTI_CreateMetadata(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, int* myVarIDs = talloc(int, FTI_Exec->nbVar); int* myRanks = talloc(int, FTI_Exec->nbVar); - uint64_t* myCounts = talloc(uint64_t, 32 * FTI_Exec->nbVar); + int64_t* myCounts = talloc(int64_t, 32 * FTI_Exec->nbVar); int* myVarTypeIDs = talloc(int, FTI_Exec->nbVar); int* myVarTypeSizes = talloc(int, FTI_Exec->nbVar); - uint64_t* myVarSizes = talloc(uint64_t, FTI_Exec->nbVar); - uint64_t* myVarPositions = talloc(uint64_t, FTI_Exec->nbVar); + int64_t* myVarSizes = talloc(int64_t, FTI_Exec->nbVar); + int64_t* myVarPositions = talloc(int64_t, FTI_Exec->nbVar); char *ArrayOfIdChars = (char *)malloc(FTI_Exec->nbVar * sizeof(char*) *FTI_BUFS); char *ArrayOfNames = (char *)malloc(FTI_Exec->nbVar * @@ -997,7 +997,7 @@ int FTI_CreateMetadata(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, myVarIDs[i] = data[i].id; myRanks[i] = data[i].attribute.dim.ndims; memcpy(&myCounts[i*32], &data[i].attribute.dim.count, - 32 * sizeof(uint64_t)); + 32 * sizeof(int64_t)); myVarPositions[i] = data[i].filePos; strncpy(&ArrayOfIdChars[i*FTI_BUFS], data[i].idChar, FTI_BUFS); strncpy(&ArrayOfNames[i*FTI_BUFS], data[i].attribute.name, FTI_BUFS); diff --git a/src/meta.h b/src/meta.h index 25261f227..323f4bb1f 100644 --- a/src/meta.h +++ b/src/meta.h @@ -28,12 +28,12 @@ int FTI_LoadMetaDataset(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, FTIT_topology* FTI_Topo, FTIT_checkpoint* FTI_Ckpt, FTIT_keymap* FTI_Data); int FTI_WriteMetadata(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, - FTIT_topology* FTI_Topo, FTIT_checkpoint* FTI_Ckpt, uint64_t* fs, - uint64_t mfs, char* fnl, char* checksums, int* allVarIDs, - int* allRanks, uint64_t* allCounts, + FTIT_topology* FTI_Topo, FTIT_checkpoint* FTI_Ckpt, int64_t* fs, + int64_t mfs, char* fnl, char* checksums, int* allVarIDs, + int* allRanks, int64_t* allCounts, int* allVarTypeIDs, int* allVarTypeSizes, - uint64_t* allVarSizes, uint64_t* allLayerSizes, char* allLayerHashes, - uint64_t *allVarPositions, char *allNames, char *allCharIds); + int64_t* allVarSizes, int64_t* allLayerSizes, char* allLayerHashes, + int64_t *allVarPositions, char *allNames, char *allCharIds); int FTI_CreateMetadata(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, FTIT_topology* FTI_Topo, FTIT_checkpoint* FTI_Ckpt, FTIT_keymap* FTI_Data); diff --git a/src/postckpt.c b/src/postckpt.c index bef191f45..3a5dfb143 100644 --- a/src/postckpt.c +++ b/src/postckpt.c @@ -357,7 +357,7 @@ int FTI_RSenc(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, int remBsize = bs; - int32_t ps = ((maxFs / bs)) * bs; + int64_t ps = ((maxFs / bs)) * bs; if (ps < maxFs) { ps = ps + bs; } @@ -368,7 +368,7 @@ int FTI_RSenc(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, MD5_Init(&mdContext); // For each block - int32_t pos = 0; + int64_t pos = 0; while (pos < ps) { if ((maxFs - pos) < bs) { remBsize = maxFs - pos; diff --git a/src/postreco.c b/src/postreco.c index 6a051e05e..fb719bce8 100644 --- a/src/postreco.c +++ b/src/postreco.c @@ -68,7 +68,7 @@ int FTI_Decode(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, int k = FTI_Topo->groupSize; int m = k; - uint64_t fs = FTI_Exec->ckptMeta.fs; + int64_t fs = FTI_Exec->ckptMeta.fs; char** data = talloc(char*, k); char** coding = talloc(char*, m); @@ -130,11 +130,10 @@ int FTI_Decode(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, } FILE *fd, *efd; - uint64_t maxFs = FTI_Exec->ckptMeta.maxFs; - uint64_t ps = ((maxFs / FTI_Conf->blockSize)) * FTI_Conf->blockSize; + int64_t maxFs = FTI_Exec->ckptMeta.maxFs; + int64_t ps = ((maxFs / FTI_Conf->blockSize)) * FTI_Conf->blockSize; if (ps < maxFs) { ps = ps + FTI_Conf->blockSize; // Calculating padding size - DBG_MSG("maxFs: %lu, ps: %lu", 0, maxFs, ps); } if (erased[FTI_Topo->groupRank] == 0) { // Resize and open files // determine file size in order to write at the end of the @@ -238,7 +237,7 @@ int FTI_Decode(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, } // Main loop, block by block - uint64_t pos = 0; + int64_t pos = 0; int remBsize = bs; MD5_CTX md5ctxRS; @@ -383,9 +382,7 @@ int FTI_Decode(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, return FTI_NSCS; } - DBG_MSG("fs: %lu",-1,fs_); - - fs = (uint64_t) fs_; + fs = (int64_t) fs_; FTI_Exec->ckptMeta.fs = fs; close(ifd); diff --git a/src/recover.c b/src/recover.c index f2125ec44..c3e575f4a 100644 --- a/src/recover.c +++ b/src/recover.c @@ -51,7 +51,7 @@ **/ /*-------------------------------------------------------------------------*/ -int FTI_CheckFile(char* fn, uint64_t fs, char* checksum) { +int FTI_CheckFile(char* fn, int64_t fs, char* checksum) { char str[FTI_BUFS]; if (access(fn, F_OK) == 0) { struct stat fileStatus; @@ -105,9 +105,9 @@ int FTI_CheckErasures(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, FTIT_topology* FTI_Topo, FTIT_checkpoint* FTI_Ckpt, int *erased) { int level = FTI_Exec->ckptMeta.level; - uint64_t fs = FTI_Exec->ckptMeta.fs; - uint64_t pfs = FTI_Exec->ckptMeta.pfs; - uint64_t maxFs = FTI_Exec->ckptMeta.maxFs; + int64_t fs = FTI_Exec->ckptMeta.fs; + int64_t pfs = FTI_Exec->ckptMeta.pfs; + int64_t maxFs = FTI_Exec->ckptMeta.maxFs; char ckptFile[FTI_BUFS]; strncpy(ckptFile, FTI_Exec->ckptMeta.ckptFile, FTI_BUFS); @@ -123,7 +123,7 @@ int FTI_CheckErasures(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, char fn[FTI_BUFS]; // Path to the checkpoint/partner file name int buf; int ckptId, rank; // Variables for proper partner file name - int (*consistency)(char *, uint64_t , char*); + int (*consistency)(char *, int64_t , char*); #ifdef ENABLE_HDF5 if (FTI_Conf->ioMode == FTI_IO_HDF5) { consistency = &FTI_CheckHDF5File; diff --git a/src/recover.h b/src/recover.h index b372a1272..839c75151 100644 --- a/src/recover.h +++ b/src/recover.h @@ -10,7 +10,7 @@ #include "interface.h" -int FTI_CheckFile(char *fn, uint64_t fs, char* checksum); +int FTI_CheckFile(char *fn, int64_t fs, char* checksum); int FTI_CheckErasures(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, FTIT_topology* FTI_Topo, FTIT_checkpoint* FTI_Ckpt, int *erased); diff --git a/src/util/failure-injection.h b/src/util/failure-injection.h index 5eb435e49..8bd872a9e 100644 --- a/src/util/failure-injection.h +++ b/src/util/failure-injection.h @@ -73,8 +73,8 @@ #include #include -static inline uint64_t get_ruint() { - uint64_t buffer; +static inline int64_t get_ruint() { + int64_t buffer; int fd = open("/dev/urandom", O_RDWR); read(fd, &buffer, 8); close(fd); @@ -89,7 +89,7 @@ unsigned int FUNCTION(const char *testFunction); #define FTI_FI_WRITE(ERR, FD, BUF, COUNT, FN) \ do { \ if (FUNCTION(__FUNCTION__)) { \ - if (get_ruint() < ((uint64_t)((double)PROBABILITY()*INT_MAX))) { \ + if (get_ruint() < ((int64_t)((double)PROBABILITY()*INT_MAX))) { \ close(FD); \ FD = open(FN, O_RDONLY); \ } \ @@ -100,7 +100,7 @@ unsigned int FUNCTION(const char *testFunction); #define FTI_FI_FWRITE(ERR, BUF, SIZE, COUNT, FSTREAM, FN) \ do { \ if (FUNCTION(__FUNCTION__)) { \ - if (get_ruint() < ((uint64_t)((double)PROBABILITY()*INT_MAX))) { \ + if (get_ruint() < ((int64_t)((double)PROBABILITY()*INT_MAX))) { \ fclose(FSTREAM); \ FSTREAM = fopen(FN, "rb"); \ } \ diff --git a/src/util/keymap.c b/src/util/keymap.c index 7956cfec9..9cb8199be 100644 --- a/src/util/keymap.c +++ b/src/util/keymap.c @@ -109,8 +109,8 @@ int FTI_KeyMapPushBack(void* new_item, int key) { return FTI_NSCS; } - uint64_t new_size = self._size; - uint64_t new_used = self._used + 1; + int64_t new_size = self._size; + int64_t new_used = self._used + 1; if (new_used > self._size) { // double container size each time limit is reached except diff --git a/src/util/macros.h b/src/util/macros.h index e639b8dcb..c2291ed22 100644 --- a/src/util/macros.h +++ b/src/util/macros.h @@ -73,7 +73,7 @@ void cleanup(char* pattern, ...); bytes = fread(buff, size, number, fd); \ if (ferror(fd)) { \ char ErrorString[400]; \ - snprintf(ErrorString, sizeof(ErrorString), "FILE %s FUNC %s:%d Error Reading File Bytes Read : %lu", __FILE__, __FUNCTION__, __LINE__, (uint64_t)bytes); \ + snprintf(ErrorString, sizeof(ErrorString), "FILE %s FUNC %s:%d Error Reading File Bytes Read : %lu", __FILE__, __FUNCTION__, __LINE__, (int64_t)bytes); \ FTI_Print(ErrorString, FTI_EROR); \ cleanup(format, __VA_ARGS__, NULL); \ fclose(fd); \ @@ -86,7 +86,7 @@ void cleanup(char* pattern, ...); bytes = fwrite(buff, size, number, fd); \ if (ferror(fd)) { \ char ErrorString[400]; \ - snprintf(ErrorString, sizeof(ErrorString), "FILE %s FUNC %s:%d Error Writing File Bytes Written : %lu", __FILE__, __FUNCTION__, __LINE__, (uint64_t)bytes); \ + snprintf(ErrorString, sizeof(ErrorString), "FILE %s FUNC %s:%d Error Writing File Bytes Written : %lu", __FILE__, __FUNCTION__, __LINE__, (int64_t)bytes); \ FTI_Print(ErrorString, FTI_EROR); \ cleanup(format, __VA_ARGS__, NULL); \ fclose(fd); \ diff --git a/src/util/tools.c b/src/util/tools.c index c6c448cb1..46cea5f18 100644 --- a/src/util/tools.c +++ b/src/util/tools.c @@ -96,7 +96,7 @@ int FTI_InitExecVars(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, = MD5_DIGEST_STRING_LENGTH + MD5_DIGEST_LENGTH + sizeof(int) - + 6*sizeof(uint64_t) + + 6*sizeof(int64_t) + sizeof(uint32_t); // TODO(leobago) RS L3 only works for even file sizes. @@ -106,13 +106,13 @@ int FTI_InitExecVars(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, FTI_dbstructsize = sizeof(int) /* numvars */ - + sizeof(uint64_t); /* dbsize */ + + sizeof(int64_t); /* dbsize */ FTI_dbvarstructsize = 2*sizeof(int) /* numvars */ + 2*sizeof(bool) + 2*sizeof(uintptr_t) - + 2*sizeof(uint64_t) + + 2*sizeof(int64_t) + MD5_DIGEST_LENGTH; // diff --git a/src/util/utility.h b/src/util/utility.h index 9629ed321..eee7ceef4 100644 --- a/src/util/utility.h +++ b/src/util/utility.h @@ -46,7 +46,7 @@ typedef struct { FTIT_configuration* FTI_Conf; // Configuration of the FTI FTIT_topology *FTI_Topo; // Topology of the nodes MPI_Offset offset; // Offset of the Rank in the file - size_t loffset; // Offset in the local file + int64_t loffset; // Offset in the local file int err; // Errors MPI_Info info; // MPI info of the file MPI_File pfh; // File descriptor @@ -56,7 +56,7 @@ typedef struct { typedef struct { FILE *f; // Posix file descriptor - size_t offset; // offset in the file + int64_t offset; // offset in the file char flag; // flags to open the file MD5_CTX integrity; // integrity of the file }WritePosixInfo_t; @@ -64,7 +64,7 @@ typedef struct { #ifdef ENABLE_IME_NATIVE typedef struct { int f; // IME native file descriptor - size_t offset; // offset in the file + int64_t offset; // offset in the file int flag; // flags to open the file mode_t mode; // mode the file has been opened MD5_CTX integrity; // integrity of the file @@ -77,12 +77,12 @@ typedef struct { FTIT_checkpoint *FTI_Ckpt; // FTI Checkpoint options FTIT_execution *FTI_Exec; // FTI execution options FTIT_topology *FTI_Topo; // FTI node topology - size_t layerSize; // size of the dcp layer + int64_t layerSize; // size of the dcp layer }WriteDCPPosixInfo_t; typedef struct { FILE *f; // Posix file descriptor - size_t offset; // offset in the file + int64_t offset; // offset in the file char flag; // flags to open the file MD5_CTX integrity; // integrity of the file FTIT_configuration *FTI_Conf; // FTI Configuration @@ -109,7 +109,7 @@ int FTI_WriteHDF5Data(FTIT_dataset * data, void *write_info); int FTI_WriteHDF5(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, FTIT_topology* FTI_Topo, FTIT_checkpoint* FTI_Ckpt, FTIT_keymap* FTI_Data); -size_t FTI_GetHDF5FilePos(void *); +int64_t FTI_GetHDF5FilePos(void *); #endif #ifdef ENABLE_SIONLIB @@ -119,7 +119,7 @@ typedef struct { int *file_map; int *ranks; int *rank_map; - size_t loffset; + int64_t loffset; sion_int64* chunkSizes; }WriteSionInfo_t; @@ -127,15 +127,15 @@ int FTI_WriteSionData(FTIT_dataset * data, void *fd); void* FTI_InitSion(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, FTIT_topology* FTI_Topo, FTIT_checkpoint *FTI_Ckpt, FTIT_keymap *FTI_Data); int FTI_SionClose(void *fileDesc); -size_t FTI_GetSionFilePos(void *fileDesc); +int64_t FTI_GetSionFilePos(void *fileDesc); #endif // Wrappers around MPIO int FTI_MPIOOpen(char *fn, void *fileDesc); int FTI_MPIOClose(void *fileDesc); -int FTI_MPIOWrite(void *src, size_t size, void *fileDesc); -int FTI_MPIORead(void *src, size_t size, void *fileDesc); -size_t FTI_GetMPIOFilePos(void *fileDesc); +int FTI_MPIOWrite(void *src, int64_t size, void *fileDesc); +int FTI_MPIORead(void *src, int64_t size, void *fileDesc); +int64_t FTI_GetMPIOFilePos(void *fileDesc); void *FTI_InitMPIO(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, FTIT_topology* FTI_Topo, FTIT_checkpoint *FTI_Ckpt, FTIT_keymap *FTI_Data); @@ -143,7 +143,7 @@ int FTI_WriteMPIOData(FTIT_dataset * data, void *write_info); // Wrappers around dcp POSIX -size_t FTI_GetDCPPosixFilePos(void *fileDesc); +int64_t FTI_GetDCPPosixFilePos(void *fileDesc); void *FTI_InitDCPPosix(FTIT_configuration* FTI_Conf, FTIT_execution* FTI_Exec, FTIT_topology* FTI_Topo, FTIT_checkpoint* FTI_Ckpt, FTIT_keymap* FTI_Data); int FTI_WritePosixDCPData(FTIT_dataset *data, void *fd); @@ -154,6 +154,6 @@ int copyDataFromDevive(FTIT_execution* FTI_Exec, FTIT_keymap* FTI_Data); #ifdef ENABLE_SIONLIB -int write_sion(void *src, size_t size, void *opaque); +int write_sion(void *src, int64_t size, void *opaque); #endif #endif // FTI_UTILITY_H_ From b3915fe916b0d6d8c43cb73d379f4f4d7d595fbc Mon Sep 17 00:00:00 2001 From: kellekai Date: Wed, 21 Jul 2021 18:41:27 +0200 Subject: [PATCH 06/11] fixes SIONlib --- src/IO/sion-fti.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/IO/sion-fti.c b/src/IO/sion-fti.c index 0d354309f..6b233b107 100644 --- a/src/IO/sion-fti.c +++ b/src/IO/sion-fti.c @@ -178,7 +178,7 @@ int FTI_WriteSionData(FTIT_dataset *data, void *fd) { } -size_t FTI_GetSionFilePos(void *fileDesc) { +int64_t FTI_GetSionFilePos(void *fileDesc) { WriteSionInfo_t *fd = (WriteSionInfo_t *) fileDesc; return fd->loffset; } From c437a58df253a129b89dd9bcfd69ae0ed3dd34fe Mon Sep 17 00:00:00 2001 From: kellekai Date: Wed, 21 Jul 2021 18:53:51 +0200 Subject: [PATCH 07/11] fixes dcp test compilation --- testing/suites/features/differentialCkpt/diff_test.h | 4 ++-- testing/suites/features/differentialCkpt/diff_test_func.c | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/testing/suites/features/differentialCkpt/diff_test.h b/testing/suites/features/differentialCkpt/diff_test.h index 42b6c575e..3354f9814 100644 --- a/testing/suites/features/differentialCkpt/diff_test.h +++ b/testing/suites/features/differentialCkpt/diff_test.h @@ -89,9 +89,9 @@ extern int numHeads; extern int finalTag; extern int headRank; -fti_id_t FTI_UI, FTI_XOR_INFO; +extern fti_id_t FTI_UI, FTI_XOR_INFO; +extern uint32_t pat; // UNUSED!! -uint32_t pat; //static double share_ratio; extern int A[1]; diff --git a/testing/suites/features/differentialCkpt/diff_test_func.c b/testing/suites/features/differentialCkpt/diff_test_func.c index 66bf2f361..9a6081446 100644 --- a/testing/suites/features/differentialCkpt/diff_test_func.c +++ b/testing/suites/features/differentialCkpt/diff_test_func.c @@ -11,6 +11,9 @@ int finalTag; int headRank; int grank; +fti_id_t FTI_UI, FTI_XOR_INFO; +uint32_t pat; + int A[1] = {100}; int B[2] = {30, 70}; int C[3] = {20, 40, 40}; From b335f216a08b11bae0074e2741682c8078d727cb Mon Sep 17 00:00:00 2001 From: kellekai Date: Thu, 22 Jul 2021 09:07:50 +0200 Subject: [PATCH 08/11] add debug flags to test apps and allow to pass a precommand in the testdriver of itf --- testing/CMakeLists.txt | 4 ++-- testing/tools/itf/modules/fti | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/CMakeLists.txt b/testing/CMakeLists.txt index a8146bad3..b08ef44ad 100644 --- a/testing/CMakeLists.txt +++ b/testing/CMakeLists.txt @@ -44,14 +44,14 @@ function(InstallTestApplication AppName) message("This is a CPP file") target_link_libraries(${AppName} fti.static ${MPI_CXX_LIBRARIES} m) target_include_directories(${AppName} PUBLIC ${CMAKE_SOURCE_DIR}/include) - set_property(TARGET ${AppName} APPEND PROPERTY COMPILE_FLAGS "${MPI_CXX_COMPILE_FLAGS}") + set_property(TARGET ${AppName} APPEND PROPERTY COMPILE_FLAGS "${MPI_CXX_COMPILE_FLAGS} ${CMAKE_CXX_FLAGS}") set_property(TARGET ${AppName} APPEND PROPERTY LINK_FLAGS ${MPI_CXX_LINK_FLAGS}) return() endif() endforeach() target_link_libraries(${AppName} fti.static ${MPI_C_LIBRARIES} m) target_include_directories(${AppName} PUBLIC ${CMAKE_SOURCE_DIR}/include) - set_property(TARGET ${AppName} APPEND PROPERTY COMPILE_FLAGS "${MPI_C_COMPILE_FLAGS}") + set_property(TARGET ${AppName} APPEND PROPERTY COMPILE_FLAGS "${MPI_C_COMPILE_FLAGS} ${CMAKE_C_FLAGS}") set_property(TARGET ${AppName} APPEND PROPERTY LINK_FLAGS ${MPI_C_LINK_FLAGS}) endfunction() diff --git a/testing/tools/itf/modules/fti b/testing/tools/itf/modules/fti index de88b98ef..bf06ebfd5 100644 --- a/testing/tools/itf/modules/fti +++ b/testing/tools/itf/modules/fti @@ -125,7 +125,7 @@ fti_run() { # If this ever needs to be changed, the testrunner will need a rework. # Otherwise, the suite file will be consumed by mpirun. - local fti_cmd="mpirun $MPIRUN_ARGS -n ${itf_cfg['fti:nranks']}" + local fti_cmd="mpirun $MPIRUN_ARGS -n ${itf_cfg['fti:nranks']} $MPIRUN_PRE_COMMAND" local app_stdout="${itf_cfg['fti:app_stdout']}" local test_stdout="${itf_cfg['core:stdout']}" From d6ca75107207d39750d70d24311f3bcad1bc437a Mon Sep 17 00:00:00 2001 From: kellekai Date: Thu, 22 Jul 2021 09:13:17 +0200 Subject: [PATCH 09/11] fixes corner cases test --- testing/suites/features/cornerCases/consistency.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/suites/features/cornerCases/consistency.c b/testing/suites/features/cornerCases/consistency.c index a77eced3b..2a4649f51 100644 --- a/testing/suites/features/cornerCases/consistency.c +++ b/testing/suites/features/cornerCases/consistency.c @@ -233,7 +233,7 @@ int initReInit() { /* check for a correct restart without the crash of the application. */ int reInit() { - //int initres = FTI_Init(configfile2, MPI_COMM_WORLD); + FTI_Init(configfile2, MPI_COMM_WORLD); int* array2 = malloc(sizeof(int) * ARRAY_SIZE); initArray(array2); From 95a72fc8425a41f6076dc0d520faeaf78dcf1198 Mon Sep 17 00:00:00 2001 From: Alessandro Barbieri Date: Sat, 11 Sep 2021 12:35:11 +0200 Subject: [PATCH 10/11] add sphinx.ext.autodoc to doc extensions --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index a9202111b..0d51e62f2 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -31,7 +31,7 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['breathe', 'sphinx.ext.autosectionlabel'] +extensions = ['breathe', 'sphinx.ext.autosectionlabel', 'sphinx.ext.autodoc'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] From 831d56609543d67460caf47213c14fb622473aad Mon Sep 17 00:00:00 2001 From: Kai Keller Date: Fri, 14 Jan 2022 17:04:18 +0100 Subject: [PATCH 11/11] checks if compilers are compatible using the cmake fortran-c interface and only issues a warning when using different compiler signatures --- CMakeLists.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 78a1d4969..6382da8f2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -212,6 +212,14 @@ if(ENABLE_FORTRAN) set(F_CC_ID "${CMAKE_Fortran_COMPILER_ID}_${F_CC_MAJOR}") string(COMPARE NOTEQUAL ${C_CC_ID} ${F_CC_ID} COMPILER_VER_DIFFER) if(${COMPILER_VER_DIFFER}) + message(WARNING + "Different compiler idetifications for Fortran and C!") + endif() + + # 2.2.f) Check if c/c++ and fortran compilers are compatible + FortranCInterface_VERIFY() + FortranCInterface_VERIFY(CXX) + if((NOT ${FortranCInterface_VERIFIED_C}) AND (NOT ${FortranCInterface_VERIFIED_CXX})) message(FATAL_ERROR "Different compiler idetifications for Fortran and C!") endif()