diff --git a/CMakeLists.txt b/CMakeLists.txt index 2ec40578..f1754e49 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -244,7 +244,7 @@ message(STATUS "========================================================") # library compilation # ######################################## include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include) -set(ALL_MODULES cachelib admission evictionC evictionCPP traceReader profiler dataStructure ds_hash utils) +set(ALL_MODULES cachelib admission prefetch evictionC evictionCPP traceReader profiler dataStructure ds_hash utils) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/libCacheSim/cache) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/libCacheSim/dataStructure) diff --git a/libCacheSim/bin/cachesim/cache_init.h b/libCacheSim/bin/cachesim/cache_init.h index 80e40a83..f0e57a69 100644 --- a/libCacheSim/bin/cachesim/cache_init.h +++ b/libCacheSim/bin/cachesim/cache_init.h @@ -1,4 +1,5 @@ +#include #define _GNU_SOURCE #include #include diff --git a/libCacheSim/bin/cachesim/cli_parser.c b/libCacheSim/bin/cachesim/cli_parser.c index c7cb7204..a2299bfc 100644 --- a/libCacheSim/bin/cachesim/cli_parser.c +++ b/libCacheSim/bin/cachesim/cli_parser.c @@ -8,6 +8,7 @@ #include "../../include/libCacheSim/const.h" #include "../../include/libCacheSim/dist.h" +#include "../../include/libCacheSim/prefetchAlgo.h" #include "../../utils/include/mystr.h" #include "../../utils/include/mysys.h" #include "../cli_reader_utils.h" @@ -43,6 +44,9 @@ enum argp_option_short { OPTION_NUM_THREAD = 0x106, OPTION_SAMPLE_RATIO = 's', OPTION_REPORT_INTERVAL = 0x108, + + OPTION_PREFETCH_ALGO = 'p', + OPTION_PREFETCH_PARAMS = 0x109, }; /* @@ -61,12 +65,17 @@ static struct argp_option options[] = { "Sample ratio, 1 means no sampling, 0.01 means sample 1% of objects", 2}, {NULL, 0, NULL, 0, "cache related parameters:", 0}, - {"eviction-params", OPTION_EVICTION_PARAMS, "\"n-seg=\"4", 0, + {"eviction-params", OPTION_EVICTION_PARAMS, "\"n-seg=4\"", 0, "optional params for each eviction algorithm, e.g., n-seg=4", 4}, {"admission", OPTION_ADMISSION_ALGO, "bloom-filter", 0, "Admission algorithm: size/bloom-filter/prob", 4}, {"admission-params", OPTION_ADMISSION_PARAMS, "\"prob=0.8\"", 0, "params for admission algorithm", 4}, + {"prefetch", OPTION_PREFETCH_ALGO, "Mithril", 0, + "Prefetching algorithm: Mithril", 4}, + {"prefetch-params", OPTION_PREFETCH_PARAMS, "\"block-size=65536\"", 0, + "optional params for each prefetching algorithm, e.g., block-size=65536", + 4}, {0, 0, 0, 0, "Other options:"}, {"ignore-obj-size", OPTION_IGNORE_OBJ_SIZE, "false", 0, @@ -85,8 +94,7 @@ static struct argp_option options[] = { "Whether consider per object metadata size in the simulated cache", 10}, {"verbose", OPTION_VERBOSE, "1", 0, "Produce verbose output", 10}, - {0} - }; + {0}}; /* PARSER. Field 2 in ARGP. @@ -113,11 +121,19 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { case OPTION_ADMISSION_ALGO: arguments->admission_algo = arg; break; + case OPTION_PREFETCH_ALGO: + arguments->prefetch_algo = arg; + break; case OPTION_ADMISSION_PARAMS: arguments->admission_params = strdup(arg); replace_char(arguments->admission_params, ';', ','); replace_char(arguments->admission_params, '_', '-'); break; + case OPTION_PREFETCH_PARAMS: + arguments->prefetch_params = strdup(arg); + replace_char(arguments->prefetch_params, ';', ','); + replace_char(arguments->prefetch_params, '_', '-'); + break; case OPTION_OUTPUT_PATH: strncpy(arguments->ofilepath, arg, OFILEPATH_LEN); break; @@ -195,7 +211,9 @@ static void init_arg(struct arguments *args) { args->trace_path = NULL; args->eviction_params = NULL; args->admission_algo = NULL; + args->prefetch_algo = NULL; args->admission_params = NULL; + args->prefetch_params = NULL; args->trace_type_str = NULL; args->trace_type_params = NULL; args->verbose = true; @@ -322,6 +340,11 @@ void parse_cmd(int argc, char *argv[], struct arguments *args) { args->caches[idx]->admissioner = create_admissioner(args->admission_algo, args->admission_params); } + + if (args->prefetch_algo != NULL) { + args->caches[idx]->prefetcher = create_prefetcher( + args->prefetch_algo, args->prefetch_params, args->cache_sizes[j]); + } } } @@ -492,6 +515,16 @@ void print_parsed_args(struct arguments *args) { n += snprintf(output_str + n, OUTPUT_STR_LEN - n - 1, ", admission-params: %s", args->admission_params); + if (args->prefetch_algo != NULL) { + n += snprintf(output_str + n, OUTPUT_STR_LEN - n - 1, ", prefetch: %s", + args->prefetch_algo); + } + + if (args->prefetch_params != NULL) { + n += snprintf(output_str + n, OUTPUT_STR_LEN - n - 1, + ", prefetch-params: %s", args->prefetch_params); + } + if (args->eviction_params != NULL) n += snprintf(output_str + n, OUTPUT_STR_LEN - n - 1, ", eviction-params: %s", args->eviction_params); diff --git a/libCacheSim/bin/cachesim/internal.h b/libCacheSim/bin/cachesim/internal.h index 52b0589f..cdf5eef4 100644 --- a/libCacheSim/bin/cachesim/internal.h +++ b/libCacheSim/bin/cachesim/internal.h @@ -25,6 +25,7 @@ struct arguments { char *eviction_algo[N_MAX_ALGO]; int n_eviction_algo; char *admission_algo; + char *prefetch_algo; uint64_t cache_sizes[N_MAX_CACHE_SIZE]; int n_cache_size; int warmup_sec; @@ -35,6 +36,7 @@ struct arguments { char *trace_type_params; char *eviction_params; char *admission_params; + char *prefetch_params; double sample_ratio; int n_thread; int64_t n_req; /* number of requests to process */ @@ -64,4 +66,3 @@ void print_parsed_args(struct arguments *args); #ifdef __cplusplus } #endif - diff --git a/libCacheSim/cache/CMakeLists.txt b/libCacheSim/cache/CMakeLists.txt index 0b4b120e..38ba2a49 100644 --- a/libCacheSim/cache/CMakeLists.txt +++ b/libCacheSim/cache/CMakeLists.txt @@ -1,9 +1,11 @@ include_directories(admission) include_directories(eviction) +include_directories(prefetch) add_subdirectory(admission) add_subdirectory(eviction) +add_subdirectory(prefetch) add_library(cachelib cache.c cacheObj.c) target_link_libraries(cachelib dataStructure) diff --git a/libCacheSim/cache/admission/adaptsize.cpp b/libCacheSim/cache/admission/adaptsize.cpp index 1a3457a6..ad5de582 100644 --- a/libCacheSim/cache/admission/adaptsize.cpp +++ b/libCacheSim/cache/admission/adaptsize.cpp @@ -62,6 +62,9 @@ void free_adaptsize_admissioner(admissioner_t *admissioner) { static_cast(admissioner->params); free(pa); + if (admissioner->init_params) { + free(admissioner->init_params); + } free(admissioner); } diff --git a/libCacheSim/cache/admission/bloomfilter.c b/libCacheSim/cache/admission/bloomfilter.c index 0dff49e2..193a3001 100644 --- a/libCacheSim/cache/admission/bloomfilter.c +++ b/libCacheSim/cache/admission/bloomfilter.c @@ -18,7 +18,8 @@ typedef struct bloomfilter_admission { bool bloomfilter_admit(admissioner_t *admissioner, const request_t *req) { bf_admission_params_t *bf = admissioner->params; gpointer key = GINT_TO_POINTER(req->obj_id); - gpointer n_times = g_hash_table_lookup(bf->seen_times, GSIZE_TO_POINTER(req->obj_id)); + gpointer n_times = + g_hash_table_lookup(bf->seen_times, GSIZE_TO_POINTER(req->obj_id)); if (n_times == NULL) { g_hash_table_insert(bf->seen_times, key, GINT_TO_POINTER(1)); return false; @@ -37,6 +38,9 @@ void free_bloomfilter_admissioner(admissioner_t *admissioner) { struct bloomfilter_admission *bf = admissioner->params; g_hash_table_destroy(bf->seen_times); free(bf); + if (admissioner->init_params) { + free(admissioner->init_params); + } free(admissioner); } diff --git a/libCacheSim/cache/admission/prob.c b/libCacheSim/cache/admission/prob.c index 2745799f..5f7a0340 100644 --- a/libCacheSim/cache/admission/prob.c +++ b/libCacheSim/cache/admission/prob.c @@ -75,6 +75,9 @@ void free_prob_admissioner(admissioner_t *admissioner) { prob_admission_params_t *pa = admissioner->params; free(pa); + if (admissioner->init_params) { + free(admissioner->init_params); + } free(admissioner); } diff --git a/libCacheSim/cache/admission/size.c b/libCacheSim/cache/admission/size.c index 35290672..a868e77b 100644 --- a/libCacheSim/cache/admission/size.c +++ b/libCacheSim/cache/admission/size.c @@ -64,6 +64,9 @@ void free_size_admissioner(admissioner_t *admissioner) { size_admission_params_t *pa = admissioner->params; free(pa); + if (admissioner->init_params) { + free(admissioner->init_params); + } free(admissioner); } diff --git a/libCacheSim/cache/cache.c b/libCacheSim/cache/cache.c index c2134880..4428ae8e 100644 --- a/libCacheSim/cache/cache.c +++ b/libCacheSim/cache/cache.c @@ -5,6 +5,7 @@ #include "../include/libCacheSim/cache.h" #include "../dataStructure/hashtable/hashtable.h" +#include "../include/libCacheSim/prefetchAlgo.h" /** this file contains both base function, which should be called by all *eviction algorithms, and the queue related functions, which should be called @@ -33,6 +34,7 @@ cache_t *cache_struct_init(const char *const cache_name, cache->cache_size = params.cache_size; cache->eviction_params = NULL; cache->admissioner = NULL; + cache->prefetcher = NULL; cache->future_stack_dist = NULL; cache->future_stack_dist_array_size = 0; cache->default_ttl = params.default_ttl; @@ -71,6 +73,7 @@ cache_t *cache_struct_init(const char *const cache_name, void cache_struct_free(cache_t *cache) { free_hashtable(cache->hashtable); if (cache->admissioner != NULL) cache->admissioner->free(cache->admissioner); + if (cache->prefetcher != NULL) cache->prefetcher->free(cache->prefetcher); my_free(sizeof(cache_t), cache); } @@ -95,6 +98,10 @@ cache_t *create_cache_with_new_size(const cache_t *old_cache, if (old_cache->admissioner != NULL) { cache->admissioner = old_cache->admissioner->clone(old_cache->admissioner); } + if (old_cache->prefetcher != NULL) { + cache->prefetcher = + old_cache->prefetcher->clone(old_cache->prefetcher, new_size); + } cache->future_stack_dist = old_cache->future_stack_dist; cache->future_stack_dist_array_size = old_cache->future_stack_dist_array_size; return cache; @@ -146,6 +153,13 @@ cache_obj_t *cache_find_base(cache_t *cache, const request_t *req, const bool update_cache) { cache_obj_t *cache_obj = hashtable_find(cache->hashtable, req); + // "update_cache = true" means that it is a real user request, use handle_find + // to update prefetcher's state + if (cache->prefetcher && cache->prefetcher->handle_find && update_cache) { + bool hit = (cache_obj != NULL); + cache->prefetcher->handle_find(cache, req, hit); + } + if (cache_obj != NULL) { #ifdef SUPPORT_TTL if (cache_obj->exp_time != 0 && cache_obj->exp_time < req->clock_time) { @@ -193,26 +207,27 @@ bool cache_get_base(cache_t *cache, const request_t *req) { cache->get_occupied_byte(cache), cache->cache_size); cache_obj_t *obj = cache->find(cache, req, true); + bool hit = (obj != NULL); - if (obj != NULL) { + if (hit) { VVERBOSE("req %ld, obj %ld --- cache hit\n", cache->n_req, req->obj_id); - return true; - } - - if (cache->can_insert(cache, req) == false) { + } else if (!cache->can_insert(cache, req)) { VVERBOSE("req %ld, obj %ld --- cache miss cannot insert\n", cache->n_req, req->obj_id); - return false; + } else { + while (cache->get_occupied_byte(cache) + req->obj_size + + cache->obj_md_size > + cache->cache_size) { + cache->evict(cache, req); + } + cache->insert(cache, req); } - while (cache->get_occupied_byte(cache) + req->obj_size + cache->obj_md_size > - cache->cache_size) { - cache->evict(cache, req); + if (cache->prefetcher && cache->prefetcher->prefetch) { + cache->prefetcher->prefetch(cache, req); } - cache->insert(cache, req); - - return false; + return hit; } /** @@ -237,8 +252,8 @@ cache_obj_t *cache_insert_base(cache_t *cache, const request_t *req) { } #endif -#if defined(TRACK_EVICTION_V_AGE) || \ - defined(TRACK_DEMOTION) || defined(TRACK_CREATE_TIME) +#if defined(TRACK_EVICTION_V_AGE) || defined(TRACK_DEMOTION) || \ + defined(TRACK_CREATE_TIME) cache_obj->create_time = CURR_TIME(cache, req); #endif @@ -263,6 +278,17 @@ void cache_evict_base(cache_t *cache, cache_obj_t *obj, record_eviction_age(cache, obj, CURR_TIME(cache, req) - obj->create_time); } #endif + if (cache->prefetcher && cache->prefetcher->handle_evict) { + request_t *check_req = new_request(); + check_req->obj_id = obj->obj_id; + check_req->obj_size = obj->obj_size; + // check_req->ttl = 0; // re-add objects should be? + cache_remove_obj_base(cache, obj, remove_from_hashtable); + cache->prefetcher->handle_evict(cache, check_req); + my_free(sizeof(request_t), check_req); + return; + } + cache_remove_obj_base(cache, obj, remove_from_hashtable); } diff --git a/libCacheSim/cache/eviction/CMakeLists.txt b/libCacheSim/cache/eviction/CMakeLists.txt index 3d46114c..45dd821f 100644 --- a/libCacheSim/cache/eviction/CMakeLists.txt +++ b/libCacheSim/cache/eviction/CMakeLists.txt @@ -37,6 +37,7 @@ set(sourceC other/S3LRU.c Sieve.c + ) if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/priv") diff --git a/libCacheSim/cache/eviction/unsupported/Mithril.c b/libCacheSim/cache/eviction/unsupported/Mithril.c index c56919bb..2a6f48c3 100644 --- a/libCacheSim/cache/eviction/unsupported/Mithril.c +++ b/libCacheSim/cache/eviction/unsupported/Mithril.c @@ -8,14 +8,16 @@ #include "Mithril.h" -//#define TRACK_BLOCK 192618l +#include + +// #define TRACK_BLOCK 192618l #ifdef __cplusplus extern "C" { #endif static inline void _Mithril_rec_min_support_one(cache_t *Mithril, - request_t *cp); + request_t *cp); /***************** verify function, not part of Mithril *******************/ @@ -23,7 +25,6 @@ static inline void _Mithril_rec_min_support_one(cache_t *Mithril, verify the integrity of recording and mining hashtable */ void verify_rm_hashtable(gpointer key, gpointer value, gpointer user_data) { - cache_t *Mithril = (cache_t *)user_data; Mithril_params_t *Mithril_params = (Mithril_params_t *)(Mithril->cache_params); @@ -69,7 +70,6 @@ void verify_rm_hashtable(gpointer key, gpointer value, gpointer user_data) { void verify_prefetched_hashtable(gpointer key, gpointer value, gpointer user_data) { - Mithril_params_t *Mithril_params = (Mithril_params_t *)user_data; GHashTable *h = ((struct LRU_params *)(Mithril_params->cache->cache_params))->hashtable; @@ -115,17 +115,15 @@ static inline gint _Mithril_get_total_num_of_ts(gint64 *row, gint row_length) { int count = 0; for (i = 1; i < row_length; i++) { t = NUM_OF_TS(row[i]); - if (t == 0) - return count; + if (t == 0) return count; count += t; } return count; } static inline gboolean _Mithril_remove_from_rmtable(obj_id_t obj_id_type, - rec_mining_t *rmtable, - gint64 *row) { - + rec_mining_t *rmtable, + gint64 *row) { if (obj_id_type == OBJ_ID_NUM) { return g_hash_table_remove(rmtable->hashtable, row); } else if (obj_id_type == OBJ_ID_STR) { @@ -140,12 +138,11 @@ static inline gboolean _Mithril_remove_from_rmtable(obj_id_t obj_id_type, check whether last request is part of a sequential access */ static inline gboolean _Mithril_check_sequential(cache_t *Mithril, - request_t *cp) { + request_t *cp) { int i; Mithril_params_t *Mithril_params = (Mithril_params_t *)(Mithril->cache_params); - if (Mithril_params->sequential_K == 0) - return FALSE; + if (Mithril_params->sequential_K == 0) return FALSE; if (Mithril->obj_id_type != OBJ_ID_NUM) ERROR("sequential prefetching only support data obj_id_type l\n"); @@ -155,13 +152,11 @@ static inline gboolean _Mithril_check_sequential(cache_t *Mithril, gpointer old_item_p = cp->obj_id_ptr; cp->obj_id_ptr = &last; gint sequential_K = Mithril_params->sequential_K; - if (sequential_K == -1) - /* when use AMP, this is -1 */ + if (sequential_K == -1) /* when use AMP, this is -1 */ sequential_K = 1; for (i = 0; i < sequential_K; i++) { last--; - if (!Mithril_params->cache->check(Mithril_params->cache, - cp)) { + if (!Mithril_params->cache->check(Mithril_params->cache, cp)) { is_sequential = FALSE; break; } @@ -185,12 +180,10 @@ static inline void _Mithril_record_entry(cache_t *Mithril, request_t *cp) { if (Mithril_params->rec_trigger != each_req) /* if it does not record at each request, check whether it is hit or miss */ - if (Mithril_params->cache->check(Mithril_params->cache, cp)) - return; + if (Mithril_params->cache->check(Mithril_params->cache, cp)) return; /* check it is sequtial or not */ - if (Mithril_params->sequential_type && - _Mithril_check_sequential(Mithril, cp)) + if (Mithril_params->sequential_type && _Mithril_check_sequential(Mithril, cp)) return; if (Mithril_params->min_support == 1) { @@ -200,8 +193,8 @@ static inline void _Mithril_record_entry(cache_t *Mithril, request_t *cp) { else { gint64 *row_in_rtable; // check the obj_id in hashtable for training - gint index = - GPOINTER_TO_INT(g_hash_table_lookup(rmtable->hashtable, cp->obj_id_ptr)); + gint index = GPOINTER_TO_INT( + g_hash_table_lookup(rmtable->hashtable, cp->obj_id_ptr)); if (index == 0) { // the node is not in the recording/mining data, should be added @@ -249,20 +242,22 @@ static inline void _Mithril_record_entry(cache_t *Mithril, request_t *cp) { if (cp->obj_id_type == OBJ_ID_STR) { if (!g_hash_table_contains(rmtable->hashtable, (char *)(row_in_rtable[0]))) - ERROR("remove old entry from recording table, " - "but it is not in recording hashtable " - "%s, pointer %ld, current ts %ld\n", - (char *)(row_in_rtable[0]), (long)rmtable->rtable_cur_row, - (long)Mithril_params->ts); + ERROR( + "remove old entry from recording table, " + "but it is not in recording hashtable " + "%s, pointer %ld, current ts %ld\n", + (char *)(row_in_rtable[0]), (long)rmtable->rtable_cur_row, + (long)Mithril_params->ts); g_hash_table_remove(rmtable->hashtable, (char *)(row_in_rtable[0])); } else if (cp->obj_id_type == OBJ_ID_NUM) { if (!g_hash_table_contains(rmtable->hashtable, row_in_rtable)) { - ERROR("remove old entry from recording table, " - "but it is not in recording hashtable, " - "block %ld, recording table pos %ld, ts %ld ", - *(long *)row_in_rtable, (long)rmtable->rtable_cur_row, - (long)Mithril_params->ts); + ERROR( + "remove old entry from recording table, " + "but it is not in recording hashtable, " + "block %ld, recording table pos %ld, ts %ld ", + *(long *)row_in_rtable, (long)rmtable->rtable_cur_row, + (long)Mithril_params->ts); long temp = rmtable->rtable_cur_row - 1; fprintf(stderr, "previous line block %ld\n", @@ -298,18 +293,20 @@ static inline void _Mithril_record_entry(cache_t *Mithril, request_t *cp) { #ifdef SANITY_CHECK if (cp->obj_id_type == OBJ_ID_NUM) { if (*(gint64 *)(cp->obj_id_ptr) != row_in_mtable[0]) { - ERROR("inconsistent entry in mtable " - "and mining hashtable current request %ld, " - "mining table %ld\n", - *(gint64 *)(cp->obj_id_ptr), row_in_mtable[0]); + ERROR( + "inconsistent entry in mtable " + "and mining hashtable current request %ld, " + "mining table %ld\n", + *(gint64 *)(cp->obj_id_ptr), row_in_mtable[0]); abort(); } } else { if (strcmp(cp->obj_id_ptr, (char *)(row_in_mtable[0])) != 0) { - ERROR("inconsistent entry in mtable " - "and mining hashtable current request %ld, " - "mining table %s %s\n", - (char *)(cp->obj_id_ptr), (char *)(row_in_mtable[0])); + ERROR( + "inconsistent entry in mtable " + "and mining hashtable current request %ld, " + "mining table %s %s\n", + (char *)(cp->obj_id_ptr), (char *)(row_in_mtable[0])); abort(); } } @@ -327,7 +324,7 @@ static inline void _Mithril_record_entry(cache_t *Mithril, request_t *cp) { if (timestamps_length == Mithril_params->max_support) { /* no timestamp added, drop this request, it is too frequent */ if (!_Mithril_remove_from_rmtable(cp->obj_id_type, rmtable, - row_in_mtable)) + row_in_mtable)) ERROR("removing from rmtable failed for mining table entry\n"); /** for dataType c, now the pointer to string has been freed, @@ -401,10 +398,11 @@ static inline void _Mithril_record_entry(cache_t *Mithril, request_t *cp) { if ((long)rmtable->mining_table->len >= Mithril_params->mtable_size) { /* if this happens, array will re-malloc, which will make * the hashtable key not reliable when obj_id_type is l */ - ERROR("mining table length reaches limit, but no mining, " - "entry %d, size %u, threshold %d\n", - rmtable->n_avail_mining, rmtable->mining_table->len, - Mithril_params->mtable_size); + ERROR( + "mining table length reaches limit, but no mining, " + "entry %d, size %u, threshold %d\n", + rmtable->n_avail_mining, rmtable->mining_table->len, + Mithril_params->mtable_size); abort(); } #endif @@ -434,8 +432,7 @@ static inline void _Mithril_record_entry(cache_t *Mithril, request_t *cp) { /** if current pointer points to 1, * then don't move it, clear the row (that moves to mining table) **/ - for (i = 0; i < rmtable->rtable_row_len; i++) - row_in_rtable[i] = 0; + for (i = 0; i < rmtable->rtable_row_len; i++) row_in_rtable[i] = 0; } gint64 *inserted_row_in_mtable = @@ -482,8 +479,7 @@ static inline void _Mithril_record_entry(cache_t *Mithril, request_t *cp) { // one entry has been moved to mining table, shrinking recording table // size by 1 - if (rmtable->rtable_cur_row >= 2) - rmtable->rtable_cur_row--; + if (rmtable->rtable_cur_row >= 2) rmtable->rtable_cur_row--; } } } @@ -497,16 +493,15 @@ static inline void _Mithril_record_entry(cache_t *Mithril, request_t *cp) { } static inline void _Mithril_rec_min_support_one(cache_t *Mithril, - request_t *cp) { - + const request_t *cp) { Mithril_params_t *Mithril_params = (Mithril_params_t *)(Mithril->cache_params); rec_mining_t *rmtable = Mithril_params->rmtable; #ifdef TRACK_BLOCK if (*(gint64 *)(cp->obj_id_ptr) == TRACK_BLOCK) { - int old_pos = - GPOINTER_TO_INT(g_hash_table_lookup(rmtable->hashtable, cp->obj_id_ptr)); + int old_pos = GPOINTER_TO_INT( + g_hash_table_lookup(rmtable->hashtable, cp->obj_id_ptr)); printf("insert %ld, old pos %d", TRACK_BLOCK, old_pos); if (old_pos == 0) printf("\n"); @@ -541,8 +536,7 @@ static inline void _Mithril_rec_min_support_one(cache_t *Mithril, array_ele[0] = *(gint64 *)(cp->obj_id_ptr); hash_key = GET_ROW_IN_MTABLE(Mithril_params, rmtable->mining_table->len); } - for (i = 1; i < rmtable->mtable_row_len; i++) - array_ele[i] = 0; + for (i = 1; i < rmtable->mtable_row_len; i++) array_ele[i] = 0; array_ele[1] = ADD_TS(array_ele[1], Mithril_params->ts); g_array_append_val(rmtable->mining_table, array_ele); @@ -603,7 +597,7 @@ static inline void _Mithril_rec_min_support_one(cache_t *Mithril, if (timestamps_length == Mithril_params->max_support) { /* no timestamp added, drop this request, it is too frequent */ if (!_Mithril_remove_from_rmtable(cp->obj_id_type, rmtable, - row_in_mtable)) + row_in_mtable)) ERROR("removing from rmtable failed for mining table entry\n"); g_array_remove_index_fast(rmtable->mining_table, index - 1); @@ -624,7 +618,6 @@ static inline void _Mithril_rec_min_support_one(cache_t *Mithril, } static inline void _Mithril_prefetch(cache_t *Mithril, request_t *cp) { - Mithril_params_t *Mithril_params = (Mithril_params_t *)(Mithril->cache_params); gint prefetch_table_index = GPOINTER_TO_INT( @@ -640,15 +633,13 @@ static inline void _Mithril_prefetch(cache_t *Mithril, request_t *cp) { int i; for (i = 1; i < Mithril_params->pf_list_size + 1; i++) { // begin from 1 because index 0 is the obj_id of originated request - if (Mithril_params->ptable_array[dim1][dim2 + i] == 0) - break; + if (Mithril_params->ptable_array[dim1][dim2 + i] == 0) break; if (cp->obj_id_type == OBJ_ID_NUM) cp->obj_id_ptr = &(Mithril_params->ptable_array[dim1][dim2 + i]); else cp->obj_id_ptr = (char *)(Mithril_params->ptable_array[dim1][dim2 + i]); - if (Mithril_params->output_statistics) - Mithril_params->num_of_check += 1; + if (Mithril_params->output_statistics) Mithril_params->num_of_check += 1; // can't use Mithril_check here if (Mithril_params->cache->check(Mithril_params->cache, cp)) { @@ -689,20 +680,20 @@ static inline void _Mithril_prefetch(cache_t *Mithril, request_t *cp) { // use this, not add because we need to record stat when evicting Mithril_params->cache->_insert(Mithril_params->cache, cp); - while ((long)Mithril_params->cache->get_used_size( - Mithril_params->cache) > Mithril_params->cache->size) + while ((long)Mithril_params->cache->get_used_size(Mithril_params->cache) > + Mithril_params->cache->size) _Mithril_evict(Mithril, cp); if (Mithril_params->output_statistics) { Mithril_params->num_of_prefetch_sequential += 1; - g_hash_table_add(Mithril_params->prefetched_hashtable_sequential, GSIZE_TO_POINTER(next)); + g_hash_table_add(Mithril_params->prefetched_hashtable_sequential, + GSIZE_TO_POINTER(next)); } cp->obj_id_ptr = old_obj_id; } } gboolean Mithril_check(cache_t *Mithril, request_t *cp) { - // check element, record entry and prefetch element Mithril_params_t *Mithril_params = (Mithril_params_t *)(Mithril->cache_params); @@ -721,31 +712,29 @@ gboolean Mithril_check(cache_t *Mithril, request_t *cp) { } } - gboolean result = - Mithril_params->cache->check(Mithril_params->cache, cp); + gboolean result = Mithril_params->cache->check(Mithril_params->cache, cp); - if (Mithril_params->rec_trigger != evict) - _Mithril_record_entry(Mithril, cp); + if (Mithril_params->rec_trigger != evict) _Mithril_record_entry(Mithril, cp); return result; } void _Mithril_update(cache_t *Mithril, request_t *cp) { - Mithril_params_t *Mithril_params = (Mithril_params_t *)(Mithril->cache_params); Mithril_params->cache->_update(Mithril_params->cache, cp); } void _Mithril_evict(cache_t *Mithril, request_t *cp) { - Mithril_params_t *Mithril_params = (Mithril_params_t *)(Mithril->cache_params); if (Mithril_params->output_statistics) { - gpointer gp = Mithril_params->cache->evict_with_return(Mithril_params->cache, cp); + gpointer gp = + Mithril_params->cache->evict_with_return(Mithril_params->cache, cp); - gint type = GPOINTER_TO_INT(g_hash_table_lookup(Mithril_params->prefetched_hashtable_Mithril, gp)); + gint type = GPOINTER_TO_INT( + g_hash_table_lookup(Mithril_params->prefetched_hashtable_Mithril, gp)); if (type != 0 && type < Mithril_params->cycle_time) { // give one more chance gpointer new_key = gp; @@ -787,8 +776,8 @@ gboolean Mithril_add(cache_t *Mithril, request_t *cp) { retval = Mithril_check(Mithril, cp); AMP_add_no_eviction(Mithril_params->cache, cp); _Mithril_prefetch(Mithril, cp); - while ((long)Mithril_params->cache->get_used_size( - Mithril_params->cache) > Mithril->size) + while ((long)Mithril_params->cache->get_used_size(Mithril_params->cache) > + Mithril->size) _Mithril_evict(Mithril, cp); } else { @@ -799,8 +788,8 @@ gboolean Mithril_add(cache_t *Mithril, request_t *cp) { } else { _Mithril_insert(Mithril, cp); _Mithril_prefetch(Mithril, cp); - while ((long)Mithril_params->cache->get_used_size( - Mithril_params->cache) > Mithril->size) + while ((long)Mithril_params->cache->get_used_size(Mithril_params->cache) > + Mithril->size) _Mithril_evict(Mithril, cp); retval = FALSE; } @@ -822,8 +811,7 @@ void _Mithril_evict_only(cache_t *Mithril, request_t *cp) { (Mithril_params_t *)(Mithril->cache_params); gpointer gp; - gp = Mithril_params->cache->evict_with_return(Mithril_params->cache, - cp); + gp = Mithril_params->cache->evict_with_return(Mithril_params->cache, cp); gint type = GPOINTER_TO_INT( g_hash_table_lookup(Mithril_params->prefetched_hashtable_Mithril, gp)); @@ -838,7 +826,7 @@ void _Mithril_evict_only(cache_t *Mithril, request_t *cp) { GINT_TO_POINTER(type + 1)); gpointer old_cp = cp->obj_id_ptr; cp->obj_id_ptr = gp; - _Mithril_insert(Mithril, cp); // insert is same + _Mithril_insert(Mithril, cp); // insert is same cp->obj_id_ptr = old_cp; _Mithril_evict_only(Mithril, cp); } else { @@ -861,18 +849,18 @@ gboolean Mithril_add_only(cache_t *Mithril, request_t *cp) { if (strcmp(Mithril_params->cache->cache_name, "AMP")) { retval = Mithril_check_only(Mithril, cp); AMP_add_only_no_eviction(Mithril_params->cache, cp); - while ((long)Mithril_params->cache->get_used_size( - Mithril_params->cache) > Mithril->size) + while ((long)Mithril_params->cache->get_used_size(Mithril_params->cache) > + Mithril->size) _Mithril_evict_only(Mithril, cp); } else { if (Mithril_check_only(Mithril, cp)) { // new - _Mithril_update(Mithril, cp); // update is same + _Mithril_update(Mithril, cp); // update is same retval = TRUE; } else { - _Mithril_insert(Mithril, cp); // insert is same - while ((long)Mithril_params->cache->get_used_size( - Mithril_params->cache) > Mithril->size) + _Mithril_insert(Mithril, cp); // insert is same + while ((long)Mithril_params->cache->get_used_size(Mithril_params->cache) > + Mithril->size) _Mithril_evict_only(Mithril, cp); retval = FALSE; } @@ -881,73 +869,77 @@ gboolean Mithril_add_only(cache_t *Mithril, request_t *cp) { return retval; } -//gboolean Mithril_add_withsize(cache_t *Mithril, request_t *cp) { -// int i; +// gboolean Mithril_add_withsize(cache_t *Mithril, request_t *cp) { +// int i; // -// gboolean ret_val; -// if (Mithril->block_size != 0 && cp->disk_sector_size != 0) { +// gboolean ret_val; +// if (Mithril->block_size != 0 && cp->disk_sector_size != 0) { // -// *(gint64 *)(cp->obj_id_ptr) = -// (gint64)(*(gint64 *)(cp->obj_id_ptr) * cp->disk_sector_size / -// Mithril->block_size); -// } +// *(gint64 *)(cp->obj_id_ptr) = +// (gint64)(*(gint64 *)(cp->obj_id_ptr) * cp->disk_sector_size / +// Mithril->block_size); +// } // -//#ifdef TRACK_BLOCK -// Mithril_params_t *Mithril_params = -// (Mithril_params_t *)(Mithril->cache_params); -// rec_mining_t *rmtable = Mithril_params->rmtable; -// static int last_found_pos = 0; -// gint64 b = TRACK_BLOCK; -// if (*(gint64 *)(cp->obj_id_ptr) == TRACK_BLOCK) { -// printf("ts %lu, track block add\n", cp->ts); -// } -// int old_pos = GPOINTER_TO_INT(g_hash_table_lookup(rmtable->hashtable, &b)); -// if (old_pos != 0) { -// if (old_pos > 0) { -// gint64 *row = GET_ROW_IN_RTABLE(Mithril_params, old_pos); -// if (*row != TRACK_BLOCK) { -// ERROR("the row (%d) from recording table does not match track block, " -// "it is %ld\n", -// old_pos, *row); -// abort(); -// } -// } else if (old_pos < 0) { -// gint64 *row = GET_ROW_IN_MTABLE(Mithril_params, -old_pos - 1); -// if (*row != TRACK_BLOCK) { -// ERROR("the row (%d) from recording table does not match track block, " -// "it is %ld\n", -// old_pos, *row); -// abort(); -// } -// } -// if (last_found_pos != old_pos) { -// ERROR("ts %lu, found track block change in hashtable, pos %d\n", cp->ts, -// old_pos); -// last_found_pos = old_pos; -// } -// } else { -// if (last_found_pos != 0) { -// printf("ts %lu, track block %ld disappeared, might because of mining\n", -// cp->ts, TRACK_BLOCK); -// } -// } -//#endif +// #ifdef TRACK_BLOCK +// Mithril_params_t *Mithril_params = +// (Mithril_params_t *)(Mithril->cache_params); +// rec_mining_t *rmtable = Mithril_params->rmtable; +// static int last_found_pos = 0; +// gint64 b = TRACK_BLOCK; +// if (*(gint64 *)(cp->obj_id_ptr) == TRACK_BLOCK) { +// printf("ts %lu, track block add\n", cp->ts); +// } +// int old_pos = GPOINTER_TO_INT(g_hash_table_lookup(rmtable->hashtable, &b)); +// if (old_pos != 0) { +// if (old_pos > 0) { +// gint64 *row = GET_ROW_IN_RTABLE(Mithril_params, old_pos); +// if (*row != TRACK_BLOCK) { +// ERROR("the row (%d) from recording table does not match track block, +// " +// "it is %ld\n", +// old_pos, *row); +// abort(); +// } +// } else if (old_pos < 0) { +// gint64 *row = GET_ROW_IN_MTABLE(Mithril_params, -old_pos - 1); +// if (*row != TRACK_BLOCK) { +// ERROR("the row (%d) from recording table does not match track block, +// " +// "it is %ld\n", +// old_pos, *row); +// abort(); +// } +// } +// if (last_found_pos != old_pos) { +// ERROR("ts %lu, found track block change in hashtable, pos %d\n", +// cp->ts, +// old_pos); +// last_found_pos = old_pos; +// } +// } else { +// if (last_found_pos != 0) { +// printf("ts %lu, track block %ld disappeared, might because of +// mining\n", +// cp->ts, TRACK_BLOCK); +// } +// } +// #endif // -// ret_val = Mithril_add(Mithril, cp); +// ret_val = Mithril_add(Mithril, cp); // -// if (Mithril->block_size != 0 && cp->disk_sector_size != 0) { -// int n = (int)ceil((double)cp->size / Mithril->block_size); +// if (Mithril->block_size != 0 && cp->disk_sector_size != 0) { +// int n = (int)ceil((double)cp->size / Mithril->block_size); // -// for (i = 0; i < n - 1; i++) { -// (*(guint64 *)(cp->obj_id_ptr))++; -// Mithril_add_only(Mithril, cp); -// } -// *(gint64 *)(cp->obj_id_ptr) -= (n - 1); -// } +// for (i = 0; i < n - 1; i++) { +// (*(guint64 *)(cp->obj_id_ptr))++; +// Mithril_add_only(Mithril, cp); +// } +// *(gint64 *)(cp->obj_id_ptr) -= (n - 1); +// } // -// Mithril->ts += 1; -// return ret_val; -//} +// Mithril->ts += 1; +// return ret_val; +// } void Mithril_destroy(cache_t *cache) { Mithril_params_t *Mithril_params = (Mithril_params_t *)(cache->cache_params); @@ -984,7 +976,7 @@ void Mithril_destroy(cache_t *cache) { g_hash_table_destroy(Mithril_params->prefetched_hashtable_Mithril); g_hash_table_destroy(Mithril_params->prefetched_hashtable_sequential); } - Mithril_params->cache->destroy(Mithril_params->cache); // 0921 + Mithril_params->cache->destroy(Mithril_params->cache); // 0921 cache_destroy(cache); } @@ -1029,7 +1021,7 @@ void Mithril_destroy_unique(cache_t *cache) { g_hash_table_destroy(Mithril_params->prefetched_hashtable_Mithril); g_hash_table_destroy(Mithril_params->prefetched_hashtable_sequential); } - Mithril_params->cache->destroy_unique(Mithril_params->cache); // 0921 + Mithril_params->cache->destroy_unique(Mithril_params->cache); // 0921 cache_destroy_unique(cache); } @@ -1050,8 +1042,8 @@ cache_t *Mithril_init(guint64 size, obj_id_t obj_id_type, void *params) { cache->_evict = _Mithril_evict; cache->get_used_size = Mithril_get_size; cache->cache_init_params = params; -// cache->add_only = Mithril_add_only; -// cache->add_withsize = Mithril_add_withsize; + // cache->add_only = Mithril_add_only; + // cache->add_withsize = Mithril_add_withsize; Mithril_init_params_t *init_params = (Mithril_init_params_t *)params; @@ -1103,13 +1095,14 @@ cache_t *Mithril_init(guint64 size, obj_id_t obj_id_type, void *params) { PREFETCH_TABLE_SHARD_SIZE * (Mithril_params->pf_list_size * 8 + 8 + 4); if (Mithril_params->max_support != 1) { - rmtable->n_rows_in_rtable = (gint)( - size * Mithril_params->block_size * RECORDING_TABLE_MAXIMAL / - ((gint)ceil((double)Mithril_params->min_support / (double)2) * 2 + 8 + - 4)); + rmtable->n_rows_in_rtable = + (gint)(size * Mithril_params->block_size * RECORDING_TABLE_MAXIMAL / + ((gint)ceil((double)Mithril_params->min_support / (double)2) * + 2 + + 8 + 4)); rmtable->recording_table = g_new0( gint64, rmtable->n_rows_in_rtable * - rmtable->rtable_row_len); // this should begins with 1 + rmtable->rtable_row_len); // this should begins with 1 Mithril_params->cur_metadata_size += (((gint)ceil((double)init_params->min_support / (double)4 + 1) * 8 + 4) * @@ -1134,7 +1127,7 @@ cache_t *Mithril_init(guint64 size, obj_id_t obj_id_type, void *params) { Mithril_params->cache = LRU_init(size, obj_id_type, NULL); else if (strcmp(init_params->cache_type, "FIFO") == 0) Mithril_params->cache = FIFO_init(size, obj_id_type, NULL); - else if (strcmp(init_params->cache_type, "LFU") == 0) // use LFU_fast + else if (strcmp(init_params->cache_type, "LFU") == 0) // use LFU_fast Mithril_params->cache = LFUFast_init(size, obj_id_type, NULL); else if (strcmp(init_params->cache_type, "AMP") == 0) { struct AMP_init_params *AMP_init_params = g_new0(struct AMP_init_params, 1); @@ -1142,8 +1135,7 @@ cache_t *Mithril_init(guint64 size, obj_id_t obj_id_type, void *params) { AMP_init_params->p_threshold = init_params->AMP_pthreshold; AMP_init_params->read_size = 1; AMP_init_params->K = init_params->sequential_K; - Mithril_params->cache = - AMP_init(size, obj_id_type, AMP_init_params); + Mithril_params->cache = AMP_init(size, obj_id_type, AMP_init_params); } else if (strcmp(init_params->cache_type, "Belady") == 0) { struct Belady_init_params *Belady_init_params = g_new(struct Belady_init_params, 1); @@ -1163,22 +1155,22 @@ cache_t *Mithril_init(guint64 size, obj_id_t obj_id_type, void *params) { g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, NULL); if (Mithril_params->output_statistics) { - Mithril_params->prefetched_hashtable_Mithril = g_hash_table_new_full( - g_direct_hash, g_direct_equal, NULL, NULL); - Mithril_params->prefetched_hashtable_sequential = g_hash_table_new_full( - g_direct_hash, g_direct_equal, NULL, NULL); + Mithril_params->prefetched_hashtable_Mithril = + g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, NULL); + Mithril_params->prefetched_hashtable_sequential = + g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, NULL); } } else if (obj_id_type == OBJ_ID_STR) { - rmtable->hashtable = g_hash_table_new_full( - g_str_hash, g_str_equal, g_free, NULL); + rmtable->hashtable = + g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL); Mithril_params->prefetch_hashtable = g_hash_table_new_full(g_str_hash, g_str_equal, NULL, NULL); if (Mithril_params->output_statistics) { - Mithril_params->prefetched_hashtable_Mithril = g_hash_table_new_full( - g_str_hash, g_str_equal, g_free, NULL); - Mithril_params->prefetched_hashtable_sequential = g_hash_table_new_full( - g_str_hash, g_str_equal, g_free, NULL); + Mithril_params->prefetched_hashtable_Mithril = + g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL); + Mithril_params->prefetched_hashtable_sequential = + g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL); } } else { ERROR("does not support given obj_id type: %c\n", obj_id_type); @@ -1239,8 +1231,7 @@ void _Mithril_mining(cache_t *Mithril) { if (GET_NTH_TS(item2, 1) - GET_NTH_TS(item1, 1) > Mithril_params->lookahead_range) break; - num_of_ts2 = - _Mithril_get_total_num_of_ts(item2, rmtable->mtable_row_len); + num_of_ts2 = _Mithril_get_total_num_of_ts(item2, rmtable->mtable_row_len); if (ABS(num_of_ts1 - num_of_ts2) > Mithril_params->confidence) { continue; @@ -1342,8 +1333,7 @@ void Mithril_add_to_prefetch_table(cache_t *Mithril, gpointer gp1, exit(1); } #endif - if ((Mithril_params->ptable_array[dim1][dim2 + i]) == 0) - break; + if ((Mithril_params->ptable_array[dim1][dim2 + i]) == 0) break; if ((Mithril_params->ptable_array[dim1][dim2 + i]) == *(gint64 *)(gp2)) /* update score here, not implemented yet */ insert = FALSE; @@ -1357,8 +1347,7 @@ void Mithril_add_to_prefetch_table(cache_t *Mithril, gpointer gp1, fprintf(stderr, "ERROR prefetch table pos wrong\n"); exit(1); } - if ((Mithril_params->ptable_array[dim1][dim2 + i]) == 0) - break; + if ((Mithril_params->ptable_array[dim1][dim2 + i]) == 0) break; if (strcmp((gchar *)(Mithril_params->ptable_array[dim1][dim2 + i]), (gchar *)gp2) == 0) /* update score here, not implemented yet */ @@ -1464,7 +1453,7 @@ void Mithril_add_to_prefetch_table(cache_t *Mithril, gpointer gp1, Mithril_params->cur_metadata_size += required_meta_data_size; Mithril_params->cache->size = Mithril->size - (gint)((Mithril_params->cur_metadata_size) / - Mithril_params->block_size); + Mithril_params->block_size); } else { Mithril_params->ptable_is_full = TRUE; Mithril_params->ptable_cur_row = 1; @@ -1477,8 +1466,7 @@ void _Mithril_aging(cache_t *Mithril) { ; } guint64 Mithril_get_size(cache_t *cache) { Mithril_params_t *Mithril_params = (Mithril_params_t *)(cache->cache_params); - return (guint64)Mithril_params->cache->get_used_size( - Mithril_params->cache); + return (guint64)Mithril_params->cache->get_used_size(Mithril_params->cache); } void prefetch_node_destroyer(gpointer data) { diff --git a/libCacheSim/cache/prefetch/CMakeLists.txt b/libCacheSim/cache/prefetch/CMakeLists.txt new file mode 100644 index 00000000..025b6409 --- /dev/null +++ b/libCacheSim/cache/prefetch/CMakeLists.txt @@ -0,0 +1,7 @@ + +add_library(prefetchC Mithril.c) + +add_library(prefetch INTERFACE) +target_link_libraries(prefetch INTERFACE prefetchC) + + diff --git a/libCacheSim/cache/prefetch/Mithril.c b/libCacheSim/cache/prefetch/Mithril.c new file mode 100644 index 00000000..9678340a --- /dev/null +++ b/libCacheSim/cache/prefetch/Mithril.c @@ -0,0 +1,1160 @@ +// +// a Mithril module that supports different obj size +// +// +// Mithril.c +// libCacheSim +// +// Created by Zhelong on 23/8/15. +// Copyright © 2023 Zhelong. All rights reserved. +// +#include "../../include/libCacheSim/prefetchAlgo/Mithril.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "../../include/libCacheSim/prefetchAlgo.h" +#include "glibconfig.h" + +#define TRACK_BLOCK 192618l +#define SANITY_CHECK 1 +#define PROFILING +// #define debug + +#ifdef __cplusplus +extern "C" { +#endif + +// *********************************************************************** +// **** **** +// **** helper function declarations **** +// **** **** +// *********************************************************************** +static inline bool _Mithril_check_sequential(cache_t *Mithril, + const request_t *req); +static inline void _Mithril_record_entry(cache_t *Mithril, + const request_t *req); +static inline void _Mithril_rec_min_support_one(cache_t *Mithril, + const request_t *req); +static inline gint _Mithril_get_total_num_of_ts(gint64 *row, gint row_length); +static void _Mithril_mining(cache_t *Mithril); + +static void _Mithril_add_to_prefetch_table(cache_t *Mithril, gpointer gp1, + gpointer gp2); + +const char *Mithril_default_params(void) { + return "lookahead-range=20, " + "max-support=8, min-support=2, confidence=1, pf-list-size=2, " + "rec-trigger=miss, block-size=1, max-metadata-size=0.1, " + "cycle-time=2, mining-threshold=5120, sequential-type=0, " + "sequential-K=-1, AMP-pthreshold=-1"; +} + +static void set_Mithril_default_init_params( + Mithril_init_params_t *init_params) { + init_params->lookahead_range = 20; + init_params->max_support = 8; + init_params->min_support = 2; + init_params->confidence = 1; + init_params->pf_list_size = 2; + init_params->rec_trigger = miss; + init_params->block_size = 1; // for general use + init_params->max_metadata_size = 0.1; + init_params->cycle_time = 2; + init_params->mining_threshold = MINING_THRESHOLD; + + init_params->sequential_type = 0; + init_params->sequential_K = -1; + + init_params->AMP_pthreshold = -1; +} + +static void Mithril_parse_init_params(const char *cache_specific_params, + Mithril_init_params_t *init_params) { + char *params_str = strdup(cache_specific_params); + + while (params_str != NULL && params_str[0] != '\0') { + char *key = strsep((char **)¶ms_str, "="); + char *value = strsep((char **)¶ms_str, ","); + while (params_str != NULL && *params_str == ' ') { + params_str++; + } + if (strcasecmp(key, "lookahead-range") == 0) { + init_params->lookahead_range = atoi(value); + } else if (strcasecmp(key, "max-support") == 0) { + init_params->max_support = atoi(value); + } else if (strcasecmp(key, "min-support") == 0) { + init_params->min_support = atoi(value); + } else if (strcasecmp(key, "confidence") == 0) { + init_params->confidence = atoi(value); + } else if (strcasecmp(key, "pf-list-size") == 0) { + init_params->pf_list_size = atoi(value); + } else if (strcasecmp(key, "rec-trigger") == 0) { + if (strcasecmp(value, "miss") == 0) { + init_params->rec_trigger = miss; + } else if (strcasecmp(value, "evict") == 0) { + init_params->rec_trigger = evict; + } else if (strcasecmp(value, "miss_evict") == 0) { + init_params->rec_trigger = miss_evict; + } else if (strcasecmp(value, "each_req") == 0) { + init_params->rec_trigger = each_req; + } else { + ERROR("Mithril's rec-trigger does not support %s \n", value); + } + } else if (strcasecmp(key, "block-size") == 0) { + init_params->block_size = (unsigned long)atoi(value); + } else if (strcasecmp(key, "max-metadata-size") == 0) { + init_params->max_metadata_size = atof(value); + } else if (strcasecmp(key, "cycle-time") == 0) { + init_params->cycle_time = atoi(value); + } else if (strcasecmp(key, "mining-threshold") == 0) { + init_params->mining_threshold = atoi(value); + } else if (strcasecmp(key, "sequential-type") == 0) { + init_params->sequential_type = atoi(value); + } else if (strcasecmp(key, "sequential-K") == 0) { + init_params->sequential_K = atoi(value); + } else if (strcasecmp(key, "AMP-pthreshold") == 0) { + init_params->AMP_pthreshold = atoi(value); + } else if (strcasecmp(key, "print") == 0 || + strcasecmp(key, "default") == 0) { + printf("default params: %s\n", Mithril_default_params()); + exit(0); + } else { + ERROR("Mithril does not have parameter %s\n", key); + printf("default params: %s\n", Mithril_default_params()); + exit(1); + } + } +} + +static void set_Mithril_params(Mithril_params_t *Mithril_params, + Mithril_init_params_t *init_params, + uint64_t cache_size) { + Mithril_params->lookahead_range = init_params->lookahead_range; + Mithril_params->max_support = init_params->max_support; + Mithril_params->min_support = init_params->min_support; + Mithril_params->confidence = init_params->confidence; + Mithril_params->cycle_time = init_params->cycle_time; + Mithril_params->pf_list_size = init_params->pf_list_size; + Mithril_params->mining_threshold = init_params->mining_threshold; + + Mithril_params->block_size = init_params->block_size; + Mithril_params->sequential_type = init_params->sequential_type; + Mithril_params->sequential_K = init_params->sequential_K; + Mithril_params->output_statistics = 1; + + Mithril_params->mtable_size = + (gint)(init_params->mining_threshold / Mithril_params->min_support); + + Mithril_params->rec_trigger = init_params->rec_trigger; + + Mithril_params->max_metadata_size = + (gint64)(init_params->block_size * cache_size * + init_params->max_metadata_size); + + gint max_num_of_shards_in_prefetch_table = + (gint)(Mithril_params->max_metadata_size / + (PREFETCH_TABLE_SHARD_SIZE * init_params->pf_list_size)); + assert(max_num_of_shards_in_prefetch_table > 0); + /* now adjust the cache size by deducting current meta data size + 8 is the size of storage for block, 4 is the size of storage for index to + array */ + Mithril_params->cur_metadata_size = + (init_params->max_support * 2 + 8 + 4) * Mithril_params->mtable_size + + max_num_of_shards_in_prefetch_table * 8 + + PREFETCH_TABLE_SHARD_SIZE * (Mithril_params->pf_list_size * 8 + 8 + 4); + + Mithril_params->rmtable = g_new0(rec_mining_t, 1); + rec_mining_t *rmtable = Mithril_params->rmtable; + rmtable->n_avail_mining = 0; + rmtable->rtable_cur_row = 1; + rmtable->rtable_row_len = + (gint)ceil((double)Mithril_params->min_support / (double)4) + 1; + rmtable->mtable_row_len = + (gint)ceil((double)Mithril_params->max_support / (double)4) + 1; + rmtable->mining_table = + g_array_sized_new(FALSE, TRUE, sizeof(int64_t) * rmtable->mtable_row_len, + Mithril_params->mtable_size); + rmtable->hashtable = + g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, NULL); + Mithril_params->prefetch_hashtable = + g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, NULL); + Mithril_params->cache_size_map = + g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, NULL); + + if (Mithril_params->output_statistics) { + Mithril_params->prefetched_hashtable_Mithril = + g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, NULL); + Mithril_params->prefetched_hashtable_sequential = + g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, NULL); + } + + Mithril_params->ptable_cur_row = 1; + Mithril_params->ptable_is_full = FALSE; + // always save to size+1 position, and enlarge table when size%shards_size == + // 0 + Mithril_params->ptable_array = + g_new0(gint64 *, max_num_of_shards_in_prefetch_table); + Mithril_params->ptable_array[0] = g_new0( + gint64, PREFETCH_TABLE_SHARD_SIZE * (Mithril_params->pf_list_size + 1)); + + Mithril_params->ts = 0; + + Mithril_params->hit_on_prefetch_Mithril = 0; + Mithril_params->hit_on_prefetch_sequential = 0; + Mithril_params->num_of_prefetch_Mithril = 0; + Mithril_params->num_of_prefetch_sequential = 0; + Mithril_params->num_of_check = 0; + + if (Mithril_params->max_support != 1) { + rmtable->n_rows_in_rtable = + (gint64)(cache_size * Mithril_params->block_size * + RECORDING_TABLE_MAXIMAL / + ((int)ceil((double)Mithril_params->min_support / (double)2) * + 2 + + 8 + 4)); + rmtable->recording_table = g_new0( + gint64, rmtable->n_rows_in_rtable * + rmtable->rtable_row_len); // this should begins with 1 + Mithril_params->cur_metadata_size += + (((gint64)ceil((double)init_params->min_support / (double)4 + 1) * 8 + + 4) * + rmtable->n_rows_in_rtable); + } +} + +// *********************************************************************** +// **** **** +// **** prefetcher interfaces **** +// **** **** +// **** create, free, clone, handle_find, handle_evict, prefetch **** +// *********************************************************************** +/** + 1. record the request in cache_size_map for being aware of prefetching object's + size in the future. + 2. record entry if rec_trigger is not evict. + + @param cache the cache struct + @param req the request containing the request + @return +*/ +static void Mithril_handle_find(cache_t *cache, const request_t *req, + bool hit) { + Mithril_params_t *Mithril_params = + (Mithril_params_t *)(cache->prefetcher->params); + + /*use cache_size_map to record the current requested obj's size*/ + g_hash_table_insert(Mithril_params->cache_size_map, + GINT_TO_POINTER(req->obj_id), + GINT_TO_POINTER(req->obj_size)); + + if (Mithril_params->output_statistics) { + if (g_hash_table_contains(Mithril_params->prefetched_hashtable_Mithril, + GINT_TO_POINTER(req->obj_id))) { + Mithril_params->hit_on_prefetch_Mithril += 1; + g_hash_table_remove(Mithril_params->prefetched_hashtable_Mithril, + GINT_TO_POINTER(req->obj_id)); + } + if (g_hash_table_contains(Mithril_params->prefetched_hashtable_sequential, + GINT_TO_POINTER(req->obj_id))) { + Mithril_params->hit_on_prefetch_sequential += 1; + g_hash_table_remove(Mithril_params->prefetched_hashtable_sequential, + GINT_TO_POINTER(req->obj_id)); + } + } + + // 1. record entry when rec_trigger is each_req. + // 2. record entry when (rec_trigger is miss or miss_evict (in other words, + // !evict)) && !hit + if ((Mithril_params->rec_trigger == each_req) || + (Mithril_params->rec_trigger != evict && !hit)) { + _Mithril_record_entry(cache, req); + } +} + +/** + evict_req->obj_id has been evict by cache_remove_base. + Now, prefetcher checks whether it can be added to cache (second chance). + + @param cache the cache struct + @param req the request containing the request + @return +*/ +void Mithril_handle_evict(cache_t *cache, const request_t *check_req) { + Mithril_params_t *Mithril_params = + (Mithril_params_t *)(cache->prefetcher->params); + + if (Mithril_params->output_statistics) { + obj_id_t check_id = check_req->obj_id; + + gint type = GPOINTER_TO_INT( + g_hash_table_lookup(Mithril_params->prefetched_hashtable_Mithril, + GINT_TO_POINTER(check_id))); + if (type != 0 && type < Mithril_params->cycle_time) { + // give one more chance + g_hash_table_insert(Mithril_params->prefetched_hashtable_Mithril, + GINT_TO_POINTER(check_id), GINT_TO_POINTER(type + 1)); + + while ((long)cache->get_occupied_byte(cache) + check_req->obj_size + + cache->obj_md_size > + (long)cache->cache_size) { + cache->evict(cache, check_req); + } + cache->insert(cache, check_req); + } else { + if (Mithril_params->rec_trigger == evict || + Mithril_params->rec_trigger == miss_evict) { + _Mithril_record_entry(cache, check_req); + } + + g_hash_table_remove(Mithril_params->prefetched_hashtable_Mithril, + GINT_TO_POINTER(check_req->obj_id)); + g_hash_table_remove(Mithril_params->prefetched_hashtable_sequential, + GINT_TO_POINTER(check_req->obj_id)); + } + } +} + +/** + prefetch some objs associated with req->obj_id by searching prefetch_hashtable + and ptable_array and evict when space is full. + + @param cache the cache struct + @param req the request containing the request + @return + */ +void Mithril_prefetch(cache_t *cache, const request_t *req) { + Mithril_params_t *Mithril_params = + (Mithril_params_t *)(cache->prefetcher->params); + + gint prefetch_table_index = GPOINTER_TO_INT(g_hash_table_lookup( + Mithril_params->prefetch_hashtable, GINT_TO_POINTER(req->obj_id))); + + gint dim1 = + (gint)floor(prefetch_table_index / (double)PREFETCH_TABLE_SHARD_SIZE); + gint dim2 = prefetch_table_index % PREFETCH_TABLE_SHARD_SIZE * + (Mithril_params->pf_list_size + 1); + + request_t *new_req = my_malloc(request_t); + memcpy(new_req, req, sizeof(request_t)); + + if (prefetch_table_index) { + int i; + for (i = 1; i < Mithril_params->pf_list_size + 1; i++) { + // begin from 1 because index 0 is the obj_id of originated request + if (Mithril_params->ptable_array[dim1][dim2 + i] == 0) { + break; + } + new_req->obj_id = Mithril_params->ptable_array[dim1][dim2 + i]; + new_req->obj_size = GPOINTER_TO_INT(g_hash_table_lookup( + Mithril_params->cache_size_map, GINT_TO_POINTER(new_req->obj_id))); + + if (Mithril_params->output_statistics) { + Mithril_params->num_of_check += 1; + } + + if (cache->find(cache, new_req, false)) { + continue; + } + + while ((long)cache->get_occupied_byte(cache) + new_req->obj_size + + cache->obj_md_size > + (long)cache->cache_size) { + cache->evict(cache, new_req); + } + cache->insert(cache, new_req); + + if (Mithril_params->output_statistics) { + Mithril_params->num_of_prefetch_Mithril += 1; + + g_hash_table_insert(Mithril_params->prefetched_hashtable_Mithril, + GINT_TO_POINTER(new_req->obj_id), + GINT_TO_POINTER(1)); + } + } + } + + // prefetch sequential + // just use in block or cache line level where obj_size is same + if (Mithril_params->sequential_type == 1 && + _Mithril_check_sequential(cache, req)) { + new_req->obj_id = req->obj_id + 1; + new_req->obj_size = req->obj_size; // same size + + if (cache->find(cache, new_req, false)) { + my_free(sizeof(request_t), new_req); + return; + } + + // use this, not add because we need to record stat when evicting + + while ((long)cache->get_occupied_byte(cache) + new_req->obj_size + + cache->obj_md_size > + cache->cache_size) { + cache->evict(cache, new_req); + } + cache->insert(cache, new_req); + + if (Mithril_params->output_statistics) { + Mithril_params->num_of_prefetch_sequential += 1; + g_hash_table_insert(Mithril_params->prefetched_hashtable_Mithril, + GINT_TO_POINTER(new_req->obj_id), GINT_TO_POINTER(1)); + } + } + my_free(sizeof(request), new_req); + + Mithril_params->ts++; +} + +void free_Mithril_prefetcher(prefetcher_t *prefetcher) { + Mithril_params_t *Mithril_params = (Mithril_params_t *)prefetcher->params; + + g_hash_table_destroy(Mithril_params->prefetch_hashtable); + g_hash_table_destroy(Mithril_params->cache_size_map); + g_hash_table_destroy(Mithril_params->rmtable->hashtable); + g_free(Mithril_params->rmtable->recording_table); + g_array_free(Mithril_params->rmtable->mining_table, TRUE); + g_free(Mithril_params->rmtable); + + int i = 0; + gint max_num_of_shards_in_prefetch_table = + (gint)(Mithril_params->max_metadata_size / + (PREFETCH_TABLE_SHARD_SIZE * Mithril_params->pf_list_size)); + + while (i < max_num_of_shards_in_prefetch_table) { + if (Mithril_params->ptable_array[i]) { + g_free(Mithril_params->ptable_array[i]); + } else { + break; + } + i++; + } + g_free(Mithril_params->ptable_array); + + if (Mithril_params->output_statistics) { + g_hash_table_destroy(Mithril_params->prefetched_hashtable_Mithril); + g_hash_table_destroy(Mithril_params->prefetched_hashtable_sequential); + } + my_free(sizeof(Mithril_params_t), Mithril_params); + if (prefetcher->init_params) { + free(prefetcher->init_params); + } + my_free(sizeof(prefetcher_t), prefetcher); +} + +prefetcher_t *clone_Mithril_prefetcher(prefetcher_t *prefetcher, + uint64_t cache_size) { + return create_Mithril_prefetcher(prefetcher->init_params, cache_size); +} + +prefetcher_t *create_Mithril_prefetcher(const char *init_params, + uint64_t cache_size) { + Mithril_init_params_t *Mithril_init_params = my_malloc(Mithril_init_params_t); + memset(Mithril_init_params, 0, sizeof(Mithril_init_params_t)); + + set_Mithril_default_init_params(Mithril_init_params); + if (init_params != NULL) { + Mithril_parse_init_params(init_params, Mithril_init_params); + check_params((Mithril_init_params)); + } + + Mithril_params_t *Mithril_params = my_malloc(Mithril_params_t); + // when all object's size is 1, cache->cache_size is the number of objects + // that can be cached, and users should set block_size in prefetching_params. + // Otherwise, cache->cache_size is the total bytes that can be cached and + // block_size is 1 in the default setting. + set_Mithril_params(Mithril_params, Mithril_init_params, cache_size); + + prefetcher_t *prefetcher = (prefetcher_t *)my_malloc(prefetcher_t); + memset(prefetcher, 0, sizeof(prefetcher_t)); + prefetcher->params = Mithril_params; + prefetcher->prefetch = Mithril_prefetch; + prefetcher->handle_find = Mithril_handle_find; + prefetcher->handle_evict = Mithril_handle_evict; + prefetcher->free = free_Mithril_prefetcher; + prefetcher->clone = clone_Mithril_prefetcher; + if (init_params) { + prefetcher->init_params = strdup(init_params); + } + + my_free(sizeof(Mithril_init_params_t), Mithril_init_params); + return prefetcher; +} + +/******************** Mithril help function ********************/ +/** + check whether last request is part of a sequential access + */ +static inline bool _Mithril_check_sequential(cache_t *cache, + const request_t *req) { + int i; + Mithril_params_t *Mithril_params = + (Mithril_params_t *)(cache->prefetcher->params); + if (Mithril_params->sequential_K == 0) return FALSE; + + request_t *new_req = my_malloc(request_t); + memcpy(new_req, req, sizeof(request_t)); + bool is_sequential = TRUE; + gint sequential_K = Mithril_params->sequential_K; + if (sequential_K == -1) { /* when use AMP, this is -1 */ + sequential_K = 1; + } + for (i = 0; i < sequential_K; i++) { + new_req->obj_id--; + if (!cache->find(cache, new_req, false)) { + is_sequential = FALSE; + break; + } + } + return is_sequential; +} + +static inline void _Mithril_rec_min_support_one(cache_t *cache, + const request_t *req) { + Mithril_params_t *Mithril_params = + (Mithril_params_t *)(cache->prefetcher->params); + rec_mining_t *rmtable = Mithril_params->rmtable; + +#ifdef TRACK_BLOCK + if (req->obj_id == TRACK_BLOCK) { + int old_pos = GPOINTER_TO_INT( + g_hash_table_lookup(rmtable->hashtable, GINT_TO_POINTER(req->obj_id))); + printf("insert %ld, old pos %d", TRACK_BLOCK, old_pos); + if (old_pos == 0) + printf("\n"); + else + printf(", block at old_pos %ld\n", + *(gint64 *)GET_ROW_IN_MTABLE(Mithril_params, old_pos - 1)); + + } else { + gint64 b = TRACK_BLOCK; + int old_pos = GPOINTER_TO_INT( + g_hash_table_lookup(rmtable->hashtable, GINT_TO_POINTER(b))); + if (old_pos != 0) { + ERROR("ts %lu, checking %ld, %ld is found at pos %d\n", + Mithril_params->ts, TRACK_BLOCK, + *(gint64 *)GET_ROW_IN_MTABLE(Mithril_params, old_pos - 1), old_pos); + abort(); + } + } +#endif + + int i; + // check the obj_id in hashtable for training + gint index = GPOINTER_TO_INT( + g_hash_table_lookup(rmtable->hashtable, GINT_TO_POINTER(req->obj_id))); + if (index == 0) { + // the node is not in the recording/mining data, should be added + gint64 array_ele[rmtable->mtable_row_len]; + // gpointer hash_key; + array_ele[0] = req->obj_id; + // hash_key = GET_ROW_IN_MTABLE(Mithril_params, + // rmtable->mining_table->len); + + for (i = 1; i < rmtable->mtable_row_len; i++) array_ele[i] = 0; + array_ele[1] = ADD_TS(array_ele[1], Mithril_params->ts); + + g_array_append_val(rmtable->mining_table, array_ele); + rmtable->n_avail_mining++; + + // all index is real row number + 1 + g_hash_table_insert(rmtable->hashtable, GINT_TO_POINTER(req->obj_id), + GINT_TO_POINTER(rmtable->mining_table->len)); + +#ifdef SANITY_CHECK + gint64 *row_in_mtable = + GET_ROW_IN_MTABLE(Mithril_params, rmtable->mining_table->len - 1); + if (req->obj_id != row_in_mtable[0]) { + ERROR("after inserting, hashtable mining not consistent %ld %ld\n", + req->obj_id, row_in_mtable[0]); + abort(); + } +#endif + } else { + /* in mining table */ + gint64 *row_in_mtable = GET_ROW_IN_MTABLE(Mithril_params, index - 1); + +#ifdef SANITY_CHECK + if (req->obj_id != row_in_mtable[0]) { + ERROR("ts %lu, hashtable mining found position not correct %ld %ld\n", + Mithril_params->ts, req->obj_id, row_in_mtable[0]); + abort(); + } +#endif + + int timestamps_length = 0; + + for (i = 1; i < rmtable->mtable_row_len; i++) { + timestamps_length += NUM_OF_TS(row_in_mtable[i]); + if (NUM_OF_TS(row_in_mtable[i]) < 4) { + row_in_mtable[i] = ADD_TS(row_in_mtable[i], Mithril_params->ts); + break; + } + } + if (timestamps_length == Mithril_params->max_support) { + /* no timestamp added, drop this request, it is too frequent */ + if (!g_hash_table_remove(rmtable->hashtable, + GINT_TO_POINTER(row_in_mtable[0]))) { + ERROR("removing from rmtable failed for mining table entry\n"); + } + + g_array_remove_index_fast(rmtable->mining_table, index - 1); + + // if array is moved, need to update hashtable + if (index - 1 != (long)rmtable->mining_table->len) { + g_hash_table_replace(rmtable->hashtable, + GINT_TO_POINTER(row_in_mtable[0]), + GINT_TO_POINTER(index)); + } + rmtable->n_avail_mining--; + } + } +} + +/** + record req to the recording table or the mining table + + @param Mithril the cache struct + @param req the request containing the request + @return + */ +static inline void _Mithril_record_entry(cache_t *cache, const request_t *req) { + Mithril_params_t *Mithril_params = + (Mithril_params_t *)(cache->prefetcher->params); + rec_mining_t *rmtable = Mithril_params->rmtable; + + int i; + + /* check it is sequtial or not */ + if (Mithril_params->sequential_type && _Mithril_check_sequential(cache, req)) + return; + + if (Mithril_params->min_support == 1) { + _Mithril_rec_min_support_one(cache, req); + } else { + gint64 *row_in_rtable; + // check the obj_id in hashtable for training + gint index = GPOINTER_TO_INT( + g_hash_table_lookup(rmtable->hashtable, GINT_TO_POINTER(req->obj_id))); + + if (index == 0) { + // the node is not in the recording/mining data, should be added + row_in_rtable = GET_CUR_ROW_IN_RTABLE(Mithril_params); + +#ifdef SANITY_CHECK + if (row_in_rtable[0] != 0) { + ERROR("recording table is not clean\n"); + abort(); + } +#endif + + row_in_rtable[0] = req->obj_id; + // row_in_rtable is a pointer to the block number + g_hash_table_insert(rmtable->hashtable, GINT_TO_POINTER(row_in_rtable[0]), + GINT_TO_POINTER(rmtable->rtable_cur_row)); + + row_in_rtable[1] = ADD_TS(row_in_rtable[1], Mithril_params->ts); + + // move cur_row to next + rmtable->rtable_cur_row++; + if (rmtable->rtable_cur_row >= rmtable->n_rows_in_rtable) { + /* recording table is full */ + rmtable->rtable_cur_row = 1; + } + + row_in_rtable = + GET_ROW_IN_RTABLE(Mithril_params, rmtable->rtable_cur_row); + + if (row_in_rtable[0] != 0) { + /** clear current row, + * this is because the recording table is full + * and we need to begin from beginning + * and current position has old resident, + * we need to remove them + **/ + if (!g_hash_table_contains(rmtable->hashtable, + GINT_TO_POINTER(row_in_rtable[0]))) { + ERROR( + "remove old entry from recording table, " + "but it is not in recording hashtable, " + "block %ld, recording table pos %ld, ts %ld ", + row_in_rtable[0], (long)rmtable->rtable_cur_row, + (long)Mithril_params->ts); + + long temp = rmtable->rtable_cur_row - 1; + fprintf(stderr, "previous line block %ld\n", + *(long *)(GET_ROW_IN_RTABLE(Mithril_params, temp))); + abort(); + } + + g_hash_table_remove(rmtable->hashtable, + GINT_TO_POINTER(row_in_rtable[0])); + + /* clear recording table */ + for (i = 0; i < rmtable->rtable_row_len; i++) { + row_in_rtable[i] = 0; + } + } + } else { + /** first check it is in recording table or mining table, + * if in mining table (index < 0), + * check how many ts it has, if equal max_support, remove it + * otherwise add to mining table; + * if in recording table (index > 0), + * check how many ts it has , + * if equal to min_support-1, add and move to mining table, + **/ + if (index < 0) { + /* in mining table */ + gint64 *row_in_mtable = GET_ROW_IN_MTABLE(Mithril_params, -index - 1); + +#ifdef SANITY_CHECK + if (req->obj_id != row_in_mtable[0]) { + ERROR( + "inconsistent entry in mtable " + "and mining hashtable current request %ld, " + "mining table %ld\n", + req->obj_id, row_in_mtable[0]); + abort(); + } +#endif + int timestamps_length = 0; + + for (i = 1; i < rmtable->mtable_row_len; i++) { + timestamps_length += NUM_OF_TS(row_in_mtable[i]); + if (NUM_OF_TS(row_in_mtable[i]) < 4) { + row_in_mtable[i] = ADD_TS(row_in_mtable[i], Mithril_params->ts); + break; + } + } + if (timestamps_length == Mithril_params->max_support) { + /* no timestamp added, drop this request, it is too frequent */ + if (!g_hash_table_remove(rmtable->hashtable, + GINT_TO_POINTER(row_in_mtable[0]))) { + ERROR("removing from rmtable failed for mining table entry\n"); + } + + /** for dataType c, now the pointer to string has been freed, + * so mining table entry is incorrent, + * but mining table entry will be deleted, so it is OK + */ + + g_array_remove_index_fast(rmtable->mining_table, -index - 1); + + /** if the removed block is not the last entry, + * g_array_remove_index_fast uses the last entry to fill in + * the old position, so we need to update its index + **/ + if (-index - 1 != (long)rmtable->mining_table->len) { + g_hash_table_replace(rmtable->hashtable, + GINT_TO_POINTER(row_in_mtable[0]), + GINT_TO_POINTER(index)); + } + rmtable->n_avail_mining--; + } + } else { + /* in recording table */ + row_in_rtable = GET_ROW_IN_RTABLE(Mithril_params, index); + gint64 *cur_row_in_rtable = + GET_ROW_IN_RTABLE(Mithril_params, rmtable->rtable_cur_row - 1); + int timestamps_length = 0; + +#ifdef SANITY_CHECK + if (req->obj_id != row_in_rtable[0]) { + ERROR("Hashtable recording found position not correct %ld %ld\n", + req->obj_id, row_in_rtable[0]); + abort(); + } +#endif + + for (i = 1; i < rmtable->rtable_row_len; i++) { + timestamps_length += NUM_OF_TS(row_in_rtable[i]); + if (NUM_OF_TS(row_in_rtable[i]) < 4) { + row_in_rtable[i] = ADD_TS(row_in_rtable[i], Mithril_params->ts); + break; + } + } + + if (timestamps_length == Mithril_params->min_support - 1) { + /* time to move to mining table */ + // gint64 *array_ele = malloc(sizeof(gint64) * + // rmtable->mtable_row_len); + gint64 array_ele[rmtable->mtable_row_len]; + memcpy(array_ele, row_in_rtable, + sizeof(TS_REPRESENTATION) * rmtable->rtable_row_len); + + /** clear the rest of array, + * this is important as + * we don't clear the content of array after mining + **/ + memset(array_ele + rmtable->rtable_row_len, 0, + sizeof(TS_REPRESENTATION) * + (rmtable->mtable_row_len - rmtable->rtable_row_len)); +#ifdef SANITY_CHECK + if ((long)rmtable->mining_table->len >= Mithril_params->mtable_size) { + /* if this happens, array will re-malloc, which will make + * the hashtable key not reliable when obj_id_type is l */ + ERROR( + "mining table length reaches limit, but no mining, " + "entry %d, size %u, threshold %d\n", + rmtable->n_avail_mining, rmtable->mining_table->len, + Mithril_params->mtable_size); + abort(); + } +#endif + g_array_append_val(rmtable->mining_table, array_ele); + rmtable->n_avail_mining++; + + if (index != rmtable->rtable_cur_row - 1 && + rmtable->rtable_cur_row >= 2) { + /** moved row is not the last entry in recording table + * move last row to current position + **/ + +#ifdef SANITY_CHECK + if (row_in_rtable == cur_row_in_rtable) + ERROR("FOUND SRC DEST same, ts %ld %p %p %ld %ld %d %ld\n", + Mithril_params->ts, row_in_rtable, cur_row_in_rtable, + *row_in_rtable, *cur_row_in_rtable, index, + rmtable->rtable_cur_row - 1); +#endif + memcpy(row_in_rtable, cur_row_in_rtable, + sizeof(TS_REPRESENTATION) * rmtable->rtable_row_len); + } + if (rmtable->rtable_cur_row >= 2) { + for (i = 0; i < rmtable->rtable_row_len; i++) { + cur_row_in_rtable[i] = 0; + } + } else { + /** if current pointer points to 1, + * then don't move it, clear the row (that moves to mining table) + **/ + for (i = 0; i < rmtable->rtable_row_len; i++) row_in_rtable[i] = 0; + } + + gint64 *inserted_row_in_mtable = + GET_ROW_IN_MTABLE(Mithril_params, rmtable->mining_table->len - 1); + +#ifdef SANITY_CHECK + if (inserted_row_in_mtable[0] != (gint64)req->obj_id) { + ERROR("current block %ld, moving mining row block %ld\n", + (gint64)req->obj_id, inserted_row_in_mtable[0]); + abort(); + } +#endif + /** because we don't want to have zero as index, + * so we add one before taking negative, + * in other words, the range of mining table index + * is -1 ~ -max_index-1, mapping to 0~max_index + */ + g_hash_table_replace( + rmtable->hashtable, GINT_TO_POINTER(inserted_row_in_mtable[0]), + GINT_TO_POINTER(-((gint)rmtable->mining_table->len - 1 + 1))); + + if (index != rmtable->rtable_cur_row - 1 && + rmtable->rtable_cur_row >= 2) + // last entry in the recording table is moved up index position + g_hash_table_replace(rmtable->hashtable, + GINT_TO_POINTER(row_in_rtable[0]), + GINT_TO_POINTER(index)); + + // one entry has been moved to mining table, shrinking recording + // table size by 1 + if (rmtable->rtable_cur_row >= 2) rmtable->rtable_cur_row--; + + // free(array_ele); + } + } + } + } + if (rmtable->n_avail_mining >= Mithril_params->mtable_size || + (Mithril_params->min_support == 1 && + rmtable->n_avail_mining > Mithril_params->mining_threshold / 8)) { + _Mithril_mining(cache); + rmtable->n_avail_mining = 0; + } +} + +static inline gint _Mithril_get_total_num_of_ts(gint64 *row, gint row_length) { + int i, t; + int count = 0; + for (i = 1; i < row_length; i++) { + t = NUM_OF_TS(row[i]); + if (t == 0) return count; + count += t; + } + return count; +} + +gint mining_table_entry_cmp(gconstpointer a, gconstpointer b) { + return (gint)GET_NTH_TS(a, 1) - (gint)GET_NTH_TS(b, 1); +} + +/* in debug */ +void print_one_line(gpointer key, gpointer value, gpointer user_data) { + gint src_key = GPOINTER_TO_INT(key); + gint prefetch_table_index = GPOINTER_TO_INT(value); + Mithril_params_t *Mithril_params = (Mithril_params_t *)user_data; + gint dim1 = + (gint)floor(prefetch_table_index / (double)PREFETCH_TABLE_SHARD_SIZE); + gint dim2 = prefetch_table_index % PREFETCH_TABLE_SHARD_SIZE * + (Mithril_params->pf_list_size + 1); + printf("src %d, prefetch ", src_key); + for (int i = 1; i < Mithril_params->pf_list_size + 1; i++) { + printf("%ld ", Mithril_params->ptable_array[dim1][dim2 + i]); + } + printf("\n"); +} + +/* in debug */ +void print_prefetch_table(Mithril_params_t *Mithril_params) { + g_hash_table_foreach(Mithril_params->prefetch_hashtable, print_one_line, + Mithril_params); +} + +/** + the mining funciton, it is called when mining table is ready + + @param Mithril the cache struct + */ +static void _Mithril_mining(cache_t *cache) { + Mithril_params_t *Mithril_params = + (Mithril_params_t *)(cache->prefetcher->params); + rec_mining_t *rmtable = Mithril_params->rmtable; + +#ifdef PROFILING + GTimer *timer = g_timer_new(); + gulong microsecond; + g_timer_start(timer); +#endif + + int i, j, k; + + /* first sort mining table, then do the mining */ + /* first remove all elements from hashtable, otherwise after sort, it will + mess up for obj_id_type l but we can't do this for dataType c, otherwise + the string will be freed during remove in hashtable + */ + gint64 *item = (gint64 *)rmtable->mining_table->data; + for (i = 0; i < (int)rmtable->mining_table->len; i++) { + g_hash_table_remove(rmtable->hashtable, GINT_TO_POINTER(*item)); + item += rmtable->mtable_row_len; + } + + g_array_sort(rmtable->mining_table, mining_table_entry_cmp); + + gboolean associated_flag, first_flag; + gint64 *item1, *item2; + gint num_of_ts1, num_of_ts2, shorter_length; + for (i = 0; i < (long)rmtable->mining_table->len - 1; i++) { + item1 = GET_ROW_IN_MTABLE(Mithril_params, i); + num_of_ts1 = _Mithril_get_total_num_of_ts(item1, rmtable->mtable_row_len); + first_flag = TRUE; + + for (j = i + 1; j < (long)rmtable->mining_table->len; j++) { + item2 = GET_ROW_IN_MTABLE(Mithril_params, j); + + // check first timestamp + if (GET_NTH_TS(item2, 1) - GET_NTH_TS(item1, 1) > + Mithril_params->lookahead_range) { + break; + } + num_of_ts2 = _Mithril_get_total_num_of_ts(item2, rmtable->mtable_row_len); + + if (ABS(num_of_ts1 - num_of_ts2) > Mithril_params->confidence) { + continue; + } + + shorter_length = MIN(num_of_ts1, num_of_ts2); + + associated_flag = FALSE; + if (first_flag) { + associated_flag = TRUE; + first_flag = FALSE; + } + // is next line useless?? + if (shorter_length == 1 && + ABS(GET_NTH_TS(item1, 1) - GET_NTH_TS(item2, 1)) == 1) { + associated_flag = TRUE; + } + + gint error = 0; + for (k = 1; k < shorter_length; k++) { + if (ABS(GET_NTH_TS(item1, k) - GET_NTH_TS(item2, k)) > + Mithril_params->lookahead_range) { + error++; + if (error > Mithril_params->confidence) { + associated_flag = FALSE; + break; + } + } + + if (ABS(GET_NTH_TS(item1, k) - GET_NTH_TS(item2, k)) == 1) { + associated_flag = TRUE; + } + } + if (associated_flag) { + // finally, add to prefetch table + _Mithril_add_to_prefetch_table(cache, GINT_TO_POINTER(item1[0]), + GINT_TO_POINTER(item2[0])); + } + } + } + + // may be just following? + rmtable->mining_table->len = 0; + +#ifdef PROFILING + printf("ts: %lu, clearing training data takes %lf seconds\n", + Mithril_params->ts, g_timer_elapsed(timer, µsecond)); + g_timer_stop(timer); + g_timer_destroy(timer); +#endif + +#ifdef debug + print_prefetch_table(Mithril_params); +#endif +} + +/** + add two associated block into prefetch table + + @param Mithril the cache struct + @param gp1 pointer to the first block + @param gp2 pointer to the second block + */ +static void _Mithril_add_to_prefetch_table(cache_t *cache, gpointer gp1, + gpointer gp2) { + /** currently prefetch table can only support up to 2^31 entries, + * and this function assumes the platform is 64 bit */ + Mithril_params_t *Mithril_params = + (Mithril_params_t *)(cache->prefetcher->params); + + gint prefetch_table_index = GPOINTER_TO_INT( + g_hash_table_lookup(Mithril_params->prefetch_hashtable, gp1)); + gint dim1 = + (gint)floor(prefetch_table_index / (double)PREFETCH_TABLE_SHARD_SIZE); + gint dim2 = prefetch_table_index % PREFETCH_TABLE_SHARD_SIZE * + (Mithril_params->pf_list_size + 1); + + // insert into prefetch hashtable + int i; + if (prefetch_table_index) { + // already have an entry in prefetch table, just add to that entry + gboolean insert = TRUE; + + for (i = 1; i < Mithril_params->pf_list_size + 1; i++) { + // if this element is already in + // the array, then don't need add + // again ATTENTION: the following + // assumes a 64 bit platform +#ifdef SANITY_CHECK + if (Mithril_params->ptable_array[dim1][dim2] != GPOINTER_TO_INT(gp1)) { + fprintf(stderr, "ERROR prefetch table pos wrong %d %ld, dim %d %d\n", + GPOINTER_TO_INT(gp1), Mithril_params->ptable_array[dim1][dim2], + dim1, dim2); + exit(1); + } +#endif + if ((Mithril_params->ptable_array[dim1][dim2 + i]) == 0) break; + if ((Mithril_params->ptable_array[dim1][dim2 + i]) == + GPOINTER_TO_INT(gp2)) { + /* update score here, not implemented yet */ + insert = FALSE; + } + } + + if (insert) { + if (i == Mithril_params->pf_list_size + 1) { + // list full, randomly pick one for replacement + // i = rand()%Mithril_params->pf_list_size + 1; + + // use FIFO + int j; + for (j = 2; j < Mithril_params->pf_list_size + 1; j++) { + Mithril_params->ptable_array[dim1][dim2 + j - 1] = + Mithril_params->ptable_array[dim1][dim2 + j]; + } + i = Mithril_params->pf_list_size; + } + // new add at position i + Mithril_params->ptable_array[dim1][dim2 + i] = GPOINTER_TO_INT(gp2); + } + } else { + // does not have entry, need to add a new entry + Mithril_params->ptable_cur_row++; + dim1 = (gint)floor(Mithril_params->ptable_cur_row / + (double)PREFETCH_TABLE_SHARD_SIZE); + dim2 = Mithril_params->ptable_cur_row % PREFETCH_TABLE_SHARD_SIZE * + (Mithril_params->pf_list_size + 1); + + /* check whether prefetch table is fully allocated, if True, we are going + to replace the entry at ptable_cur_row by set the entry it points to as + 0, delete from prefetch_hashtable and add new entry */ + if (Mithril_params->ptable_is_full) { + g_hash_table_remove( + Mithril_params->prefetch_hashtable, + GINT_TO_POINTER(Mithril_params->ptable_array[dim1][dim2])); + + memset(&(Mithril_params->ptable_array[dim1][dim2]), 0, + sizeof(gint64) * (Mithril_params->pf_list_size + 1)); + } + + Mithril_params->ptable_array[dim1][dim2 + 1] = GPOINTER_TO_INT(gp2); + Mithril_params->ptable_array[dim1][dim2] = GPOINTER_TO_INT(gp1); + +#ifdef SANITY_CHECK + // make sure gp1 is not in prefetch_hashtable + if (g_hash_table_contains(Mithril_params->prefetch_hashtable, gp1)) { + gpointer gp = + g_hash_table_lookup(Mithril_params->prefetch_hashtable, gp1); + printf("contains %d, value %d, %d\n", GPOINTER_TO_INT(gp1), + GPOINTER_TO_INT(gp), prefetch_table_index); + } +#endif + + g_hash_table_insert(Mithril_params->prefetch_hashtable, gp1, + GINT_TO_POINTER(Mithril_params->ptable_cur_row)); + + // check current shard is full or not + if ((Mithril_params->ptable_cur_row + 1) % PREFETCH_TABLE_SHARD_SIZE == 0) { + /* need to allocate a new shard for prefetch table */ + if (Mithril_params->cur_metadata_size + + PREFETCH_TABLE_SHARD_SIZE * + (Mithril_params->pf_list_size * 8 + 8 + 4) < + Mithril_params->max_metadata_size) { + Mithril_params->ptable_array[dim1 + 1] = + g_new0(gint64, PREFETCH_TABLE_SHARD_SIZE * + (Mithril_params->pf_list_size + 1)); + gint required_meta_data_size = + PREFETCH_TABLE_SHARD_SIZE * + (Mithril_params->pf_list_size * 8 + 8 + 4); + Mithril_params->cur_metadata_size += required_meta_data_size; + + // For the general purpose, it has been decided not to consider the + // metadata overhead of the prefetcher + + // if(consider_metasize) { + // Mithril->cache_size = + // Mithril->cache_size - + // (gint)((Mithril_params->cur_metadata_size) / + // Mithril_params->block_size); + // cache->cache_size = Mithril->cache_size; + // // delay the eviction + // } + } else { + Mithril_params->ptable_is_full = TRUE; + Mithril_params->ptable_cur_row = 1; + } + } + } +} + +#ifdef __cplusplus +} +#endif diff --git a/libCacheSim/include/libCacheSim/cache.h b/libCacheSim/include/libCacheSim/cache.h index e3fbc4c9..9a2adedf 100644 --- a/libCacheSim/include/libCacheSim/cache.h +++ b/libCacheSim/include/libCacheSim/cache.h @@ -26,6 +26,9 @@ extern "C" { struct cache; typedef struct cache cache_t; +struct prefetcher; +typedef struct prefetcher prefetcher_t; + typedef struct { uint64_t cache_size; uint64_t default_ttl; @@ -107,6 +110,8 @@ struct cache { admissioner_t *admissioner; + prefetcher_t *prefetcher; + void *eviction_params; // other name: logical_time, virtual_time, reference_count @@ -305,7 +310,8 @@ static inline void print_cache_stat(const cache_t *cache) { * @param cache * @param age */ -static inline void record_log2_eviction_age(cache_t *cache, const unsigned long long age) { +static inline void record_log2_eviction_age(cache_t *cache, + const unsigned long long age) { int age_log2 = age == 0 ? 0 : LOG2_ULL(age); cache->log_eviction_age_cnt[age_log2] += 1; } diff --git a/libCacheSim/include/libCacheSim/evictionAlgo.h b/libCacheSim/include/libCacheSim/evictionAlgo.h index dfede091..2f4a1b83 100644 --- a/libCacheSim/include/libCacheSim/evictionAlgo.h +++ b/libCacheSim/include/libCacheSim/evictionAlgo.h @@ -119,10 +119,10 @@ cache_t *WTinyLFU_init(const common_cache_params_t ccache_params, const char *cache_specific_params); cache_t *FIFO_Merge_init(const common_cache_params_t ccache_params, - const char *cache_specific_params); + const char *cache_specific_params); cache_t *FIFO_Reinsertion_init(const common_cache_params_t ccache_params, - const char *cache_specific_params); + const char *cache_specific_params); cache_t *flashProb_init(const common_cache_params_t ccache_params, const char *cache_specific_params); @@ -135,7 +135,6 @@ cache_t *nop_init(const common_cache_params_t ccache_params, cache_t *QDLP_init(const common_cache_params_t ccache_params, const char *cache_specific_params); - cache_t *S3LRU_init(const common_cache_params_t ccache_params, const char *cache_specific_params); diff --git a/libCacheSim/include/libCacheSim/prefetchAlgo.h b/libCacheSim/include/libCacheSim/prefetchAlgo.h new file mode 100644 index 00000000..5a0c3c78 --- /dev/null +++ b/libCacheSim/include/libCacheSim/prefetchAlgo.h @@ -0,0 +1,50 @@ +#ifndef PREFETCHINGALGO_H +#define PREFETCHINGALGO_H + +#include "cache.h" +#include "request.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct prefetcher; +typedef struct prefetcher *(*prefetcher_create_func_ptr)(const char *); +typedef void (*prefetcher_prefetch_func_ptr)(cache_t *, const request_t *); +typedef void (*prefetcher_handle_find_func_ptr)(cache_t *, const request_t *, + bool); +typedef void (*prefetcher_handle_evict_func_ptr)(cache_t *, const request_t *); +typedef void (*prefetcher_free_func_ptr)(struct prefetcher *); +typedef struct prefetcher *(*prefetcher_clone_func_ptr)(struct prefetcher *, + uint64_t); + +typedef struct prefetcher { + void *params; + void *init_params; + prefetcher_prefetch_func_ptr prefetch; + prefetcher_handle_find_func_ptr handle_find; + prefetcher_handle_evict_func_ptr handle_evict; + prefetcher_free_func_ptr free; + prefetcher_clone_func_ptr clone; +} prefetcher_t; + +prefetcher_t *create_Mithril_prefetcher(const char *init_paramsm, + uint64_t cache_size); + +static inline prefetcher_t *create_prefetcher(const char *prefetching_algo, + const char *prefetching_params, + uint64_t cache_size) { + prefetcher_t *prefetcher = NULL; + if (strcasecmp(prefetching_algo, "Mithril") == 0) { + prefetcher = create_Mithril_prefetcher(prefetching_params, cache_size); + } else { + ERROR("prefetching algo %s not supported\n", prefetching_algo); + } + + return prefetcher; +} + +#ifdef __cplusplus +} +#endif +#endif \ No newline at end of file diff --git a/libCacheSim/include/libCacheSim/prefetchAlgo/Mithril.h b/libCacheSim/include/libCacheSim/prefetchAlgo/Mithril.h new file mode 100644 index 00000000..2d8d981a --- /dev/null +++ b/libCacheSim/include/libCacheSim/prefetchAlgo/Mithril.h @@ -0,0 +1,355 @@ +// +// Mithril.h +// Mithrilcache +// +// Created by Juncheng on 9/2/16. +// Copyright © 2016 Juncheng. All rights reserved. +// + +/* 201912 due to the remove of sector_size and block_size, AMP, Mithril does not + * work out-of-box now, new changes are needed to make them correct */ + +// Modified by Zhelong on 2023/8/15 + +#ifndef Mithril_h +#define Mithril_h + +#include +#include +#include +#include +#include + +#include "../cache.h" + +/** related to mining table size, + * mining_table_size = mining_threshold/min_support unit: entries + **/ +#define MINING_THRESHOLD 5120 + +/* use shards for prefetch table because + * the size of prefetch table cannot be predicted before hand */ +#define PREFETCH_TABLE_SHARD_SIZE 2000 + +/* recording table size, unit: percentage of cache size */ +#define RECORDING_TABLE_MAXIMAL 0.02 + +/* check params vaild */ +#define check_params(params) \ + (assert(params->lookahead_range > 0 && params->lookahead_range <= 100 && \ + params->max_support > 0 && params->min_support > 0 && \ + params->max_support >= params->min_support && \ + params->confidence > 0 && params->confidence <= 100 && \ + params->pf_list_size > 0 && params->pf_list_size <= 100 && \ + params->max_metadata_size > 0 && params->max_metadata_size < 1 && \ + params->cycle_time > 0 && params->cycle_time <= 100 && \ + params->block_size > 0 && params->sequential_type >= 0 && \ + params->sequential_type <= 2 && params->output_statistics >= 0 && \ + params->output_statistics <= 1)) + +/** + retrieve the row_num row in the recording table + + @param param Mithril_params + @param row_num the number of row + @return a pointer to the beginning of the row + */ +#define GET_ROW_IN_RTABLE(param, row_num) \ + ((param)->rmtable->recording_table + \ + (row_num) * (param)->rmtable->rtable_row_len) + +/** + retrieve current row in the recording table + + @param param Mithril_params + @return a pointer to current row + */ +#define GET_CUR_ROW_IN_RTABLE(param) \ + ((param)->rmtable->recording_table + \ + (param)->rmtable->rtable_cur_row * (param)->rmtable->rtable_row_len) + +/** + retrieve the row_num row in the mining table + + @param param Mithril_params + @param row_num the order of row + @return a pointer to the beginning of the row + */ +#define GET_ROW_IN_MTABLE(param, row_num) \ + (((TS_REPRESENTATION *)(param)->rmtable->mining_table->data) + \ + (param)->rmtable->mtable_row_len * (row_num)) + +/**************************************************************************** + * macros for getting and setting time stamp in one 64-bit int + * format of timestamps are stored in the 64-bit int + * the first 4 bits: the number of timestamps 0~4 + * each next 15 bits: one timestamp, if timestamp larger than the max, round + over + * the first timestamp is stored in the 4-18 bit + * the second timestamp is stored in the 19-33 bit + * similarly, 34-48, 49-63 bit for the last two timestamps + + + *****************************************************************************/ + +/** + calculate the number of timestamps stored in one integer + + @param param the 64-bit integer + @return the number of timestamps + */ +#define NUM_OF_TS(param) ((gint)((param) >> 60)) + +/** + retrieve the Nth timestamp in the 64-bit integer + + @param param the 64-bit integer + @param n the order, range 1-4 + @return the timestamp + */ +#define _GET_NTH_TS(param, n) \ + (((param) >> (15 * (4 - (n)))) & ((1UL << 15) - 1)) + +/** + retrieve the Nth timestamp from a row in the table + + @param row a pointer to the row + @param n the order + @return the timestamp + */ +#define GET_NTH_TS(row, n) \ + ((gint)_GET_NTH_TS((*((gint64 *)(row) + 1 + (gint)((n) / 5))), ((n) % 4))) +// ((gint)_GET_NTH_TS((*((gint64 *)(row) + 1 + (gint)(floor((n) / 4)))), \ + // ((n) % 4))) + +/** + clear and set the Nth timestamp in the 64-bit integer + + @param list the 64-bit integer + @param n the order + @param ts timestamp to set, only lower 15 bit is used + */ +#define SET_NTH_TS(list, n, ts) \ + (((list) & (~(((1UL << 15) - 1) << (15 * (4 - (n)))))) | \ + (((ts) & ((1UL << 15) - 1)) << (15 * (4 - (n))))) + +/** + append a timestamp to a 64-bit integer + + @param list the 64-bit integer + @param ts timestamp to set + */ +#define ADD_TS(list, ts) \ + (((SET_NTH_TS((list), (NUM_OF_TS((list)) + 1), (ts))) & ((1UL << 60) - 1)) | \ + ((NUM_OF_TS((list)) + 1UL) << 60)) + +#ifdef __cplusplus +extern "C" { +#endif + +/* the representation of timestamp */ +typedef gint64 TS_REPRESENTATION; + +/* when recording takes place */ +typedef enum _recording_loc { + miss = 0, // this is the default, change order will have effect + evict, + miss_evict, + each_req, +} rec_trigger_e; + +/* the data for Mithril initialization */ +typedef struct { + /** when we say two obj/blocks are associated, + * how far away they can be separated away, + * the more parallelism, the larger this value should be + **/ + gint lookahead_range; + + /* max_support is the threshold for a block to become frequent block */ + gint max_support; + /** min_support is the threshold + * when a block to become mature for mining, + * it is also the threshold for migrating a block from recording table + * to mining table, and it is the row size of the recording table + **/ + gint min_support; + + /* this is now always set at 1 for precision */ + gint confidence; + + /* the number of associations can be stored in prefetching table*/ + gint pf_list_size; + + /* when to record */ + rec_trigger_e rec_trigger; + + /** size of block + * when all object's size is 1, shoule be set in eviction_params. + * Otherwise, should be 1 (default). + **/ + guint64 block_size; + + /* the maximum allocated metadata size */ + gdouble max_metadata_size; + + /** when a prefetched element is going to be evicted, + * if it hasn't been accessed, + * we give it several (cycle_time-1) more chances, + * each time, we put it back to MRU end, + * the range of cycle_time is 1-3 in most cases. + * the idea comes from AMP. + * 1: no cycle, no more chance, otherwise give cycle_time-1 chances + **/ + gint cycle_time; + + /** related to mining table size, + * mining_table_size = mining_threshold/min_support unit: entries + **/ + int mining_threshold; + + /** the embedded sequential prefetching method. just use in block or cache + * line level where obj_size is same. + * 0: no sequential_prefetching, 1: simple + * sequential_prefetching, 2: AMP + */ + gint sequential_type; + + /** this is the K parameter in sequential prefetching, + * it dictates when we think it is accessing sequentially + * if there have been sequential_K sequential blocks, + * then it is in sequential mode and activates sequential prefetching. + **/ + gint sequential_K; + + /** this is the p parameter of AMP, + only use it when using AMP as sequential prefetching algorithm **/ + gint AMP_pthreshold; + + /** this is the control knob for whether printing out statistics **/ + gint output_statistics; + +} Mithril_init_params_t; + +typedef struct { + /** the hash table for storing block related info, + * currently key is the block number + * value is the order of row in recording table or mining table, + * if the value is positive, it is pointing to recording table, + * if it is negative, it is pointing to mining table + **/ + GHashTable *hashtable; + + /** this is the location for storing recording table, + * recording table is N*(min_support/4+1) array, + * where 4 is the number of timestamps stored in one 64bit integer, + * N is number of entries in recording table, plus one is for label + **/ + gint64 *recording_table; + + /** row len of the recording table, + * which is min_suuport/4 + 1 for now + **/ + gint8 rtable_row_len; + + /** the number of rows in recording table, + * it decides how many blocks can be stored in the recording table, + * the larger recording table, the more blocks we can keep, + * however, if a block stays in recording table for too long, + * it means it is rare, as it has not accumulated enough accesses + * to be migrated to mining table + * and keeping such block is not reasonable. + * Therefore, giving too much space to recording table is not useful. + **/ + gint64 n_rows_in_rtable; + + /** this is the current row number of recording table **/ + gint64 rtable_cur_row; + + /** the row len of mining table, + * which is max_support/4 + 1 for now + **/ + gint8 mtable_row_len; + + /** location for mining table, + * use GArray for quick appending + **/ + GArray *mining_table; + + /** this is the counter for how many obj/blocks + * in the mining table are ready for mining **/ + gint n_avail_mining; +} rec_mining_t; + +typedef struct { + /* see Mithril_init_params_t */ + gint lookahead_range; + gint max_support; + gint min_support; + gint confidence; + gint cycle_time; + gint pf_list_size; + gint mining_threshold; + + gint block_size; + gint sequential_type; + gint sequential_K; + gint output_statistics; + + /* mining table size */ + gint mtable_size; + + /** when to trigger recording, miss/evict/miss_evict/each_req **/ + rec_trigger_e rec_trigger; + + /** similar to the one in init_params, + * but this one uses byts, not percentage + **/ + gint64 max_metadata_size; + + /** current used metadata_size (unit: byte) **/ + gint64 cur_metadata_size; + + /* recording and mining table */ + rec_mining_t *rmtable; + + /* prefetch hashtable block -> index in ptable_array*/ + GHashTable *prefetch_hashtable; + + /* the number of current row in prefetch table */ + gint32 ptable_cur_row; + + /** a flag indicates whether prefetch table is fully allocated or not + * if it is full, we are replacing using a FIFO policy, + * and we don't allocate prefetch table shard when it + * reaches end of current shard + **/ + gboolean ptable_is_full; + + /** prefetch table, + * this is a two dimension array for storing associations + **/ + gint64 **ptable_array; + + /** timestamp, currently reference number **/ + guint64 ts; + + // for statistics + GHashTable *prefetched_hashtable_Mithril; + guint64 hit_on_prefetch_Mithril; + guint64 num_of_prefetch_Mithril; + + GHashTable *prefetched_hashtable_sequential; + guint64 hit_on_prefetch_sequential; + guint64 num_of_prefetch_sequential; + + guint64 num_of_check; + + GHashTable *cache_size_map; +} Mithril_params_t; + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index b41c7ca0..b9a05b82 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -16,12 +16,16 @@ target_link_libraries(testSimulator ${coreLib}) add_executable(testEvictionAlgo test_evictionAlgo.c) target_link_libraries(testEvictionAlgo ${coreLib}) +add_executable(testPrefetchAlgo test_prefetchAlgo.c) +target_link_libraries(testPrefetchAlgo ${coreLib}) + add_test(NAME testReader COMMAND testReader WORKING_DIRECTORY .) add_test(NAME testDistUtils COMMAND testDistUtils WORKING_DIRECTORY .) add_test(NAME testProfilerLRU COMMAND testProfilerLRU WORKING_DIRECTORY .) add_test(NAME testSimulator COMMAND testSimulator WORKING_DIRECTORY .) add_test(NAME testEvictionAlgo COMMAND testEvictionAlgo WORKING_DIRECTORY .) +add_test(NAME testPrefetchAlgo COMMAND testPrefetchAlgo WORKING_DIRECTORY .) # if (ENABLE_GLCACHE) # add_executable(testGLCache test_glcache.c) diff --git a/test/common.h b/test/common.h index 08a89528..abf6e50d 100644 --- a/test/common.h +++ b/test/common.h @@ -8,9 +8,11 @@ #include #include #include +#include #include #include "../libCacheSim/include/libCacheSim.h" +#include "../libCacheSim/include/libCacheSim/prefetchAlgo.h" #define BLOCK_UNIT_SIZE 0 // 16 * 1024 #define DISK_SECTOR_SIZE 0 // 512 @@ -232,6 +234,10 @@ static cache_t *create_test_cache(const char *alg_name, cache = S3FIFO_init(cc_params, "move-to-main-threshold=2"); } else if (strcasecmp(alg_name, "Sieve") == 0) { cache = Sieve_init(cc_params, NULL); + } else if (strcasecmp(alg_name, "Mithril") == 0) { + cache = LRU_init(cc_params, NULL); + cache->prefetcher = + create_prefetcher("Mithril", NULL, cc_params.cache_size); } else { printf("cannot recognize algorithm %s\n", alg_name); exit(1); diff --git a/test/test_prefetchAlgo.c b/test/test_prefetchAlgo.c new file mode 100644 index 00000000..190c6ec0 --- /dev/null +++ b/test/test_prefetchAlgo.c @@ -0,0 +1,85 @@ +// +// Created by Zhelong Zhao on 2023.08.16. +// +#include "../libCacheSim/utils/include/mymath.h" +#include "common.h" + +// static const uint64_t req_cnt_true = 113872, req_byte_true = 4205978112; +static const uint64_t req_cnt_true = 113872, req_byte_true = 4368040448; + +static void _verify_profiler_results(const cache_stat_t *res, + uint64_t num_of_sizes, + uint64_t req_cnt_true, + const uint64_t *miss_cnt_true, + uint64_t req_byte_true, + const uint64_t *miss_byte_true) { + for (uint64_t i = 0; i < num_of_sizes; i++) { + g_assert_cmpuint(req_cnt_true, ==, res[i].n_req); + g_assert_cmpuint(miss_cnt_true[i], ==, res[i].n_miss); + g_assert_cmpuint(req_byte_true, ==, res[i].n_req_byte); + g_assert_cmpuint(miss_byte_true[i], ==, res[i].n_miss_byte); + } +} + +static void print_results(const cache_t *cache, const cache_stat_t *res) { + printf("%s uint64_t cache_size[] = {", cache->cache_name); + printf("%ld", res[0].cache_size); + for (uint64_t i = 1; i < CACHE_SIZE / STEP_SIZE; i++) { + printf(", %ld", res[i].cache_size); + } + printf("};\n"); + + printf("uint64_t miss_cnt_true[] = {"); + printf("%ld", res[0].n_miss); + for (uint64_t i = 1; i < CACHE_SIZE / STEP_SIZE; i++) { + printf(", %ld", res[i].n_miss); + } + printf("};\n"); + + printf("uint64_t miss_byte_true[] = {"); + printf("%ld", res[0].n_miss_byte); + for (uint64_t i = 1; i < CACHE_SIZE / STEP_SIZE; i++) { + printf(", %ld", res[i].n_miss_byte); + } + printf("};\n"); +} + +static void test_Mithril(gconstpointer user_data) { + uint64_t miss_cnt_true[] = {79796, 78480, 76126, 75256, + 72336, 72062, 71936, 71667}; + uint64_t miss_byte_true[] = {3471357440, 3399726080, 3285093888, 3245231616, + 3092759040, 3077801472, 3075234816, 3061489664}; + + reader_t *reader = (reader_t *)user_data; + common_cache_params_t cc_params = { + .cache_size = CACHE_SIZE, .hashpower = 20, .default_ttl = DEFAULT_TTL}; + cache_t *cache = create_test_cache("Mithril", cc_params, reader, NULL); + g_assert_true(cache != NULL); + cache_stat_t *res = simulate_at_multi_sizes_with_step_size( + reader, cache, STEP_SIZE, NULL, 0, 0, _n_cores()); + + print_results(cache, res); + _verify_profiler_results(res, CACHE_SIZE / STEP_SIZE, req_cnt_true, + miss_cnt_true, req_byte_true, miss_byte_true); + cache->cache_free(cache); + my_free(sizeof(cache_stat_t), res); +} + +int main(int argc, char *argv[]) { + g_test_init(&argc, &argv, NULL); + srand(0); // for reproducibility + set_rand_seed(rand()); + + reader_t *reader; + + // do not use these two because object size change over time and + // not all algorithms can handle the object size change correctly + // reader = setup_csv_reader_obj_num(); + // reader = setup_vscsi_reader(); + + reader = setup_oracleGeneralBin_reader(); + // reader = setup_vscsi_reader_with_ignored_obj_size(); + g_test_add_data_func("/libCacheSim/cacheAlgo_Mithril", reader, test_Mithril); + + return g_test_run(); +} \ No newline at end of file