diff --git a/include/abstract_graph_store.h b/include/abstract_graph_store.h index 00f237ec1..afa5a6d96 100644 --- a/include/abstract_graph_store.h +++ b/include/abstract_graph_store.h @@ -17,8 +17,8 @@ class AbstractGraphStore { } - virtual int load(const std::string &index_path_prefix) = 0; - virtual int store(const std::string &index_path_prefix, const size_t num_points) = 0; + virtual int load(const std::string &index_path_prefix, const size_t num_points) = 0; + virtual int store(const std::string &index_path_prefix, const size_t num_points, const size_t num_fz_points) = 0; // not synchronised, user should use lock when necvessary. virtual std::vector &get_neighbours(const location_t i) = 0; diff --git a/include/in_mem_graph_store.h b/include/in_mem_graph_store.h index 3b8f6cbfd..9e92e428e 100644 --- a/include/in_mem_graph_store.h +++ b/include/in_mem_graph_store.h @@ -11,10 +11,10 @@ namespace diskann class InMemGraphStore : public AbstractGraphStore { public: - InMemGraphStore(const size_t total_pts, const size_t num_frozen_points); + InMemGraphStore(const size_t total_pts); - int load(const std::string &index_path_prefix); - int store(const std::string &index_path_prefix, const size_t num_points); + int load(const std::string &index_path_prefix, const size_t num_points); + int store(const std::string &index_path_prefix, const size_t num_points, const size_t num_froxen_points); virtual std::vector &get_neighbours(const location_t i) override; virtual void set_neighbours(const location_t i, std::vector &neighbors) override; @@ -36,13 +36,13 @@ class InMemGraphStore : public AbstractGraphStore virtual location_t load_impl(AlignedFileReader &reader, size_t expected_num_points); #endif - int save_graph(const std::string &index_path_prefix, const size_t active_points); + int save_graph(const std::string &index_path_prefix, const size_t active_points, const size_t num_frozen_points); private: size_t _max_range_of_graph = 0; uint32_t _max_observed_degree = 0; uint32_t _start = 0; - size_t _num_frozen_pts; + std::vector> _graph; }; diff --git a/include/index.h b/include/index.h index 422bf8dd7..3d7a7fb28 100644 --- a/include/index.h +++ b/include/index.h @@ -351,7 +351,7 @@ template clas // externally and won't be returned by search. At least 1 frozen point is // needed for a dynamic index. The frozen points have consecutive locations. // See also _start below. - size_t _num_frozen_pts; + size_t _num_frozen_pts = 0; size_t _node_size; size_t _data_len; size_t _neighbor_len; diff --git a/include/index_factory.h b/include/index_factory.h index 5e5e97ec1..3d1eb7992 100644 --- a/include/index_factory.h +++ b/include/index_factory.h @@ -17,8 +17,7 @@ class IndexFactory std::unique_ptr> construct_datastore(DataStoreStrategy stratagy, size_t num_points, size_t dimension); - std::unique_ptr construct_graphstore(GraphStoreStrategy stratagy, size_t size, - size_t frozen_points); + std::unique_ptr construct_graphstore(GraphStoreStrategy stratagy, size_t size); template std::unique_ptr create_instance(); diff --git a/src/in_mem_graph_store.cpp b/src/in_mem_graph_store.cpp index 80cafec26..281574af7 100644 --- a/src/in_mem_graph_store.cpp +++ b/src/in_mem_graph_store.cpp @@ -7,18 +7,18 @@ namespace diskann { -InMemGraphStore::InMemGraphStore(const size_t total_pts, const size_t num_frozen_pts) - : AbstractGraphStore(total_pts), _num_frozen_pts(num_frozen_pts) +InMemGraphStore::InMemGraphStore(const size_t total_pts) : AbstractGraphStore(total_pts) { } -int InMemGraphStore::load(const std::string &index_path_prefix) +int InMemGraphStore::load(const std::string &index_path_prefix, const size_t num_points) { - return load_impl(index_path_prefix, get_total_points()); + return load_impl(index_path_prefix, num_points); } -int InMemGraphStore::store(const std::string &index_path_prefix, const size_t num_points) +int InMemGraphStore::store(const std::string &index_path_prefix, const size_t num_points, + const size_t num_frozen_points) { - return save_graph(index_path_prefix, num_points); + return save_graph(index_path_prefix, num_points, num_frozen_points); } std::vector &InMemGraphStore::get_neighbours(const location_t i) { @@ -46,7 +46,7 @@ void InMemGraphStore::clear_graph() location_t InMemGraphStore::load_impl(const std::string &filename, size_t expected_num_points) { size_t expected_file_size; - size_t file_frozen_pts; + auto max_points = get_max_points(); int header_size = 2 * sizeof(size_t) + 2 * sizeof(uint32_t); std::unique_ptr header = std::make_unique(header_size); @@ -146,38 +146,14 @@ location_t InMemGraphStore::load_impl(const std::string &filename, size_t expect << ", _max_observed_degree: " << _max_observed_degree << ", _start: " << _start << ", file_frozen_pts: " << file_frozen_pts << std::endl; - if (file_frozen_pts != _num_frozen_pts) - { - std::stringstream stream; - if (file_frozen_pts == 1) - { - stream << "ERROR: When loading graph, detected dynamic index, but " - "constructor asks for static index. Exitting." - << std::endl; - } - else - { - stream << "ERROR: When loading index, detected static index, but " - "constructor asks for dynamic index. Exitting." - << std::endl; - } - diskann::cerr << stream.str() << std::endl; - throw diskann::ANNException(stream.str(), -1, __FUNCSIG__, __FILE__, __LINE__); - } - diskann::cout << "Loading vamana graph " << filename << "..." << std::flush; - const size_t expected_max_points = expected_num_points - file_frozen_pts; - // If user provides more points than max_points // resize the _graph to the larger size. - auto max_points = get_total_points() - _num_frozen_pts; // from parent class holding max_pts - if (max_points < expected_max_points) + if (get_total_points() < expected_num_points) { - diskann::cout << "Number of points in data: " << expected_max_points - << " is greater than max_points: " << max_points - << " Setting max points to: " << expected_max_points << std::endl; - _graph.resize(expected_max_points + file_frozen_pts); + diskann::cout << "resizing graph to " << expected_num_points << std::endl; + _graph.resize(expected_num_points); } size_t bytes_read = vamana_metadata_size; @@ -213,7 +189,8 @@ location_t InMemGraphStore::load_impl(const std::string &filename, size_t expect return nodes_read; } -int InMemGraphStore::save_graph(const std::string &index_path_prefix, const size_t num_points) +int InMemGraphStore::save_graph(const std::string &index_path_prefix, const size_t num_points, + const size_t num_frozen_points) { std::ofstream out; open_file_to_write(out, index_path_prefix); @@ -226,7 +203,7 @@ int InMemGraphStore::save_graph(const std::string &index_path_prefix, const size out.write((char *)&_max_observed_degree, sizeof(uint32_t)); uint32_t ep_u32 = _start; out.write((char *)&ep_u32, sizeof(uint32_t)); - out.write((char *)&_num_frozen_pts, sizeof(size_t)); + out.write((char *)&num_frozen_points, sizeof(size_t)); // Note: num_points = _nd + _num_frozen_points for (uint32_t i = 0; i < num_points; i++) diff --git a/src/index.cpp b/src/index.cpp index c1d9a1dab..b478823ed 100644 --- a/src/index.cpp +++ b/src/index.cpp @@ -123,7 +123,7 @@ Index::Index(Metric m, const size_t dim, const size_t max_point _data_store = std::make_unique>((location_t)total_internal_points, _dim, this->_distance); // init graph store => TODO : graph store should be injected - _graph_store = std::make_unique(total_internal_points, _num_frozen_pts); + _graph_store = std::make_unique(total_internal_points); _graph_store->set_start((uint32_t)_max_points); _graph_store->resize_graph(total_internal_points); } @@ -271,7 +271,7 @@ template size_t Index size_t Index::save_graph(std::string graph_file) { - return _graph_store->store(graph_file, _nd + _num_frozen_pts); + return _graph_store->store(graph_file, _nd + _num_frozen_pts, _num_frozen_pts); } template @@ -686,7 +686,7 @@ size_t Index::load_graph(std::string filename, size_t expected_ { #endif - return _graph_store->load(filename); + return _graph_store->load(filename, expected_num_points); } template diff --git a/src/index_factory.cpp b/src/index_factory.cpp index b5ea3d30b..39f9adaaa 100644 --- a/src/index_factory.cpp +++ b/src/index_factory.cpp @@ -75,13 +75,12 @@ std::unique_ptr> IndexFactory::construct_datastore(DataStor return nullptr; } -std::unique_ptr IndexFactory::construct_graphstore(GraphStoreStrategy strategy, size_t size, - size_t frozen_points) +std::unique_ptr IndexFactory::construct_graphstore(GraphStoreStrategy strategy, size_t size) { switch (strategy) { case GraphStoreStrategy::MEMORY: - return std::make_unique(size, frozen_points); + return std::make_unique(size); default: throw ANNException("Error : Current GraphStoreStratagy is not supported.", -1); } @@ -93,7 +92,7 @@ std::unique_ptr IndexFactory::create_instance() size_t num_points = _config->max_points; size_t dim = _config->dimension; auto data_store = construct_datastore(_config->data_strategy, num_points, dim); - auto graph_store = construct_graphstore(_config->graph_strategy, num_points, _config->num_frozen_pts); + auto graph_store = construct_graphstore(_config->graph_strategy, num_points + _config->num_frozen_pts); return std::make_unique>(*_config, std::move(data_store), std::move(graph_store)); }