From 8f2016b199b06f8cbc4aaf104a7f45623b665f5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Holm=20Gj=C3=B8rup?= Date: Thu, 15 Feb 2024 20:54:32 +0100 Subject: [PATCH 01/36] Add quick in-memory version of state-btree --- concordium-std/src/impls.rs | 193 ++++++++++++++++++++++ concordium-std/src/test_infrastructure.rs | 119 ++++++++++++- concordium-std/src/types.rs | 17 ++ 3 files changed, 328 insertions(+), 1 deletion(-) diff --git a/concordium-std/src/impls.rs b/concordium-std/src/impls.rs index 7e5b0a05..2df8933a 100644 --- a/concordium-std/src/impls.rs +++ b/concordium-std/src/impls.rs @@ -2298,6 +2298,11 @@ where StateMap::open(state_api, prefix) } + pub fn new_btree_map(&mut self) -> StateBTreeMap { + let (state_api, prefix) = self.new_state_container(); + StateBTreeMap::new(state_api, prefix) + } + /// Create a new empty [`StateSet`]. pub fn new_set(&mut self) -> StateSet { let (state_api, prefix) = self.new_state_container(); @@ -3114,6 +3119,194 @@ impl Deserial for MetadataUrl { } } +impl StateBTreeMap { + pub(crate) fn new(state_api: S, prefix: StateItemPrefix) -> Self { + Self { + _marker_key: Default::default(), + _marker_value: Default::default(), + root: None, + len: 0, + state_api, + prefix, + } + } +} + +impl StateBTreeMap { + pub fn insert(&mut self, key: K, value: V) -> Option { + let Some(root_node) = &mut self.root else { + let node = StateBTreeNode { + keys: vec![key], + values: vec![value], + children: Vec::new(), + }; + self.root = Some(node); + self.len = 1; + return None; + }; + + if !root_node.is_full() { + let out = root_node.insert_non_full(key, value); + if out.is_none() { + self.len += 1; + } + return out; + } + + let mut new_root = StateBTreeNode { + keys: Vec::new(), + values: Vec::new(), + children: vec![self.root.take().unwrap_abort()], /* Safe to unwrap, since we checked + * above that this is Some. */ + }; + new_root.split_child(0); + let out = new_root.insert_non_full(key, value); + self.root = Some(new_root); + if out.is_none() { + self.len += 1; + } + out + } + + pub fn higher(&self, key: &K) -> Option<&K> { + if let Some(root_node) = &self.root { + root_node.higher(key) + } else { + None + } + } + + pub fn lower(&self, key: &K) -> Option<&K> { + if let Some(root_node) = &self.root { + root_node.lower(key) + } else { + None + } + } + + pub fn get(&self, key: &K) -> Option<&V> { + if let Some(root_node) = &self.root { + root_node.get(key) + } else { + None + } + } + + pub fn len(&self) -> usize { self.len } + + pub fn is_empty(&self) -> bool { self.root.is_none() } +} + +impl StateBTreeNode { + fn is_full(&self) -> bool { self.keys.len() == M } + + fn is_leaf(&self) -> bool { self.children.is_empty() } + + fn get(&self, key: &K) -> Option<&V> { + match self.keys.binary_search(key) { + Ok(matched_key_index) => Some(&self.values[matched_key_index]), + Err(above_key_index) => { + if self.is_leaf() { + None + } else { + self.children[above_key_index].get(key) + } + } + } + } + + fn insert_non_full(&mut self, key: K, value: V) -> Option { + let mut i = match self.keys.binary_search(&key) { + Ok(index) => { + let value = mem::replace(&mut self.values[index], value); + return Some(value); + } + Err(index) => index, + }; + if self.is_leaf() { + self.keys.insert(i, key); + self.values.insert(i, value); + None + } else { + if self.children[i].is_full() { + self.split_child(i); + if self.keys[i] < key { + i += 1; + } + } + self.children[i].insert_non_full(key, value) + } + } + + fn split_child(&mut self, child_index: usize) { + let left = &mut self.children[child_index]; + let split_index = (M + 1) / 2; + let right = StateBTreeNode { + keys: left.keys.split_off(split_index), + values: left.values.split_off(split_index), + children: if left.is_leaf() { + Vec::new() + } else { + left.children.split_off(split_index) + }, + }; + + let key = left.keys.pop().unwrap_abort(); + let value = left.values.pop().unwrap_abort(); + self.children.insert(child_index + 1, right); + self.keys.insert(child_index, key); + self.values.insert(child_index, value); + } + + fn higher(&self, key: &K) -> Option<&K> { + let higher_key_index = match self.keys.binary_search(key) { + Ok(index) => index + 1, + Err(index) => index, + }; + + if self.is_leaf() { + if higher_key_index < self.keys.len() { + Some(&self.keys[higher_key_index]) + } else { + None + } + } else { + self.children[higher_key_index].higher(key).or_else(|| { + if higher_key_index < self.keys.len() { + Some(&self.keys[higher_key_index]) + } else { + None + } + }) + } + } + + fn lower(&self, key: &K) -> Option<&K> { + let lower_key_index = match self.keys.binary_search(key) { + Ok(index) => index, + Err(index) => index, + }; + + if self.is_leaf() { + if key <= &self.keys[0] { + None + } else { + // lower_key_index cannot be 0 in this case, since the binary seach will only + // return 0 in the true branch above. + Some(&self.keys[lower_key_index - 1]) + } + } else { + self.children[lower_key_index].lower(key).or_else(|| { + if lower_key_index > 0 { + Some(&self.keys[lower_key_index - 1]) + } else { + None + } + }) + } + } +} + #[cfg(test)] mod tests { diff --git a/concordium-std/src/test_infrastructure.rs b/concordium-std/src/test_infrastructure.rs index 052e0867..2331063b 100644 --- a/concordium-std/src/test_infrastructure.rs +++ b/concordium-std/src/test_infrastructure.rs @@ -1934,7 +1934,7 @@ mod test { cell::RefCell, rc::Rc, test_infrastructure::{TestStateBuilder, TestStateEntry}, - Deletable, EntryRaw, HasStateApi, HasStateEntry, StateMap, StateSet, + Deletable, EntryRaw, HasStateApi, HasStateEntry, StateBTreeMap, StateMap, StateSet, INITIAL_NEXT_ITEM_PREFIX, }; use concordium_contracts_common::{to_bytes, Deserial, Read, Seek, SeekFrom, Write}; @@ -2486,4 +2486,121 @@ mod test { state.lookup_entry(&[]).expect("Lookup failed").size().expect("Getting size failed"); assert_eq!(expected_size as u32, actual_size); } + + #[test] + fn test_btree_m5_insert_6() { + let mut state_builder = TestStateBuilder::new(); + let mut map: StateBTreeMap<5, u32, &str, _> = state_builder.new_btree_map(); + map.insert(0, "zero"); + map.insert(1, "one"); + map.insert(2, "two"); + map.insert(3, "three"); + map.insert(4, "four"); + map.insert(5, "five"); + let &s = map.get(&5).expect("to find key"); + + assert_eq!(s, "five") + } + + #[test] + fn test_btree_m3_insert_7() { + let mut state_builder = TestStateBuilder::new(); + let mut map: StateBTreeMap<3, u32, &str, _> = state_builder.new_btree_map(); + map.insert(0, "zero"); + map.insert(1, "one"); + map.insert(2, "two"); + map.insert(3, "three"); + map.insert(4, "four"); + map.insert(5, "five"); + map.insert(6, "six"); + map.insert(7, "seven"); + let &s = map.get(&7).expect("to find key"); + assert_eq!(s, "seven") + } + + #[test] + fn test_btree_m3_insert_7_no_order() { + let mut state_builder = TestStateBuilder::new(); + let mut map: StateBTreeMap<3, u32, &str, _> = state_builder.new_btree_map(); + map.insert(0, "zero"); + map.insert(1, "one"); + map.insert(2, "two"); + map.insert(3, "three"); + map.insert(7, "seven"); + map.insert(6, "six"); + map.insert(5, "five"); + map.insert(4, "four"); + let &s = map.get(&7).expect("to find key"); + assert_eq!(s, "seven") + } + + #[test] + fn test_btree_m3_higher() { + let mut state_builder = TestStateBuilder::new(); + let mut map: StateBTreeMap<3, u32, &str, _> = state_builder.new_btree_map(); + map.insert(1, "one"); + map.insert(2, "two"); + map.insert(3, "three"); + map.insert(4, "four"); + map.insert(5, "five"); + map.insert(7, "seven"); + let &s = map.higher(&3).expect("to find higher key"); + assert_eq!(s, 4); + let &s = map.higher(&5).expect("to find higher key"); + assert_eq!(s, 7); + let s = map.higher(&7); + assert_eq!(s, None) + } + + #[test] + fn test_btree_m3_lower() { + let mut state_builder = TestStateBuilder::new(); + let mut map: StateBTreeMap<3, u32, &str, _> = state_builder.new_btree_map(); + map.insert(1, "one"); + map.insert(2, "two"); + map.insert(3, "three"); + map.insert(4, "four"); + map.insert(5, "five"); + map.insert(7, "seven"); + let &s = map.lower(&3).expect("to find lower key"); + assert_eq!(s, 2); + let &s = map.lower(&7).expect("to find lower key"); + assert_eq!(s, 5); + let s = map.lower(&1); + assert_eq!(s, None) + } + + #[test] + fn test_btree_m3_insert_10000() { + let mut state_builder = TestStateBuilder::new(); + let mut map: StateBTreeMap<3, u32, String, _> = state_builder.new_btree_map(); + for (n, s) in (0..5000).into_iter().map(|n| (n, n.to_string())) { + map.insert(n, s); + } + + for (n, s) in (5000..10000).into_iter().rev().map(|n| (n, n.to_string())) { + map.insert(n, s); + } + + for n in 0..10000 { + let s = map.get(&n); + assert_eq!(s, Some(&n.to_string())); + } + + assert_eq!(map.len(), 10000) + } + + #[test] + fn test_btree_m3_7_get_8() { + let mut state_builder = TestStateBuilder::new(); + let mut map: StateBTreeMap<3, u32, String, _> = state_builder.new_btree_map(); + for (n, s) in (0..=7).into_iter().map(|n| (n, n.to_string())) { + map.insert(n, s); + } + + println!("{:#?}", map.root); + let s = map.get(&8); + + assert_eq!(s, None); + } } diff --git a/concordium-std/src/types.rs b/concordium-std/src/types.rs index ee3e35f5..0dfb4855 100644 --- a/concordium-std/src/types.rs +++ b/concordium-std/src/types.rs @@ -1258,3 +1258,20 @@ pub struct MetadataUrl { /// A optional hash of the content. pub hash: Option<[u8; 32]>, } + +#[derive(Debug)] +pub struct StateBTreeMap { + pub(crate) _marker_key: PhantomData, + pub(crate) _marker_value: PhantomData, + pub(crate) root: Option>, + pub(crate) len: usize, + pub(crate) state_api: S, + pub(crate) prefix: StateItemPrefix, +} + +#[derive(Debug)] +pub(crate) struct StateBTreeNode { + pub(crate) keys: Vec, // never empty, sorted + pub(crate) values: Vec, // never empty + pub(crate) children: Vec>, // Empty means leaf. +} From 3779adadb1f805fb3f891602b4a3b6f029828eb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Holm=20Gj=C3=B8rup?= Date: Tue, 20 Feb 2024 16:10:31 +0100 Subject: [PATCH 02/36] Move Btree nodes and values into the contract state --- concordium-std/src/impls.rs | 455 ++++++++++++++++------ concordium-std/src/test_infrastructure.rs | 112 +++--- concordium-std/src/types.rs | 33 +- 3 files changed, 403 insertions(+), 197 deletions(-) diff --git a/concordium-std/src/impls.rs b/concordium-std/src/impls.rs index 2df8933a..7ddb8fc6 100644 --- a/concordium-std/src/impls.rs +++ b/concordium-std/src/impls.rs @@ -7,7 +7,7 @@ use crate::{ marker::PhantomData, mem, num, num::NonZeroU32, - prims, + prims, state_btree_internals, traits::*, types::*, vec::Vec, @@ -3119,6 +3119,82 @@ impl Deserial for MetadataUrl { } } +impl Serial for StateBTreeMap { + fn serial(&self, out: &mut W) -> Result<(), W::Err> { + self.prefix.serial(out)?; + self.root.serial(out)?; + self.len.serial(out)?; + self.next_node_id.serial(out)?; + self.next_value_id.serial(out) + } +} + +impl DeserialWithState for StateBTreeMap { + fn deserial_with_state(state: &S, source: &mut R) -> ParseResult { + let prefix = source.get()?; + let root = source.get()?; + let len = source.get()?; + let next_node_id = source.get()?; + let next_value_id = source.get()?; + + Ok(StateBTreeMap { + _marker_key: Default::default(), + _marker_value: Default::default(), + root, + len, + prefix, + next_node_id, + next_value_id, + state_api: state.clone(), + }) + } +} + +impl Serial for state_btree_internals::StateBTreeNode { + fn serial(&self, out: &mut W) -> Result<(), W::Err> { + self.keys.serial(out)?; + self.values.serial(out)?; + self.children.serial(out) + } +} + +impl Deserial for state_btree_internals::StateBTreeNode { + fn deserial(source: &mut R) -> ParseResult { + let keys = source.get()?; + let values = source.get()?; + let children = source.get()?; + Ok(Self { + keys, + values, + children, + }) + } +} + +impl Serial for state_btree_internals::NodeId { + fn serial(&self, out: &mut W) -> Result<(), W::Err> { self.id.serial(out) } +} + +impl Deserial for state_btree_internals::NodeId { + fn deserial(source: &mut R) -> ParseResult { + Ok(Self { + id: source.get()?, + }) + } +} + +impl Serial for state_btree_internals::ValueId { + fn serial(&self, out: &mut W) -> Result<(), W::Err> { self.id.serial(out) } +} + +impl Deserial for state_btree_internals::ValueId { + fn deserial(source: &mut R) -> ParseResult { + Ok(Self { + id: source.get()?, + }) + } +} + impl StateBTreeMap { pub(crate) fn new(state_api: S, prefix: StateItemPrefix) -> Self { Self { @@ -3126,184 +3202,309 @@ impl StateBTreeMap { _marker_value: Default::default(), root: None, len: 0, + next_node_id: state_btree_internals::NodeId { + id: 0, + }, + next_value_id: state_btree_internals::ValueId { + id: 0, + }, state_api, prefix, } } } -impl StateBTreeMap { +impl + StateBTreeMap +{ + pub fn higher(&self, key: &K) -> Option { + let Some(root_node_id) = self.root else { + return None; + }; + + let mut node = self.get_node(root_node_id); + let mut higher_so_far: Option = None; + loop { + let higher_key_index = match node.keys.binary_search(key) { + Ok(index) => index + 1, + Err(index) => index, + }; + + if node.is_leaf() { + return if higher_key_index < node.keys.len() { + Some(node.keys[higher_key_index].clone()) + } else { + higher_so_far + }; + } else { + if higher_key_index < node.keys.len() { + higher_so_far = Some(node.keys[higher_key_index].clone()) + } + + let child_node_id = node.children[higher_key_index]; + node = self.get_node(child_node_id); + } + } + } + + pub fn lower(&self, key: &K) -> Option { + let Some(root_node_id) = self.root else { + return None; + }; + + let mut node = self.get_node(root_node_id); + let mut lower_so_far: Option = None; + loop { + let lower_key_index = match node.keys.binary_search(key) { + Ok(index) => index, + Err(index) => index, + }; + + if node.is_leaf() { + return if key <= &node.keys[0] { + lower_so_far + } else { + // lower_key_index cannot be 0 in this case, since the binary search will only + // return 0 in the true branch above. + Some(node.keys[lower_key_index - 1].clone()) + }; + } else { + if lower_key_index > 0 { + lower_so_far = Some(node.keys[lower_key_index - 1].clone()); + } + let child_node_id = node.children[lower_key_index]; + node = self.get_node(child_node_id) + } + } + } +} + +impl StateBTreeMap { + fn get_node<'a, 'b>( + &'a self, + node_id: state_btree_internals::NodeId, + ) -> StateRef<'b, state_btree_internals::StateBTreeNode> { + let key = self.node_key(node_id); + let mut entry = self.state_api.lookup_entry(&key).unwrap_abort(); + StateRef::new(entry.get().unwrap_abort()) + } + + fn node_key(&self, node_id: state_btree_internals::NodeId) -> Vec { + let mut key = self.prefix.to_vec(); + key.push(0u8); // Node key discriminant + node_id.serial(&mut key).unwrap_abort(); + key + } + + fn value_key(&self, value_id: state_btree_internals::ValueId) -> Vec { + let mut key = self.prefix.to_vec(); + key.push(1u8); // Node key discriminant + value_id.serial(&mut key).unwrap_abort(); + key + } +} + +impl StateBTreeMap { pub fn insert(&mut self, key: K, value: V) -> Option { - let Some(root_node) = &mut self.root else { - let node = StateBTreeNode { - keys: vec![key], - values: vec![value], - children: Vec::new(), + let Some(root_id) = self.root else { + let node_id = { + let value_id = self.create_value(value); + let (node_id, _node) = self.create_node(vec![key], vec![value_id], Vec::new()); + node_id }; - self.root = Some(node); + self.root = Some(node_id); self.len = 1; return None; }; - if !root_node.is_full() { - let out = root_node.insert_non_full(key, value); - if out.is_none() { - self.len += 1; + { + let root_node = self.get_node_mut(root_id); + if !root_node.is_full() { + let out = self.insert_non_full(root_node, key, value); + if out.is_none() { + self.len += 1; + } + return out; } - return out; } - let mut new_root = StateBTreeNode { - keys: Vec::new(), - values: Vec::new(), - children: vec![self.root.take().unwrap_abort()], /* Safe to unwrap, since we checked - * above that this is Some. */ - }; - new_root.split_child(0); - let out = new_root.insert_non_full(key, value); - self.root = Some(new_root); + let (new_root_id, mut new_root) = self.create_node(Vec::new(), Vec::new(), vec![root_id]); + self.split_child(&mut new_root, 0); + let out = self.insert_non_full(new_root, key, value); + self.root = Some(new_root_id); if out.is_none() { self.len += 1; } out } - pub fn higher(&self, key: &K) -> Option<&K> { - if let Some(root_node) = &self.root { - root_node.higher(key) - } else { - None - } - } - - pub fn lower(&self, key: &K) -> Option<&K> { - if let Some(root_node) = &self.root { - root_node.lower(key) - } else { - None - } - } + pub fn get<'a>(&'a self, key: &K) -> Option> { + let Some(root_node) = self.root else { + return None; + }; - pub fn get(&self, key: &K) -> Option<&V> { - if let Some(root_node) = &self.root { - root_node.get(key) - } else { - None + let mut node = self.get_node(root_node); + loop { + match node.keys.binary_search(key) { + Ok(matched_key_index) => { + let value_id = node.values[matched_key_index]; + let value = self.get_value(value_id); + return Some(StateRef::new(value)); + } + Err(above_key_index) => { + if node.is_leaf() { + return None; + } else { + node = self.get_node(node.children[above_key_index]); + } + } + } } } - pub fn len(&self) -> usize { self.len } + pub fn len(&self) -> u32 { self.len } pub fn is_empty(&self) -> bool { self.root.is_none() } -} -impl StateBTreeNode { - fn is_full(&self) -> bool { self.keys.len() == M } + fn create_value(&mut self, value: V) -> state_btree_internals::ValueId { + let value_id = self.next_value_id; + self.next_value_id = state_btree_internals::ValueId { + id: value_id.id + 1, + }; + let key = self.value_key(value_id); + let mut entry = self.state_api.create_entry(&key).unwrap_abort(); + value.serial(&mut entry).unwrap_abort(); + value_id + } - fn is_leaf(&self) -> bool { self.children.is_empty() } + fn get_value(&self, value_id: state_btree_internals::ValueId) -> V { + let key = self.value_key(value_id); + let mut entry = self.state_api.lookup_entry(&key).unwrap_abort(); + entry.get().unwrap_abort() + } - fn get(&self, key: &K) -> Option<&V> { - match self.keys.binary_search(key) { - Ok(matched_key_index) => Some(&self.values[matched_key_index]), - Err(above_key_index) => { - if self.is_leaf() { - None - } else { - self.children[above_key_index].get(key) - } + fn set_value(&mut self, value_id: state_btree_internals::ValueId, value: V) -> Option { + let key = self.value_key(value_id); + match self.state_api.entry(key) { + EntryRaw::Vacant(_) => None, + EntryRaw::Occupied(mut raw_entry) => { + let entry = raw_entry.get_mut(); + let old_value = entry.get().unwrap_abort(); + value.serial(entry).unwrap_abort(); + Some(old_value) } } } - fn insert_non_full(&mut self, key: K, value: V) -> Option { - let mut i = match self.keys.binary_search(&key) { - Ok(index) => { - let value = mem::replace(&mut self.values[index], value); - return Some(value); - } - Err(index) => index, + fn create_node<'a, 'b>( + &'a mut self, + keys: Vec, + values: Vec, + children: Vec, + ) -> ( + state_btree_internals::NodeId, + StateRefMut<'b, state_btree_internals::StateBTreeNode, S>, + ) { + let node_id = self.next_node_id; + self.next_node_id = state_btree_internals::NodeId { + id: node_id.id + 1, }; - if self.is_leaf() { - self.keys.insert(i, key); - self.values.insert(i, value); - None - } else { - if self.children[i].is_full() { - self.split_child(i); - if self.keys[i] < key { - i += 1; + let node = state_btree_internals::StateBTreeNode { + keys, + values, + children, + }; + let entry = self.state_api.create_entry(&self.node_key(node_id)).unwrap_abort(); + let mut ref_mut: StateRefMut<'_, state_btree_internals::StateBTreeNode, S> = + StateRefMut::new(entry, self.state_api.clone()); + ref_mut.set(node); + (node_id, ref_mut) + } + + fn get_node_mut<'a, 'b>( + &'a mut self, + node_id: state_btree_internals::NodeId, + ) -> StateRefMut<'b, state_btree_internals::StateBTreeNode, S> { + let key = self.node_key(node_id); + let entry = self.state_api.entry(key); + match entry { + EntryRaw::Vacant(_) => crate::trap(), + EntryRaw::Occupied(entry) => StateRefMut::new(entry.get(), self.state_api.clone()), + } + } + + fn insert_non_full( + &mut self, + initial_node: StateRefMut, S>, + key: K, + value: V, + ) -> Option { + let mut node = initial_node; + loop { + let mut i = match node.keys.binary_search(&key) { + Ok(index) => { + let value_id = node.values[index]; + let value = self.set_value(value_id, value); + return value; + } + Err(index) => index, + }; + if node.is_leaf() { + node.keys.insert(i, key); + let value_id = self.create_value(value); + node.values.insert(i, value_id); + return None; + } else { + let child = self.get_node(node.children[i]); + if child.is_full() { + self.split_child(&mut node, i); + if node.keys[i] < key { + i += 1; + } } + node = self.get_node_mut(node.children[i]); } - self.children[i].insert_non_full(key, value) } } - fn split_child(&mut self, child_index: usize) { - let left = &mut self.children[child_index]; + fn split_child<'a, 'b>( + &'a mut self, + node: &'b mut StateRefMut, S>, + child_index: usize, + ) { + let mut left = self.get_node_mut(node.children[child_index]); let split_index = (M + 1) / 2; - let right = StateBTreeNode { - keys: left.keys.split_off(split_index), - values: left.values.split_off(split_index), - children: if left.is_leaf() { + let (right_id, _right) = self.create_node( + left.keys.split_off(split_index), + left.values.split_off(split_index), + if left.is_leaf() { Vec::new() } else { left.children.split_off(split_index) }, - }; - + ); let key = left.keys.pop().unwrap_abort(); let value = left.values.pop().unwrap_abort(); - self.children.insert(child_index + 1, right); - self.keys.insert(child_index, key); - self.values.insert(child_index, value); + node.children.insert(child_index + 1, right_id); + node.keys.insert(child_index, key); + node.values.insert(child_index, value); } +} - fn higher(&self, key: &K) -> Option<&K> { - let higher_key_index = match self.keys.binary_search(key) { - Ok(index) => index + 1, - Err(index) => index, - }; +impl state_btree_internals::StateBTreeNode { + fn is_full(&self) -> bool { self.keys.len() == M } - if self.is_leaf() { - if higher_key_index < self.keys.len() { - Some(&self.keys[higher_key_index]) - } else { - None - } - } else { - self.children[higher_key_index].higher(key).or_else(|| { - if higher_key_index < self.keys.len() { - Some(&self.keys[higher_key_index]) - } else { - None - } - }) - } - } + fn is_leaf(&self) -> bool { self.children.is_empty() } +} - fn lower(&self, key: &K) -> Option<&K> { - let lower_key_index = match self.keys.binary_search(key) { - Ok(index) => index, - Err(index) => index, - }; +impl Deletable for StateBTreeMap +where + S: HasStateApi, +{ + fn delete(self) { + todo!("clear the map"); - if self.is_leaf() { - if key <= &self.keys[0] { - None - } else { - // lower_key_index cannot be 0 in this case, since the binary seach will only - // return 0 in the true branch above. - Some(&self.keys[lower_key_index - 1]) - } - } else { - self.children[lower_key_index].lower(key).or_else(|| { - if lower_key_index > 0 { - Some(&self.keys[lower_key_index - 1]) - } else { - None - } - }) - } + //self.clear(); } } diff --git a/concordium-std/src/test_infrastructure.rs b/concordium-std/src/test_infrastructure.rs index 2331063b..59320534 100644 --- a/concordium-std/src/test_infrastructure.rs +++ b/concordium-std/src/test_infrastructure.rs @@ -1929,6 +1929,8 @@ pub fn concordium_qc(num_tests: u64, f: A) { #[cfg(test)] mod test { + use core::ops::Deref; + use super::TestStateApi; use crate::{ cell::RefCell, @@ -2490,90 +2492,77 @@ mod test { #[test] fn test_btree_m5_insert_6() { let mut state_builder = TestStateBuilder::new(); - let mut map: StateBTreeMap<5, u32, &str, _> = state_builder.new_btree_map(); - map.insert(0, "zero"); - map.insert(1, "one"); - map.insert(2, "two"); - map.insert(3, "three"); - map.insert(4, "four"); - map.insert(5, "five"); - let &s = map.get(&5).expect("to find key"); + let mut map: StateBTreeMap<5, u32, _, _> = state_builder.new_btree_map(); + for (n, s) in (0..=5).into_iter().map(|n| (n, n.to_string())) { + map.insert(n, s); + } + let s = map.get(&5).expect("to find key"); - assert_eq!(s, "five") + assert_eq!(s.as_str(), "5") } #[test] - fn test_btree_m3_insert_7() { + fn test_btree_m3_insert_0_7() { let mut state_builder = TestStateBuilder::new(); - let mut map: StateBTreeMap<3, u32, &str, _> = state_builder.new_btree_map(); - map.insert(0, "zero"); - map.insert(1, "one"); - map.insert(2, "two"); - map.insert(3, "three"); - map.insert(4, "four"); - map.insert(5, "five"); - map.insert(6, "six"); - map.insert(7, "seven"); - let &s = map.get(&7).expect("to find key"); - assert_eq!(s, "seven") + let mut map: StateBTreeMap<3, u32, String, _> = state_builder.new_btree_map(); + for (n, s) in (0..=7).into_iter().map(|n| (n, n.to_string())) { + map.insert(n, s); + } + let s = map.get(&7).expect("to find key"); + assert_eq!(s.as_str(), "7") } #[test] fn test_btree_m3_insert_7_no_order() { let mut state_builder = TestStateBuilder::new(); - let mut map: StateBTreeMap<3, u32, &str, _> = state_builder.new_btree_map(); - map.insert(0, "zero"); - map.insert(1, "one"); - map.insert(2, "two"); - map.insert(3, "three"); - map.insert(7, "seven"); - map.insert(6, "six"); - map.insert(5, "five"); - map.insert(4, "four"); - let &s = map.get(&7).expect("to find key"); - assert_eq!(s, "seven") + let mut map: StateBTreeMap<3, u32, _, _> = state_builder.new_btree_map(); + + map.insert(0, "zero".to_string()); + map.insert(1, "one".to_string()); + map.insert(2, "two".to_string()); + map.insert(3, "three".to_string()); + map.insert(7, "seven".to_string()); + map.insert(6, "six".to_string()); + map.insert(5, "five".to_string()); + map.insert(4, "four".to_string()); + let s = map.get(&7).expect("to find key"); + assert_eq!(s.as_str(), "seven") } #[test] fn test_btree_m3_higher() { let mut state_builder = TestStateBuilder::new(); - let mut map: StateBTreeMap<3, u32, &str, _> = state_builder.new_btree_map(); - map.insert(1, "one"); - map.insert(2, "two"); - map.insert(3, "three"); - map.insert(4, "four"); - map.insert(5, "five"); - map.insert(7, "seven"); - let &s = map.higher(&3).expect("to find higher key"); - assert_eq!(s, 4); - let &s = map.higher(&5).expect("to find higher key"); - assert_eq!(s, 7); - let s = map.higher(&7); - assert_eq!(s, None) + let mut map: StateBTreeMap<3, u32, _, _> = state_builder.new_btree_map(); + map.insert(1, "one".to_string()); + map.insert(2, "two".to_string()); + map.insert(3, "three".to_string()); + map.insert(4, "four".to_string()); + map.insert(5, "five".to_string()); + map.insert(7, "seven".to_string()); + assert_eq!(map.higher(&3), Some(4)); + assert_eq!(map.higher(&5), Some(7)); + assert_eq!(map.higher(&7), None) } #[test] fn test_btree_m3_lower() { let mut state_builder = TestStateBuilder::new(); - let mut map: StateBTreeMap<3, u32, &str, _> = state_builder.new_btree_map(); - map.insert(1, "one"); - map.insert(2, "two"); - map.insert(3, "three"); - map.insert(4, "four"); - map.insert(5, "five"); - map.insert(7, "seven"); - let &s = map.lower(&3).expect("to find lower key"); - assert_eq!(s, 2); - let &s = map.lower(&7).expect("to find lower key"); - assert_eq!(s, 5); - let s = map.lower(&1); - assert_eq!(s, None) + let mut map: StateBTreeMap<3, u32, _, _> = state_builder.new_btree_map(); + map.insert(1, "one".to_string()); + map.insert(2, "two".to_string()); + map.insert(3, "three".to_string()); + map.insert(4, "four".to_string()); + map.insert(5, "five".to_string()); + map.insert(7, "seven".to_string()); + assert_eq!(map.lower(&3), Some(2)); + assert_eq!(map.lower(&7), Some(5)); + assert_eq!(map.lower(&1), None) } #[test] fn test_btree_m3_insert_10000() { let mut state_builder = TestStateBuilder::new(); - let mut map: StateBTreeMap<3, u32, String, _> = state_builder.new_btree_map(); + let mut map: StateBTreeMap<3, u32, _, _> = state_builder.new_btree_map(); for (n, s) in (0..5000).into_iter().map(|n| (n, n.to_string())) { map.insert(n, s); } @@ -2584,7 +2573,7 @@ mod test { for n in 0..10000 { let s = map.get(&n); - assert_eq!(s, Some(&n.to_string())); + assert_eq!(s.as_deref(), Some(&n.to_string())); } assert_eq!(map.len(), 10000) @@ -2598,9 +2587,8 @@ mod test { map.insert(n, s); } - println!("{:#?}", map.root); let s = map.get(&8); - assert_eq!(s, None); + assert_eq!(s.as_deref(), None); } } diff --git a/concordium-std/src/types.rs b/concordium-std/src/types.rs index 0dfb4855..57ce8b91 100644 --- a/concordium-std/src/types.rs +++ b/concordium-std/src/types.rs @@ -1263,15 +1263,32 @@ pub struct MetadataUrl { pub struct StateBTreeMap { pub(crate) _marker_key: PhantomData, pub(crate) _marker_value: PhantomData, - pub(crate) root: Option>, - pub(crate) len: usize, - pub(crate) state_api: S, + pub(crate) root: Option, + pub(crate) len: u32, pub(crate) prefix: StateItemPrefix, + pub(crate) next_node_id: state_btree_internals::NodeId, + pub(crate) next_value_id: state_btree_internals::ValueId, + pub(crate) state_api: S, } -#[derive(Debug)] -pub(crate) struct StateBTreeNode { - pub(crate) keys: Vec, // never empty, sorted - pub(crate) values: Vec, // never empty - pub(crate) children: Vec>, // Empty means leaf. +pub(crate) mod state_btree_internals { + #[derive(Debug, Copy, Clone)] + #[repr(transparent)] + pub(crate) struct NodeId { + pub(crate) id: u32, + } + + #[derive(Debug, Copy, Clone)] + #[repr(transparent)] + pub(crate) struct ValueId { + pub(crate) id: u32, + } + + #[derive(Debug)] + pub(crate) struct StateBTreeNode { + pub(crate) keys: Vec, // never empty, sorted. + pub(crate) values: Vec, // never empty, order corresponds to the keys. + /// Either empty or keys.len() + 1. + pub(crate) children: Vec, + } } From 5e9d818692b968699e231de23502b5c8ae648d4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Holm=20Gj=C3=B8rup?= Date: Tue, 20 Feb 2024 16:52:08 +0100 Subject: [PATCH 03/36] Btree: use keys instead of value ID --- concordium-std/src/impls.rs | 106 ++++++++++-------------------------- concordium-std/src/types.rs | 10 +--- 2 files changed, 30 insertions(+), 86 deletions(-) diff --git a/concordium-std/src/impls.rs b/concordium-std/src/impls.rs index 7ddb8fc6..45d30605 100644 --- a/concordium-std/src/impls.rs +++ b/concordium-std/src/impls.rs @@ -3124,8 +3124,7 @@ impl Serial for StateBTreeMap { self.prefix.serial(out)?; self.root.serial(out)?; self.len.serial(out)?; - self.next_node_id.serial(out)?; - self.next_value_id.serial(out) + self.next_node_id.serial(out) } } @@ -3135,7 +3134,6 @@ impl DeserialWithState for StateBTreeMa let root = source.get()?; let len = source.get()?; let next_node_id = source.get()?; - let next_value_id = source.get()?; Ok(StateBTreeMap { _marker_key: Default::default(), @@ -3144,7 +3142,6 @@ impl DeserialWithState for StateBTreeMa len, prefix, next_node_id, - next_value_id, state_api: state.clone(), }) } @@ -3153,7 +3150,6 @@ impl DeserialWithState for StateBTreeMa impl Serial for state_btree_internals::StateBTreeNode { fn serial(&self, out: &mut W) -> Result<(), W::Err> { self.keys.serial(out)?; - self.values.serial(out)?; self.children.serial(out) } } @@ -3161,11 +3157,9 @@ impl Serial for state_btree_internals::StateBTreeNode impl Deserial for state_btree_internals::StateBTreeNode { fn deserial(source: &mut R) -> ParseResult { let keys = source.get()?; - let values = source.get()?; let children = source.get()?; Ok(Self { keys, - values, children, }) } @@ -3183,18 +3177,6 @@ impl Deserial for state_btree_internals::NodeId { } } -impl Serial for state_btree_internals::ValueId { - fn serial(&self, out: &mut W) -> Result<(), W::Err> { self.id.serial(out) } -} - -impl Deserial for state_btree_internals::ValueId { - fn deserial(source: &mut R) -> ParseResult { - Ok(Self { - id: source.get()?, - }) - } -} - impl StateBTreeMap { pub(crate) fn new(state_api: S, prefix: StateItemPrefix) -> Self { Self { @@ -3205,16 +3187,13 @@ impl StateBTreeMap { next_node_id: state_btree_internals::NodeId { id: 0, }, - next_value_id: state_btree_internals::ValueId { - id: 0, - }, state_api, prefix, } } } -impl +impl StateBTreeMap { pub fn higher(&self, key: &K) -> Option { @@ -3279,7 +3258,7 @@ impl } } -impl StateBTreeMap { +impl StateBTreeMap { fn get_node<'a, 'b>( &'a self, node_id: state_btree_internals::NodeId, @@ -3296,11 +3275,11 @@ impl StateBTreeMap Vec { - let mut key = self.prefix.to_vec(); - key.push(1u8); // Node key discriminant - value_id.serial(&mut key).unwrap_abort(); - key + fn value_key(&self, key: &K) -> Vec { + let mut prefixed = self.prefix.to_vec(); + prefixed.push(1u8); // Node key discriminant + key.serial(&mut prefixed).unwrap_abort(); + prefixed } } @@ -3308,8 +3287,8 @@ impl StateBTre pub fn insert(&mut self, key: K, value: V) -> Option { let Some(root_id) = self.root else { let node_id = { - let value_id = self.create_value(value); - let (node_id, _node) = self.create_node(vec![key], vec![value_id], Vec::new()); + self.create_value(&key, value); + let (node_id, _node) = self.create_node(vec![key], Vec::new()); node_id }; self.root = Some(node_id); @@ -3328,7 +3307,7 @@ impl StateBTre } } - let (new_root_id, mut new_root) = self.create_node(Vec::new(), Vec::new(), vec![root_id]); + let (new_root_id, mut new_root) = self.create_node(Vec::new(), vec![root_id]); self.split_child(&mut new_root, 0); let out = self.insert_non_full(new_root, key, value); self.root = Some(new_root_id); @@ -3339,26 +3318,10 @@ impl StateBTre } pub fn get<'a>(&'a self, key: &K) -> Option> { - let Some(root_node) = self.root else { - return None; - }; - - let mut node = self.get_node(root_node); - loop { - match node.keys.binary_search(key) { - Ok(matched_key_index) => { - let value_id = node.values[matched_key_index]; - let value = self.get_value(value_id); - return Some(StateRef::new(value)); - } - Err(above_key_index) => { - if node.is_leaf() { - return None; - } else { - node = self.get_node(node.children[above_key_index]); - } - } - } + if self.root.is_none() { + None + } else { + self.get_value(key).map(StateRef::new) } } @@ -3366,26 +3329,22 @@ impl StateBTre pub fn is_empty(&self) -> bool { self.root.is_none() } - fn create_value(&mut self, value: V) -> state_btree_internals::ValueId { - let value_id = self.next_value_id; - self.next_value_id = state_btree_internals::ValueId { - id: value_id.id + 1, - }; - let key = self.value_key(value_id); - let mut entry = self.state_api.create_entry(&key).unwrap_abort(); + fn create_value(&mut self, key: &K, value: V) { + let prefixed_key = self.value_key(key); + let mut entry = self.state_api.create_entry(&prefixed_key).unwrap_abort(); value.serial(&mut entry).unwrap_abort(); - value_id } - fn get_value(&self, value_id: state_btree_internals::ValueId) -> V { - let key = self.value_key(value_id); - let mut entry = self.state_api.lookup_entry(&key).unwrap_abort(); - entry.get().unwrap_abort() + fn get_value(&self, key: &K) -> Option { + let prefixed_key = self.value_key(key); + self.state_api + .lookup_entry(&prefixed_key) + .map(|mut entry| V::deserial_with_state(&self.state_api, &mut entry).unwrap_abort()) } - fn set_value(&mut self, value_id: state_btree_internals::ValueId, value: V) -> Option { - let key = self.value_key(value_id); - match self.state_api.entry(key) { + fn set_value(&mut self, key: &K, value: V) -> Option { + let prefixed_key = self.value_key(key); + match self.state_api.entry(prefixed_key) { EntryRaw::Vacant(_) => None, EntryRaw::Occupied(mut raw_entry) => { let entry = raw_entry.get_mut(); @@ -3399,7 +3358,6 @@ impl StateBTre fn create_node<'a, 'b>( &'a mut self, keys: Vec, - values: Vec, children: Vec, ) -> ( state_btree_internals::NodeId, @@ -3411,7 +3369,6 @@ impl StateBTre }; let node = state_btree_internals::StateBTreeNode { keys, - values, children, }; let entry = self.state_api.create_entry(&self.node_key(node_id)).unwrap_abort(); @@ -3442,17 +3399,15 @@ impl StateBTre let mut node = initial_node; loop { let mut i = match node.keys.binary_search(&key) { - Ok(index) => { - let value_id = node.values[index]; - let value = self.set_value(value_id, value); + Ok(_) => { + let value = self.set_value(&key, value); return value; } Err(index) => index, }; if node.is_leaf() { + self.create_value(&key, value); node.keys.insert(i, key); - let value_id = self.create_value(value); - node.values.insert(i, value_id); return None; } else { let child = self.get_node(node.children[i]); @@ -3476,7 +3431,6 @@ impl StateBTre let split_index = (M + 1) / 2; let (right_id, _right) = self.create_node( left.keys.split_off(split_index), - left.values.split_off(split_index), if left.is_leaf() { Vec::new() } else { @@ -3484,10 +3438,8 @@ impl StateBTre }, ); let key = left.keys.pop().unwrap_abort(); - let value = left.values.pop().unwrap_abort(); node.children.insert(child_index + 1, right_id); node.keys.insert(child_index, key); - node.values.insert(child_index, value); } } diff --git a/concordium-std/src/types.rs b/concordium-std/src/types.rs index 57ce8b91..7f6e589d 100644 --- a/concordium-std/src/types.rs +++ b/concordium-std/src/types.rs @@ -1267,7 +1267,6 @@ pub struct StateBTreeMap { pub(crate) len: u32, pub(crate) prefix: StateItemPrefix, pub(crate) next_node_id: state_btree_internals::NodeId, - pub(crate) next_value_id: state_btree_internals::ValueId, pub(crate) state_api: S, } @@ -1278,16 +1277,9 @@ pub(crate) mod state_btree_internals { pub(crate) id: u32, } - #[derive(Debug, Copy, Clone)] - #[repr(transparent)] - pub(crate) struct ValueId { - pub(crate) id: u32, - } - #[derive(Debug)] pub(crate) struct StateBTreeNode { - pub(crate) keys: Vec, // never empty, sorted. - pub(crate) values: Vec, // never empty, order corresponds to the keys. + pub(crate) keys: Vec, // never empty, sorted. /// Either empty or keys.len() + 1. pub(crate) children: Vec, } From 25d5673800e6ec50e2c105d32dfa66da15fff8b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Holm=20Gj=C3=B8rup?= Date: Thu, 22 Feb 2024 11:29:12 +0100 Subject: [PATCH 04/36] BTree: add more docs and use maybeuninit for node keys --- concordium-std/src/impls.rs | 378 ++++++++++++++++++++++-------------- concordium-std/src/types.rs | 47 ++++- 2 files changed, 272 insertions(+), 153 deletions(-) diff --git a/concordium-std/src/impls.rs b/concordium-std/src/impls.rs index 45d30605..33e8fc82 100644 --- a/concordium-std/src/impls.rs +++ b/concordium-std/src/impls.rs @@ -3138,23 +3138,23 @@ impl DeserialWithState for StateBTreeMa Ok(StateBTreeMap { _marker_key: Default::default(), _marker_value: Default::default(), + prefix, + state_api: state.clone(), root, len, - prefix, next_node_id, - state_api: state.clone(), }) } } -impl Serial for state_btree_internals::StateBTreeNode { +impl Serial for state_btree_internals::Node { fn serial(&self, out: &mut W) -> Result<(), W::Err> { self.keys.serial(out)?; self.children.serial(out) } } -impl Deserial for state_btree_internals::StateBTreeNode { +impl Deserial for state_btree_internals::Node { fn deserial(source: &mut R) -> ParseResult { let keys = source.get()?; let children = source.get()?; @@ -3178,25 +3178,95 @@ impl Deserial for state_btree_internals::NodeId { } impl StateBTreeMap { + /// Construct a new [`StateBTreeMap`] given a unique prefix to use in the + /// key-value store. pub(crate) fn new(state_api: S, prefix: StateItemPrefix) -> Self { Self { _marker_key: Default::default(), _marker_value: Default::default(), + prefix, + state_api, root: None, len: 0, next_node_id: state_btree_internals::NodeId { id: 0, }, - state_api, - prefix, } } -} -impl - StateBTreeMap -{ - pub fn higher(&self, key: &K) -> Option { + /// Insert a key-value pair into the map. + pub fn insert(&mut self, key: K, value: V) -> Option + where + S: HasStateApi, + K: Serialize + Ord, + V: Serial + DeserialWithState, { + let Some(root_id) = self.root else { + let node_id = { + self.create_value(&key, value); + let (node_id, _node) = self.create_node(vec![key], Vec::new()); + node_id + }; + self.root = Some(node_id); + self.len = 1; + return None; + }; + + { + let root_node = self.get_node_mut(root_id); + if !root_node.is_full() { + let out = self.insert_non_full(root_node, key, value); + if out.is_none() { + self.len += 1; + } + return out; + } + } + + let (new_root_id, mut new_root) = self.create_node(Vec::new(), vec![root_id]); + self.split_child(&mut new_root, 0); + let out = self.insert_non_full(new_root, key, value); + self.root = Some(new_root_id); + if out.is_none() { + self.len += 1; + } + out + } + + pub fn get(&self, key: &K) -> Option> + where + K: Serial, + S: HasStateApi, + V: DeserialWithState, { + if self.root.is_none() { + None + } else { + let mut entry = self.get_value_entry(key)?; + let value = V::deserial_with_state(&self.state_api, &mut entry).unwrap_abort(); + Some(StateRef::new(value)) + } + } + + pub fn get_mut(&mut self, key: &K) -> Option> + where + K: Serialize, + S: HasStateApi, + V: Serial + DeserialWithState, { + if self.root.is_none() { + None + } else { + let entry = self.get_value_entry(key)?; + Some(StateRefMut::new(entry, self.state_api.clone())) + } + } + + pub fn len(&self) -> u32 { self.len } + + pub fn is_empty(&self) -> bool { self.root.is_none() } + + pub fn higher(&self, key: &K) -> Option + where + S: HasStateApi, + K: Serialize + Ord, { let Some(root_node_id) = self.root else { return None; }; @@ -3211,13 +3281,13 @@ impl if node.is_leaf() { return if higher_key_index < node.keys.len() { - Some(node.keys[higher_key_index].clone()) + Some(node.keys.swap_remove(higher_key_index)) } else { higher_so_far }; } else { if higher_key_index < node.keys.len() { - higher_so_far = Some(node.keys[higher_key_index].clone()) + higher_so_far = Some(node.keys.swap_remove(higher_key_index)) } let child_node_id = node.children[higher_key_index]; @@ -3226,7 +3296,11 @@ impl } } - pub fn lower(&self, key: &K) -> Option { + /// + pub fn lower(&self, key: &K) -> Option + where + S: HasStateApi, + K: Serialize + Ord, { let Some(root_node_id) = self.root else { return None; }; @@ -3245,188 +3319,125 @@ impl } else { // lower_key_index cannot be 0 in this case, since the binary search will only // return 0 in the true branch above. - Some(node.keys[lower_key_index - 1].clone()) + Some(node.keys.swap_remove(lower_key_index - 1)) }; } else { if lower_key_index > 0 { - lower_so_far = Some(node.keys[lower_key_index - 1].clone()); + lower_so_far = Some(node.keys.swap_remove(lower_key_index - 1)); } let child_node_id = node.children[lower_key_index]; node = self.get_node(child_node_id) } } } -} - -impl StateBTreeMap { - fn get_node<'a, 'b>( - &'a self, - node_id: state_btree_internals::NodeId, - ) -> StateRef<'b, state_btree_internals::StateBTreeNode> { - let key = self.node_key(node_id); - let mut entry = self.state_api.lookup_entry(&key).unwrap_abort(); - StateRef::new(entry.get().unwrap_abort()) - } - - fn node_key(&self, node_id: state_btree_internals::NodeId) -> Vec { - let mut key = self.prefix.to_vec(); - key.push(0u8); // Node key discriminant - node_id.serial(&mut key).unwrap_abort(); - key - } - - fn value_key(&self, key: &K) -> Vec { - let mut prefixed = self.prefix.to_vec(); - prefixed.push(1u8); // Node key discriminant - key.serial(&mut prefixed).unwrap_abort(); - prefixed - } -} - -impl StateBTreeMap { - pub fn insert(&mut self, key: K, value: V) -> Option { - let Some(root_id) = self.root else { - let node_id = { - self.create_value(&key, value); - let (node_id, _node) = self.create_node(vec![key], Vec::new()); - node_id - }; - self.root = Some(node_id); - self.len = 1; - return None; - }; - - { - let root_node = self.get_node_mut(root_id); - if !root_node.is_full() { - let out = self.insert_non_full(root_node, key, value); - if out.is_none() { - self.len += 1; - } - return out; - } - } - - let (new_root_id, mut new_root) = self.create_node(Vec::new(), vec![root_id]); - self.split_child(&mut new_root, 0); - let out = self.insert_non_full(new_root, key, value); - self.root = Some(new_root_id); - if out.is_none() { - self.len += 1; - } - out - } - - pub fn get<'a>(&'a self, key: &K) -> Option> { - if self.root.is_none() { - None - } else { - self.get_value(key).map(StateRef::new) - } - } - pub fn len(&self) -> u32 { self.len } - - pub fn is_empty(&self) -> bool { self.root.is_none() } - - fn create_value(&mut self, key: &K, value: V) { + /// Internal function to write a value to a key in the map. + fn create_value(&mut self, key: &K, value: V) + where + S: HasStateApi, + K: Serial, + V: Serial, { let prefixed_key = self.value_key(key); let mut entry = self.state_api.create_entry(&prefixed_key).unwrap_abort(); value.serial(&mut entry).unwrap_abort(); } - fn get_value(&self, key: &K) -> Option { + /// Internal function to lookup an entry holding a value. + fn get_value_entry(&self, key: &K) -> Option + where + K: Serial, + S: HasStateApi, { let prefixed_key = self.value_key(key); - self.state_api - .lookup_entry(&prefixed_key) - .map(|mut entry| V::deserial_with_state(&self.state_api, &mut entry).unwrap_abort()) + self.state_api.lookup_entry(&prefixed_key) } - fn set_value(&mut self, key: &K, value: V) -> Option { - let prefixed_key = self.value_key(key); - match self.state_api.entry(prefixed_key) { - EntryRaw::Vacant(_) => None, - EntryRaw::Occupied(mut raw_entry) => { - let entry = raw_entry.get_mut(); - let old_value = entry.get().unwrap_abort(); - value.serial(entry).unwrap_abort(); - Some(old_value) - } - } + /// Internal function to write value to a key. The key must already be + /// present in the map. Returns the old value. + fn overwrite_value(&mut self, key: &K, value: V) -> V + where + S: HasStateApi, + K: Serial, + V: Serial + DeserialWithState, { + let mut entry = self.get_value_entry(key).unwrap_abort(); + let old_value = + DeserialWithState::deserial_with_state(&self.state_api, &mut entry).unwrap_abort(); + value.serial(&mut entry).unwrap_abort(); + old_value } + /// Internal function for constructing a node. It will incrementing the next + /// node ID and create an entry in the smart contract key-value store. fn create_node<'a, 'b>( &'a mut self, keys: Vec, children: Vec, - ) -> ( - state_btree_internals::NodeId, - StateRefMut<'b, state_btree_internals::StateBTreeNode, S>, - ) { - let node_id = self.next_node_id; - self.next_node_id = state_btree_internals::NodeId { - id: node_id.id + 1, - }; - let node = state_btree_internals::StateBTreeNode { + ) -> (state_btree_internals::NodeId, StateRefMut<'b, state_btree_internals::Node, S>) + where + K: Serialize, + S: HasStateApi, { + let node_id = self.next_node_id.copy_then_increment(); + let node = state_btree_internals::Node { keys, children, }; let entry = self.state_api.create_entry(&self.node_key(node_id)).unwrap_abort(); - let mut ref_mut: StateRefMut<'_, state_btree_internals::StateBTreeNode, S> = + let mut ref_mut: StateRefMut<'_, state_btree_internals::Node, S> = StateRefMut::new(entry, self.state_api.clone()); ref_mut.set(node); (node_id, ref_mut) } - fn get_node_mut<'a, 'b>( - &'a mut self, - node_id: state_btree_internals::NodeId, - ) -> StateRefMut<'b, state_btree_internals::StateBTreeNode, S> { - let key = self.node_key(node_id); - let entry = self.state_api.entry(key); - match entry { - EntryRaw::Vacant(_) => crate::trap(), - EntryRaw::Occupied(entry) => StateRefMut::new(entry.get(), self.state_api.clone()), - } - } - + /// Internal function for inserting into a subtree. The given node must not + /// be full. fn insert_non_full( &mut self, - initial_node: StateRefMut, S>, + initial_node: StateRefMut, S>, key: K, value: V, - ) -> Option { + ) -> Option + where + K: Serialize + Ord, + S: HasStateApi, + V: Serial + DeserialWithState, { let mut node = initial_node; loop { - let mut i = match node.keys.binary_search(&key) { - Ok(_) => { - let value = self.set_value(&key, value); - return value; - } - Err(index) => index, + let Err(mut insert_index) = node.keys.binary_search(&key) else { + // The key is already in this node, so we just have to overwrite the value. + let value = self.overwrite_value(&key, value); + return Some(value); }; + // The key is not this node. if node.is_leaf() { + // Since the node is not full and this is a leaf, we can just insert here. self.create_value(&key, value); - node.keys.insert(i, key); + node.keys.insert(insert_index, key); return None; - } else { - let child = self.get_node(node.children[i]); - if child.is_full() { - self.split_child(&mut node, i); - if node.keys[i] < key { - i += 1; - } + } + + // The node is not a leaf, so we want to insert in the relevant child node. + let child = self.get_node(node.children[insert_index]); + if child.is_full() { + self.split_child(&mut node, insert_index); + // Since the child is now split into two, we have to update the insert_index to + // the relevant one of them. + if node.keys[insert_index] < key { + insert_index += 1; } - node = self.get_node_mut(node.children[i]); } + node = self.get_node_mut(node.children[insert_index]); } } + /// Internal function for splitting the child node at a given index for a + /// given node. This will also mutate the given node adding a new key + /// and child after the provided child_index. fn split_child<'a, 'b>( &'a mut self, - node: &'b mut StateRefMut, S>, + node: &'b mut StateRefMut, S>, child_index: usize, - ) { + ) where + K: Serialize + Ord, + S: HasStateApi, { let mut left = self.get_node_mut(node.children[child_index]); let split_index = (M + 1) / 2; let (right_id, _right) = self.create_node( @@ -3441,14 +3452,89 @@ impl StateBTre node.children.insert(child_index + 1, right_id); node.keys.insert(child_index, key); } + + /// Internal function for looking up a node in the tree. + /// This assumes the node is present and traps if this is not the case. + fn get_node<'a, 'b>( + &'a self, + node_id: state_btree_internals::NodeId, + ) -> state_btree_internals::Node + where + K: Deserial, + S: HasStateApi, { + let key = self.node_key(node_id); + let mut entry = self.state_api.lookup_entry(&key).unwrap_abort(); + entry.get().unwrap_abort() + } + + /// Internal function for looking up a node, providing mutable access. + /// This assumes the node is present and traps if this is not the case. + fn get_node_mut<'a, 'b>( + &'a mut self, + node_id: state_btree_internals::NodeId, + ) -> StateRefMut<'b, state_btree_internals::Node, S> + where + K: Serial, + S: HasStateApi, { + let key = self.node_key(node_id); + let entry = self.state_api.lookup_entry(&key).unwrap_abort(); + StateRefMut::new(entry, self.state_api.clone()) + } + + /// Construct the key for the node in the key-value store from the node ID. + fn node_key(&self, node_id: state_btree_internals::NodeId) -> [u8; BTREE_NODE_KEY_SIZE] { + // Create an uninitialized array of `MaybeUninit`. The `assume_init` is + // safe because the type we are claiming to have initialized here is a + // bunch of `MaybeUninit`s, which do not require initialization. + let mut prefixed: [MaybeUninit; BTREE_NODE_KEY_SIZE] = + unsafe { MaybeUninit::uninit().assume_init() }; + + for i in 0..STATE_ITEM_PREFIX_SIZE { + prefixed[i].write(self.prefix[i]); + } + prefixed[STATE_ITEM_PREFIX_SIZE].write(0u8); // Node key discriminant + let id_bytes = node_id.id.to_le_bytes(); + for i in 0..id_bytes.len() { + prefixed[STATE_ITEM_PREFIX_SIZE + 1 + i].write(id_bytes[i]); + } + // Transmuting away the maybeuninit is safe since we have initialized all of + // them. + unsafe { mem::transmute(prefixed) } + } + + /// Construct the key for the node in the key-value store from the node ID. + fn value_key(&self, key: &K) -> Vec + where + K: Serial, { + let mut prefixed = self.prefix.to_vec(); + prefixed.push(1u8); // Node key discriminant + key.serial(&mut prefixed).unwrap_abort(); + prefixed + } } -impl state_btree_internals::StateBTreeNode { +/// Byte size of the key used to store a BTree internal node in the smart +/// contract key-value store. +// 1 from byte discriminant and 4 for node ID. +const BTREE_NODE_KEY_SIZE: usize = STATE_ITEM_PREFIX_SIZE + 1 + 4; + +impl state_btree_internals::Node { + /// Check if the node holds the maximum number of keys. fn is_full(&self) -> bool { self.keys.len() == M } + /// Check if the node is representing a leaf in the tree. fn is_leaf(&self) -> bool { self.children.is_empty() } } +impl state_btree_internals::NodeId { + /// Return a copy of the NodeId, then increments itself. + pub(crate) fn copy_then_increment(&mut self) -> Self { + let current = self.clone(); + self.id += 1; + current + } +} + impl Deletable for StateBTreeMap where S: HasStateApi, diff --git a/concordium-std/src/types.rs b/concordium-std/src/types.rs index 7f6e589d..b4bb0329 100644 --- a/concordium-std/src/types.rs +++ b/concordium-std/src/types.rs @@ -414,7 +414,8 @@ pub struct ExternStateIter { pub(crate) type StateEntryId = u64; pub(crate) type StateIteratorId = u64; -pub(crate) type StateItemPrefix = [u8; 8]; +pub(crate) const STATE_ITEM_PREFIX_SIZE: usize = 8; +pub(crate) type StateItemPrefix = [u8; STATE_ITEM_PREFIX_SIZE]; /// Type of keys that index into the contract state. pub type Key = Vec; @@ -1259,28 +1260,60 @@ pub struct MetadataUrl { pub hash: Option<[u8; 32]>, } -#[derive(Debug)] +/// An ordered map based on [B-Tree](https://en.wikipedia.org/wiki/B-tree), where +/// each node is stored separately in the low-level key-value store. +/// +/// It can be seen as an extension adding the tracking of ordering on top of +/// [`StateMap`] providing functions such as [`Self::higher`] and +/// [`Self::lower`]. +/// This adds overhead when mutating the map. +/// +/// TODO Document how to construct it. +/// TODO Document size of the serialized key matters. +/// TODO Document the meaning of generics and restrictions on M. +/// TODO Document the complexity of the basic operations. pub struct StateBTreeMap { + /// Type marker for the key. pub(crate) _marker_key: PhantomData, + /// Type marker for the value. pub(crate) _marker_value: PhantomData, + /// The unique prefix to use for this map in the key-value store. + pub(crate) prefix: StateItemPrefix, + /// The API for interacting with the low-level state. + pub(crate) state_api: S, + /// The ID of the root node of the tree, where None represents the tree is + /// empty. pub(crate) root: Option, + /// Tracking the number of items in the tree. pub(crate) len: u32, - pub(crate) prefix: StateItemPrefix, + /// Tracking the next available ID for a new node. pub(crate) next_node_id: state_btree_internals::NodeId, - pub(crate) state_api: S, } +/// Module with types used internally in [`StateBTreeMap`]. pub(crate) mod state_btree_internals { + /// Identifier for a node in the tree. Used to construct the key, where this + /// node is store in the smart contract key-value store. #[derive(Debug, Copy, Clone)] #[repr(transparent)] pub(crate) struct NodeId { pub(crate) id: u32, } + /// Type representing the a node in the [`StateBTreeMap`]. + /// Each node is stored separately in the smart contract key-value store. #[derive(Debug)] - pub(crate) struct StateBTreeNode { - pub(crate) keys: Vec, // never empty, sorted. - /// Either empty or keys.len() + 1. + pub(crate) struct Node { + /// List of sorted keys tracked by this node. + /// This list should never be empty and contain at most `M` elements. + pub(crate) keys: Vec, + /// List of nodes which are children of this node in the tree. + /// + /// This list is empty when this node is representing a leaf. + /// When not empty it will contain exactly `keys.len() + 1` elements. + /// + /// The elements are ordered such that the node `children[i]` contains + /// keys that are strictly smaller than `keys[i]`. pub(crate) children: Vec, } } From 54c57149d32210e13773297d66277b3d6a93e060 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Holm=20Gj=C3=B8rup?= Date: Tue, 27 Feb 2024 17:03:37 +0100 Subject: [PATCH 05/36] Implement remove and fix a few insert bugs --- concordium-std/src/impls.rs | 599 ++++++++++++++++++---- concordium-std/src/test_infrastructure.rs | 225 +++++--- concordium-std/src/types.rs | 29 +- 3 files changed, 680 insertions(+), 173 deletions(-) diff --git a/concordium-std/src/impls.rs b/concordium-std/src/impls.rs index 33e8fc82..4803bd56 100644 --- a/concordium-std/src/impls.rs +++ b/concordium-std/src/impls.rs @@ -919,8 +919,10 @@ where /// Inserts the value with the given key. If a value already exists at the /// given key it is replaced and the old value is returned. - pub fn insert(&mut self, key: K, value: V) -> Option { - let key_bytes = self.key_with_map_prefix(&key); + /// This only borrows the key, needed internally to avoid the need to clone + /// it first. + pub(crate) fn insert_borrowed(&mut self, key: &K, value: V) -> Option { + let key_bytes = self.key_with_map_prefix(key); // Unwrapping is safe because iter() holds a reference to the stateset. match self.state_api.entry(key_bytes) { EntryRaw::Vacant(vac) => { @@ -937,6 +939,10 @@ where } } + /// Inserts the value with the given key. If a value already exists at the + /// given key it is replaced and the old value is returned. + pub fn insert(&mut self, key: K, value: V) -> Option { self.insert_borrowed(&key, value) } + /// Get an entry for the given key. pub fn entry(&mut self, key: K) -> Entry<'_, K, V, S> { let key_bytes = self.key_with_map_prefix(&key); @@ -1155,24 +1161,20 @@ impl<'a, S: HasStateApi, V: Serial + DeserialWithState> crate::ops::DerefMut /// When dropped, the value, `V`, is written to the entry in the contract state. impl<'a, V: Serial, S: HasStateApi> Drop for StateRefMut<'a, V, S> { - fn drop(&mut self) { - if let Some(value) = self.lazy_value.get_mut() { - let entry = self.entry.get_mut(); - entry.move_to_start(); - value.serial(entry).unwrap_abort() - } - } + fn drop(&mut self) { self.store_mutations() } } impl<'a, V, S> StateRefMut<'a, V, S> where - V: Serial + DeserialWithState, + V: Serial, S: HasStateApi, { /// Get a shared reference to the value. Note that [StateRefMut](Self) also /// implements [Deref](crate::ops::Deref) so this conversion can happen /// implicitly. - pub fn get(&self) -> &V { + pub fn get(&self) -> &V + where + V: DeserialWithState, { let lv = unsafe { &mut *self.lazy_value.get() }; if let Some(v) = lv { v @@ -1184,7 +1186,9 @@ where /// Get a unique reference to the value. Note that [StateRefMut](Self) also /// implements [DerefMut](crate::ops::DerefMut) so this conversion can /// happen implicitly. - pub fn get_mut(&mut self) -> &mut V { + pub fn get_mut(&mut self) -> &mut V + where + V: DeserialWithState, { let lv = unsafe { &mut *self.lazy_value.get() }; if let Some(v) = lv { v @@ -1194,7 +1198,9 @@ where } /// Load the value referenced by the entry from the chain data. - fn load_value(&self) -> V { + fn load_value(&self) -> V + where + V: DeserialWithState, { let entry = unsafe { &mut *self.entry.get() }; entry.move_to_start(); V::deserial_with_state(&self.state_api, entry).unwrap_abort() @@ -1211,6 +1217,7 @@ where /// Update the existing value with the given function. pub fn update(&mut self, f: F) where + V: DeserialWithState, F: FnOnce(&mut V), { let lv = self.lazy_value.get_mut(); let entry = self.entry.get_mut(); @@ -1227,6 +1234,21 @@ where entry.move_to_start(); value.serial(entry).unwrap_abort() } + + /// Write to the state entry if the value is loaded. + pub(crate) fn store_mutations(&mut self) { + if let Some(value) = self.lazy_value.get_mut() { + let entry = self.entry.get_mut(); + entry.move_to_start(); + value.serial(entry).unwrap_abort(); + } + } + + /// Get the inner value if loaded, while consuming the [`StateRefMut`]. + /// Mutations will not be written to the smart contract key-store when this + /// is dropped. Neither will they be written as part of this, so make sure + /// to run `store_mutations` first. + pub(crate) fn into_inner_value(mut self) -> Option { self.lazy_value.get_mut().take() } } impl Serial for StateMap { @@ -2298,9 +2320,16 @@ where StateMap::open(state_api, prefix) } - pub fn new_btree_map(&mut self) -> StateBTreeMap { + pub fn new_btree_set(&mut self) -> StateBTreeSet { let (state_api, prefix) = self.new_state_container(); - StateBTreeMap::new(state_api, prefix) + StateBTreeSet::new(state_api, prefix) + } + + pub fn new_btree_map(&mut self) -> StateBTreeMap { + StateBTreeMap { + map: self.new_map(), + ordered_set: self.new_btree_set(), + } } /// Create a new empty [`StateSet`]. @@ -3119,7 +3148,53 @@ impl Deserial for MetadataUrl { } } +// impl Serial for StateBTreeMap { +// fn serial(&self, out: &mut W) -> Result<(), W::Err> { +// self.prefix.serial(out)?; +// self.root.serial(out)?; +// self.len.serial(out)?; +// self.next_node_id.serial(out) +// } +// } + +// impl DeserialWithState for +// StateBTreeMap { fn deserial_with_state(state: &S, +// source: &mut R) -> ParseResult { let prefix = source.get()?; +// let root = source.get()?; +// let len = source.get()?; +// let next_node_id = source.get()?; + +// Ok(Self { +// _marker_key: Default::default(), +// _marker_value: Default::default(), +// prefix, +// state_api: state.clone(), +// root, +// len, +// next_node_id, +// }) +// } +// } + impl Serial for StateBTreeMap { + fn serial(&self, out: &mut W) -> Result<(), W::Err> { + self.map.serial(out)?; + self.ordered_set.serial(out) + } +} + +impl DeserialWithState for StateBTreeMap { + fn deserial_with_state(state: &S, source: &mut R) -> ParseResult { + let map = DeserialWithState::deserial_with_state(state, source)?; + let ordered_set = DeserialWithState::deserial_with_state(state, source)?; + Ok(Self { + map, + ordered_set, + }) + } +} + +impl Serial for StateBTreeSet { fn serial(&self, out: &mut W) -> Result<(), W::Err> { self.prefix.serial(out)?; self.root.serial(out)?; @@ -3128,16 +3203,15 @@ impl Serial for StateBTreeMap { } } -impl DeserialWithState for StateBTreeMap { +impl DeserialWithState for StateBTreeSet { fn deserial_with_state(state: &S, source: &mut R) -> ParseResult { let prefix = source.get()?; let root = source.get()?; let len = source.get()?; let next_node_id = source.get()?; - Ok(StateBTreeMap { + Ok(Self { _marker_key: Default::default(), - _marker_value: Default::default(), prefix, state_api: state.clone(), root, @@ -3178,12 +3252,92 @@ impl Deserial for state_btree_internals::NodeId { } impl StateBTreeMap { + /// Insert a key-value pair into the map. + pub fn insert(&mut self, key: K, value: V) -> Option + where + S: HasStateApi, + K: Serialize + Ord + fmt::Debug, + V: Serial + DeserialWithState, { + let old_value_option = self.map.insert_borrowed(&key, value); + if old_value_option.is_none() { + if !self.ordered_set.insert(key) { + // Inconsistency between the map and ordered_set. + crate::trap(); + } + } + return old_value_option; + } + + pub fn remove_and_get(&mut self, key: &K) -> Option + where + S: HasStateApi, + K: Serialize + Ord + fmt::Debug, + V: Serial + DeserialWithState + Deletable, { + let v = self.map.remove_and_get(key); + if v.is_some() && !self.ordered_set.remove(key) { + // Inconsistency between the map and ordered_set. + crate::trap(); + } + v + } + + /// Get a reference to the value corresponding to the key. + pub fn get(&self, key: &K) -> Option> + where + K: Serialize, + S: HasStateApi, + V: Serial + DeserialWithState, { + if self.ordered_set.is_empty() { + None + } else { + self.map.get(key) + } + } + + /// Get a mutable reference to the value corresponding to the key. + pub fn get_mut(&mut self, key: &K) -> Option> + where + K: Serialize, + S: HasStateApi, + V: Serial + DeserialWithState, { + if self.ordered_set.is_empty() { + None + } else { + self.map.get_mut(key) + } + } + + /// Returns the smallest key in the map, which is strictly larger than the + /// provided key. `None` meaning no such key is present in the map. + pub fn higher(&self, key: &K) -> Option + where + S: HasStateApi, + K: Serialize + Ord, { + self.ordered_set.higher(key) + } + + /// Returns the largest key in the map, which is strictly smaller than the + /// provided key. `None` meaning no such key is present in the map. + pub fn lower(&self, key: &K) -> Option + where + S: HasStateApi, + K: Serialize + Ord, { + self.ordered_set.lower(key) + } + + /// Return the number of elements in the map. + pub fn len(&self) -> u32 { self.ordered_set.len() } + + /// Returns `true` is the map contains no elements. + pub fn is_empty(&self) -> bool { self.ordered_set.is_empty() } +} + +impl StateBTreeSet { /// Construct a new [`StateBTreeMap`] given a unique prefix to use in the /// key-value store. pub(crate) fn new(state_api: S, prefix: StateItemPrefix) -> Self { Self { _marker_key: Default::default(), - _marker_value: Default::default(), prefix, state_api, root: None, @@ -3194,75 +3348,80 @@ impl StateBTreeMap { } } - /// Insert a key-value pair into the map. - pub fn insert(&mut self, key: K, value: V) -> Option + /// Insert a key into the set. + /// Returns true if the key is new in the collection. + pub fn insert(&mut self, key: K) -> bool where S: HasStateApi, - K: Serialize + Ord, - V: Serial + DeserialWithState, { + K: Serialize + Ord + fmt::Debug, { let Some(root_id) = self.root else { let node_id = { - self.create_value(&key, value); let (node_id, _node) = self.create_node(vec![key], Vec::new()); node_id }; self.root = Some(node_id); self.len = 1; - return None; + return false; }; { let root_node = self.get_node_mut(root_id); if !root_node.is_full() { - let out = self.insert_non_full(root_node, key, value); - if out.is_none() { + let new = self.insert_non_full(root_node, key); + if new { self.len += 1; } - return out; + return new; + } else if root_node.keys.binary_search(&key).is_ok() { + // TODO: The error result of the binary search can probably be reused below. + return false; } } let (new_root_id, mut new_root) = self.create_node(Vec::new(), vec![root_id]); + self.split_child(&mut new_root, 0); - let out = self.insert_non_full(new_root, key, value); + // new_root should now contain one key and two children, so we need to know + // which one to insert into. + let child_index = usize::from(new_root.keys[0] < key); + let child = self.get_node_mut(new_root.children[child_index]); + let new = self.insert_non_full(child, key); self.root = Some(new_root_id); - if out.is_none() { + if new { self.len += 1; } - out + new } - pub fn get(&self, key: &K) -> Option> + /// Returns `true` if the set contains an element equal to the value. + pub fn contains(&self, key: &K) -> bool where - K: Serial, S: HasStateApi, - V: DeserialWithState, { - if self.root.is_none() { - None - } else { - let mut entry = self.get_value_entry(key)?; - let value = V::deserial_with_state(&self.state_api, &mut entry).unwrap_abort(); - Some(StateRef::new(value)) - } - } - - pub fn get_mut(&mut self, key: &K) -> Option> - where - K: Serialize, - S: HasStateApi, - V: Serial + DeserialWithState, { - if self.root.is_none() { - None - } else { - let entry = self.get_value_entry(key)?; - Some(StateRefMut::new(entry, self.state_api.clone())) + K: Serialize + Ord, { + let Some(root_node_id) = self.root else { + return false; + }; + let mut node = self.get_node(root_node_id); + loop { + let Err(child_index) = node.keys.binary_search(key) else { + return true; + }; + if node.is_leaf() { + return false; + } + let child_node_id = node.children[child_index]; + node = self.get_node(child_node_id); } } + /// Return the number of elements in the map. pub fn len(&self) -> u32 { self.len } + /// Returns `true` is the map contains no elements. pub fn is_empty(&self) -> bool { self.root.is_none() } + /// Returns the smallest key in the map, which is strictly larger than the + /// provided key. `None` meaning no such key is present in the map. pub fn higher(&self, key: &K) -> Option where S: HasStateApi, @@ -3296,7 +3455,8 @@ impl StateBTreeMap { } } - /// + /// Returns the largest key in the map, which is strictly smaller than the + /// provided key. `None` meaning no such key is present in the map. pub fn lower(&self, key: &K) -> Option where S: HasStateApi, @@ -3331,38 +3491,219 @@ impl StateBTreeMap { } } - /// Internal function to write a value to a key in the map. - fn create_value(&mut self, key: &K, value: V) + pub fn remove(&mut self, key: &K) -> bool where - S: HasStateApi, - K: Serial, - V: Serial, { - let prefixed_key = self.value_key(key); - let mut entry = self.state_api.create_entry(&prefixed_key).unwrap_abort(); - value.serial(&mut entry).unwrap_abort(); + K: Ord + Serialize + fmt::Debug, + S: HasStateApi, { + let Some(root_node_id) = self.root else { + return false; + }; + + let mut overwrite_key_to_delete = None; + let mut node = self.get_node_mut(root_node_id); + let deleted_something = loop { + let key_to_delete = overwrite_key_to_delete.as_ref().unwrap_or(key); + let index = match node.keys.binary_search(key_to_delete) { + Ok(index) => { + if node.is_leaf() { + // Found the key in this node and the node is a leaf, meaning we simply + // remove it. + // This will not violate the minimum keys invariant, since a node ensures a + // child can spare a key before iteration and the root node is not part of + // the invariant. + node.keys.remove(index); + break true; + } + // Found the key in this node, but the node is not a leaf. + let left_child = self.get_node_mut(node.children[index]); + if !left_child.is_at_min() { + // If the child with smaller keys can spare a key, we take the highest + // key from it, changing the loop to be deleting its highest key from + // this child. + node.keys[index] = self.get_highest_key(&mut node, index); + let new_key_to_delete = { + // To avoid having to clone the key, we store changes to the node + // and unwrap it from the StateRefMut, disabling further mutations + // from being written to the key-value store, such that we can use + // the same in-memory key. + node.store_mutations(); + let mut inner_node = node.into_inner_value().unwrap_abort(); + inner_node.keys.swap_remove(index) + }; + overwrite_key_to_delete = Some(new_key_to_delete); + node = left_child; + continue; + } + + let right_child = self.get_node_mut(node.children[index + 1]); + if !right_child.is_at_min() { + // If the child with larger keys can spare a key, we take the lowest + // key from it, changing the loop to be deleting its highest key from + // this child. + node.keys[index] = self.get_lowest_key(&mut node, index + 1); + let new_key_to_delete = { + // To avoid having to clone the key, we store changes to the node + // and unwrap it from the StateRefMut, disabling further mutations + // from being written to the key-value store, such that we can use + // the same in-memory key. + node.store_mutations(); + let mut inner_node = node.into_inner_value().unwrap_abort(); + inner_node.keys.swap_remove(index) + }; + overwrite_key_to_delete = Some(new_key_to_delete); + node = right_child; + continue; + } + // No child on either side of the key can spare a key, so we merge them into + // one child, moving the key into the child and try to remove from this. + self.merge(&mut node, index); + node = self.get_node_mut(node.children[index]); + // FIXME: For some reason this is needed to cause loading the node for the next + // iteration. + let _ = node.get(); + continue; + } + Err(index) => index, + }; + // Node did not contain the key. + if node.is_leaf() { + break false; + } + + // Node did not contain the key and is not a leaf. + let mut child = self.get_node_mut(node.children[index]); + let has_smaller_sibling = 0 < index; + let has_larger_sibling = index < node.children.len() - 1; + // Check and proactively prepare if the child which could contain the key to + // spare loosing a key. + if child.is_at_min() { + // The child is at minimum keys, so first attempt to take a key from either + // sibling, otherwise merge with one of them. + 'increase_child: { + if has_smaller_sibling { + let mut smaller_sibling = self.get_node_mut(node.children[index - 1]); + if !smaller_sibling.is_at_min() { + // The smaller sibling can spare a key, so we replace the largest key + // from the sibling, put it in the parent + // and take a key from the parent. + let largest_key_sibling = smaller_sibling.keys.pop().unwrap_abort(); + let swapped_node_key = + mem::replace(&mut node.keys[index - 1], largest_key_sibling); + child.keys.insert(0, swapped_node_key); + if !child.is_leaf() { + child + .children + .insert(0, smaller_sibling.children.pop().unwrap_abort()); + } + break 'increase_child; + } + } + if has_larger_sibling { + let mut larger_sibling = self.get_node_mut(node.children[index + 1]); + if !larger_sibling.is_at_min() { + // The larger sibling can spare a key, so we replace the smallest key + // from the sibling, put it in the parent + // and take a key from the parent. + let first_key_sibling = larger_sibling.keys.remove(0); + let swapped_node_key = + mem::replace(&mut node.keys[index], first_key_sibling); + child.keys.push(swapped_node_key); + + if !child.is_leaf() { + child.children.push(larger_sibling.children.remove(0)); + } + break 'increase_child; + } + } + + if has_larger_sibling { + self.merge(&mut node, index); + } else { + self.merge(&mut node, index - 1); + } + } + } + // Merging two children moves a key from the node down as well, so if this + // happened above and index was pointing to the last child before, update + // it. + let node_lost_key = index > node.len(); + let next_index = index - usize::from(!has_larger_sibling && node_lost_key); + drop(child); // FIXME: Dropped in order to store changes before reading the child again from + // state. This should reuse the already loaded stateref instead. + node = self.get_node_mut(node.children[next_index]); + // FIXME: For some reason this is needed to cause loading the node for the next + // iteration. + let _ = node.get(); + }; + + // If something was deleted, we update the length and make sure to remove the + // root node if needed. + if deleted_something { + self.len -= 1; + if self.len == 0 { + // Remote the root node if tree is empty. + self.delete_node(root_node_id); + self.root = None; + } else { + // If the root is empty but the tree is not, point the only child of the root as + // the root. + let root = self.get_node(root_node_id); + if root.keys.is_empty() { + self.root = Some(root.children[0]); + self.delete_node(root_node_id); + } + } + } + deleted_something } - /// Internal function to lookup an entry holding a value. - fn get_value_entry(&self, key: &K) -> Option + /// Internal function for getting the highest key in a subtree. + fn get_highest_key(&self, node: &state_btree_internals::Node, child_index: usize) -> K where - K: Serial, + K: Ord + Serialize, S: HasStateApi, { - let prefixed_key = self.value_key(key); - self.state_api.lookup_entry(&prefixed_key) + let mut node = self.get_node(node.children[child_index]); + while !node.is_leaf() { + let child_node_id = node.children.last().unwrap_abort(); + node = self.get_node(*child_node_id); + } + // This does not mutate the node in the end, just the representation in memory + // which is freed after the call. + node.keys.pop().unwrap_abort() } - /// Internal function to write value to a key. The key must already be - /// present in the map. Returns the old value. - fn overwrite_value(&mut self, key: &K, value: V) -> V + /// Internal function for getting the lowest key in a subtree. + fn get_lowest_key(&self, node: &state_btree_internals::Node, child_index: usize) -> K where - S: HasStateApi, - K: Serial, - V: Serial + DeserialWithState, { - let mut entry = self.get_value_entry(key).unwrap_abort(); - let old_value = - DeserialWithState::deserial_with_state(&self.state_api, &mut entry).unwrap_abort(); - value.serial(&mut entry).unwrap_abort(); - old_value + K: Ord + Serialize, + S: HasStateApi, { + let mut node = self.get_node(node.children[child_index]); + while !node.is_leaf() { + let child_node_id = node.children.first().unwrap_abort(); + node = self.get_node(*child_node_id); + } + // This does not mutate the node in the end, just the representation in memory + // which is freed after the call. + node.keys.swap_remove(0) + } + + /// Moving key at `index` from the node to the lower child and then merges + /// this child with its higher sibling. + /// + /// Assumes: + /// - `node` have a child at `index` and `index + 1`. + /// - Both children are at minimum number of keys. + fn merge(&mut self, node: &mut state_btree_internals::Node, index: usize) + where + K: Ord + Serialize, + S: HasStateApi, { + let mut left = self.get_node_mut(node.children[index]); + let mut right = self.get_node(node.children[index + 1]); + left.keys.push(node.keys.remove(index)); + left.keys.append(&mut right.keys); + left.children.append(&mut right.children); + let removed_node_id = node.children.remove(index + 1); + self.delete_node(removed_node_id); } /// Internal function for constructing a node. It will incrementing the next @@ -3387,31 +3728,38 @@ impl StateBTreeMap { (node_id, ref_mut) } + /// Internal function for deleting a node, removing the entry in the smart + /// contract key-value store. Traps if no node was present. + fn delete_node(&mut self, node_id: state_btree_internals::NodeId) + where + S: HasStateApi, { + let key = self.node_key(node_id); + if !self.state_api.delete_prefix(&key).unwrap_abort() { + crate::trap(); + } + } + /// Internal function for inserting into a subtree. The given node must not /// be full. fn insert_non_full( &mut self, initial_node: StateRefMut, S>, key: K, - value: V, - ) -> Option + ) -> bool where - K: Serialize + Ord, - S: HasStateApi, - V: Serial + DeserialWithState, { + K: Serialize + Ord + fmt::Debug, + S: HasStateApi, { let mut node = initial_node; loop { let Err(mut insert_index) = node.keys.binary_search(&key) else { - // The key is already in this node, so we just have to overwrite the value. - let value = self.overwrite_value(&key, value); - return Some(value); + // We find the key in this node, so we do nothing. + return false; }; // The key is not this node. if node.is_leaf() { // Since the node is not full and this is a leaf, we can just insert here. - self.create_value(&key, value); node.keys.insert(insert_index, key); - return None; + return true; } // The node is not a leaf, so we want to insert in the relevant child node. @@ -3420,9 +3768,7 @@ impl StateBTreeMap { self.split_child(&mut node, insert_index); // Since the child is now split into two, we have to update the insert_index to // the relevant one of them. - if node.keys[insert_index] < key { - insert_index += 1; - } + insert_index += usize::from(node.keys[insert_index] < key); } node = self.get_node_mut(node.children[insert_index]); } @@ -3433,13 +3779,13 @@ impl StateBTreeMap { /// and child after the provided child_index. fn split_child<'a, 'b>( &'a mut self, - node: &'b mut StateRefMut, S>, + node: &'b mut state_btree_internals::Node, child_index: usize, ) where K: Serialize + Ord, S: HasStateApi, { let mut left = self.get_node_mut(node.children[child_index]); - let split_index = (M + 1) / 2; + let split_index = state_btree_internals::Node::::MINIMUM_KEY_LEN + 1; let (right_id, _right) = self.create_node( left.keys.split_off(split_index), if left.is_leaf() { @@ -3492,41 +3838,84 @@ impl StateBTreeMap { for i in 0..STATE_ITEM_PREFIX_SIZE { prefixed[i].write(self.prefix[i]); } - prefixed[STATE_ITEM_PREFIX_SIZE].write(0u8); // Node key discriminant let id_bytes = node_id.id.to_le_bytes(); for i in 0..id_bytes.len() { - prefixed[STATE_ITEM_PREFIX_SIZE + 1 + i].write(id_bytes[i]); + prefixed[STATE_ITEM_PREFIX_SIZE + i].write(id_bytes[i]); } // Transmuting away the maybeuninit is safe since we have initialized all of // them. unsafe { mem::transmute(prefixed) } } - /// Construct the key for the node in the key-value store from the node ID. - fn value_key(&self, key: &K) -> Vec + pub(crate) fn debug(&self) -> String where - K: Serial, { - let mut prefixed = self.prefix.to_vec(); - prefixed.push(1u8); // Node key discriminant - key.serial(&mut prefixed).unwrap_abort(); - prefixed + S: HasStateApi, + K: Deserial + fmt::Debug, { + let Some(root_id) = self.root else { + return "Empty".to_owned(); + }; + + let mut out = String::new(); + let mut stack = vec![root_id]; + while let Some(node_id) = stack.pop() { + let node = self.get_node(node_id); + out.push_str(format!("{} [\n", node_id.id).as_str()); + out.push_str(node.debug().as_str()); + out.push_str("]\n"); + + stack.extend(&node.children) + } + out } } /// Byte size of the key used to store a BTree internal node in the smart /// contract key-value store. -// 1 from byte discriminant and 4 for node ID. -const BTREE_NODE_KEY_SIZE: usize = STATE_ITEM_PREFIX_SIZE + 1 + 4; +const BTREE_NODE_KEY_SIZE: usize = + STATE_ITEM_PREFIX_SIZE + state_btree_internals::NodeId::SERIALIZED_BYTE_SIZE; impl state_btree_internals::Node { + /// The max length of the child list. + const MAXIMUM_CHILD_LEN: usize = 2 * M; + /// The max length of the key list. + const MAXIMUM_KEY_LEN: usize = Self::MAXIMUM_CHILD_LEN - 1; + /// The min length of the child list, when the node is not a leaf node. + const MINIMUM_CHILD_LEN: usize = M; + /// The min length of the key list, except when the node is root. + const MINIMUM_KEY_LEN: usize = Self::MINIMUM_CHILD_LEN - 1; + + /// The number of keys stored in this node. + fn len(&self) -> usize { self.keys.len() } + /// Check if the node holds the maximum number of keys. - fn is_full(&self) -> bool { self.keys.len() == M } + fn is_full(&self) -> bool { self.len() == Self::MAXIMUM_KEY_LEN } /// Check if the node is representing a leaf in the tree. fn is_leaf(&self) -> bool { self.children.is_empty() } + + /// Check if the node holds the minimum number of keys. + fn is_at_min(&self) -> bool { self.len() == Self::MINIMUM_KEY_LEN } + + fn debug(&self) -> String + where + K: fmt::Debug, { + let mut out = String::new(); + if self.is_leaf() { + out.push_str(format!("Leaf with {:?}\n", self.keys).as_str()); + } else { + for i in 0..self.len() { + out.push_str(format!("Child Id: {}\n", self.children[i].id).as_str()); + out.push_str(format!("Key: {:?}\n", self.keys[i]).as_str()); + } + out.push_str(format!("Child Id: {}\n", self.children[self.len()].id).as_str()); + } + out + } } impl state_btree_internals::NodeId { + const SERIALIZED_BYTE_SIZE: usize = 4; + /// Return a copy of the NodeId, then increments itself. pub(crate) fn copy_then_increment(&mut self) -> Self { let current = self.clone(); diff --git a/concordium-std/src/test_infrastructure.rs b/concordium-std/src/test_infrastructure.rs index 59320534..0bb61cfb 100644 --- a/concordium-std/src/test_infrastructure.rs +++ b/concordium-std/src/test_infrastructure.rs @@ -1929,14 +1929,13 @@ pub fn concordium_qc(num_tests: u64, f: A) { #[cfg(test)] mod test { - use core::ops::Deref; use super::TestStateApi; use crate::{ cell::RefCell, rc::Rc, test_infrastructure::{TestStateBuilder, TestStateEntry}, - Deletable, EntryRaw, HasStateApi, HasStateEntry, StateBTreeMap, StateMap, StateSet, + Deletable, EntryRaw, HasStateApi, HasStateEntry, StateBTreeSet, StateMap, StateSet, INITIAL_NEXT_ITEM_PREFIX, }; use concordium_contracts_common::{to_bytes, Deserial, Read, Seek, SeekFrom, Write}; @@ -2492,103 +2491,211 @@ mod test { #[test] fn test_btree_m5_insert_6() { let mut state_builder = TestStateBuilder::new(); - let mut map: StateBTreeMap<5, u32, _, _> = state_builder.new_btree_map(); - for (n, s) in (0..=5).into_iter().map(|n| (n, n.to_string())) { - map.insert(n, s); + let mut tree: StateBTreeSet<5, u32, _> = state_builder.new_btree_set(); + for n in 0..=5 { + tree.insert(n); + } + for n in 0..=5 { + assert!(tree.contains(&n)); } - let s = map.get(&5).expect("to find key"); - - assert_eq!(s.as_str(), "5") } #[test] fn test_btree_m3_insert_0_7() { let mut state_builder = TestStateBuilder::new(); - let mut map: StateBTreeMap<3, u32, String, _> = state_builder.new_btree_map(); - for (n, s) in (0..=7).into_iter().map(|n| (n, n.to_string())) { - map.insert(n, s); + let mut tree: StateBTreeSet<3, u32, _> = state_builder.new_btree_set(); + for n in 0..=7 { + tree.insert(n); + } + for n in 0..=7 { + assert!(tree.contains(&n)); } - let s = map.get(&7).expect("to find key"); - assert_eq!(s.as_str(), "7") } #[test] fn test_btree_m3_insert_7_no_order() { let mut state_builder = TestStateBuilder::new(); - let mut map: StateBTreeMap<3, u32, _, _> = state_builder.new_btree_map(); + let mut tree: StateBTreeSet<3, u32, _> = state_builder.new_btree_set(); - map.insert(0, "zero".to_string()); - map.insert(1, "one".to_string()); - map.insert(2, "two".to_string()); - map.insert(3, "three".to_string()); - map.insert(7, "seven".to_string()); - map.insert(6, "six".to_string()); - map.insert(5, "five".to_string()); - map.insert(4, "four".to_string()); - let s = map.get(&7).expect("to find key"); - assert_eq!(s.as_str(), "seven") + tree.insert(0); + tree.insert(1); + tree.insert(2); + tree.insert(3); + tree.insert(7); + tree.insert(6); + tree.insert(5); + tree.insert(4); + + for n in 0..=7 { + assert!(tree.contains(&n)); + } } #[test] fn test_btree_m3_higher() { let mut state_builder = TestStateBuilder::new(); - let mut map: StateBTreeMap<3, u32, _, _> = state_builder.new_btree_map(); - map.insert(1, "one".to_string()); - map.insert(2, "two".to_string()); - map.insert(3, "three".to_string()); - map.insert(4, "four".to_string()); - map.insert(5, "five".to_string()); - map.insert(7, "seven".to_string()); - assert_eq!(map.higher(&3), Some(4)); - assert_eq!(map.higher(&5), Some(7)); - assert_eq!(map.higher(&7), None) + let mut tree: StateBTreeSet<3, u32, _> = state_builder.new_btree_set(); + tree.insert(1); + tree.insert(2); + tree.insert(3); + tree.insert(4); + tree.insert(5); + tree.insert(7); + assert_eq!(tree.higher(&3), Some(4)); + assert_eq!(tree.higher(&5), Some(7)); + assert_eq!(tree.higher(&7), None) } #[test] fn test_btree_m3_lower() { let mut state_builder = TestStateBuilder::new(); - let mut map: StateBTreeMap<3, u32, _, _> = state_builder.new_btree_map(); - map.insert(1, "one".to_string()); - map.insert(2, "two".to_string()); - map.insert(3, "three".to_string()); - map.insert(4, "four".to_string()); - map.insert(5, "five".to_string()); - map.insert(7, "seven".to_string()); - assert_eq!(map.lower(&3), Some(2)); - assert_eq!(map.lower(&7), Some(5)); - assert_eq!(map.lower(&1), None) + let mut tree: StateBTreeSet<3, u32, _> = state_builder.new_btree_set(); + tree.insert(1); + tree.insert(2); + tree.insert(3); + tree.insert(4); + tree.insert(5); + tree.insert(7); + assert_eq!(tree.lower(&3), Some(2)); + assert_eq!(tree.lower(&7), Some(5)); + assert_eq!(tree.lower(&1), None) } #[test] - fn test_btree_m3_insert_10000() { + fn test_btree_m3_insert_1000() { let mut state_builder = TestStateBuilder::new(); - let mut map: StateBTreeMap<3, u32, _, _> = state_builder.new_btree_map(); - for (n, s) in (0..5000).into_iter().map(|n| (n, n.to_string())) { - map.insert(n, s); + let mut tree: StateBTreeSet<3, u32, _> = state_builder.new_btree_set(); + for n in 0..500 { + tree.insert(n); } - for (n, s) in (5000..10000).into_iter().rev().map(|n| (n, n.to_string())) { - map.insert(n, s); + for n in (500..1000).into_iter().rev() { + tree.insert(n); } - for n in 0..10000 { - let s = map.get(&n); - assert_eq!(s.as_deref(), Some(&n.to_string())); + for n in 0..1000 { + assert!(tree.contains(&n)) } - assert_eq!(map.len(), 10000) + assert_eq!(tree.len(), 1000) } #[test] fn test_btree_m3_7_get_8() { let mut state_builder = TestStateBuilder::new(); - let mut map: StateBTreeMap<3, u32, String, _> = state_builder.new_btree_map(); - for (n, s) in (0..=7).into_iter().map(|n| (n, n.to_string())) { - map.insert(n, s); + let mut tree: StateBTreeSet<3, u32, _> = state_builder.new_btree_set(); + for n in 0..=7 { + tree.insert(n); + } + + assert!(!tree.contains(&8)); + } + + #[test] + fn test_btree_m3_remove_from_one_node_tree() { + let mut state_builder = TestStateBuilder::new(); + let mut tree: StateBTreeSet<3, u32, _> = state_builder.new_btree_set(); + for n in 0..=1 { + tree.insert(n); + } + assert!(tree.contains(&1)); + assert!(tree.remove(&1)); + assert!(!tree.contains(&1)); + } + + #[test] + fn test_btree_remove_only_key_lower_leaf_in_three_node() { + let mut state_builder = TestStateBuilder::new(); + let mut tree: StateBTreeSet<2, u32, _> = state_builder.new_btree_set(); + for n in 0..4 { + tree.insert(n); + } + tree.remove(&3); + + assert!(tree.contains(&0)); + assert!(tree.remove(&0)); + assert!(!tree.contains(&0)); + assert!(tree.contains(&1)); + assert!(tree.contains(&2)); + } + + #[test] + fn test_btree_remove_only_key_higher_leaf_in_three_node() { + let mut state_builder = TestStateBuilder::new(); + let mut tree: StateBTreeSet<2, u32, _> = state_builder.new_btree_set(); + for n in (0..4).into_iter().rev() { + tree.insert(n); + } + tree.remove(&3); + assert!(tree.contains(&2)); + assert!(tree.remove(&2)); + assert!(tree.contains(&0)); + assert!(tree.contains(&1)); + assert!(!tree.contains(&2)); + } + + #[test] + fn test_btree_remove_from_higher_leaf_in_three_node_taking_from_sibling() { + let mut state_builder = TestStateBuilder::new(); + let mut tree: StateBTreeSet<2, u32, _> = state_builder.new_btree_set(); + for n in (0..4).into_iter().rev() { + tree.insert(n); } + assert!(tree.contains(&3)); + assert!(tree.remove(&3)); + assert!(!tree.contains(&3)); + } - let s = map.get(&8); + #[test] + fn test_btree_remove_from_lower_leaf_in_three_node_taking_from_sibling() { + let mut state_builder = TestStateBuilder::new(); + let mut tree: StateBTreeSet<2, u32, _> = state_builder.new_btree_set(); + for n in 0..4 { + tree.insert(n); + } + assert!(tree.contains(&0)); + assert!(tree.remove(&0)); + assert!(!tree.contains(&0)); + assert!(tree.contains(&1)); + assert!(tree.contains(&2)); + assert!(tree.contains(&3)); + } - assert_eq!(s.as_deref(), None); + #[test] + fn test_btree_remove_from_root_in_three_node_causing_merge() { + let mut state_builder = TestStateBuilder::new(); + let mut tree: StateBTreeSet<2, u32, _> = state_builder.new_btree_set(); + for n in 0..4 { + tree.insert(n); + } + tree.remove(&3); + + assert!(tree.contains(&1)); + assert!(tree.remove(&1)); + assert!(!tree.contains(&1)); + } + + #[test] + fn test_btree_remove_from_root_in_three_node_taking_key_from_higher_child() { + let mut state_builder = TestStateBuilder::new(); + let mut tree: StateBTreeSet<2, u32, _> = state_builder.new_btree_set(); + for n in 0..4 { + tree.insert(n); + } + assert!(tree.contains(&1)); + assert!(tree.remove(&1)); + assert!(!tree.contains(&1)); + } + + #[test] + fn test_btree_remove_from_root_in_three_node_taking_key_from_lower_child() { + let mut state_builder = TestStateBuilder::new(); + let mut tree: StateBTreeSet<2, u32, _> = state_builder.new_btree_set(); + for n in (0..4).into_iter().rev() { + tree.insert(n); + } + assert!(tree.contains(&2)); + assert!(tree.remove(&2)); + assert!(!tree.contains(&2)); } } diff --git a/concordium-std/src/types.rs b/concordium-std/src/types.rs index b4bb0329..3d617140 100644 --- a/concordium-std/src/types.rs +++ b/concordium-std/src/types.rs @@ -1273,21 +1273,31 @@ pub struct MetadataUrl { /// TODO Document the meaning of generics and restrictions on M. /// TODO Document the complexity of the basic operations. pub struct StateBTreeMap { + pub(crate) map: StateMap, + pub(crate) ordered_set: StateBTreeSet, +} + +/// An ordered set based on [B-Tree](https://en.wikipedia.org/wiki/B-tree), where +/// each node is stored separately in the low-level key-value store. +/// +/// TODO Document how to construct it. +/// TODO Document size of the serialized key matters. +/// TODO Document the meaning of generics and restrictions on M. +/// TODO Document the complexity of the basic operations. +pub struct StateBTreeSet { /// Type marker for the key. - pub(crate) _marker_key: PhantomData, - /// Type marker for the value. - pub(crate) _marker_value: PhantomData, + pub(crate) _marker_key: PhantomData, /// The unique prefix to use for this map in the key-value store. - pub(crate) prefix: StateItemPrefix, + pub(crate) prefix: StateItemPrefix, /// The API for interacting with the low-level state. - pub(crate) state_api: S, + pub(crate) state_api: S, /// The ID of the root node of the tree, where None represents the tree is /// empty. - pub(crate) root: Option, + pub(crate) root: Option, /// Tracking the number of items in the tree. - pub(crate) len: u32, + pub(crate) len: u32, /// Tracking the next available ID for a new node. - pub(crate) next_node_id: state_btree_internals::NodeId, + pub(crate) next_node_id: state_btree_internals::NodeId, } /// Module with types used internally in [`StateBTreeMap`]. @@ -1305,7 +1315,8 @@ pub(crate) mod state_btree_internals { #[derive(Debug)] pub(crate) struct Node { /// List of sorted keys tracked by this node. - /// This list should never be empty and contain at most `M` elements. + /// This list should never be empty and contain between `M - 1` and `2M + /// - 1` elements. pub(crate) keys: Vec, /// List of nodes which are children of this node in the tree. /// From 39b0a7d9f39cbcc1b0f682bbe3c406a31970498e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Holm=20Gj=C3=B8rup?= Date: Wed, 28 Feb 2024 11:25:28 +0100 Subject: [PATCH 06/36] Reuse already loaded nodes when deleting, fixing some issues aswell --- concordium-std/src/impls.rs | 81 +++++++++++++---------- concordium-std/src/test_infrastructure.rs | 8 ++- 2 files changed, 50 insertions(+), 39 deletions(-) diff --git a/concordium-std/src/impls.rs b/concordium-std/src/impls.rs index 4803bd56..ae30cb60 100644 --- a/concordium-std/src/impls.rs +++ b/concordium-std/src/impls.rs @@ -3515,7 +3515,7 @@ impl StateBTreeSet { break true; } // Found the key in this node, but the node is not a leaf. - let left_child = self.get_node_mut(node.children[index]); + let mut left_child = self.get_node_mut(node.children[index]); if !left_child.is_at_min() { // If the child with smaller keys can spare a key, we take the highest // key from it, changing the loop to be deleting its highest key from @@ -3556,8 +3556,8 @@ impl StateBTreeSet { } // No child on either side of the key can spare a key, so we merge them into // one child, moving the key into the child and try to remove from this. - self.merge(&mut node, index); - node = self.get_node_mut(node.children[index]); + self.merge(&mut node, index, &mut left_child, right_child); + node = left_child; // FIXME: For some reason this is needed to cause loading the node for the next // iteration. let _ = node.get(); @@ -3574,13 +3574,15 @@ impl StateBTreeSet { let mut child = self.get_node_mut(node.children[index]); let has_smaller_sibling = 0 < index; let has_larger_sibling = index < node.children.len() - 1; - // Check and proactively prepare if the child which could contain the key to - // spare loosing a key. - if child.is_at_min() { + // Check and proactively prepare the child to be able to delete a key. + node = if !child.is_at_min() { + child + } else { // The child is at minimum keys, so first attempt to take a key from either // sibling, otherwise merge with one of them. 'increase_child: { - if has_smaller_sibling { + // Labeled block, to be able to return early. + let smaller_sibling = if has_smaller_sibling { let mut smaller_sibling = self.get_node_mut(node.children[index - 1]); if !smaller_sibling.is_at_min() { // The smaller sibling can spare a key, so we replace the largest key @@ -3595,10 +3597,14 @@ impl StateBTreeSet { .children .insert(0, smaller_sibling.children.pop().unwrap_abort()); } - break 'increase_child; + //drop(child); + break 'increase_child child; } - } - if has_larger_sibling { + Some(smaller_sibling) + } else { + None + }; + let larger_sibling = if has_larger_sibling { let mut larger_sibling = self.get_node_mut(node.children[index + 1]); if !larger_sibling.is_at_min() { // The larger sibling can spare a key, so we replace the smallest key @@ -3612,28 +3618,26 @@ impl StateBTreeSet { if !child.is_leaf() { child.children.push(larger_sibling.children.remove(0)); } - break 'increase_child; + //drop(child); + break 'increase_child child; } - } - - if has_larger_sibling { - self.merge(&mut node, index); + Some(larger_sibling) + } else { + None + }; + + if let Some(sibling) = larger_sibling { + self.merge(&mut node, index, &mut child, sibling); + child + } else if let Some(mut sibling) = smaller_sibling { + self.merge(&mut node, index - 1, &mut sibling, child); + sibling } else { - self.merge(&mut node, index - 1); + // Unreachable code. + crate::trap(); } } - } - // Merging two children moves a key from the node down as well, so if this - // happened above and index was pointing to the last child before, update - // it. - let node_lost_key = index > node.len(); - let next_index = index - usize::from(!has_larger_sibling && node_lost_key); - drop(child); // FIXME: Dropped in order to store changes before reading the child again from - // state. This should reuse the already loaded stateref instead. - node = self.get_node_mut(node.children[next_index]); - // FIXME: For some reason this is needed to cause loading the node for the next - // iteration. - let _ = node.get(); + }; }; // If something was deleted, we update the length and make sure to remove the @@ -3693,16 +3697,21 @@ impl StateBTreeSet { /// Assumes: /// - `node` have a child at `index` and `index + 1`. /// - Both children are at minimum number of keys. - fn merge(&mut self, node: &mut state_btree_internals::Node, index: usize) - where + fn merge( + &mut self, + node: &mut state_btree_internals::Node, // parent node. + child_index: usize, /* index of the smaller child in the + * parent node. */ + child: &mut state_btree_internals::Node, // smaller child. + larger_child: StateRefMut, S>, // smaller child. + ) where K: Ord + Serialize, S: HasStateApi, { - let mut left = self.get_node_mut(node.children[index]); - let mut right = self.get_node(node.children[index + 1]); - left.keys.push(node.keys.remove(index)); - left.keys.append(&mut right.keys); - left.children.append(&mut right.children); - let removed_node_id = node.children.remove(index + 1); + let mut larger_child = larger_child.into_inner_value().unwrap_abort(); + child.keys.push(node.keys.remove(child_index)); + child.keys.append(&mut larger_child.keys); + child.children.append(&mut larger_child.children); + let removed_node_id = node.children.remove(child_index + 1); self.delete_node(removed_node_id); } diff --git a/concordium-std/src/test_infrastructure.rs b/concordium-std/src/test_infrastructure.rs index 0bb61cfb..a7db1000 100644 --- a/concordium-std/src/test_infrastructure.rs +++ b/concordium-std/src/test_infrastructure.rs @@ -2592,15 +2592,17 @@ mod test { } #[test] - fn test_btree_m3_remove_from_one_node_tree() { + fn test_btree_remove_from_one_node_tree() { let mut state_builder = TestStateBuilder::new(); - let mut tree: StateBTreeSet<3, u32, _> = state_builder.new_btree_set(); - for n in 0..=1 { + let mut tree: StateBTreeSet<2, u32, _> = state_builder.new_btree_set(); + for n in 0..3 { tree.insert(n); } assert!(tree.contains(&1)); assert!(tree.remove(&1)); + assert!(tree.contains(&0)); assert!(!tree.contains(&1)); + assert!(tree.contains(&2)); } #[test] From 2ef335431bac9460493e5df31b3fd7617e8bee07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Holm=20Gj=C3=B8rup?= Date: Thu, 29 Feb 2024 15:45:33 +0100 Subject: [PATCH 07/36] Reduce the number of state lookups when deleting --- concordium-std/src/impls.rs | 433 +++++++++++----------- concordium-std/src/test_infrastructure.rs | 26 +- concordium-std/src/types.rs | 4 +- 3 files changed, 231 insertions(+), 232 deletions(-) diff --git a/concordium-std/src/impls.rs b/concordium-std/src/impls.rs index ae30cb60..b8c375ff 100644 --- a/concordium-std/src/impls.rs +++ b/concordium-std/src/impls.rs @@ -5,7 +5,8 @@ use crate::{ convert::{self, TryInto}, fmt, marker::PhantomData, - mem, num, + mem::{self, MaybeUninit}, + num, num::NonZeroU32, prims, state_btree_internals, traits::*, @@ -14,7 +15,6 @@ use crate::{ String, }; pub(crate) use concordium_contracts_common::*; -use mem::MaybeUninit; /// Mapped to i32::MIN + 1. impl convert::From<()> for Reject { @@ -1201,14 +1201,14 @@ where fn load_value(&self) -> V where V: DeserialWithState, { - let entry = unsafe { &mut *self.entry.get() }; + let entry = unsafe { &mut *self.entry.get() }.as_mut().unwrap_abort(); entry.move_to_start(); V::deserial_with_state(&self.state_api, entry).unwrap_abort() } /// Set the value. Overwrites the existing one. pub fn set(&mut self, new_val: V) { - let entry = self.entry.get_mut(); + let entry = self.entry.get_mut().as_mut().unwrap_abort(); entry.move_to_start(); new_val.serial(entry).unwrap_abort(); let _ = self.lazy_value.get_mut().insert(new_val); @@ -1220,7 +1220,7 @@ where V: DeserialWithState, F: FnOnce(&mut V), { let lv = self.lazy_value.get_mut(); - let entry = self.entry.get_mut(); + let entry = self.entry.get_mut().as_mut().unwrap_abort(); let value = if let Some(v) = lv { v } else { @@ -1238,17 +1238,22 @@ where /// Write to the state entry if the value is loaded. pub(crate) fn store_mutations(&mut self) { if let Some(value) = self.lazy_value.get_mut() { - let entry = self.entry.get_mut(); + let entry = self.entry.get_mut().as_mut().unwrap_abort(); entry.move_to_start(); value.serial(entry).unwrap_abort(); } } - /// Get the inner value if loaded, while consuming the [`StateRefMut`]. - /// Mutations will not be written to the smart contract key-store when this - /// is dropped. Neither will they be written as part of this, so make sure - /// to run `store_mutations` first. - pub(crate) fn into_inner_value(mut self) -> Option { self.lazy_value.get_mut().take() } + /// Get the inner entry and value if loaded, while consuming the + /// [`StateRefMut`]. Mutations will not be written to the smart contract + /// key-store when this is dropped. + /// Neither will they be written as part of this, so make sure to run + /// `store_mutations` first. + pub(crate) fn into_raw_parts(mut self) -> (Option, S::EntryType) { + let value = self.lazy_value.get_mut().take(); + let entry = self.entry.get_mut().take().unwrap_abort(); + (value, entry) + } } impl Serial for StateMap { @@ -3148,34 +3153,6 @@ impl Deserial for MetadataUrl { } } -// impl Serial for StateBTreeMap { -// fn serial(&self, out: &mut W) -> Result<(), W::Err> { -// self.prefix.serial(out)?; -// self.root.serial(out)?; -// self.len.serial(out)?; -// self.next_node_id.serial(out) -// } -// } - -// impl DeserialWithState for -// StateBTreeMap { fn deserial_with_state(state: &S, -// source: &mut R) -> ParseResult { let prefix = source.get()?; -// let root = source.get()?; -// let len = source.get()?; -// let next_node_id = source.get()?; - -// Ok(Self { -// _marker_key: Default::default(), -// _marker_value: Default::default(), -// prefix, -// state_api: state.clone(), -// root, -// len, -// next_node_id, -// }) -// } -// } - impl Serial for StateBTreeMap { fn serial(&self, out: &mut W) -> Result<(), W::Err> { self.map.serial(out)?; @@ -3281,6 +3258,16 @@ impl StateBTreeMap { v } + pub fn remove(&mut self, key: &K) + where + S: HasStateApi, + K: Serialize + Ord + fmt::Debug, + V: Serial + DeserialWithState + Deletable, { + if self.ordered_set.remove(key) { + self.map.remove(key); + } + } + /// Get a reference to the value corresponding to the key. pub fn get(&self, key: &K) -> Option> where @@ -3361,32 +3348,33 @@ impl StateBTreeSet { }; self.root = Some(node_id); self.len = 1; - return false; + return true; }; - { - let root_node = self.get_node_mut(root_id); - if !root_node.is_full() { - let new = self.insert_non_full(root_node, key); - if new { - self.len += 1; - } - return new; - } else if root_node.keys.binary_search(&key).is_ok() { - // TODO: The error result of the binary search can probably be reused below. - return false; + let root_node = self.get_node_mut(root_id); + if !root_node.is_full() { + let new = self.insert_non_full(root_node, key); + if new { + self.len += 1; } + return new; + } else if root_node.keys.binary_search(&key).is_ok() { + return false; } - + // The root node is full, so we construct a new root node. let (new_root_id, mut new_root) = self.create_node(Vec::new(), vec![root_id]); - - self.split_child(&mut new_root, 0); + self.root = Some(new_root_id); + // The old root node is now a child node. + let mut child = root_node; + let new_larger_child = self.split_child(&mut new_root, 0, &mut child); // new_root should now contain one key and two children, so we need to know // which one to insert into. - let child_index = usize::from(new_root.keys[0] < key); - let child = self.get_node_mut(new_root.children[child_index]); + let child = if new_root.keys[0] < key { + new_larger_child + } else { + child + }; let new = self.insert_non_full(child, key); - self.root = Some(new_root_id); if new { self.len += 1; } @@ -3499,162 +3487,167 @@ impl StateBTreeSet { return false; }; - let mut overwrite_key_to_delete = None; - let mut node = self.get_node_mut(root_node_id); - let deleted_something = loop { - let key_to_delete = overwrite_key_to_delete.as_ref().unwrap_or(key); - let index = match node.keys.binary_search(key_to_delete) { - Ok(index) => { - if node.is_leaf() { - // Found the key in this node and the node is a leaf, meaning we simply - // remove it. - // This will not violate the minimum keys invariant, since a node ensures a - // child can spare a key before iteration and the root node is not part of - // the invariant. - node.keys.remove(index); - break true; - } - // Found the key in this node, but the node is not a leaf. - let mut left_child = self.get_node_mut(node.children[index]); - if !left_child.is_at_min() { - // If the child with smaller keys can spare a key, we take the highest - // key from it, changing the loop to be deleting its highest key from - // this child. - node.keys[index] = self.get_highest_key(&mut node, index); - let new_key_to_delete = { - // To avoid having to clone the key, we store changes to the node - // and unwrap it from the StateRefMut, disabling further mutations - // from being written to the key-value store, such that we can use - // the same in-memory key. - node.store_mutations(); - let mut inner_node = node.into_inner_value().unwrap_abort(); - inner_node.keys.swap_remove(index) - }; - overwrite_key_to_delete = Some(new_key_to_delete); - node = left_child; - continue; - } + let deleted_something = { + let mut overwrite_key_to_delete = None; + let mut node = self.get_node_mut(root_node_id); + loop { + let key_to_delete = overwrite_key_to_delete.as_ref().unwrap_or(key); + let index = match node.keys.binary_search(key_to_delete) { + Ok(index) => { + if node.is_leaf() { + // Found the key in this node and the node is a leaf, meaning we simply + // remove it. + // This will not violate the minimum keys invariant, since a node + // ensures a child can spare a key before + // iteration and the root node is not part of + // the invariant. + node.keys.remove(index); + break true; + } + // Found the key in this node, but the node is not a leaf. + let mut left_child = self.get_node_mut(node.children[index]); + if !left_child.is_at_min() { + // If the child with smaller keys can spare a key, we take the highest + // key from it, changing the loop to be deleting its highest key from + // this child. + node.keys[index] = self.get_highest_key(&mut node, index); + let new_key_to_delete = { + // To avoid having to clone the key, we store changes to the node + // and unwrap it from the StateRefMut, disabling further mutations + // from being written to the key-value store, such that we can take + // the in-memory key. + node.store_mutations(); + let mut inner_node = node.into_raw_parts().0.unwrap_abort(); + inner_node.keys.swap_remove(index) + }; + overwrite_key_to_delete = Some(new_key_to_delete); + node = left_child; + continue; + } - let right_child = self.get_node_mut(node.children[index + 1]); - if !right_child.is_at_min() { - // If the child with larger keys can spare a key, we take the lowest - // key from it, changing the loop to be deleting its highest key from - // this child. - node.keys[index] = self.get_lowest_key(&mut node, index + 1); - let new_key_to_delete = { - // To avoid having to clone the key, we store changes to the node - // and unwrap it from the StateRefMut, disabling further mutations - // from being written to the key-value store, such that we can use - // the same in-memory key. - node.store_mutations(); - let mut inner_node = node.into_inner_value().unwrap_abort(); - inner_node.keys.swap_remove(index) - }; - overwrite_key_to_delete = Some(new_key_to_delete); - node = right_child; + let right_child = self.get_node_mut(node.children[index + 1]); + if !right_child.is_at_min() { + // If the child with larger keys can spare a key, we take the lowest + // key from it, changing the loop to be deleting its highest key from + // this child. + node.keys[index] = self.get_lowest_key(&mut node, index + 1); + let new_key_to_delete = { + // To avoid having to clone the key, we store changes to the node + // and unwrap it from the StateRefMut, disabling further mutations + // from being written to the key-value store, such that we can use + // the same in-memory key. + node.store_mutations(); + let mut inner_node = node.into_raw_parts().0.unwrap_abort(); + inner_node.keys.swap_remove(index) + }; + overwrite_key_to_delete = Some(new_key_to_delete); + node = right_child; + continue; + } + // No child on either side of the key can spare a key, so we merge them into + // one child, moving the key into the child and try to remove from this. + self.merge(&mut node, index, &mut left_child, right_child); + node = left_child; + // FIXME: For some reason this is needed to cause loading the node for the + // next iteration. + let _ = node.get(); continue; } - // No child on either side of the key can spare a key, so we merge them into - // one child, moving the key into the child and try to remove from this. - self.merge(&mut node, index, &mut left_child, right_child); - node = left_child; - // FIXME: For some reason this is needed to cause loading the node for the next - // iteration. - let _ = node.get(); - continue; + Err(index) => index, + }; + // Node did not contain the key. + if node.is_leaf() { + break false; } - Err(index) => index, - }; - // Node did not contain the key. - if node.is_leaf() { - break false; - } - // Node did not contain the key and is not a leaf. - let mut child = self.get_node_mut(node.children[index]); - let has_smaller_sibling = 0 < index; - let has_larger_sibling = index < node.children.len() - 1; - // Check and proactively prepare the child to be able to delete a key. - node = if !child.is_at_min() { - child - } else { - // The child is at minimum keys, so first attempt to take a key from either - // sibling, otherwise merge with one of them. - 'increase_child: { - // Labeled block, to be able to return early. - let smaller_sibling = if has_smaller_sibling { - let mut smaller_sibling = self.get_node_mut(node.children[index - 1]); - if !smaller_sibling.is_at_min() { - // The smaller sibling can spare a key, so we replace the largest key - // from the sibling, put it in the parent - // and take a key from the parent. - let largest_key_sibling = smaller_sibling.keys.pop().unwrap_abort(); - let swapped_node_key = - mem::replace(&mut node.keys[index - 1], largest_key_sibling); - child.keys.insert(0, swapped_node_key); - if !child.is_leaf() { - child - .children - .insert(0, smaller_sibling.children.pop().unwrap_abort()); + // Node did not contain the key and is not a leaf. + let mut child = self.get_node_mut(node.children[index]); + let has_smaller_sibling = 0 < index; + let has_larger_sibling = index < node.children.len() - 1; + // Check and proactively prepare the child to be able to delete a key. + node = if !child.is_at_min() { + child + } else { + // The child is at minimum keys, so first attempt to take a key from either + // sibling, otherwise merge with one of them. + 'increase_child: { + // Labeled block, to be able to return early. + let smaller_sibling = if has_smaller_sibling { + let mut smaller_sibling = self.get_node_mut(node.children[index - 1]); + if !smaller_sibling.is_at_min() { + // The smaller sibling can spare a key, so we replace the largest + // key from the sibling, put it in + // the parent and take a key from + // the parent. + let largest_key_sibling = smaller_sibling.keys.pop().unwrap_abort(); + let swapped_node_key = + mem::replace(&mut node.keys[index - 1], largest_key_sibling); + child.keys.insert(0, swapped_node_key); + if !child.is_leaf() { + child + .children + .insert(0, smaller_sibling.children.pop().unwrap_abort()); + } + //drop(child); + break 'increase_child child; } - //drop(child); - break 'increase_child child; - } - Some(smaller_sibling) - } else { - None - }; - let larger_sibling = if has_larger_sibling { - let mut larger_sibling = self.get_node_mut(node.children[index + 1]); - if !larger_sibling.is_at_min() { - // The larger sibling can spare a key, so we replace the smallest key - // from the sibling, put it in the parent - // and take a key from the parent. - let first_key_sibling = larger_sibling.keys.remove(0); - let swapped_node_key = - mem::replace(&mut node.keys[index], first_key_sibling); - child.keys.push(swapped_node_key); - - if !child.is_leaf() { - child.children.push(larger_sibling.children.remove(0)); + Some(smaller_sibling) + } else { + None + }; + let larger_sibling = if has_larger_sibling { + let mut larger_sibling = self.get_node_mut(node.children[index + 1]); + if !larger_sibling.is_at_min() { + // The larger sibling can spare a key, so we replace the smallest + // key from the sibling, put it in + // the parent and take a key from + // the parent. + let first_key_sibling = larger_sibling.keys.remove(0); + let swapped_node_key = + mem::replace(&mut node.keys[index], first_key_sibling); + child.keys.push(swapped_node_key); + + if !child.is_leaf() { + child.children.push(larger_sibling.children.remove(0)); + } + //drop(child); + break 'increase_child child; } - //drop(child); - break 'increase_child child; + Some(larger_sibling) + } else { + None + }; + + if let Some(sibling) = larger_sibling { + self.merge(&mut node, index, &mut child, sibling); + child + } else if let Some(mut sibling) = smaller_sibling { + self.merge(&mut node, index - 1, &mut sibling, child); + sibling + } else { + // Unreachable code. + crate::trap(); } - Some(larger_sibling) - } else { - None - }; - - if let Some(sibling) = larger_sibling { - self.merge(&mut node, index, &mut child, sibling); - child - } else if let Some(mut sibling) = smaller_sibling { - self.merge(&mut node, index - 1, &mut sibling, child); - sibling - } else { - // Unreachable code. - crate::trap(); } - } - }; + }; + } }; // If something was deleted, we update the length and make sure to remove the // root node if needed. if deleted_something { self.len -= 1; + let root = self.get_node_mut(root_node_id); if self.len == 0 { // Remote the root node if tree is empty. - self.delete_node(root_node_id); self.root = None; + self.delete_node(root); } else { // If the root is empty but the tree is not, point the only child of the root as // the root. - let root = self.get_node(root_node_id); if root.keys.is_empty() { self.root = Some(root.children[0]); - self.delete_node(root_node_id); + self.delete_node(root); } } } @@ -3692,7 +3685,7 @@ impl StateBTreeSet { } /// Moving key at `index` from the node to the lower child and then merges - /// this child with its higher sibling. + /// this child with the content of its higher sibling, deleting the sibling. /// /// Assumes: /// - `node` have a child at `index` and `index + 1`. @@ -3703,16 +3696,15 @@ impl StateBTreeSet { child_index: usize, /* index of the smaller child in the * parent node. */ child: &mut state_btree_internals::Node, // smaller child. - larger_child: StateRefMut, S>, // smaller child. + mut larger_child: StateRefMut, S>, // smaller child. ) where K: Ord + Serialize, S: HasStateApi, { - let mut larger_child = larger_child.into_inner_value().unwrap_abort(); child.keys.push(node.keys.remove(child_index)); child.keys.append(&mut larger_child.keys); child.children.append(&mut larger_child.children); - let removed_node_id = node.children.remove(child_index + 1); - self.delete_node(removed_node_id); + node.children.remove(child_index + 1); + self.delete_node(larger_child); } /// Internal function for constructing a node. It will incrementing the next @@ -3739,13 +3731,11 @@ impl StateBTreeSet { /// Internal function for deleting a node, removing the entry in the smart /// contract key-value store. Traps if no node was present. - fn delete_node(&mut self, node_id: state_btree_internals::NodeId) + fn delete_node(&mut self, node: StateRefMut, S>) where + K: Serial, S: HasStateApi, { - let key = self.node_key(node_id); - if !self.state_api.delete_prefix(&key).unwrap_abort() { - crate::trap(); - } + self.state_api.delete_entry(node.into_raw_parts().1).unwrap_abort() } /// Internal function for inserting into a subtree. The given node must not @@ -3760,7 +3750,7 @@ impl StateBTreeSet { S: HasStateApi, { let mut node = initial_node; loop { - let Err(mut insert_index) = node.keys.binary_search(&key) else { + let Err(insert_index) = node.keys.binary_search(&key) else { // We find the key in this node, so we do nothing. return false; }; @@ -3772,40 +3762,49 @@ impl StateBTreeSet { } // The node is not a leaf, so we want to insert in the relevant child node. - let child = self.get_node(node.children[insert_index]); - if child.is_full() { - self.split_child(&mut node, insert_index); - // Since the child is now split into two, we have to update the insert_index to + let mut child = self.get_node_mut(node.children[insert_index]); + node = if child.is_full() { + let larger_child = self.split_child(&mut node, insert_index, &mut child); + // Since the child is now split into two, we have to update the insert into // the relevant one of them. - insert_index += usize::from(node.keys[insert_index] < key); - } - node = self.get_node_mut(node.children[insert_index]); + if node.keys[insert_index] < key { + larger_child + } else { + child + } + } else { + child + }; } } /// Internal function for splitting the child node at a given index for a /// given node. This will also mutate the given node adding a new key /// and child after the provided child_index. + /// + /// Returns the newly created node. fn split_child<'a, 'b>( &'a mut self, - node: &'b mut state_btree_internals::Node, + node: &mut state_btree_internals::Node, child_index: usize, - ) where + child: &mut state_btree_internals::Node, + ) -> StateRefMut<'b, state_btree_internals::Node, S> + where K: Serialize + Ord, S: HasStateApi, { - let mut left = self.get_node_mut(node.children[child_index]); let split_index = state_btree_internals::Node::::MINIMUM_KEY_LEN + 1; - let (right_id, _right) = self.create_node( - left.keys.split_off(split_index), - if left.is_leaf() { + let (new_larger_sibling_id, new_larger_sibling) = self.create_node( + child.keys.split_off(split_index), + if child.is_leaf() { Vec::new() } else { - left.children.split_off(split_index) + child.children.split_off(split_index) }, ); - let key = left.keys.pop().unwrap_abort(); - node.children.insert(child_index + 1, right_id); + let key = child.keys.pop().unwrap_abort(); + node.children.insert(child_index + 1, new_larger_sibling_id); node.keys.insert(child_index, key); + new_larger_sibling } /// Internal function for looking up a node in the tree. diff --git a/concordium-std/src/test_infrastructure.rs b/concordium-std/src/test_infrastructure.rs index a7db1000..5bcec151 100644 --- a/concordium-std/src/test_infrastructure.rs +++ b/concordium-std/src/test_infrastructure.rs @@ -2489,7 +2489,7 @@ mod test { } #[test] - fn test_btree_m5_insert_6() { + fn test_btree_insert_6() { let mut state_builder = TestStateBuilder::new(); let mut tree: StateBTreeSet<5, u32, _> = state_builder.new_btree_set(); for n in 0..=5 { @@ -2501,9 +2501,9 @@ mod test { } #[test] - fn test_btree_m3_insert_0_7() { + fn test_btree_insert_0_7() { let mut state_builder = TestStateBuilder::new(); - let mut tree: StateBTreeSet<3, u32, _> = state_builder.new_btree_set(); + let mut tree: StateBTreeSet<2, u32, _> = state_builder.new_btree_set(); for n in 0..=7 { tree.insert(n); } @@ -2513,9 +2513,9 @@ mod test { } #[test] - fn test_btree_m3_insert_7_no_order() { + fn test_btree_insert_7_no_order() { let mut state_builder = TestStateBuilder::new(); - let mut tree: StateBTreeSet<3, u32, _> = state_builder.new_btree_set(); + let mut tree: StateBTreeSet<2, u32, _> = state_builder.new_btree_set(); tree.insert(0); tree.insert(1); @@ -2532,9 +2532,9 @@ mod test { } #[test] - fn test_btree_m3_higher() { + fn test_btree_higher() { let mut state_builder = TestStateBuilder::new(); - let mut tree: StateBTreeSet<3, u32, _> = state_builder.new_btree_set(); + let mut tree: StateBTreeSet<2, u32, _> = state_builder.new_btree_set(); tree.insert(1); tree.insert(2); tree.insert(3); @@ -2547,9 +2547,9 @@ mod test { } #[test] - fn test_btree_m3_lower() { + fn test_btree_lower() { let mut state_builder = TestStateBuilder::new(); - let mut tree: StateBTreeSet<3, u32, _> = state_builder.new_btree_set(); + let mut tree: StateBTreeSet<2, u32, _> = state_builder.new_btree_set(); tree.insert(1); tree.insert(2); tree.insert(3); @@ -2562,9 +2562,9 @@ mod test { } #[test] - fn test_btree_m3_insert_1000() { + fn test_btree_insert_1000() { let mut state_builder = TestStateBuilder::new(); - let mut tree: StateBTreeSet<3, u32, _> = state_builder.new_btree_set(); + let mut tree: StateBTreeSet<2, u32, _> = state_builder.new_btree_set(); for n in 0..500 { tree.insert(n); } @@ -2581,9 +2581,9 @@ mod test { } #[test] - fn test_btree_m3_7_get_8() { + fn test_btree_7_get_8() { let mut state_builder = TestStateBuilder::new(); - let mut tree: StateBTreeSet<3, u32, _> = state_builder.new_btree_set(); + let mut tree: StateBTreeSet<2, u32, _> = state_builder.new_btree_set(); for n in 0..=7 { tree.insert(n); } diff --git a/concordium-std/src/types.rs b/concordium-std/src/types.rs index 3d617140..0fb5f420 100644 --- a/concordium-std/src/types.rs +++ b/concordium-std/src/types.rs @@ -384,7 +384,7 @@ impl<'a, V> crate::ops::Deref for StateRef<'a, V> { /// that the value is properly stored in the contract state maintained by the /// node. pub struct StateRefMut<'a, V: Serial, S: HasStateApi> { - pub(crate) entry: UnsafeCell, + pub(crate) entry: UnsafeCell>, pub(crate) state_api: S, pub(crate) lazy_value: UnsafeCell>, pub(crate) _marker_lifetime: PhantomData<&'a mut V>, @@ -394,7 +394,7 @@ impl<'a, V: Serial, S: HasStateApi> StateRefMut<'a, V, S> { #[inline(always)] pub(crate) fn new(entry: S::EntryType, state_api: S) -> Self { Self { - entry: UnsafeCell::new(entry), + entry: UnsafeCell::new(Some(entry)), state_api, lazy_value: UnsafeCell::new(None), _marker_lifetime: PhantomData, From 692a51dc4bd722d689e2ce2e3d0b727c27a6abad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Holm=20Gj=C3=B8rup?= Date: Fri, 1 Mar 2024 11:15:49 +0100 Subject: [PATCH 08/36] Implement `iter` and `clear` --- concordium-std/src/impls.rs | 257 +++++++++++++++------- concordium-std/src/test_infrastructure.rs | 12 + concordium-std/src/types.rs | 42 ++++ 3 files changed, 236 insertions(+), 75 deletions(-) diff --git a/concordium-std/src/impls.rs b/concordium-std/src/impls.rs index b8c375ff..8f4ef599 100644 --- a/concordium-std/src/impls.rs +++ b/concordium-std/src/impls.rs @@ -8,7 +8,7 @@ use crate::{ mem::{self, MaybeUninit}, num, num::NonZeroU32, - prims, state_btree_internals, + prims, traits::*, types::*, vec::Vec, @@ -3233,7 +3233,7 @@ impl StateBTreeMap { pub fn insert(&mut self, key: K, value: V) -> Option where S: HasStateApi, - K: Serialize + Ord + fmt::Debug, + K: Serialize + Ord, V: Serial + DeserialWithState, { let old_value_option = self.map.insert_borrowed(&key, value); if old_value_option.is_none() { @@ -3245,10 +3245,18 @@ impl StateBTreeMap { return old_value_option; } + /// Remove a key from the map, returning the value at the key if the key was + /// previously in the map. + /// + /// *Caution*: If `V` is a [StateBox], [StateMap], then it is + /// important to call [`Deletable::delete`] on the value returned when + /// you're finished with it. Otherwise, it will remain in the contract + /// state. + #[must_use] pub fn remove_and_get(&mut self, key: &K) -> Option where S: HasStateApi, - K: Serialize + Ord + fmt::Debug, + K: Serialize + Ord, V: Serial + DeserialWithState + Deletable, { let v = self.map.remove_and_get(key); if v.is_some() && !self.ordered_set.remove(key) { @@ -3258,10 +3266,12 @@ impl StateBTreeMap { v } + /// Remove a key from the map. + /// This also deletes the value in the state. pub fn remove(&mut self, key: &K) where S: HasStateApi, - K: Serialize + Ord + fmt::Debug, + K: Serialize + Ord, V: Serial + DeserialWithState + Deletable, { if self.ordered_set.remove(key) { self.map.remove(key); @@ -3317,6 +3327,47 @@ impl StateBTreeMap { /// Returns `true` is the map contains no elements. pub fn is_empty(&self) -> bool { self.ordered_set.is_empty() } + + /// Create an iterator over the entries of [`StateBTreeMap`]. + /// Ordered by `K`. + pub fn iter(&self) -> StateBTreeMapIter + where + S: HasStateApi, { + StateBTreeMapIter { + key_iter: self.ordered_set.iter(), + map: &self.map, + } + } + + /// Clears the map, removing all key-value pairs. + /// This also includes values pointed at, if `V`, for example, is a + /// [StateBox]. **If applicable use [`clear_flat`](Self::clear_flat) + /// instead.** + pub fn clear(&mut self) + where + S: HasStateApi, + K: Serialize, + V: Serial + DeserialWithState + Deletable, { + self.map.clear(); + self.ordered_set.clear(); + } + + /// Clears the map, removing all key-value pairs. + /// **This should be used over [`clear`](Self::clear) if it is + /// applicable.** It avoids recursive deletion of values since the + /// values are required to be _flat_. + /// + /// Unfortunately it is not possible to automatically choose between these + /// implementations. Once Rust gets trait specialization then this might + /// be possible. + pub fn clear_flat(&mut self) + where + S: HasStateApi, + K: Serialize, + V: Serialize, { + self.map.clear_flat(); + self.ordered_set.clear(); + } } impl StateBTreeSet { @@ -3340,7 +3391,7 @@ impl StateBTreeSet { pub fn insert(&mut self, key: K) -> bool where S: HasStateApi, - K: Serialize + Ord + fmt::Debug, { + K: Serialize + Ord, { let Some(root_id) = self.root else { let node_id = { let (node_id, _node) = self.create_node(vec![key], Vec::new()); @@ -3408,6 +3459,35 @@ impl StateBTreeSet { /// Returns `true` is the map contains no elements. pub fn is_empty(&self) -> bool { self.root.is_none() } + /// Get an iterator over the elements in the `StateBTreeSet`. The iterator + /// returns elements in increasing order. + pub fn iter(&self) -> StateBTreeSetIter + where + S: HasStateApi, { + StateBTreeSetIter { + length: self.len.try_into().unwrap_abort(), + next_node: self.root, + depth_first_stack: Vec::new(), + tree: &self, + _marker_lifetime: Default::default(), + } + } + + /// Clears the map, removing all elements. + pub fn clear(&mut self) + where + S: HasStateApi, { + // Reset the information. + self.root = None; + self.next_node_id = state_btree_internals::NodeId { + id: 0, + }; + self.len = 0; + // Then delete every node store in the state. + // Unwrapping is safe when only using the high-level API. + self.state_api.delete_prefix(&self.prefix).unwrap_abort(); + } + /// Returns the smallest key in the map, which is strictly larger than the /// provided key. `None` meaning no such key is present in the map. pub fn higher(&self, key: &K) -> Option @@ -3481,7 +3561,7 @@ impl StateBTreeSet { pub fn remove(&mut self, key: &K) -> bool where - K: Ord + Serialize + fmt::Debug, + K: Ord + Serialize, S: HasStateApi, { let Some(root_node_id) = self.root else { return false; @@ -3722,7 +3802,7 @@ impl StateBTreeSet { keys, children, }; - let entry = self.state_api.create_entry(&self.node_key(node_id)).unwrap_abort(); + let entry = self.state_api.create_entry(&node_id.as_key(&self.prefix)).unwrap_abort(); let mut ref_mut: StateRefMut<'_, state_btree_internals::Node, S> = StateRefMut::new(entry, self.state_api.clone()); ref_mut.set(node); @@ -3746,7 +3826,7 @@ impl StateBTreeSet { key: K, ) -> bool where - K: Serialize + Ord + fmt::Debug, + K: Serialize + Ord, S: HasStateApi, { let mut node = initial_node; loop { @@ -3809,14 +3889,14 @@ impl StateBTreeSet { /// Internal function for looking up a node in the tree. /// This assumes the node is present and traps if this is not the case. - fn get_node<'a, 'b>( + fn get_node<'a, 'b, Key>( &'a self, node_id: state_btree_internals::NodeId, - ) -> state_btree_internals::Node + ) -> state_btree_internals::Node where - K: Deserial, + Key: Deserial, S: HasStateApi, { - let key = self.node_key(node_id); + let key = node_id.as_key(&self.prefix); let mut entry = self.state_api.lookup_entry(&key).unwrap_abort(); entry.get().unwrap_abort() } @@ -3830,51 +3910,10 @@ impl StateBTreeSet { where K: Serial, S: HasStateApi, { - let key = self.node_key(node_id); + let key = node_id.as_key(&self.prefix); let entry = self.state_api.lookup_entry(&key).unwrap_abort(); StateRefMut::new(entry, self.state_api.clone()) } - - /// Construct the key for the node in the key-value store from the node ID. - fn node_key(&self, node_id: state_btree_internals::NodeId) -> [u8; BTREE_NODE_KEY_SIZE] { - // Create an uninitialized array of `MaybeUninit`. The `assume_init` is - // safe because the type we are claiming to have initialized here is a - // bunch of `MaybeUninit`s, which do not require initialization. - let mut prefixed: [MaybeUninit; BTREE_NODE_KEY_SIZE] = - unsafe { MaybeUninit::uninit().assume_init() }; - - for i in 0..STATE_ITEM_PREFIX_SIZE { - prefixed[i].write(self.prefix[i]); - } - let id_bytes = node_id.id.to_le_bytes(); - for i in 0..id_bytes.len() { - prefixed[STATE_ITEM_PREFIX_SIZE + i].write(id_bytes[i]); - } - // Transmuting away the maybeuninit is safe since we have initialized all of - // them. - unsafe { mem::transmute(prefixed) } - } - - pub(crate) fn debug(&self) -> String - where - S: HasStateApi, - K: Deserial + fmt::Debug, { - let Some(root_id) = self.root else { - return "Empty".to_owned(); - }; - - let mut out = String::new(); - let mut stack = vec![root_id]; - while let Some(node_id) = stack.pop() { - let node = self.get_node(node_id); - out.push_str(format!("{} [\n", node_id.id).as_str()); - out.push_str(node.debug().as_str()); - out.push_str("]\n"); - - stack.extend(&node.children) - } - out - } } /// Byte size of the key used to store a BTree internal node in the smart @@ -3903,26 +3942,11 @@ impl state_btree_internals::Node { /// Check if the node holds the minimum number of keys. fn is_at_min(&self) -> bool { self.len() == Self::MINIMUM_KEY_LEN } - - fn debug(&self) -> String - where - K: fmt::Debug, { - let mut out = String::new(); - if self.is_leaf() { - out.push_str(format!("Leaf with {:?}\n", self.keys).as_str()); - } else { - for i in 0..self.len() { - out.push_str(format!("Child Id: {}\n", self.children[i].id).as_str()); - out.push_str(format!("Key: {:?}\n", self.keys[i]).as_str()); - } - out.push_str(format!("Child Id: {}\n", self.children[self.len()].id).as_str()); - } - out - } } impl state_btree_internals::NodeId { - const SERIALIZED_BYTE_SIZE: usize = 4; + /// Byte size of `NodeId` when serialized. + pub(crate) const SERIALIZED_BYTE_SIZE: usize = 4; /// Return a copy of the NodeId, then increments itself. pub(crate) fn copy_then_increment(&mut self) -> Self { @@ -3930,17 +3954,100 @@ impl state_btree_internals::NodeId { self.id += 1; current } + + /// Construct the key for the node in the key-value store from the node ID. + fn as_key(&self, prefix: &StateItemPrefix) -> [u8; BTREE_NODE_KEY_SIZE] { + // Create an uninitialized array of `MaybeUninit`. The `assume_init` is + // safe because the type we are claiming to have initialized here is a + // bunch of `MaybeUninit`s, which do not require initialization. + let mut prefixed: [MaybeUninit; BTREE_NODE_KEY_SIZE] = + unsafe { MaybeUninit::uninit().assume_init() }; + for i in 0..STATE_ITEM_PREFIX_SIZE { + prefixed[i].write(prefix[i]); + } + let id_bytes = self.id.to_le_bytes(); + for i in 0..id_bytes.len() { + prefixed[STATE_ITEM_PREFIX_SIZE + i].write(id_bytes[i]); + } + // Transmuting away the maybeuninit is safe since we have initialized all of + // them. + unsafe { mem::transmute(prefixed) } + } } -impl Deletable for StateBTreeMap +impl Deserial for state_btree_internals::KeyWrapper { + fn deserial(source: &mut R) -> ParseResult { + let key = K::deserial(source)?; + Ok(Self { + key: Some(key), + }) + } +} + +impl<'a, 'b, const M: usize, K, S> Iterator for StateBTreeSetIter<'a, 'b, M, K, S> where + 'a: 'b, + K: Deserial, S: HasStateApi, { - fn delete(self) { - todo!("clear the map"); + type Item = StateRef<'b, K>; + + fn next(&mut self) -> Option { + while let Some(id) = self.next_node.take() { + let node = self.tree.get_node(id); + if !node.is_leaf() { + self.next_node = Some(node.children[0]); + } + self.depth_first_stack.push((node, 0)); + } - //self.clear(); + if let Some((node, index)) = self.depth_first_stack.last_mut() { + let key = node.keys[*index].key.take().unwrap_abort(); + *index += 1; + let no_more_keys = index == &node.keys.len(); + if !node.is_leaf() { + let child_id = node.children[*index]; + self.next_node = Some(child_id); + } + if no_more_keys { + self.depth_first_stack.pop(); + } + self.length -= 1; + Some(StateRef::new(key)) + } else { + None + } } + + fn size_hint(&self) -> (usize, Option) { (self.length, Some(self.length)) } +} + +impl<'a, 'b, const M: usize, K, V, S> Iterator for StateBTreeMapIter<'a, 'b, M, K, V, S> +where + 'a: 'b, + K: Serialize, + V: Serial + DeserialWithState + 'b, + S: HasStateApi, +{ + type Item = (StateRef<'b, K>, StateRef<'b, V>); + + fn next(&mut self) -> Option { + let next_key = self.key_iter.next()?; + let value = self.map.get(&next_key).unwrap_abort(); + // Unwrap is safe, otherwise the map and the set have inconsistencies. + Some((next_key, value)) + } + + fn size_hint(&self) -> (usize, Option) { self.key_iter.size_hint() } +} + +impl Deletable for StateBTreeMap +where + S: HasStateApi, + K: Serialize, + V: Serial + DeserialWithState + Deletable, +{ + fn delete(mut self) { self.clear(); } } #[cfg(test)] diff --git a/concordium-std/src/test_infrastructure.rs b/concordium-std/src/test_infrastructure.rs index 5bcec151..59a658b5 100644 --- a/concordium-std/src/test_infrastructure.rs +++ b/concordium-std/src/test_infrastructure.rs @@ -2700,4 +2700,16 @@ mod test { assert!(tree.remove(&2)); assert!(!tree.contains(&2)); } + + #[test] + fn test_btree_iter() { + let mut state_builder = TestStateBuilder::new(); + let mut tree: StateBTreeSet<2, u32, _> = state_builder.new_btree_set(); + let keys: Vec = (0..15).into_iter().collect(); + for &k in &keys { + tree.insert(k); + } + let iter_keys: Vec = tree.iter().map(|k| k.clone()).collect(); + assert_eq!(keys, iter_keys); + } } diff --git a/concordium-std/src/types.rs b/concordium-std/src/types.rs index 0fb5f420..9ce4b8a6 100644 --- a/concordium-std/src/types.rs +++ b/concordium-std/src/types.rs @@ -1327,4 +1327,46 @@ pub(crate) mod state_btree_internals { /// keys that are strictly smaller than `keys[i]`. pub(crate) children: Vec, } + + /// Wrapper implement the exact same deserial as K, but wraps it in an + /// option in memory. This is used, to allow taking a key from a mutable + /// reference to a node, without cloning the key, during iteration of the + /// set. + #[repr(transparent)] + pub(crate) struct KeyWrapper { + pub(crate) key: Option, + } +} + +/// An iterator over the entries of a [`StateBTreeSet`]. +/// +/// Ordered by `K`. +/// +/// This `struct` is created by the [`iter`][StateBTreeSet::iter] method on +/// [`StateBTreeSet`]. See its documentation for more. +pub struct StateBTreeSetIter<'a, 'b, const M: usize, K, S> { + /// The number of elements left to iterate. + pub(crate) length: usize, + /// Reference to a node in the tree to load and iterate before the current + /// node. + pub(crate) next_node: Option, + /// Tracking the nodes depth first, which are currently being iterated. + pub(crate) depth_first_stack: + Vec<(state_btree_internals::Node>, usize)>, + /// Reference to the set, needed for looking up the nodes. + pub(crate) tree: &'a StateBTreeSet, + pub(crate) _marker_lifetime: PhantomData<&'b K>, +} + +/// An iterator over the entries of a [`StateBTreeMap`]. +/// +/// Ordered by `K`. +/// +/// This `struct` is created by the [`iter`][StateBTreeMap::iter] method on +/// [`StateBTreeMap`]. See its documentation for more. +pub struct StateBTreeMapIter<'a, 'b, const M: usize, K, V, S> { + /// Iterator over the keys in the map. + pub(crate) key_iter: StateBTreeSetIter<'a, 'b, M, K, S>, + /// Reference to the map holding the values. + pub(crate) map: &'a StateMap, } From 461c6f0ca462a1cc35f3638361d3d1b29e44ebe1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Holm=20Gj=C3=B8rup?= Date: Fri, 1 Mar 2024 15:01:03 +0100 Subject: [PATCH 09/36] Improve documentation of statebtreemap --- concordium-std/src/impls.rs | 28 +++--- concordium-std/src/test_infrastructure.rs | 32 +++---- concordium-std/src/types.rs | 109 +++++++++++++++++++--- 3 files changed, 127 insertions(+), 42 deletions(-) diff --git a/concordium-std/src/impls.rs b/concordium-std/src/impls.rs index 8f4ef599..e4a75f5f 100644 --- a/concordium-std/src/impls.rs +++ b/concordium-std/src/impls.rs @@ -2325,12 +2325,14 @@ where StateMap::open(state_api, prefix) } - pub fn new_btree_set(&mut self) -> StateBTreeSet { + /// Create a new empty [`StateBTreeSet`]. + pub fn new_btree_set(&mut self) -> StateBTreeSet { let (state_api, prefix) = self.new_state_container(); StateBTreeSet::new(state_api, prefix) } - pub fn new_btree_map(&mut self) -> StateBTreeMap { + /// Create a new empty [`StateBTreeMap`]. + pub fn new_btree_map(&mut self) -> StateBTreeMap { StateBTreeMap { map: self.new_map(), ordered_set: self.new_btree_set(), @@ -3153,14 +3155,14 @@ impl Deserial for MetadataUrl { } } -impl Serial for StateBTreeMap { +impl Serial for StateBTreeMap { fn serial(&self, out: &mut W) -> Result<(), W::Err> { self.map.serial(out)?; self.ordered_set.serial(out) } } -impl DeserialWithState for StateBTreeMap { +impl DeserialWithState for StateBTreeMap { fn deserial_with_state(state: &S, source: &mut R) -> ParseResult { let map = DeserialWithState::deserial_with_state(state, source)?; let ordered_set = DeserialWithState::deserial_with_state(state, source)?; @@ -3171,7 +3173,7 @@ impl DeserialWithState for StateBTreeMa } } -impl Serial for StateBTreeSet { +impl Serial for StateBTreeSet { fn serial(&self, out: &mut W) -> Result<(), W::Err> { self.prefix.serial(out)?; self.root.serial(out)?; @@ -3180,7 +3182,7 @@ impl Serial for StateBTreeSet { } } -impl DeserialWithState for StateBTreeSet { +impl DeserialWithState for StateBTreeSet { fn deserial_with_state(state: &S, source: &mut R) -> ParseResult { let prefix = source.get()?; let root = source.get()?; @@ -3228,7 +3230,7 @@ impl Deserial for state_btree_internals::NodeId { } } -impl StateBTreeMap { +impl StateBTreeMap { /// Insert a key-value pair into the map. pub fn insert(&mut self, key: K, value: V) -> Option where @@ -3330,7 +3332,7 @@ impl StateBTreeMap { /// Create an iterator over the entries of [`StateBTreeMap`]. /// Ordered by `K`. - pub fn iter(&self) -> StateBTreeMapIter + pub fn iter(&self) -> StateBTreeMapIter where S: HasStateApi, { StateBTreeMapIter { @@ -3370,7 +3372,7 @@ impl StateBTreeMap { } } -impl StateBTreeSet { +impl StateBTreeSet { /// Construct a new [`StateBTreeMap`] given a unique prefix to use in the /// key-value store. pub(crate) fn new(state_api: S, prefix: StateItemPrefix) -> Self { @@ -3461,7 +3463,7 @@ impl StateBTreeSet { /// Get an iterator over the elements in the `StateBTreeSet`. The iterator /// returns elements in increasing order. - pub fn iter(&self) -> StateBTreeSetIter + pub fn iter(&self) -> StateBTreeSetIter where S: HasStateApi, { StateBTreeSetIter { @@ -3984,7 +3986,7 @@ impl Deserial for state_btree_internals::KeyWrapper { } } -impl<'a, 'b, const M: usize, K, S> Iterator for StateBTreeSetIter<'a, 'b, M, K, S> +impl<'a, 'b, const M: usize, K, S> Iterator for StateBTreeSetIter<'a, 'b, K, S, M> where 'a: 'b, K: Deserial, @@ -4022,7 +4024,7 @@ where fn size_hint(&self) -> (usize, Option) { (self.length, Some(self.length)) } } -impl<'a, 'b, const M: usize, K, V, S> Iterator for StateBTreeMapIter<'a, 'b, M, K, V, S> +impl<'a, 'b, const M: usize, K, V, S> Iterator for StateBTreeMapIter<'a, 'b, K, V, S, M> where 'a: 'b, K: Serialize, @@ -4041,7 +4043,7 @@ where fn size_hint(&self) -> (usize, Option) { self.key_iter.size_hint() } } -impl Deletable for StateBTreeMap +impl Deletable for StateBTreeMap where S: HasStateApi, K: Serialize, diff --git a/concordium-std/src/test_infrastructure.rs b/concordium-std/src/test_infrastructure.rs index 59a658b5..a688b169 100644 --- a/concordium-std/src/test_infrastructure.rs +++ b/concordium-std/src/test_infrastructure.rs @@ -2491,7 +2491,7 @@ mod test { #[test] fn test_btree_insert_6() { let mut state_builder = TestStateBuilder::new(); - let mut tree: StateBTreeSet<5, u32, _> = state_builder.new_btree_set(); + let mut tree: StateBTreeSet = state_builder.new_btree_set(); for n in 0..=5 { tree.insert(n); } @@ -2503,7 +2503,7 @@ mod test { #[test] fn test_btree_insert_0_7() { let mut state_builder = TestStateBuilder::new(); - let mut tree: StateBTreeSet<2, u32, _> = state_builder.new_btree_set(); + let mut tree: StateBTreeSet = state_builder.new_btree_set(); for n in 0..=7 { tree.insert(n); } @@ -2515,7 +2515,7 @@ mod test { #[test] fn test_btree_insert_7_no_order() { let mut state_builder = TestStateBuilder::new(); - let mut tree: StateBTreeSet<2, u32, _> = state_builder.new_btree_set(); + let mut tree: StateBTreeSet = state_builder.new_btree_set(); tree.insert(0); tree.insert(1); @@ -2534,7 +2534,7 @@ mod test { #[test] fn test_btree_higher() { let mut state_builder = TestStateBuilder::new(); - let mut tree: StateBTreeSet<2, u32, _> = state_builder.new_btree_set(); + let mut tree: StateBTreeSet = state_builder.new_btree_set(); tree.insert(1); tree.insert(2); tree.insert(3); @@ -2549,7 +2549,7 @@ mod test { #[test] fn test_btree_lower() { let mut state_builder = TestStateBuilder::new(); - let mut tree: StateBTreeSet<2, u32, _> = state_builder.new_btree_set(); + let mut tree: StateBTreeSet = state_builder.new_btree_set(); tree.insert(1); tree.insert(2); tree.insert(3); @@ -2564,7 +2564,7 @@ mod test { #[test] fn test_btree_insert_1000() { let mut state_builder = TestStateBuilder::new(); - let mut tree: StateBTreeSet<2, u32, _> = state_builder.new_btree_set(); + let mut tree: StateBTreeSet = state_builder.new_btree_set(); for n in 0..500 { tree.insert(n); } @@ -2583,7 +2583,7 @@ mod test { #[test] fn test_btree_7_get_8() { let mut state_builder = TestStateBuilder::new(); - let mut tree: StateBTreeSet<2, u32, _> = state_builder.new_btree_set(); + let mut tree: StateBTreeSet = state_builder.new_btree_set(); for n in 0..=7 { tree.insert(n); } @@ -2594,7 +2594,7 @@ mod test { #[test] fn test_btree_remove_from_one_node_tree() { let mut state_builder = TestStateBuilder::new(); - let mut tree: StateBTreeSet<2, u32, _> = state_builder.new_btree_set(); + let mut tree: StateBTreeSet = state_builder.new_btree_set(); for n in 0..3 { tree.insert(n); } @@ -2608,7 +2608,7 @@ mod test { #[test] fn test_btree_remove_only_key_lower_leaf_in_three_node() { let mut state_builder = TestStateBuilder::new(); - let mut tree: StateBTreeSet<2, u32, _> = state_builder.new_btree_set(); + let mut tree: StateBTreeSet = state_builder.new_btree_set(); for n in 0..4 { tree.insert(n); } @@ -2624,7 +2624,7 @@ mod test { #[test] fn test_btree_remove_only_key_higher_leaf_in_three_node() { let mut state_builder = TestStateBuilder::new(); - let mut tree: StateBTreeSet<2, u32, _> = state_builder.new_btree_set(); + let mut tree: StateBTreeSet = state_builder.new_btree_set(); for n in (0..4).into_iter().rev() { tree.insert(n); } @@ -2639,7 +2639,7 @@ mod test { #[test] fn test_btree_remove_from_higher_leaf_in_three_node_taking_from_sibling() { let mut state_builder = TestStateBuilder::new(); - let mut tree: StateBTreeSet<2, u32, _> = state_builder.new_btree_set(); + let mut tree: StateBTreeSet = state_builder.new_btree_set(); for n in (0..4).into_iter().rev() { tree.insert(n); } @@ -2651,7 +2651,7 @@ mod test { #[test] fn test_btree_remove_from_lower_leaf_in_three_node_taking_from_sibling() { let mut state_builder = TestStateBuilder::new(); - let mut tree: StateBTreeSet<2, u32, _> = state_builder.new_btree_set(); + let mut tree: StateBTreeSet = state_builder.new_btree_set(); for n in 0..4 { tree.insert(n); } @@ -2666,7 +2666,7 @@ mod test { #[test] fn test_btree_remove_from_root_in_three_node_causing_merge() { let mut state_builder = TestStateBuilder::new(); - let mut tree: StateBTreeSet<2, u32, _> = state_builder.new_btree_set(); + let mut tree: StateBTreeSet = state_builder.new_btree_set(); for n in 0..4 { tree.insert(n); } @@ -2680,7 +2680,7 @@ mod test { #[test] fn test_btree_remove_from_root_in_three_node_taking_key_from_higher_child() { let mut state_builder = TestStateBuilder::new(); - let mut tree: StateBTreeSet<2, u32, _> = state_builder.new_btree_set(); + let mut tree: StateBTreeSet = state_builder.new_btree_set(); for n in 0..4 { tree.insert(n); } @@ -2692,7 +2692,7 @@ mod test { #[test] fn test_btree_remove_from_root_in_three_node_taking_key_from_lower_child() { let mut state_builder = TestStateBuilder::new(); - let mut tree: StateBTreeSet<2, u32, _> = state_builder.new_btree_set(); + let mut tree: StateBTreeSet = state_builder.new_btree_set(); for n in (0..4).into_iter().rev() { tree.insert(n); } @@ -2704,7 +2704,7 @@ mod test { #[test] fn test_btree_iter() { let mut state_builder = TestStateBuilder::new(); - let mut tree: StateBTreeSet<2, u32, _> = state_builder.new_btree_set(); + let mut tree: StateBTreeSet = state_builder.new_btree_set(); let keys: Vec = (0..15).into_iter().collect(); for &k in &keys { tree.insert(k); diff --git a/concordium-std/src/types.rs b/concordium-std/src/types.rs index 9ce4b8a6..36aef696 100644 --- a/concordium-std/src/types.rs +++ b/concordium-std/src/types.rs @@ -384,6 +384,11 @@ impl<'a, V> crate::ops::Deref for StateRef<'a, V> { /// that the value is properly stored in the contract state maintained by the /// node. pub struct StateRefMut<'a, V: Serial, S: HasStateApi> { + /// This is set as an `UnsafeCell`, to be able to get a mutable reference to + /// the entry without `StateRefMut` being mutable. + /// The `Option` allows for having an internal method destroying the + /// `StateRefMut` into its raw parst without `Drop` causing a write to the + /// contract state. pub(crate) entry: UnsafeCell>, pub(crate) state_api: S, pub(crate) lazy_value: UnsafeCell>, @@ -1263,18 +1268,96 @@ pub struct MetadataUrl { /// An ordered map based on [B-Tree](https://en.wikipedia.org/wiki/B-tree), where /// each node is stored separately in the low-level key-value store. /// -/// It can be seen as an extension adding the tracking of ordering on top of -/// [`StateMap`] providing functions such as [`Self::higher`] and +/// It can be seen as an extension adding the tracking the ordering of the keys +/// on top of [`StateMap`] providing functions such as [`Self::higher`] and /// [`Self::lower`]. -/// This adds overhead when mutating the map. +/// This adds some overhead when inserting and deleting entries from the map +/// compared to [`StateMap`] and [`StateMap`] is prefered if ordering is not +/// needed. +/// +/// The byte size of the serialized keys (`K`) influences costs of operations, +/// and it is more cost-efficent when keys are kept as few bytes as possible. +/// +/// New maps can be constructed using the +/// [`new_btree_map`][StateBuilder::new_btree_map] method on the +/// [`StateBuilder`]. +/// +/// ``` +/// # use concordium_std::*; +/// # use concordium_std::test_infrastructure::*; +/// # let mut state_builder = TestStateBuilder::new(); +/// /// In an init method: +/// let mut map1 = state_builder.new_btree_map(); +/// # map1.insert(0u8, 1u8); // Specifies type of map. +/// +/// # let mut host = TestHost::new((), state_builder); +/// /// In a receive method: +/// let mut map2 = host.state_builder().new_btree_map(); +/// # map2.insert(0u16, 1u16); +/// ``` +/// +/// ## Type parameters +/// +/// The map `StateBTreeMap` is parametrized by the types: +/// - `K`: Keys used in the map. Most operations on the map require this to +/// implement [`Serialize`](crate::Serialize). Keys cannot contain references +/// to the low-level state, such as types containing [`StateBox`], +/// [`StateMap`] and [`StateSet`]. +/// - `V`: Values stored in the map. Most operations on the map require this to +/// implement [`Serial`](crate::Serial) and +/// [`DeserialWithState`](crate::DeserialWithState). +/// - `S`: The low-level state implementation used, this allows for mocking the +/// state API in unit tests, see +/// [`TestStateApi`](crate::test_infrastructure::TestStateApi). +/// - `M`: A `const usize` determining the _minimum degree_ of the B-tree. +/// _Must_ be a value of `2` or above for the tree to be work. This can be +/// used to tweak the height of the tree vs size of each node in the tree. The +/// default is set based on benchmarks. /// -/// TODO Document how to construct it. -/// TODO Document size of the serialized key matters. -/// TODO Document the meaning of generics and restrictions on M. /// TODO Document the complexity of the basic operations. -pub struct StateBTreeMap { +/// +/// ## **Caution** +/// +/// `StateBTreeMap`s must be explicitly deleted when they are no longer needed, +/// otherwise they will remain in the contract's state, albeit unreachable. +/// +/// ```no_run +/// # use concordium_std::*; +/// struct MyState { +/// inner: StateBTreeMap, +/// } +/// fn incorrect_replace(state_builder: &mut StateBuilder, state: &mut MyState) { +/// // The following is incorrect. The old value of `inner` is not properly deleted. +/// // from the state. +/// state.inner = state_builder.new_btree_map(); // ⚠️ +/// } +/// ``` +/// Instead, either the map should be [cleared](StateBTreeMap::clear) or +/// explicitly deleted. +/// +/// ```no_run +/// # use concordium_std::*; +/// # struct MyState { +/// # inner: StateBTreeMap +/// # } +/// fn correct_replace(state_builder: &mut StateBuilder, state: &mut MyState) { +/// state.inner.clear_flat(); +/// } +/// ``` +/// Or alternatively +/// ```no_run +/// # use concordium_std::*; +/// # struct MyState { +/// # inner: StateBTreeMap +/// # } +/// fn correct_replace(state_builder: &mut StateBuilder, state: &mut MyState) { +/// let old_map = mem::replace(&mut state.inner, state_builder.new_map()); +/// old_map.delete() +/// } +/// ``` +pub struct StateBTreeMap { pub(crate) map: StateMap, - pub(crate) ordered_set: StateBTreeSet, + pub(crate) ordered_set: StateBTreeSet, } /// An ordered set based on [B-Tree](https://en.wikipedia.org/wiki/B-tree), where @@ -1284,7 +1367,7 @@ pub struct StateBTreeMap { /// TODO Document size of the serialized key matters. /// TODO Document the meaning of generics and restrictions on M. /// TODO Document the complexity of the basic operations. -pub struct StateBTreeSet { +pub struct StateBTreeSet { /// Type marker for the key. pub(crate) _marker_key: PhantomData, /// The unique prefix to use for this map in the key-value store. @@ -1344,7 +1427,7 @@ pub(crate) mod state_btree_internals { /// /// This `struct` is created by the [`iter`][StateBTreeSet::iter] method on /// [`StateBTreeSet`]. See its documentation for more. -pub struct StateBTreeSetIter<'a, 'b, const M: usize, K, S> { +pub struct StateBTreeSetIter<'a, 'b, K, S, const M: usize> { /// The number of elements left to iterate. pub(crate) length: usize, /// Reference to a node in the tree to load and iterate before the current @@ -1354,7 +1437,7 @@ pub struct StateBTreeSetIter<'a, 'b, const M: usize, K, S> { pub(crate) depth_first_stack: Vec<(state_btree_internals::Node>, usize)>, /// Reference to the set, needed for looking up the nodes. - pub(crate) tree: &'a StateBTreeSet, + pub(crate) tree: &'a StateBTreeSet, pub(crate) _marker_lifetime: PhantomData<&'b K>, } @@ -1364,9 +1447,9 @@ pub struct StateBTreeSetIter<'a, 'b, const M: usize, K, S> { /// /// This `struct` is created by the [`iter`][StateBTreeMap::iter] method on /// [`StateBTreeMap`]. See its documentation for more. -pub struct StateBTreeMapIter<'a, 'b, const M: usize, K, V, S> { +pub struct StateBTreeMapIter<'a, 'b, K, V, S, const M: usize> { /// Iterator over the keys in the map. - pub(crate) key_iter: StateBTreeSetIter<'a, 'b, M, K, S>, + pub(crate) key_iter: StateBTreeSetIter<'a, 'b, K, S, M>, /// Reference to the map holding the values. pub(crate) map: &'a StateMap, } From 7b676ddf98ac29bfee89a16daecf73f537995629 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Holm=20Gj=C3=B8rup?= Date: Fri, 1 Mar 2024 21:56:51 +0100 Subject: [PATCH 10/36] Document StateBTreeMap and StateBTreeSet + add few more methods --- concordium-std/src/impls.rs | 105 +++++++++++++---- concordium-std/src/test_infrastructure.rs | 12 +- concordium-std/src/types.rs | 134 +++++++++++++++++----- 3 files changed, 194 insertions(+), 57 deletions(-) diff --git a/concordium-std/src/impls.rs b/concordium-std/src/impls.rs index e4a75f5f..fb074d15 100644 --- a/concordium-std/src/impls.rs +++ b/concordium-std/src/impls.rs @@ -3306,9 +3306,17 @@ impl StateBTreeMap { } } + /// Returns `true` if the map contains a value for the specified key. + pub fn contains_key(&self, key: &K) -> bool + where + K: Serialize + Ord, + S: HasStateApi, { + self.ordered_set.contains(key) + } + /// Returns the smallest key in the map, which is strictly larger than the /// provided key. `None` meaning no such key is present in the map. - pub fn higher(&self, key: &K) -> Option + pub fn higher(&self, key: &K) -> Option> where S: HasStateApi, K: Serialize + Ord, { @@ -3317,13 +3325,31 @@ impl StateBTreeMap { /// Returns the largest key in the map, which is strictly smaller than the /// provided key. `None` meaning no such key is present in the map. - pub fn lower(&self, key: &K) -> Option + pub fn lower(&self, key: &K) -> Option> where S: HasStateApi, K: Serialize + Ord, { self.ordered_set.lower(key) } + /// Returns a reference to the first key in the map, if any. This key is + /// always the minimum of all keys in the map. + pub fn first_key(&self) -> Option> + where + S: HasStateApi, + K: Serialize + Ord, { + self.ordered_set.first() + } + + /// Returns a reference to the last key in the map, if any. This key is + /// always the maximum of all keys in the map. + pub fn last_key(&self) -> Option> + where + S: HasStateApi, + K: Serialize + Ord, { + self.ordered_set.last() + } + /// Return the number of elements in the map. pub fn len(&self) -> u32 { self.ordered_set.len() } @@ -3373,7 +3399,7 @@ impl StateBTreeMap { } impl StateBTreeSet { - /// Construct a new [`StateBTreeMap`] given a unique prefix to use in the + /// Construct a new [`StateBTreeSet`] given a unique prefix to use in the /// key-value store. pub(crate) fn new(state_api: S, prefix: StateItemPrefix) -> Self { Self { @@ -3455,10 +3481,10 @@ impl StateBTreeSet { } } - /// Return the number of elements in the map. + /// Return the number of elements in the set. pub fn len(&self) -> u32 { self.len } - /// Returns `true` is the map contains no elements. + /// Returns `true` is the set contains no elements. pub fn is_empty(&self) -> bool { self.root.is_none() } /// Get an iterator over the elements in the `StateBTreeSet`. The iterator @@ -3475,7 +3501,7 @@ impl StateBTreeSet { } } - /// Clears the map, removing all elements. + /// Clears the set, removing all elements. pub fn clear(&mut self) where S: HasStateApi, { @@ -3490,9 +3516,9 @@ impl StateBTreeSet { self.state_api.delete_prefix(&self.prefix).unwrap_abort(); } - /// Returns the smallest key in the map, which is strictly larger than the - /// provided key. `None` meaning no such key is present in the map. - pub fn higher(&self, key: &K) -> Option + /// Returns the smallest key in the set, which is strictly larger than the + /// provided key. `None` meaning no such key is present in the set. + pub fn higher(&self, key: &K) -> Option> where S: HasStateApi, K: Serialize + Ord, { @@ -3501,7 +3527,7 @@ impl StateBTreeSet { }; let mut node = self.get_node(root_node_id); - let mut higher_so_far: Option = None; + let mut higher_so_far = None; loop { let higher_key_index = match node.keys.binary_search(key) { Ok(index) => index + 1, @@ -3510,13 +3536,13 @@ impl StateBTreeSet { if node.is_leaf() { return if higher_key_index < node.keys.len() { - Some(node.keys.swap_remove(higher_key_index)) + Some(StateRef::new(node.keys.swap_remove(higher_key_index))) } else { higher_so_far }; } else { if higher_key_index < node.keys.len() { - higher_so_far = Some(node.keys.swap_remove(higher_key_index)) + higher_so_far = Some(StateRef::new(node.keys.swap_remove(higher_key_index))) } let child_node_id = node.children[higher_key_index]; @@ -3525,9 +3551,9 @@ impl StateBTreeSet { } } - /// Returns the largest key in the map, which is strictly smaller than the - /// provided key. `None` meaning no such key is present in the map. - pub fn lower(&self, key: &K) -> Option + /// Returns the largest key in the set, which is strictly smaller than the + /// provided key. `None` meaning no such key is present in the set. + pub fn lower(&self, key: &K) -> Option> where S: HasStateApi, K: Serialize + Ord, { @@ -3536,7 +3562,7 @@ impl StateBTreeSet { }; let mut node = self.get_node(root_node_id); - let mut lower_so_far: Option = None; + let mut lower_so_far = None; loop { let lower_key_index = match node.keys.binary_search(key) { Ok(index) => index, @@ -3549,11 +3575,11 @@ impl StateBTreeSet { } else { // lower_key_index cannot be 0 in this case, since the binary search will only // return 0 in the true branch above. - Some(node.keys.swap_remove(lower_key_index - 1)) + Some(StateRef::new(node.keys.swap_remove(lower_key_index - 1))) }; } else { if lower_key_index > 0 { - lower_so_far = Some(node.keys.swap_remove(lower_key_index - 1)); + lower_so_far = Some(StateRef::new(node.keys.swap_remove(lower_key_index - 1))); } let child_node_id = node.children[lower_key_index]; node = self.get_node(child_node_id) @@ -3561,6 +3587,42 @@ impl StateBTreeSet { } } + /// Returns a reference to the first key in the set, if any. This key is + /// always the minimum of all keys in the set. + pub fn first(&self) -> Option> + where + S: HasStateApi, + K: Serialize + Ord, { + let Some(root_node_id) = self.root else { + return None; + }; + let mut root = self.get_node(root_node_id); + if root.is_leaf() { + Some(StateRef::new(root.keys.swap_remove(0))) + } else { + Some(StateRef::new(self.get_lowest_key(&root, 0))) + } + } + + /// Returns a reference to the last key in the set, if any. This key is + /// always the maximum of all keys in the set. + pub fn last(&self) -> Option> + where + S: HasStateApi, + K: Serialize + Ord, { + let Some(root_node_id) = self.root else { + return None; + }; + let mut root = self.get_node(root_node_id); + if root.is_leaf() { + Some(StateRef::new(root.keys.pop().unwrap_abort())) + } else { + Some(StateRef::new(self.get_highest_key(&root, root.children.len() - 1))) + } + } + + /// Remove a key from the set. + /// Returns whether such an element was present. pub fn remove(&mut self, key: &K) -> bool where K: Ord + Serialize, @@ -3630,9 +3692,6 @@ impl StateBTreeSet { // one child, moving the key into the child and try to remove from this. self.merge(&mut node, index, &mut left_child, right_child); node = left_child; - // FIXME: For some reason this is needed to cause loading the node for the - // next iteration. - let _ = node.get(); continue; } Err(index) => index, @@ -3737,6 +3796,8 @@ impl StateBTreeSet { } /// Internal function for getting the highest key in a subtree. + // FIXME: In the context where this function is called, the child node is + // already loaded and should be reused, probably resulting in `K: Clone`. fn get_highest_key(&self, node: &state_btree_internals::Node, child_index: usize) -> K where K: Ord + Serialize, @@ -3752,6 +3813,8 @@ impl StateBTreeSet { } /// Internal function for getting the lowest key in a subtree. + // FIXME: In the context where this function is called, the child node is + // already loaded and should be reused, probably resulting in `K: Clone`. fn get_lowest_key(&self, node: &state_btree_internals::Node, child_index: usize) -> K where K: Ord + Serialize, diff --git a/concordium-std/src/test_infrastructure.rs b/concordium-std/src/test_infrastructure.rs index a688b169..a4d7eae1 100644 --- a/concordium-std/src/test_infrastructure.rs +++ b/concordium-std/src/test_infrastructure.rs @@ -2541,9 +2541,9 @@ mod test { tree.insert(4); tree.insert(5); tree.insert(7); - assert_eq!(tree.higher(&3), Some(4)); - assert_eq!(tree.higher(&5), Some(7)); - assert_eq!(tree.higher(&7), None) + assert_eq!(tree.higher(&3).as_deref(), Some(&4)); + assert_eq!(tree.higher(&5).as_deref(), Some(&7)); + assert_eq!(tree.higher(&7).as_deref(), None) } #[test] @@ -2556,9 +2556,9 @@ mod test { tree.insert(4); tree.insert(5); tree.insert(7); - assert_eq!(tree.lower(&3), Some(2)); - assert_eq!(tree.lower(&7), Some(5)); - assert_eq!(tree.lower(&1), None) + assert_eq!(tree.lower(&3).as_deref(), Some(&2)); + assert_eq!(tree.lower(&7).as_deref(), Some(&5)); + assert_eq!(tree.lower(&1).as_deref(), None) } #[test] diff --git a/concordium-std/src/types.rs b/concordium-std/src/types.rs index 36aef696..af4bd56d 100644 --- a/concordium-std/src/types.rs +++ b/concordium-std/src/types.rs @@ -1269,32 +1269,20 @@ pub struct MetadataUrl { /// each node is stored separately in the low-level key-value store. /// /// It can be seen as an extension adding the tracking the ordering of the keys -/// on top of [`StateMap`] providing functions such as [`Self::higher`] and -/// [`Self::lower`]. +/// on top of [`StateMap`] providing functions such as [`higher`](Self::higher) +/// and [`lower`](Self::lower). /// This adds some overhead when inserting and deleting entries from the map -/// compared to [`StateMap`] and [`StateMap`] is prefered if ordering is not -/// needed. +/// compared to [`StateMap`]. /// -/// The byte size of the serialized keys (`K`) influences costs of operations, -/// and it is more cost-efficent when keys are kept as few bytes as possible. +/// | Operation | Performance | +/// |-------------------------------------------------|---------------| +/// | [`get`](Self::get) / [`get_mut`](Self::get_mut) | O(k) | +/// | [`insert`](Self::insert) | O(k + log(n)) | +/// | [`remove`](Self::remove) | O(k + log(n)) | +/// | [`higher`](Self::higher)/[`lower`](Self::lower) | O(k + log(n)) | /// -/// New maps can be constructed using the -/// [`new_btree_map`][StateBuilder::new_btree_map] method on the -/// [`StateBuilder`]. -/// -/// ``` -/// # use concordium_std::*; -/// # use concordium_std::test_infrastructure::*; -/// # let mut state_builder = TestStateBuilder::new(); -/// /// In an init method: -/// let mut map1 = state_builder.new_btree_map(); -/// # map1.insert(0u8, 1u8); // Specifies type of map. -/// -/// # let mut host = TestHost::new((), state_builder); -/// /// In a receive method: -/// let mut map2 = host.state_builder().new_btree_map(); -/// # map2.insert(0u16, 1u16); -/// ``` +/// Where `k` is the byte size of the serialized keys and `n` is the number of +/// entries in the map. /// /// ## Type parameters /// @@ -1314,9 +1302,27 @@ pub struct MetadataUrl { /// used to tweak the height of the tree vs size of each node in the tree. The /// default is set based on benchmarks. /// -/// TODO Document the complexity of the basic operations. +/// ## Usage /// -/// ## **Caution** +/// New maps can be constructed using the +/// [`new_btree_map`][StateBuilder::new_btree_map] method on the +/// [`StateBuilder`]. +/// +/// ``` +/// # use concordium_std::*; +/// # use concordium_std::test_infrastructure::*; +/// # let mut state_builder = TestStateBuilder::new(); +/// /// In an init method: +/// let mut map1 = state_builder.new_btree_map(); +/// # map1.insert(0u8, 1u8); // Specifies type of map. +/// +/// # let mut host = TestHost::new((), state_builder); +/// /// In a receive method: +/// let mut map2 = host.state_builder().new_btree_map(); +/// # map2.insert(0u16, 1u16); +/// ``` +/// +/// ### **Caution** /// /// `StateBTreeMap`s must be explicitly deleted when they are no longer needed, /// otherwise they will remain in the contract's state, albeit unreachable. @@ -1351,7 +1357,7 @@ pub struct MetadataUrl { /// # inner: StateBTreeMap /// # } /// fn correct_replace(state_builder: &mut StateBuilder, state: &mut MyState) { -/// let old_map = mem::replace(&mut state.inner, state_builder.new_map()); +/// let old_map = mem::replace(&mut state.inner, state_builder.new_btree_map()); /// old_map.delete() /// } /// ``` @@ -1363,10 +1369,78 @@ pub struct StateBTreeMap { /// An ordered set based on [B-Tree](https://en.wikipedia.org/wiki/B-tree), where /// each node is stored separately in the low-level key-value store. /// -/// TODO Document how to construct it. -/// TODO Document size of the serialized key matters. -/// TODO Document the meaning of generics and restrictions on M. -/// TODO Document the complexity of the basic operations. +/// | Operation | Performance | +/// |-------------------------------------------------|---------------| +/// | [`contains`](Self::contains) | O(k + log(n)) | +/// | [`insert`](Self::insert) | O(k + log(n)) | +/// | [`remove`](Self::remove) | O(k + log(n)) | +/// | [`higher`](Self::higher)/[`lower`](Self::lower) | O(k + log(n)) | +/// +/// Where `k` is the byte size of the serialized keys and `n` is the number of +/// entries in the map. +/// +/// ## Type parameters +/// +/// The map `StateBTreeSet` is parametrized by the types: +/// - `K`: Keys used in the set. Most operations on the set require this to +/// implement [`Serialize`](crate::Serialize). Keys cannot contain references +/// to the low-level state, such as types containing [`StateBox`], +/// [`StateMap`] and [`StateSet`]. +/// - `S`: The low-level state implementation used, this allows for mocking the +/// state API in unit tests, see +/// [`TestStateApi`](crate::test_infrastructure::TestStateApi). +/// - `M`: A `const usize` determining the _minimum degree_ of the B-tree. +/// _Must_ be a value of `2` or above for the tree to be work. This can be +/// used to tweak the height of the tree vs size of each node in the tree. The +/// default is set based on benchmarks. +/// +/// ## Usage +/// +/// New sets can be constructed using the +/// [`new_btree_set`][StateBuilder::new_btree_set] method on the +/// [`StateBuilder`]. +/// +/// ``` +/// # use concordium_std::*; +/// # use concordium_std::test_infrastructure::*; +/// # let mut state_builder = TestStateBuilder::new(); +/// /// In an init method: +/// let mut map1 = state_builder.new_btree_set(); +/// # map1.insert(0u8); // Specifies type of map. +/// +/// # let mut host = TestHost::new((), state_builder); +/// /// In a receive method: +/// let mut map2 = host.state_builder().new_btree_set(); +/// # map2.insert(0u16); +/// ``` +/// +/// ### **Caution** +/// +/// `StateBTreeSet`s must be explicitly deleted when they are no longer needed, +/// otherwise they will remain in the contract's state, albeit unreachable. +/// +/// ```no_run +/// # use concordium_std::*; +/// struct MyState { +/// inner: StateBTreeSet, +/// } +/// fn incorrect_replace(state_builder: &mut StateBuilder, state: &mut MyState) { +/// // The following is incorrect. The old value of `inner` is not properly deleted. +/// // from the state. +/// state.inner = state_builder.new_btree_set(); // ⚠️ +/// } +/// ``` +/// Instead, the set should be [cleared](StateBTreeSet::clear): +/// +/// ```no_run +/// # use concordium_std::*; +/// # struct MyState { +/// # inner: StateBTreeSet +/// # } +/// fn correct_replace(state_builder: &mut StateBuilder, state: &mut MyState) { +/// state.inner.clear(); +/// } +/// ``` pub struct StateBTreeSet { /// Type marker for the key. pub(crate) _marker_key: PhantomData, From 78352b0f49c052e756a8d61a294842c3cbb659f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Holm=20Gj=C3=B8rup?= Date: Mon, 4 Mar 2024 09:13:17 +0100 Subject: [PATCH 11/36] Fix clippy feedback --- concordium-std/src/impls.rs | 93 +++++++++++++---------- concordium-std/src/test_infrastructure.rs | 32 ++++---- concordium-std/src/types.rs | 2 + 3 files changed, 71 insertions(+), 56 deletions(-) diff --git a/concordium-std/src/impls.rs b/concordium-std/src/impls.rs index fb074d15..7375ed24 100644 --- a/concordium-std/src/impls.rs +++ b/concordium-std/src/impls.rs @@ -2326,19 +2326,35 @@ where } /// Create a new empty [`StateBTreeSet`]. - pub fn new_btree_set(&mut self) -> StateBTreeSet { + pub fn new_btree_set(&mut self) -> StateBTreeSet { let (state_api, prefix) = self.new_state_container(); StateBTreeSet::new(state_api, prefix) } /// Create a new empty [`StateBTreeMap`]. - pub fn new_btree_map(&mut self) -> StateBTreeMap { + pub fn new_btree_map(&mut self) -> StateBTreeMap { StateBTreeMap { map: self.new_map(), ordered_set: self.new_btree_set(), } } + /// Create a new empty [`StateBTreeSet`], setting the minimum degree `M` of + /// the B-Tree explicitly. + pub fn new_btree_set_degree(&mut self) -> StateBTreeSet { + let (state_api, prefix) = self.new_state_container(); + StateBTreeSet::new(state_api, prefix) + } + + /// Create a new empty [`StateBTreeMap`], setting the minimum degree `M` of + /// the B-Tree explicitly. + pub fn new_btree_map_degree(&mut self) -> StateBTreeMap { + StateBTreeMap { + map: self.new_map(), + ordered_set: self.new_btree_set_degree(), + } + } + /// Create a new empty [`StateSet`]. pub fn new_set(&mut self) -> StateSet { let (state_api, prefix) = self.new_state_container(); @@ -3238,13 +3254,11 @@ impl StateBTreeMap { K: Serialize + Ord, V: Serial + DeserialWithState, { let old_value_option = self.map.insert_borrowed(&key, value); - if old_value_option.is_none() { - if !self.ordered_set.insert(key) { - // Inconsistency between the map and ordered_set. - crate::trap(); - } + if old_value_option.is_none() && !self.ordered_set.insert(key) { + // Inconsistency between the map and ordered_set. + crate::trap(); } - return old_value_option; + old_value_option } /// Remove a key from the map, returning the value at the key if the key was @@ -3422,7 +3436,7 @@ impl StateBTreeSet { K: Serialize + Ord, { let Some(root_id) = self.root else { let node_id = { - let (node_id, _node) = self.create_node(vec![key], Vec::new()); + let (node_id, _node) = self.create_node(crate::vec![key], Vec::new()); node_id }; self.root = Some(node_id); @@ -3441,7 +3455,7 @@ impl StateBTreeSet { return false; } // The root node is full, so we construct a new root node. - let (new_root_id, mut new_root) = self.create_node(Vec::new(), vec![root_id]); + let (new_root_id, mut new_root) = self.create_node(Vec::new(), crate::vec![root_id]); self.root = Some(new_root_id); // The old root node is now a child node. let mut child = root_node; @@ -3496,7 +3510,7 @@ impl StateBTreeSet { length: self.len.try_into().unwrap_abort(), next_node: self.root, depth_first_stack: Vec::new(), - tree: &self, + tree: self, _marker_lifetime: Default::default(), } } @@ -3642,9 +3656,8 @@ impl StateBTreeSet { // Found the key in this node and the node is a leaf, meaning we simply // remove it. // This will not violate the minimum keys invariant, since a node - // ensures a child can spare a key before - // iteration and the root node is not part of - // the invariant. + // ensures a child can spare a key before iteration and the root node is + // not part of the invariant. node.keys.remove(index); break true; } @@ -3654,7 +3667,7 @@ impl StateBTreeSet { // If the child with smaller keys can spare a key, we take the highest // key from it, changing the loop to be deleting its highest key from // this child. - node.keys[index] = self.get_highest_key(&mut node, index); + node.keys[index] = self.get_highest_key(&node, index); let new_key_to_delete = { // To avoid having to clone the key, we store changes to the node // and unwrap it from the StateRefMut, disabling further mutations @@ -3674,7 +3687,7 @@ impl StateBTreeSet { // If the child with larger keys can spare a key, we take the lowest // key from it, changing the loop to be deleting its highest key from // this child. - node.keys[index] = self.get_lowest_key(&mut node, index + 1); + node.keys[index] = self.get_lowest_key(&node, index + 1); let new_key_to_delete = { // To avoid having to clone the key, we store changes to the node // and unwrap it from the StateRefMut, disabling further mutations @@ -3688,8 +3701,9 @@ impl StateBTreeSet { node = right_child; continue; } - // No child on either side of the key can spare a key, so we merge them into - // one child, moving the key into the child and try to remove from this. + // No child on either side of the key can spare a key at this point, so we + // merge them into one child, moving the key into the merged child and try + // to remove from this. self.merge(&mut node, index, &mut left_child, right_child); node = left_child; continue; @@ -3729,7 +3743,6 @@ impl StateBTreeSet { .children .insert(0, smaller_sibling.children.pop().unwrap_abort()); } - //drop(child); break 'increase_child child; } Some(smaller_sibling) @@ -3751,7 +3764,6 @@ impl StateBTreeSet { if !child.is_leaf() { child.children.push(larger_sibling.children.remove(0)); } - //drop(child); break 'increase_child child; } Some(larger_sibling) @@ -3766,7 +3778,8 @@ impl StateBTreeSet { self.merge(&mut node, index - 1, &mut sibling, child); sibling } else { - // Unreachable code. + // Unreachable code, since M must be 2 or larger (the minimum degree), a + // child node must have at least one sibling. crate::trap(); } } @@ -3830,32 +3843,32 @@ impl StateBTreeSet { } /// Moving key at `index` from the node to the lower child and then merges - /// this child with the content of its higher sibling, deleting the sibling. + /// this child with the content of its larger sibling, deleting the sibling. /// /// Assumes: - /// - `node` have a child at `index` and `index + 1`. - /// - Both children are at minimum number of keys. + /// - `parent_node` have a child at `index` and `index + 1`. + /// - Both children are at minimum number of keys (`M - 1`). fn merge( &mut self, - node: &mut state_btree_internals::Node, // parent node. - child_index: usize, /* index of the smaller child in the - * parent node. */ - child: &mut state_btree_internals::Node, // smaller child. - mut larger_child: StateRefMut, S>, // smaller child. + parent_node: &mut state_btree_internals::Node, + index: usize, + child: &mut state_btree_internals::Node, + mut larger_child: StateRefMut, S>, ) where K: Ord + Serialize, S: HasStateApi, { - child.keys.push(node.keys.remove(child_index)); + let parent_key = parent_node.keys.remove(index); + parent_node.children.remove(index + 1); + child.keys.push(parent_key); child.keys.append(&mut larger_child.keys); child.children.append(&mut larger_child.children); - node.children.remove(child_index + 1); self.delete_node(larger_child); } /// Internal function for constructing a node. It will incrementing the next /// node ID and create an entry in the smart contract key-value store. - fn create_node<'a, 'b>( - &'a mut self, + fn create_node<'b>( + &mut self, keys: Vec, children: Vec, ) -> (state_btree_internals::NodeId, StateRefMut<'b, state_btree_internals::Node, S>) @@ -3928,8 +3941,8 @@ impl StateBTreeSet { /// and child after the provided child_index. /// /// Returns the newly created node. - fn split_child<'a, 'b>( - &'a mut self, + fn split_child<'b>( + &mut self, node: &mut state_btree_internals::Node, child_index: usize, child: &mut state_btree_internals::Node, @@ -3954,8 +3967,8 @@ impl StateBTreeSet { /// Internal function for looking up a node in the tree. /// This assumes the node is present and traps if this is not the case. - fn get_node<'a, 'b, Key>( - &'a self, + fn get_node( + &self, node_id: state_btree_internals::NodeId, ) -> state_btree_internals::Node where @@ -3968,8 +3981,8 @@ impl StateBTreeSet { /// Internal function for looking up a node, providing mutable access. /// This assumes the node is present and traps if this is not the case. - fn get_node_mut<'a, 'b>( - &'a mut self, + fn get_node_mut<'b>( + &mut self, node_id: state_btree_internals::NodeId, ) -> StateRefMut<'b, state_btree_internals::Node, S> where @@ -4015,7 +4028,7 @@ impl state_btree_internals::NodeId { /// Return a copy of the NodeId, then increments itself. pub(crate) fn copy_then_increment(&mut self) -> Self { - let current = self.clone(); + let current = *self; self.id += 1; current } diff --git a/concordium-std/src/test_infrastructure.rs b/concordium-std/src/test_infrastructure.rs index a4d7eae1..92762040 100644 --- a/concordium-std/src/test_infrastructure.rs +++ b/concordium-std/src/test_infrastructure.rs @@ -2491,7 +2491,7 @@ mod test { #[test] fn test_btree_insert_6() { let mut state_builder = TestStateBuilder::new(); - let mut tree: StateBTreeSet = state_builder.new_btree_set(); + let mut tree = state_builder.new_btree_set_degree::<5, _>(); for n in 0..=5 { tree.insert(n); } @@ -2503,7 +2503,7 @@ mod test { #[test] fn test_btree_insert_0_7() { let mut state_builder = TestStateBuilder::new(); - let mut tree: StateBTreeSet = state_builder.new_btree_set(); + let mut tree = state_builder.new_btree_set_degree::<2, _>(); for n in 0..=7 { tree.insert(n); } @@ -2515,7 +2515,7 @@ mod test { #[test] fn test_btree_insert_7_no_order() { let mut state_builder = TestStateBuilder::new(); - let mut tree: StateBTreeSet = state_builder.new_btree_set(); + let mut tree = state_builder.new_btree_set_degree::<2, _>(); tree.insert(0); tree.insert(1); @@ -2534,7 +2534,7 @@ mod test { #[test] fn test_btree_higher() { let mut state_builder = TestStateBuilder::new(); - let mut tree: StateBTreeSet = state_builder.new_btree_set(); + let mut tree = state_builder.new_btree_set_degree::<2, _>(); tree.insert(1); tree.insert(2); tree.insert(3); @@ -2549,7 +2549,7 @@ mod test { #[test] fn test_btree_lower() { let mut state_builder = TestStateBuilder::new(); - let mut tree: StateBTreeSet = state_builder.new_btree_set(); + let mut tree = state_builder.new_btree_set_degree::<2, _>(); tree.insert(1); tree.insert(2); tree.insert(3); @@ -2564,7 +2564,7 @@ mod test { #[test] fn test_btree_insert_1000() { let mut state_builder = TestStateBuilder::new(); - let mut tree: StateBTreeSet = state_builder.new_btree_set(); + let mut tree = state_builder.new_btree_set_degree::<2, _>(); for n in 0..500 { tree.insert(n); } @@ -2583,7 +2583,7 @@ mod test { #[test] fn test_btree_7_get_8() { let mut state_builder = TestStateBuilder::new(); - let mut tree: StateBTreeSet = state_builder.new_btree_set(); + let mut tree = state_builder.new_btree_set_degree::<2, _>(); for n in 0..=7 { tree.insert(n); } @@ -2594,7 +2594,7 @@ mod test { #[test] fn test_btree_remove_from_one_node_tree() { let mut state_builder = TestStateBuilder::new(); - let mut tree: StateBTreeSet = state_builder.new_btree_set(); + let mut tree = state_builder.new_btree_set_degree::<2, _>(); for n in 0..3 { tree.insert(n); } @@ -2608,7 +2608,7 @@ mod test { #[test] fn test_btree_remove_only_key_lower_leaf_in_three_node() { let mut state_builder = TestStateBuilder::new(); - let mut tree: StateBTreeSet = state_builder.new_btree_set(); + let mut tree = state_builder.new_btree_set_degree::<2, _>(); for n in 0..4 { tree.insert(n); } @@ -2624,7 +2624,7 @@ mod test { #[test] fn test_btree_remove_only_key_higher_leaf_in_three_node() { let mut state_builder = TestStateBuilder::new(); - let mut tree: StateBTreeSet = state_builder.new_btree_set(); + let mut tree = state_builder.new_btree_set_degree::<2, _>(); for n in (0..4).into_iter().rev() { tree.insert(n); } @@ -2639,7 +2639,7 @@ mod test { #[test] fn test_btree_remove_from_higher_leaf_in_three_node_taking_from_sibling() { let mut state_builder = TestStateBuilder::new(); - let mut tree: StateBTreeSet = state_builder.new_btree_set(); + let mut tree = state_builder.new_btree_set_degree::<2, _>(); for n in (0..4).into_iter().rev() { tree.insert(n); } @@ -2651,7 +2651,7 @@ mod test { #[test] fn test_btree_remove_from_lower_leaf_in_three_node_taking_from_sibling() { let mut state_builder = TestStateBuilder::new(); - let mut tree: StateBTreeSet = state_builder.new_btree_set(); + let mut tree = state_builder.new_btree_set_degree::<2, _>(); for n in 0..4 { tree.insert(n); } @@ -2666,7 +2666,7 @@ mod test { #[test] fn test_btree_remove_from_root_in_three_node_causing_merge() { let mut state_builder = TestStateBuilder::new(); - let mut tree: StateBTreeSet = state_builder.new_btree_set(); + let mut tree = state_builder.new_btree_set_degree::<2, _>(); for n in 0..4 { tree.insert(n); } @@ -2680,7 +2680,7 @@ mod test { #[test] fn test_btree_remove_from_root_in_three_node_taking_key_from_higher_child() { let mut state_builder = TestStateBuilder::new(); - let mut tree: StateBTreeSet = state_builder.new_btree_set(); + let mut tree = state_builder.new_btree_set_degree::<2, _>(); for n in 0..4 { tree.insert(n); } @@ -2692,7 +2692,7 @@ mod test { #[test] fn test_btree_remove_from_root_in_three_node_taking_key_from_lower_child() { let mut state_builder = TestStateBuilder::new(); - let mut tree: StateBTreeSet = state_builder.new_btree_set(); + let mut tree = state_builder.new_btree_set_degree::<2, _>(); for n in (0..4).into_iter().rev() { tree.insert(n); } @@ -2704,7 +2704,7 @@ mod test { #[test] fn test_btree_iter() { let mut state_builder = TestStateBuilder::new(); - let mut tree: StateBTreeSet = state_builder.new_btree_set(); + let mut tree = state_builder.new_btree_set_degree::<2, _>(); let keys: Vec = (0..15).into_iter().collect(); for &k in &keys { tree.insert(k); diff --git a/concordium-std/src/types.rs b/concordium-std/src/types.rs index af4bd56d..18df832e 100644 --- a/concordium-std/src/types.rs +++ b/concordium-std/src/types.rs @@ -1459,6 +1459,8 @@ pub struct StateBTreeSet { /// Module with types used internally in [`StateBTreeMap`]. pub(crate) mod state_btree_internals { + use super::*; + /// Identifier for a node in the tree. Used to construct the key, where this /// node is store in the smart contract key-value store. #[derive(Debug, Copy, Clone)] From c9ae82b6b32e0223bb114a23a9bb4de3d6eb2b08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Holm=20Gj=C3=B8rup?= Date: Mon, 4 Mar 2024 10:03:55 +0100 Subject: [PATCH 12/36] Update changelog --- concordium-std/CHANGELOG.md | 1 + concordium-std/src/impls.rs | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/concordium-std/CHANGELOG.md b/concordium-std/CHANGELOG.md index 71466611..b10dd32f 100644 --- a/concordium-std/CHANGELOG.md +++ b/concordium-std/CHANGELOG.md @@ -6,6 +6,7 @@ via the `HasHost::contract_module_reference` and `HasHost::contract_name` functions. These are only available from protocol version 7, and as such are guarded by the `p7` feature flag. +- Add two ordered collections: `StateBTreeMap` and `StateBTreeSet`. These are based on [B-Tree](https://en.wikipedia.org/wiki/B-tree), but where each node are store in the low-level smart contract key-value store. ## concordium-std 10.0.0 (2024-02-22) diff --git a/concordium-std/src/impls.rs b/concordium-std/src/impls.rs index 7375ed24..f0c72448 100644 --- a/concordium-std/src/impls.rs +++ b/concordium-std/src/impls.rs @@ -1201,6 +1201,8 @@ where fn load_value(&self) -> V where V: DeserialWithState, { + // Safe to unwrap below, since the entry can only be `None`, using methods which + // are consuming self. let entry = unsafe { &mut *self.entry.get() }.as_mut().unwrap_abort(); entry.move_to_start(); V::deserial_with_state(&self.state_api, entry).unwrap_abort() @@ -1208,6 +1210,8 @@ where /// Set the value. Overwrites the existing one. pub fn set(&mut self, new_val: V) { + // Safe to unwrap below, since the entry can only be `None`, using methods which + // are consuming self. let entry = self.entry.get_mut().as_mut().unwrap_abort(); entry.move_to_start(); new_val.serial(entry).unwrap_abort(); @@ -1220,6 +1224,8 @@ where V: DeserialWithState, F: FnOnce(&mut V), { let lv = self.lazy_value.get_mut(); + // Safe to unwrap below, since the entry can only be `None`, using methods which + // are consuming self. let entry = self.entry.get_mut().as_mut().unwrap_abort(); let value = if let Some(v) = lv { v @@ -1238,6 +1244,8 @@ where /// Write to the state entry if the value is loaded. pub(crate) fn store_mutations(&mut self) { if let Some(value) = self.lazy_value.get_mut() { + // Safe to unwrap below, since the entry can only be `None`, using methods which + // are consuming self. let entry = self.entry.get_mut().as_mut().unwrap_abort(); entry.move_to_start(); value.serial(entry).unwrap_abort(); From cb19bea3a6b82d6113049083ebfd9ec88c15a3f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Holm=20Gj=C3=B8rup?= Date: Tue, 5 Mar 2024 10:03:21 +0100 Subject: [PATCH 13/36] StateBTreeSet::remove now avoids searching ahead for replacement key --- concordium-std/src/impls.rs | 167 ++++++++++++++++++++++-------------- 1 file changed, 101 insertions(+), 66 deletions(-) diff --git a/concordium-std/src/impls.rs b/concordium-std/src/impls.rs index f0c72448..9984d155 100644 --- a/concordium-std/src/impls.rs +++ b/concordium-std/src/impls.rs @@ -3653,75 +3653,114 @@ impl StateBTreeSet { return false; }; + /// The goal of the loop below. + enum Mode<'b, const M: usize, K: Serial, S: HasStateApi> { + /// The goal is to remove the key provided by the user. + RemoveKey, + /// The key was removed from a node, which now need some key to be + /// moved up. + MoveUp { + /// Whether the key is taken from the smaller child, meaning the + /// largest key should be moved up. + from_left_child: bool, + /// The node which deleted a key and now is waiting for a new + /// one. + receiving_node: StateRefMut<'b, state_btree_internals::Node, S>, + /// The index to store the key to. + key_index: usize, + }, + } + let deleted_something = { - let mut overwrite_key_to_delete = None; let mut node = self.get_node_mut(root_node_id); + let mut mode = Mode::RemoveKey; loop { - let key_to_delete = overwrite_key_to_delete.as_ref().unwrap_or(key); - let index = match node.keys.binary_search(key_to_delete) { - Ok(index) => { - if node.is_leaf() { - // Found the key in this node and the node is a leaf, meaning we simply - // remove it. - // This will not violate the minimum keys invariant, since a node - // ensures a child can spare a key before iteration and the root node is - // not part of the invariant. - node.keys.remove(index); - break true; - } - // Found the key in this node, but the node is not a leaf. - let mut left_child = self.get_node_mut(node.children[index]); - if !left_child.is_at_min() { - // If the child with smaller keys can spare a key, we take the highest - // key from it, changing the loop to be deleting its highest key from - // this child. - node.keys[index] = self.get_highest_key(&node, index); - let new_key_to_delete = { - // To avoid having to clone the key, we store changes to the node - // and unwrap it from the StateRefMut, disabling further mutations - // from being written to the key-value store, such that we can take - // the in-memory key. - node.store_mutations(); - let mut inner_node = node.into_raw_parts().0.unwrap_abort(); - inner_node.keys.swap_remove(index) - }; - overwrite_key_to_delete = Some(new_key_to_delete); + let index = if let Mode::MoveUp { + from_left_child, + ref mut receiving_node, + key_index, + } = mode + { + if node.is_leaf() { + // The node is a leaf, meaning we simply remove it. + // This will not violate the minimum keys invariant, since a node + // ensures a child can spare a key before iteration. + receiving_node.keys[key_index] = if from_left_child { + node.keys.remove(0) + } else { + node.keys.pop().unwrap_abort() + }; + break true; + } + // Node is not a leaf, so we move further down this subtree. + if from_left_child { + // If we are taking a key from the left child, it must be the largest key. + // So we traverse down the largest child. + node.children.len() - 1 + } else { + // If we are taking a key from the right child, it must be the smallest key. + // So we traverse down the smallest child. + 0 + } + } else { + match node.keys.binary_search(key) { + Ok(index) => { + if node.is_leaf() { + // Found the key in this node and the node is a leaf, meaning we + // simply remove it. + // This will not violate the minimum keys invariant, since a node + // ensures a child can spare a key before iteration and the root + // node is not part of the + // invariant. + node.keys.remove(index); + break true; + } + // Found the key in this node, but the node is not a leaf. + let mut left_child = self.get_node_mut(node.children[index]); + if !left_child.is_at_min() { + // If the child with smaller keys can spare a key, we take the + // highest key from it, changing the + // loop to be deleting its highest key from + // this child. + mode = Mode::MoveUp { + from_left_child: true, + receiving_node: node, + key_index: index, + }; + node = left_child; + continue; + } + + let right_child = self.get_node_mut(node.children[index + 1]); + if !right_child.is_at_min() { + // If the child with larger keys can spare a key, we take the lowest + // key from it, changing the loop to be deleting its highest key + // from this child. + mode = Mode::MoveUp { + from_left_child: false, + receiving_node: node, + key_index: index, + }; + node = right_child; + continue; + } + // No child on either side of the key can spare a key at this point, so + // we merge them into one child, moving the + // key into the merged child and try + // to remove from this. + self.merge(&mut node, index, &mut left_child, right_child); node = left_child; continue; } - - let right_child = self.get_node_mut(node.children[index + 1]); - if !right_child.is_at_min() { - // If the child with larger keys can spare a key, we take the lowest - // key from it, changing the loop to be deleting its highest key from - // this child. - node.keys[index] = self.get_lowest_key(&node, index + 1); - let new_key_to_delete = { - // To avoid having to clone the key, we store changes to the node - // and unwrap it from the StateRefMut, disabling further mutations - // from being written to the key-value store, such that we can use - // the same in-memory key. - node.store_mutations(); - let mut inner_node = node.into_raw_parts().0.unwrap_abort(); - inner_node.keys.swap_remove(index) - }; - overwrite_key_to_delete = Some(new_key_to_delete); - node = right_child; - continue; + Err(index) => { + // Node did not contain the key. + if node.is_leaf() { + break false; + } + index } - // No child on either side of the key can spare a key at this point, so we - // merge them into one child, moving the key into the merged child and try - // to remove from this. - self.merge(&mut node, index, &mut left_child, right_child); - node = left_child; - continue; } - Err(index) => index, }; - // Node did not contain the key. - if node.is_leaf() { - break false; - } // Node did not contain the key and is not a leaf. let mut child = self.get_node_mut(node.children[index]); @@ -3805,8 +3844,8 @@ impl StateBTreeSet { self.root = None; self.delete_node(root); } else { - // If the root is empty but the tree is not, point the only child of the root as - // the root. + // If the root is empty but the tree is not, point to the only child of the root + // as the root. if root.keys.is_empty() { self.root = Some(root.children[0]); self.delete_node(root); @@ -3817,8 +3856,6 @@ impl StateBTreeSet { } /// Internal function for getting the highest key in a subtree. - // FIXME: In the context where this function is called, the child node is - // already loaded and should be reused, probably resulting in `K: Clone`. fn get_highest_key(&self, node: &state_btree_internals::Node, child_index: usize) -> K where K: Ord + Serialize, @@ -3834,8 +3871,6 @@ impl StateBTreeSet { } /// Internal function for getting the lowest key in a subtree. - // FIXME: In the context where this function is called, the child node is - // already loaded and should be reused, probably resulting in `K: Clone`. fn get_lowest_key(&self, node: &state_btree_internals::Node, child_index: usize) -> K where K: Ord + Serialize, From 50dd73b9114b630886ebe8ba0cefcceb4043252f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Holm=20Gj=C3=B8rup?= Date: Wed, 6 Mar 2024 17:00:30 +0100 Subject: [PATCH 14/36] Address review comments --- concordium-std/src/impls.rs | 338 ++++++++++++++++++------------------ concordium-std/src/types.rs | 12 +- 2 files changed, 171 insertions(+), 179 deletions(-) diff --git a/concordium-std/src/impls.rs b/concordium-std/src/impls.rs index 9984d155..059c88e5 100644 --- a/concordium-std/src/impls.rs +++ b/concordium-std/src/impls.rs @@ -3256,6 +3256,7 @@ impl Deserial for state_btree_internals::NodeId { impl StateBTreeMap { /// Insert a key-value pair into the map. + /// Returns the previous value if the key was already in the map. pub fn insert(&mut self, key: K, value: V) -> Option where S: HasStateApi, @@ -3592,7 +3593,7 @@ impl StateBTreeSet { }; if node.is_leaf() { - return if key <= &node.keys[0] { + return if lower_key_index == 0 { lower_so_far } else { // lower_key_index cannot be 0 in this case, since the binary search will only @@ -3653,182 +3654,53 @@ impl StateBTreeSet { return false; }; - /// The goal of the loop below. - enum Mode<'b, const M: usize, K: Serial, S: HasStateApi> { - /// The goal is to remove the key provided by the user. - RemoveKey, - /// The key was removed from a node, which now need some key to be - /// moved up. - MoveUp { - /// Whether the key is taken from the smaller child, meaning the - /// largest key should be moved up. - from_left_child: bool, - /// The node which deleted a key and now is waiting for a new - /// one. - receiving_node: StateRefMut<'b, state_btree_internals::Node, S>, - /// The index to store the key to. - key_index: usize, - }, - } - let deleted_something = { let mut node = self.get_node_mut(root_node_id); - let mut mode = Mode::RemoveKey; loop { - let index = if let Mode::MoveUp { - from_left_child, - ref mut receiving_node, - key_index, - } = mode - { - if node.is_leaf() { - // The node is a leaf, meaning we simply remove it. - // This will not violate the minimum keys invariant, since a node - // ensures a child can spare a key before iteration. - receiving_node.keys[key_index] = if from_left_child { - node.keys.remove(0) - } else { - node.keys.pop().unwrap_abort() - }; - break true; - } - // Node is not a leaf, so we move further down this subtree. - if from_left_child { - // If we are taking a key from the left child, it must be the largest key. - // So we traverse down the largest child. - node.children.len() - 1 - } else { - // If we are taking a key from the right child, it must be the smallest key. - // So we traverse down the smallest child. - 0 - } - } else { - match node.keys.binary_search(key) { - Ok(index) => { - if node.is_leaf() { - // Found the key in this node and the node is a leaf, meaning we - // simply remove it. - // This will not violate the minimum keys invariant, since a node - // ensures a child can spare a key before iteration and the root - // node is not part of the - // invariant. - node.keys.remove(index); - break true; - } - // Found the key in this node, but the node is not a leaf. - let mut left_child = self.get_node_mut(node.children[index]); - if !left_child.is_at_min() { - // If the child with smaller keys can spare a key, we take the - // highest key from it, changing the - // loop to be deleting its highest key from - // this child. - mode = Mode::MoveUp { - from_left_child: true, - receiving_node: node, - key_index: index, - }; - node = left_child; - continue; - } - - let right_child = self.get_node_mut(node.children[index + 1]); - if !right_child.is_at_min() { - // If the child with larger keys can spare a key, we take the lowest - // key from it, changing the loop to be deleting its highest key - // from this child. - mode = Mode::MoveUp { - from_left_child: false, - receiving_node: node, - key_index: index, - }; - node = right_child; - continue; - } - // No child on either side of the key can spare a key at this point, so - // we merge them into one child, moving the - // key into the merged child and try - // to remove from this. - self.merge(&mut node, index, &mut left_child, right_child); - node = left_child; - continue; + match node.keys.binary_search(key) { + Ok(index) => { + if node.is_leaf() { + // Found the key in this node and the node is a leaf, meaning we + // simply remove it. + // This will not violate the minimum keys invariant, since a node + // ensures a child can spare a key before iteration and the root + // node is not part of the + // invariant. + node.keys.remove(index); + break true; } - Err(index) => { - // Node did not contain the key. - if node.is_leaf() { - break false; - } - index + // Found the key in this node, but the node is not a leaf. + let mut left_child = self.get_node_mut(node.children[index]); + if !left_child.is_at_min() { + // If the child with smaller keys can spare a key, we take the + // highest key from it. + node.keys[index] = self.remove_largest_key(left_child); + break true; } - } - }; - // Node did not contain the key and is not a leaf. - let mut child = self.get_node_mut(node.children[index]); - let has_smaller_sibling = 0 < index; - let has_larger_sibling = index < node.children.len() - 1; - // Check and proactively prepare the child to be able to delete a key. - node = if !child.is_at_min() { - child - } else { - // The child is at minimum keys, so first attempt to take a key from either - // sibling, otherwise merge with one of them. - 'increase_child: { - // Labeled block, to be able to return early. - let smaller_sibling = if has_smaller_sibling { - let mut smaller_sibling = self.get_node_mut(node.children[index - 1]); - if !smaller_sibling.is_at_min() { - // The smaller sibling can spare a key, so we replace the largest - // key from the sibling, put it in - // the parent and take a key from - // the parent. - let largest_key_sibling = smaller_sibling.keys.pop().unwrap_abort(); - let swapped_node_key = - mem::replace(&mut node.keys[index - 1], largest_key_sibling); - child.keys.insert(0, swapped_node_key); - if !child.is_leaf() { - child - .children - .insert(0, smaller_sibling.children.pop().unwrap_abort()); - } - break 'increase_child child; - } - Some(smaller_sibling) - } else { - None - }; - let larger_sibling = if has_larger_sibling { - let mut larger_sibling = self.get_node_mut(node.children[index + 1]); - if !larger_sibling.is_at_min() { - // The larger sibling can spare a key, so we replace the smallest - // key from the sibling, put it in - // the parent and take a key from - // the parent. - let first_key_sibling = larger_sibling.keys.remove(0); - let swapped_node_key = - mem::replace(&mut node.keys[index], first_key_sibling); - child.keys.push(swapped_node_key); - - if !child.is_leaf() { - child.children.push(larger_sibling.children.remove(0)); - } - break 'increase_child child; - } - Some(larger_sibling) - } else { - None - }; - - if let Some(sibling) = larger_sibling { - self.merge(&mut node, index, &mut child, sibling); - child - } else if let Some(mut sibling) = smaller_sibling { - self.merge(&mut node, index - 1, &mut sibling, child); - sibling - } else { - // Unreachable code, since M must be 2 or larger (the minimum degree), a - // child node must have at least one sibling. - crate::trap(); + let right_child = self.get_node_mut(node.children[index + 1]); + if !right_child.is_at_min() { + // If the child with larger keys can spare a key, we take the lowest + // key from it. + node.keys[index] = self.remove_smallest_key(right_child); + break true; } + // No child on either side of the key can spare a key at this point, so + // we merge them into one child, moving the + // key into the merged child and try + // to remove from this. + self.merge(&mut node, index, &mut left_child, right_child); + node = left_child; + continue; + } + Err(index) => { + // Node did not contain the key. + if node.is_leaf() { + break false; + } + // Node did not contain the key and is not a leaf. + // Check and proactively prepare the child to be able to delete a key. + node = self.prepare_child_for_key_removal(node, index); } }; } @@ -3855,6 +3727,125 @@ impl StateBTreeSet { deleted_something } + /// Internal function for taking the largest key in a subtree. + /// + /// Assumes: + /// - The provided node is not the root. + /// - The node contain more than minimum number of keys. + fn remove_largest_key<'a>( + &mut self, + mut node: StateRefMut<'a, state_btree_internals::Node, S>, + ) -> K + where + K: Ord + Serialize, + S: HasStateApi, { + while !node.is_leaf() { + // Node is not a leaf, so we move further down this subtree. + let child_index = node.children.len() - 1; + node = self.prepare_child_for_key_removal(node, child_index); + } + // The node is a leaf, meaning we simply remove it. + // This will not violate the minimum keys invariant, since a node + // ensures a child can spare a key before iteration. + node.keys.pop().unwrap_abort() + } + + /// Internal function for taking the smallest key in a subtree. + /// + /// Assumes: + /// - The provided node is not the root. + /// - The node contain more than minimum number of keys. + fn remove_smallest_key<'a>( + &mut self, + mut node: StateRefMut<'a, state_btree_internals::Node, S>, + ) -> K + where + K: Ord + Serialize, + S: HasStateApi, { + while !node.is_leaf() { + // Node is not a leaf, so we move further down this subtree. + let child_index = 0; + node = self.prepare_child_for_key_removal(node, child_index); + } + // The node is a leaf, meaning we simply remove it. + // This will not violate the minimum keys invariant, since a node + // ensures a child can spare a key before iteration. + node.keys.remove(0) + } + + /// Internal function for key rotation, preparing a node for deleting a key. + /// Assumes: + /// - The provided `node` is not a leaf and has a child at `index`. + /// - + fn prepare_child_for_key_removal<'b, 'c>( + &mut self, + mut node: StateRefMut<'b, state_btree_internals::Node, S>, + index: usize, + ) -> StateRefMut<'c, state_btree_internals::Node, S> + where + K: Ord + Serialize, + S: HasStateApi, { + let mut child = self.get_node_mut(node.children[index]); + if !child.is_at_min() { + return child; + } + // The child is at minimum keys, so first attempt to take a key from either + // sibling, otherwise merge with one of them. + let has_smaller_sibling = 0 < index; + let has_larger_sibling = index < node.children.len() - 1; + let smaller_sibling = if has_smaller_sibling { + let mut smaller_sibling = self.get_node_mut(node.children[index - 1]); + if !smaller_sibling.is_at_min() { + // The smaller sibling can spare a key, so we replace the largest + // key from the sibling, put it in + // the parent and take a key from + // the parent. + let largest_key_sibling = smaller_sibling.keys.pop().unwrap_abort(); + let swapped_node_key = mem::replace(&mut node.keys[index - 1], largest_key_sibling); + child.keys.insert(0, swapped_node_key); + if !child.is_leaf() { + child.children.insert(0, smaller_sibling.children.pop().unwrap_abort()); + } + return child; + } + Some(smaller_sibling) + } else { + None + }; + let larger_sibling = if has_larger_sibling { + let mut larger_sibling = self.get_node_mut(node.children[index + 1]); + if !larger_sibling.is_at_min() { + // The larger sibling can spare a key, so we replace the smallest + // key from the sibling, put it in + // the parent and take a key from + // the parent. + let first_key_sibling = larger_sibling.keys.remove(0); + let swapped_node_key = mem::replace(&mut node.keys[index], first_key_sibling); + child.keys.push(swapped_node_key); + + if !child.is_leaf() { + child.children.push(larger_sibling.children.remove(0)); + } + return child; + } + Some(larger_sibling) + } else { + None + }; + + if let Some(sibling) = larger_sibling { + self.merge(&mut node, index, &mut child, sibling); + child + } else if let Some(mut sibling) = smaller_sibling { + self.merge(&mut node, index - 1, &mut sibling, child); + sibling + } else { + // Unreachable code, since M must be 2 or larger (the minimum degree), a + // child node must have at least one sibling. + crate::trap(); + } + } + /// Internal function for getting the highest key in a subtree. fn get_highest_key(&self, node: &state_btree_internals::Node, child_index: usize) -> K where @@ -3889,7 +3880,8 @@ impl StateBTreeSet { /// this child with the content of its larger sibling, deleting the sibling. /// /// Assumes: - /// - `parent_node` have a child at `index` and `index + 1`. + /// - `parent_node` has children `child` at `index` and `larger_child` at + /// `index + 1`. /// - Both children are at minimum number of keys (`M - 1`). fn merge( &mut self, @@ -3908,7 +3900,7 @@ impl StateBTreeSet { self.delete_node(larger_child); } - /// Internal function for constructing a node. It will incrementing the next + /// Internal function for constructing a node. It will increment the next /// node ID and create an entry in the smart contract key-value store. fn create_node<'b>( &mut self, @@ -3955,7 +3947,7 @@ impl StateBTreeSet { // We find the key in this node, so we do nothing. return false; }; - // The key is not this node. + // The key is not in this node. if node.is_leaf() { // Since the node is not full and this is a leaf, we can just insert here. node.keys.insert(insert_index, key); diff --git a/concordium-std/src/types.rs b/concordium-std/src/types.rs index 18df832e..40e4e7c3 100644 --- a/concordium-std/src/types.rs +++ b/concordium-std/src/types.rs @@ -387,7 +387,7 @@ pub struct StateRefMut<'a, V: Serial, S: HasStateApi> { /// This is set as an `UnsafeCell`, to be able to get a mutable reference to /// the entry without `StateRefMut` being mutable. /// The `Option` allows for having an internal method destroying the - /// `StateRefMut` into its raw parst without `Drop` causing a write to the + /// `StateRefMut` into its raw parts without `Drop` causing a write to the /// contract state. pub(crate) entry: UnsafeCell>, pub(crate) state_api: S, @@ -1298,8 +1298,8 @@ pub struct MetadataUrl { /// state API in unit tests, see /// [`TestStateApi`](crate::test_infrastructure::TestStateApi). /// - `M`: A `const usize` determining the _minimum degree_ of the B-tree. -/// _Must_ be a value of `2` or above for the tree to be work. This can be -/// used to tweak the height of the tree vs size of each node in the tree. The +/// _Must_ be a value of `2` or above for the tree to work. This can be used +/// to tweak the height of the tree vs size of each node in the tree. The /// default is set based on benchmarks. /// /// ## Usage @@ -1390,8 +1390,8 @@ pub struct StateBTreeMap { /// state API in unit tests, see /// [`TestStateApi`](crate::test_infrastructure::TestStateApi). /// - `M`: A `const usize` determining the _minimum degree_ of the B-tree. -/// _Must_ be a value of `2` or above for the tree to be work. This can be -/// used to tweak the height of the tree vs size of each node in the tree. The +/// _Must_ be a value of `2` or above for the tree to work. This can be used +/// to tweak the height of the tree vs size of each node in the tree. The /// default is set based on benchmarks. /// /// ## Usage @@ -1469,7 +1469,7 @@ pub(crate) mod state_btree_internals { pub(crate) id: u32, } - /// Type representing the a node in the [`StateBTreeMap`]. + /// Type representing a node in the [`StateBTreeMap`]. /// Each node is stored separately in the smart contract key-value store. #[derive(Debug)] pub(crate) struct Node { From 44275119648c171e83bfff0b04d4fa642ef3898d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Holm=20Gj=C3=B8rup?= Date: Fri, 8 Mar 2024 15:18:31 +0100 Subject: [PATCH 15/36] Address some more review comments --- concordium-std/CHANGELOG.md | 2 +- concordium-std/Cargo.toml | 2 +- concordium-std/src/impls.rs | 56 +++++++++++++++++++++++++++++-------- concordium-std/src/types.rs | 17 +++++------ 4 files changed, 55 insertions(+), 22 deletions(-) diff --git a/concordium-std/CHANGELOG.md b/concordium-std/CHANGELOG.md index b10dd32f..6ea668de 100644 --- a/concordium-std/CHANGELOG.md +++ b/concordium-std/CHANGELOG.md @@ -6,7 +6,7 @@ via the `HasHost::contract_module_reference` and `HasHost::contract_name` functions. These are only available from protocol version 7, and as such are guarded by the `p7` feature flag. -- Add two ordered collections: `StateBTreeMap` and `StateBTreeSet`. These are based on [B-Tree](https://en.wikipedia.org/wiki/B-tree), but where each node are store in the low-level smart contract key-value store. +- Add two ordered collections: `StateBTreeMap` and `StateBTreeSet`. These are based on [B-Tree](https://en.wikipedia.org/wiki/B-tree), but where each node is stored in the low-level smart contract key-value store. ## concordium-std 10.0.0 (2024-02-22) diff --git a/concordium-std/Cargo.toml b/concordium-std/Cargo.toml index 4530c354..7804d777 100644 --- a/concordium-std/Cargo.toml +++ b/concordium-std/Cargo.toml @@ -38,7 +38,7 @@ bump_alloc = [] p7 = [] [lib] -crate-type = ["rlib"] +crate-type = ["cdylib", "rlib"] [profile.release] # Tell `rustc` to optimize for small code size. diff --git a/concordium-std/src/impls.rs b/concordium-std/src/impls.rs index 059c88e5..aee3d6ed 100644 --- a/concordium-std/src/impls.rs +++ b/concordium-std/src/impls.rs @@ -3483,7 +3483,7 @@ impl StateBTreeSet { new } - /// Returns `true` if the set contains an element equal to the value. + /// Returns `true` if the set contains an element equal to the key. pub fn contains(&self, key: &K) -> bool where S: HasStateApi, @@ -3559,12 +3559,16 @@ impl StateBTreeSet { if node.is_leaf() { return if higher_key_index < node.keys.len() { + // This does not mutate the node in the end, just the representation in memory + // which is freed after the call. Some(StateRef::new(node.keys.swap_remove(higher_key_index))) } else { higher_so_far }; } else { if higher_key_index < node.keys.len() { + // This does not mutate the node in the end, just the representation in memory + // which is freed after the call. higher_so_far = Some(StateRef::new(node.keys.swap_remove(higher_key_index))) } @@ -3602,6 +3606,8 @@ impl StateBTreeSet { }; } else { if lower_key_index > 0 { + // This does not mutate the node in the end, just the representation in memory + // which is freed after the call. lower_so_far = Some(StateRef::new(node.keys.swap_remove(lower_key_index - 1))); } let child_node_id = node.children[lower_key_index]; @@ -3754,7 +3760,7 @@ impl StateBTreeSet { /// /// Assumes: /// - The provided node is not the root. - /// - The node contain more than minimum number of keys. + /// - The provided `node` contain more than minimum number of keys. fn remove_smallest_key<'a>( &mut self, mut node: StateRefMut<'a, state_btree_internals::Node, S>, @@ -3774,9 +3780,10 @@ impl StateBTreeSet { } /// Internal function for key rotation, preparing a node for deleting a key. + /// Returns the now prepared child at `index`. /// Assumes: /// - The provided `node` is not a leaf and has a child at `index`. - /// - + /// - The minimum degree `M` is at least 2 or more. fn prepare_child_for_key_removal<'b, 'c>( &mut self, mut node: StateRefMut<'b, state_btree_internals::Node, S>, @@ -3847,6 +3854,8 @@ impl StateBTreeSet { } /// Internal function for getting the highest key in a subtree. + /// Assumes the provided `node` is not a leaf and has a child at + /// `child_index`. fn get_highest_key(&self, node: &state_btree_internals::Node, child_index: usize) -> K where K: Ord + Serialize, @@ -3862,6 +3871,8 @@ impl StateBTreeSet { } /// Internal function for getting the lowest key in a subtree. + /// Assumes the provided `node` is not a leaf and has a child at + /// `child_index`. fn get_lowest_key(&self, node: &state_btree_internals::Node, child_index: usize) -> K where K: Ord + Serialize, @@ -3931,8 +3942,8 @@ impl StateBTreeSet { self.state_api.delete_entry(node.into_raw_parts().1).unwrap_abort() } - /// Internal function for inserting into a subtree. The given node must not - /// be full. + /// Internal function for inserting into a subtree. + /// Assumes the given node is not full. fn insert_non_full( &mut self, initial_node: StateRefMut, S>, @@ -3949,24 +3960,25 @@ impl StateBTreeSet { }; // The key is not in this node. if node.is_leaf() { - // Since the node is not full and this is a leaf, we can just insert here. + // Since we can assume the node is not full and this is a leaf, we can just + // insert here. node.keys.insert(insert_index, key); return true; } // The node is not a leaf, so we want to insert in the relevant child node. let mut child = self.get_node_mut(node.children[insert_index]); - node = if child.is_full() { + node = if !child.is_full() { + child + } else { let larger_child = self.split_child(&mut node, insert_index, &mut child); - // Since the child is now split into two, we have to update the insert into - // the relevant one of them. + // Since the child is now split into two, we have to check which one to insert + // into. if node.keys[insert_index] < key { larger_child } else { child } - } else { - child }; } } @@ -3974,8 +3986,11 @@ impl StateBTreeSet { /// Internal function for splitting the child node at a given index for a /// given node. This will also mutate the given node adding a new key /// and child after the provided child_index. - /// /// Returns the newly created node. + /// + /// Assumes: + /// - Node is not a leaf and has `child` as child at `child_index`. + /// - `child` is at maximum keys (`2 * M - 1`). fn split_child<'b>( &mut self, node: &mut state_btree_internals::Node, @@ -4180,3 +4195,20 @@ mod tests { t.compile_fail("tests/state/map-multiple-state-ref-mut.rs"); } } + +#[concordium_cfg_test] +mod wasm_test { + use concordium_contracts_common::concordium_test; + + use crate::{claim_eq, StateApi, StateBuilder}; + + #[concordium_test] + fn test() { + let mut state_builder = StateBuilder::open(StateApi::open()); + let mut map = state_builder.new_map(); + + map.insert(0, "0".to_string()); + + claim_eq!(map.get(&0).as_deref(), Some(&"1".to_string())); + } +} diff --git a/concordium-std/src/types.rs b/concordium-std/src/types.rs index 40e4e7c3..c31906bb 100644 --- a/concordium-std/src/types.rs +++ b/concordium-std/src/types.rs @@ -1268,11 +1268,10 @@ pub struct MetadataUrl { /// An ordered map based on [B-Tree](https://en.wikipedia.org/wiki/B-tree), where /// each node is stored separately in the low-level key-value store. /// -/// It can be seen as an extension adding the tracking the ordering of the keys -/// on top of [`StateMap`] providing functions such as [`higher`](Self::higher) -/// and [`lower`](Self::lower). -/// This adds some overhead when inserting and deleting entries from the map -/// compared to [`StateMap`]. +/// It can be seen as an extension adding tracking of the keys ordering on top +/// of [`StateMap`] to provide functions such as [`higher`](Self::higher) and +/// [`lower`](Self::lower). This results in some overhead when inserting and +/// deleting entries from the map compared to using [`StateMap`]. /// /// | Operation | Performance | /// |-------------------------------------------------|---------------| @@ -1480,10 +1479,11 @@ pub(crate) mod state_btree_internals { /// List of nodes which are children of this node in the tree. /// /// This list is empty when this node is representing a leaf. - /// When not empty it will contain exactly `keys.len() + 1` elements. + /// When not a leaf, it will contain exactly `keys.len() + 1` elements. /// - /// The elements are ordered such that the node `children[i]` contains - /// keys that are strictly smaller than `keys[i]`. + /// The elements are ordered such that for a key `keys[i]`: + /// - `children[i]` is a subtree containing strictly smaller keys. + /// - `children[i + 1]` is a subtree containing strictly larger keys. pub(crate) children: Vec, } @@ -1514,6 +1514,7 @@ pub struct StateBTreeSetIter<'a, 'b, K, S, const M: usize> { Vec<(state_btree_internals::Node>, usize)>, /// Reference to the set, needed for looking up the nodes. pub(crate) tree: &'a StateBTreeSet, + /// Marker for tracking the lifetime of the key. pub(crate) _marker_lifetime: PhantomData<&'b K>, } From 1365d243a60dd36bc6822f843bb2b040d8a69a11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Holm=20Gj=C3=B8rup?= Date: Mon, 11 Mar 2024 12:40:27 +0100 Subject: [PATCH 16/36] Restructure StateBTreeSet/Map into a separate module --- concordium-std/src/impls.rs | 1431 ++++++-------------- concordium-std/src/lib.rs | 2 + concordium-std/src/state_btree.rs | 1481 +++++++++++++++++++++ concordium-std/src/test_infrastructure.rs | 636 --------- concordium-std/src/types.rs | 266 ---- 5 files changed, 1902 insertions(+), 1914 deletions(-) create mode 100644 concordium-std/src/state_btree.rs diff --git a/concordium-std/src/impls.rs b/concordium-std/src/impls.rs index aee3d6ed..c19145e1 100644 --- a/concordium-std/src/impls.rs +++ b/concordium-std/src/impls.rs @@ -8,7 +8,7 @@ use crate::{ mem::{self, MaybeUninit}, num, num::NonZeroU32, - prims, + prims, state_btree, traits::*, types::*, vec::Vec, @@ -774,8 +774,6 @@ where /// "allocator"/state builder stores "next location". The values stored at this /// location are 64-bit integers. const NEXT_ITEM_PREFIX_KEY: [u8; 8] = 0u64.to_le_bytes(); -#[cfg(test)] -const GENERIC_MAP_PREFIX: u64 = 1; /// Initial location to store in [NEXT_ITEM_PREFIX_KEY]. For example, the /// initial call to "new_state_box" will allocate the box at this location. pub(crate) const INITIAL_NEXT_ITEM_PREFIX: [u8; 8] = 2u64.to_le_bytes(); @@ -2334,14 +2332,14 @@ where } /// Create a new empty [`StateBTreeSet`]. - pub fn new_btree_set(&mut self) -> StateBTreeSet { + pub fn new_btree_set(&mut self) -> state_btree::StateBTreeSet { let (state_api, prefix) = self.new_state_container(); - StateBTreeSet::new(state_api, prefix) + state_btree::StateBTreeSet::new(state_api, prefix) } /// Create a new empty [`StateBTreeMap`]. - pub fn new_btree_map(&mut self) -> StateBTreeMap { - StateBTreeMap { + pub fn new_btree_map(&mut self) -> state_btree::StateBTreeMap { + state_btree::StateBTreeMap { map: self.new_map(), ordered_set: self.new_btree_set(), } @@ -2349,15 +2347,19 @@ where /// Create a new empty [`StateBTreeSet`], setting the minimum degree `M` of /// the B-Tree explicitly. - pub fn new_btree_set_degree(&mut self) -> StateBTreeSet { + pub fn new_btree_set_degree( + &mut self, + ) -> state_btree::StateBTreeSet { let (state_api, prefix) = self.new_state_container(); - StateBTreeSet::new(state_api, prefix) + state_btree::StateBTreeSet::new(state_api, prefix) } /// Create a new empty [`StateBTreeMap`], setting the minimum degree `M` of /// the B-Tree explicitly. - pub fn new_btree_map_degree(&mut self) -> StateBTreeMap { - StateBTreeMap { + pub fn new_btree_map_degree( + &mut self, + ) -> state_btree::StateBTreeMap { + state_btree::StateBTreeMap { map: self.new_map(), ordered_set: self.new_btree_set_degree(), } @@ -2438,51 +2440,6 @@ where } } -#[cfg(test)] -/// Some helper methods that are used for internal tests. -impl StateBuilder -where - S: HasStateApi, -{ - /// Get a value from the generic map. - /// `Some(Err(_))` means that something exists in the state with that key, - /// but it isn't of type `V`. - pub(crate) fn get>(&self, key: K) -> Option> { - let key_with_map_prefix = Self::prepend_generic_map_key(key); - - self.state_api - .lookup_entry(&key_with_map_prefix) - .map(|mut entry| V::deserial_with_state(&self.state_api, &mut entry)) - } - - /// Inserts a value in the generic map. - /// The key and value are serialized before insert. - pub(crate) fn insert( - &mut self, - key: K, - value: V, - ) -> Result<(), StateError> { - let key_with_map_prefix = Self::prepend_generic_map_key(key); - match self.state_api.entry(key_with_map_prefix) { - EntryRaw::Vacant(vac) => { - let _ = vac.insert(&value); - } - EntryRaw::Occupied(mut occ) => occ.insert(&value), - } - Ok(()) - } - - /// Serializes the key and prepends [GENERIC_MAP_PREFIX]. - /// This is similar to how [StateMap] works, where a unique prefix is - /// prepended onto keys. Since there is only one generic map, the prefix - /// is a constant. - fn prepend_generic_map_key(key: K) -> Vec { - let mut key_with_map_prefix = to_bytes(&GENERIC_MAP_PREFIX); - key_with_map_prefix.append(&mut to_bytes(&key)); - key_with_map_prefix - } -} - impl HasHost for ExternHost where S: Serial + DeserialWithState, @@ -2612,7 +2569,6 @@ where (&mut self.state, &mut self.state_builder) } } - impl HasHost for ExternLowLevelHost { type ReturnValueType = ExternCallResponse; type StateApiType = ExternStateApi; @@ -3179,1036 +3135,487 @@ impl Deserial for MetadataUrl { } } -impl Serial for StateBTreeMap { - fn serial(&self, out: &mut W) -> Result<(), W::Err> { - self.map.serial(out)?; - self.ordered_set.serial(out) - } -} - -impl DeserialWithState for StateBTreeMap { - fn deserial_with_state(state: &S, source: &mut R) -> ParseResult { - let map = DeserialWithState::deserial_with_state(state, source)?; - let ordered_set = DeserialWithState::deserial_with_state(state, source)?; - Ok(Self { - map, - ordered_set, - }) - } -} - -impl Serial for StateBTreeSet { - fn serial(&self, out: &mut W) -> Result<(), W::Err> { - self.prefix.serial(out)?; - self.root.serial(out)?; - self.len.serial(out)?; - self.next_node_id.serial(out) - } -} - -impl DeserialWithState for StateBTreeSet { - fn deserial_with_state(state: &S, source: &mut R) -> ParseResult { - let prefix = source.get()?; - let root = source.get()?; - let len = source.get()?; - let next_node_id = source.get()?; - - Ok(Self { - _marker_key: Default::default(), - prefix, - state_api: state.clone(), - root, - len, - next_node_id, - }) - } -} - -impl Serial for state_btree_internals::Node { - fn serial(&self, out: &mut W) -> Result<(), W::Err> { - self.keys.serial(out)?; - self.children.serial(out) - } -} +#[cfg(test)] +mod tests { -impl Deserial for state_btree_internals::Node { - fn deserial(source: &mut R) -> ParseResult { - let keys = source.get()?; - let children = source.get()?; - Ok(Self { - keys, - children, - }) + /// Check that you cannot have multiple active entries from a statemap at + /// the same time. See the test file for details. + #[test] + fn statemap_multiple_entries_not_allowed() { + let t = trybuild::TestCases::new(); + t.compile_fail("tests/state/map-multiple-entries.rs"); } -} - -impl Serial for state_btree_internals::NodeId { - fn serial(&self, out: &mut W) -> Result<(), W::Err> { self.id.serial(out) } -} -impl Deserial for state_btree_internals::NodeId { - fn deserial(source: &mut R) -> ParseResult { - Ok(Self { - id: source.get()?, - }) + #[test] + fn statemap_multiple_state_ref_mut_not_allowed() { + let t = trybuild::TestCases::new(); + t.compile_fail("tests/state/map-multiple-state-ref-mut.rs"); } } -impl StateBTreeMap { - /// Insert a key-value pair into the map. - /// Returns the previous value if the key was already in the map. - pub fn insert(&mut self, key: K, value: V) -> Option - where - S: HasStateApi, - K: Serialize + Ord, - V: Serial + DeserialWithState, { - let old_value_option = self.map.insert_borrowed(&key, value); - if old_value_option.is_none() && !self.ordered_set.insert(key) { - // Inconsistency between the map and ordered_set. - crate::trap(); - } - old_value_option - } - - /// Remove a key from the map, returning the value at the key if the key was - /// previously in the map. - /// - /// *Caution*: If `V` is a [StateBox], [StateMap], then it is - /// important to call [`Deletable::delete`] on the value returned when - /// you're finished with it. Otherwise, it will remain in the contract - /// state. - #[must_use] - pub fn remove_and_get(&mut self, key: &K) -> Option - where - S: HasStateApi, - K: Serialize + Ord, - V: Serial + DeserialWithState + Deletable, { - let v = self.map.remove_and_get(key); - if v.is_some() && !self.ordered_set.remove(key) { - // Inconsistency between the map and ordered_set. - crate::trap(); - } - v - } +/// This test module rely on the runtime providing host functions and can only +/// be run using `cargo concordium test`. +#[cfg(feature = "wasm-test")] +mod wasm_test { + use crate::{ + claim, claim_eq, concordium_test, to_bytes, Deletable, Deserial, DeserialWithState, + EntryRaw, HasStateApi, HasStateEntry, ParseResult, Serial, StateApi, StateBuilder, + StateError, StateMap, StateSet, INITIAL_NEXT_ITEM_PREFIX, + }; - /// Remove a key from the map. - /// This also deletes the value in the state. - pub fn remove(&mut self, key: &K) - where - S: HasStateApi, - K: Serialize + Ord, - V: Serial + DeserialWithState + Deletable, { - if self.ordered_set.remove(key) { - self.map.remove(key); - } - } + const GENERIC_MAP_PREFIX: u64 = 1; - /// Get a reference to the value corresponding to the key. - pub fn get(&self, key: &K) -> Option> + /// Some helper methods that are used for internal tests. + impl StateBuilder where - K: Serialize, S: HasStateApi, - V: Serial + DeserialWithState, { - if self.ordered_set.is_empty() { - None - } else { - self.map.get(key) + { + /// Get a value from the generic map. + /// `Some(Err(_))` means that something exists in the state with that + /// key, but it isn't of type `V`. + pub(crate) fn get>( + &self, + key: K, + ) -> Option> { + let key_with_map_prefix = Self::prepend_generic_map_key(key); + + self.state_api + .lookup_entry(&key_with_map_prefix) + .map(|mut entry| V::deserial_with_state(&self.state_api, &mut entry)) + } + + /// Inserts a value in the generic map. + /// The key and value are serialized before insert. + pub(crate) fn insert( + &mut self, + key: K, + value: V, + ) -> Result<(), StateError> { + let key_with_map_prefix = Self::prepend_generic_map_key(key); + match self.state_api.entry(key_with_map_prefix) { + EntryRaw::Vacant(vac) => { + let _ = vac.insert(&value); + } + EntryRaw::Occupied(mut occ) => occ.insert(&value), + } + Ok(()) } - } - /// Get a mutable reference to the value corresponding to the key. - pub fn get_mut(&mut self, key: &K) -> Option> - where - K: Serialize, - S: HasStateApi, - V: Serial + DeserialWithState, { - if self.ordered_set.is_empty() { - None - } else { - self.map.get_mut(key) + /// Serializes the key and prepends [GENERIC_MAP_PREFIX]. + /// This is similar to how [StateMap] works, where a unique prefix is + /// prepended onto keys. Since there is only one generic map, the prefix + /// is a constant. + fn prepend_generic_map_key(key: K) -> Vec { + let mut key_with_map_prefix = to_bytes(&GENERIC_MAP_PREFIX); + key_with_map_prefix.append(&mut to_bytes(&key)); + key_with_map_prefix } } - /// Returns `true` if the map contains a value for the specified key. - pub fn contains_key(&self, key: &K) -> bool - where - K: Serialize + Ord, - S: HasStateApi, { - self.ordered_set.contains(key) - } - - /// Returns the smallest key in the map, which is strictly larger than the - /// provided key. `None` meaning no such key is present in the map. - pub fn higher(&self, key: &K) -> Option> - where - S: HasStateApi, - K: Serialize + Ord, { - self.ordered_set.higher(key) - } - - /// Returns the largest key in the map, which is strictly smaller than the - /// provided key. `None` meaning no such key is present in the map. - pub fn lower(&self, key: &K) -> Option> - where - S: HasStateApi, - K: Serialize + Ord, { - self.ordered_set.lower(key) - } - - /// Returns a reference to the first key in the map, if any. This key is - /// always the minimum of all keys in the map. - pub fn first_key(&self) -> Option> - where - S: HasStateApi, - K: Serialize + Ord, { - self.ordered_set.first() - } - - /// Returns a reference to the last key in the map, if any. This key is - /// always the maximum of all keys in the map. - pub fn last_key(&self) -> Option> - where - S: HasStateApi, - K: Serialize + Ord, { - self.ordered_set.last() + #[concordium_test] + fn high_level_insert_get() { + let expected_value: u64 = 123123123; + let mut state_builder = StateBuilder::open(StateApi::open()); + state_builder.insert(0, expected_value).expect("Insert failed"); + let actual_value: u64 = state_builder.get(0).expect("Not found").expect("Not a valid u64"); + claim_eq!(expected_value, actual_value); } - /// Return the number of elements in the map. - pub fn len(&self) -> u32 { self.ordered_set.len() } - - /// Returns `true` is the map contains no elements. - pub fn is_empty(&self) -> bool { self.ordered_set.is_empty() } - - /// Create an iterator over the entries of [`StateBTreeMap`]. - /// Ordered by `K`. - pub fn iter(&self) -> StateBTreeMapIter - where - S: HasStateApi, { - StateBTreeMapIter { - key_iter: self.ordered_set.iter(), - map: &self.map, + #[concordium_test] + fn low_level_entry() { + let expected_value: u64 = 123123123; + let key = to_bytes(&42u64); + let mut state = StateApi::open(); + state + .entry(&key[..]) + .or_insert_raw(&to_bytes(&expected_value)) + .expect("No iterators, so insertion should work."); + + match state.entry(key) { + EntryRaw::Vacant(_) => panic!("Unexpected vacant entry."), + EntryRaw::Occupied(occ) => { + claim_eq!(u64::deserial(&mut occ.get()), Ok(expected_value)) + } } } - /// Clears the map, removing all key-value pairs. - /// This also includes values pointed at, if `V`, for example, is a - /// [StateBox]. **If applicable use [`clear_flat`](Self::clear_flat) - /// instead.** - pub fn clear(&mut self) - where - S: HasStateApi, - K: Serialize, - V: Serial + DeserialWithState + Deletable, { - self.map.clear(); - self.ordered_set.clear(); - } - - /// Clears the map, removing all key-value pairs. - /// **This should be used over [`clear`](Self::clear) if it is - /// applicable.** It avoids recursive deletion of values since the - /// values are required to be _flat_. - /// - /// Unfortunately it is not possible to automatically choose between these - /// implementations. Once Rust gets trait specialization then this might - /// be possible. - pub fn clear_flat(&mut self) - where - S: HasStateApi, - K: Serialize, - V: Serialize, { - self.map.clear_flat(); - self.ordered_set.clear(); - } -} + #[concordium_test] + fn high_level_statemap() { + let my_map_key = "my_map"; + let mut state_builder = StateBuilder::open(StateApi::open()); -impl StateBTreeSet { - /// Construct a new [`StateBTreeSet`] given a unique prefix to use in the - /// key-value store. - pub(crate) fn new(state_api: S, prefix: StateItemPrefix) -> Self { - Self { - _marker_key: Default::default(), - prefix, - state_api, - root: None, - len: 0, - next_node_id: state_btree_internals::NodeId { - id: 0, - }, - } + let map_to_insert = state_builder.new_map::(); + state_builder.insert(my_map_key, map_to_insert).expect("Insert failed"); + + let mut my_map: StateMap = state_builder + .get(my_map_key) + .expect("Could not get statemap") + .expect("Deserializing statemap failed"); + my_map.insert("abc".to_string(), "hello, world".to_string()); + my_map.insert("def".to_string(), "hallo, Weld".to_string()); + my_map.insert("ghi".to_string(), "hej, verden".to_string()); + claim_eq!(*my_map.get(&"abc".to_string()).unwrap(), "hello, world".to_string()); + + let mut iter = my_map.iter(); + let (k1, v1) = iter.next().unwrap(); + claim_eq!(*k1, "abc".to_string()); + claim_eq!(*v1, "hello, world".to_string()); + let (k2, v2) = iter.next().unwrap(); + claim_eq!(*k2, "def".to_string()); + claim_eq!(*v2, "hallo, Weld".to_string()); + let (k3, v3) = iter.next().unwrap(); + claim_eq!(*k3, "ghi".to_string()); + claim_eq!(*v3, "hej, verden".to_string()); + claim!(iter.next().is_none()); } - /// Insert a key into the set. - /// Returns true if the key is new in the collection. - pub fn insert(&mut self, key: K) -> bool - where - S: HasStateApi, - K: Serialize + Ord, { - let Some(root_id) = self.root else { - let node_id = { - let (node_id, _node) = self.create_node(crate::vec![key], Vec::new()); - node_id - }; - self.root = Some(node_id); - self.len = 1; - return true; - }; - - let root_node = self.get_node_mut(root_id); - if !root_node.is_full() { - let new = self.insert_non_full(root_node, key); - if new { - self.len += 1; - } - return new; - } else if root_node.keys.binary_search(&key).is_ok() { - return false; - } - // The root node is full, so we construct a new root node. - let (new_root_id, mut new_root) = self.create_node(Vec::new(), crate::vec![root_id]); - self.root = Some(new_root_id); - // The old root node is now a child node. - let mut child = root_node; - let new_larger_child = self.split_child(&mut new_root, 0, &mut child); - // new_root should now contain one key and two children, so we need to know - // which one to insert into. - let child = if new_root.keys[0] < key { - new_larger_child - } else { - child - }; - let new = self.insert_non_full(child, key); - if new { - self.len += 1; - } - new + #[concordium_test] + fn statemap_insert_remove() { + let mut state_builder = StateBuilder::open(StateApi::open()); + let mut map = state_builder.new_map(); + let value = String::from("hello"); + let _ = map.insert(42, value.clone()); + claim_eq!(*map.get(&42).unwrap(), value); + map.remove(&42); + claim!(map.get(&42).is_none()); } - /// Returns `true` if the set contains an element equal to the key. - pub fn contains(&self, key: &K) -> bool - where - S: HasStateApi, - K: Serialize + Ord, { - let Some(root_node_id) = self.root else { - return false; - }; - let mut node = self.get_node(root_node_id); - loop { - let Err(child_index) = node.keys.binary_search(key) else { - return true; - }; - if node.is_leaf() { - return false; - } - let child_node_id = node.children[child_index]; - node = self.get_node(child_node_id); - } + #[concordium_test] + fn statemap_clear() { + let mut state_builder = StateBuilder::open(StateApi::open()); + let mut map = state_builder.new_map(); + let _ = map.insert(1, 2); + let _ = map.insert(2, 3); + let _ = map.insert(3, 4); + map.clear(); + claim!(map.is_empty()); } - /// Return the number of elements in the set. - pub fn len(&self) -> u32 { self.len } + #[concordium_test] + fn high_level_nested_statemaps() { + let inner_map_key = 0u8; + let key_to_value = 77u8; + let value = 255u8; + let mut state_builder = StateBuilder::open(StateApi::open()); + let mut outer_map = state_builder.new_map::>(); + let mut inner_map = state_builder.new_map::(); - /// Returns `true` is the set contains no elements. - pub fn is_empty(&self) -> bool { self.root.is_none() } + inner_map.insert(key_to_value, value); + outer_map.insert(inner_map_key, inner_map); - /// Get an iterator over the elements in the `StateBTreeSet`. The iterator - /// returns elements in increasing order. - pub fn iter(&self) -> StateBTreeSetIter - where - S: HasStateApi, { - StateBTreeSetIter { - length: self.len.try_into().unwrap_abort(), - next_node: self.root, - depth_first_stack: Vec::new(), - tree: self, - _marker_lifetime: Default::default(), - } + claim_eq!(*outer_map.get(&inner_map_key).unwrap().get(&key_to_value).unwrap(), value); } - /// Clears the set, removing all elements. - pub fn clear(&mut self) - where - S: HasStateApi, { - // Reset the information. - self.root = None; - self.next_node_id = state_btree_internals::NodeId { - id: 0, - }; - self.len = 0; - // Then delete every node store in the state. - // Unwrapping is safe when only using the high-level API. - self.state_api.delete_prefix(&self.prefix).unwrap_abort(); + #[concordium_test] + fn statemap_iter_mut_works() { + let mut state_builder = StateBuilder::open(StateApi::open()); + let mut map = state_builder.new_map(); + map.insert(0u8, 1u8); + map.insert(1u8, 2u8); + map.insert(2u8, 3u8); + for (_, mut v) in map.iter_mut() { + v.update(|old_value| *old_value += 10); + } + let mut iter = map.iter(); + let (k1, v1) = iter.next().unwrap(); + claim_eq!(*k1, 0); + claim_eq!(*v1, 11); + let (k2, v2) = iter.next().unwrap(); + claim_eq!(*k2, 1); + claim_eq!(*v2, 12); + let (k3, v3) = iter.next().unwrap(); + claim_eq!(*k3, 2); + claim_eq!(*v3, 13); + claim!(iter.next().is_none()); } - /// Returns the smallest key in the set, which is strictly larger than the - /// provided key. `None` meaning no such key is present in the set. - pub fn higher(&self, key: &K) -> Option> - where - S: HasStateApi, - K: Serialize + Ord, { - let Some(root_node_id) = self.root else { - return None; - }; - - let mut node = self.get_node(root_node_id); - let mut higher_so_far = None; - loop { - let higher_key_index = match node.keys.binary_search(key) { - Ok(index) => index + 1, - Err(index) => index, - }; - - if node.is_leaf() { - return if higher_key_index < node.keys.len() { - // This does not mutate the node in the end, just the representation in memory - // which is freed after the call. - Some(StateRef::new(node.keys.swap_remove(higher_key_index))) - } else { - higher_so_far - }; - } else { - if higher_key_index < node.keys.len() { - // This does not mutate the node in the end, just the representation in memory - // which is freed after the call. - higher_so_far = Some(StateRef::new(node.keys.swap_remove(higher_key_index))) + #[concordium_test] + fn iter_mut_works_on_nested_statemaps() { + let mut state_builder = StateBuilder::open(StateApi::open()); + let mut outer_map = state_builder.new_map(); + let mut inner_map = state_builder.new_map(); + inner_map.insert(0u8, 1u8); + inner_map.insert(1u8, 2u8); + outer_map.insert(99u8, inner_map); + for (_, mut v_map) in outer_map.iter_mut() { + v_map.update(|v_map| { + for (_, mut inner_v) in v_map.iter_mut() { + inner_v.update(|inner_v| *inner_v += 10); } - - let child_node_id = node.children[higher_key_index]; - node = self.get_node(child_node_id); - } + }); } - } - /// Returns the largest key in the set, which is strictly smaller than the - /// provided key. `None` meaning no such key is present in the set. - pub fn lower(&self, key: &K) -> Option> - where - S: HasStateApi, - K: Serialize + Ord, { - let Some(root_node_id) = self.root else { - return None; - }; + // Check the outer map. + let mut outer_iter = outer_map.iter(); + let (inner_map_key, inner_map) = outer_iter.next().unwrap(); + claim_eq!(*inner_map_key, 99); + claim!(outer_iter.next().is_none()); - let mut node = self.get_node(root_node_id); - let mut lower_so_far = None; - loop { - let lower_key_index = match node.keys.binary_search(key) { - Ok(index) => index, - Err(index) => index, - }; - - if node.is_leaf() { - return if lower_key_index == 0 { - lower_so_far - } else { - // lower_key_index cannot be 0 in this case, since the binary search will only - // return 0 in the true branch above. - Some(StateRef::new(node.keys.swap_remove(lower_key_index - 1))) - }; - } else { - if lower_key_index > 0 { - // This does not mutate the node in the end, just the representation in memory - // which is freed after the call. - lower_so_far = Some(StateRef::new(node.keys.swap_remove(lower_key_index - 1))); - } - let child_node_id = node.children[lower_key_index]; - node = self.get_node(child_node_id) - } - } + // Check the inner map. + let mut inner_iter = inner_map.iter(); + let (k1, v1) = inner_iter.next().unwrap(); + claim_eq!(*k1, 0); + claim_eq!(*v1, 11); + let (k2, v2) = inner_iter.next().unwrap(); + claim_eq!(*k2, 1); + claim_eq!(*v2, 12); + claim!(inner_iter.next().is_none()); } - /// Returns a reference to the first key in the set, if any. This key is - /// always the minimum of all keys in the set. - pub fn first(&self) -> Option> - where - S: HasStateApi, - K: Serialize + Ord, { - let Some(root_node_id) = self.root else { - return None; - }; - let mut root = self.get_node(root_node_id); - if root.is_leaf() { - Some(StateRef::new(root.keys.swap_remove(0))) - } else { - Some(StateRef::new(self.get_lowest_key(&root, 0))) - } + #[concordium_test] + fn statemap_iterator_unlocks_tree_once_dropped() { + let mut state_builder = StateBuilder::open(StateApi::open()); + let mut map = state_builder.new_map(); + map.insert(0u8, 1u8); + map.insert(1u8, 2u8); + { + let _iter = map.iter(); + // Uncommenting these two lines (and making iter mutable) should + // give a compile error: + // + // map.insert(2u8, 3u8); + // let n = iter.next(); + } // iter is dropped here, unlocking the subtree. + map.insert(2u8, 3u8); } - /// Returns a reference to the last key in the set, if any. This key is - /// always the maximum of all keys in the set. - pub fn last(&self) -> Option> - where - S: HasStateApi, - K: Serialize + Ord, { - let Some(root_node_id) = self.root else { - return None; - }; - let mut root = self.get_node(root_node_id); - if root.is_leaf() { - Some(StateRef::new(root.keys.pop().unwrap_abort())) - } else { - Some(StateRef::new(self.get_highest_key(&root, root.children.len() - 1))) - } - } + #[concordium_test] + fn high_level_stateset() { + let my_set_key = "my_set"; + let mut state_builder = StateBuilder::open(StateApi::open()); - /// Remove a key from the set. - /// Returns whether such an element was present. - pub fn remove(&mut self, key: &K) -> bool - where - K: Ord + Serialize, - S: HasStateApi, { - let Some(root_node_id) = self.root else { - return false; - }; + let mut set = state_builder.new_set::(); + claim!(set.insert(0)); + claim!(set.insert(1)); + claim!(!set.insert(1)); + claim!(set.insert(2)); + claim!(set.remove(&2)); + state_builder.insert(my_set_key, set).expect("Insert failed"); - let deleted_something = { - let mut node = self.get_node_mut(root_node_id); - loop { - match node.keys.binary_search(key) { - Ok(index) => { - if node.is_leaf() { - // Found the key in this node and the node is a leaf, meaning we - // simply remove it. - // This will not violate the minimum keys invariant, since a node - // ensures a child can spare a key before iteration and the root - // node is not part of the - // invariant. - node.keys.remove(index); - break true; - } - // Found the key in this node, but the node is not a leaf. - let mut left_child = self.get_node_mut(node.children[index]); - if !left_child.is_at_min() { - // If the child with smaller keys can spare a key, we take the - // highest key from it. - node.keys[index] = self.remove_largest_key(left_child); - break true; - } - - let right_child = self.get_node_mut(node.children[index + 1]); - if !right_child.is_at_min() { - // If the child with larger keys can spare a key, we take the lowest - // key from it. - node.keys[index] = self.remove_smallest_key(right_child); - break true; - } - // No child on either side of the key can spare a key at this point, so - // we merge them into one child, moving the - // key into the merged child and try - // to remove from this. - self.merge(&mut node, index, &mut left_child, right_child); - node = left_child; - continue; - } - Err(index) => { - // Node did not contain the key. - if node.is_leaf() { - break false; - } - // Node did not contain the key and is not a leaf. - // Check and proactively prepare the child to be able to delete a key. - node = self.prepare_child_for_key_removal(node, index); - } - }; - } - }; + claim!(state_builder.get::<_, StateSet>(my_set_key).unwrap().unwrap().contains(&0),); + claim!(!state_builder.get::<_, StateSet>(my_set_key).unwrap().unwrap().contains(&2),); - // If something was deleted, we update the length and make sure to remove the - // root node if needed. - if deleted_something { - self.len -= 1; - let root = self.get_node_mut(root_node_id); - if self.len == 0 { - // Remote the root node if tree is empty. - self.root = None; - self.delete_node(root); - } else { - // If the root is empty but the tree is not, point to the only child of the root - // as the root. - if root.keys.is_empty() { - self.root = Some(root.children[0]); - self.delete_node(root); - } - } - } - deleted_something + let set = state_builder.get::<_, StateSet>(my_set_key).unwrap().unwrap(); + let mut iter = set.iter(); + claim_eq!(*iter.next().unwrap(), 0); + claim_eq!(*iter.next().unwrap(), 1); + claim!(iter.next().is_none()); } - /// Internal function for taking the largest key in a subtree. - /// - /// Assumes: - /// - The provided node is not the root. - /// - The node contain more than minimum number of keys. - fn remove_largest_key<'a>( - &mut self, - mut node: StateRefMut<'a, state_btree_internals::Node, S>, - ) -> K - where - K: Ord + Serialize, - S: HasStateApi, { - while !node.is_leaf() { - // Node is not a leaf, so we move further down this subtree. - let child_index = node.children.len() - 1; - node = self.prepare_child_for_key_removal(node, child_index); - } - // The node is a leaf, meaning we simply remove it. - // This will not violate the minimum keys invariant, since a node - // ensures a child can spare a key before iteration. - node.keys.pop().unwrap_abort() - } - - /// Internal function for taking the smallest key in a subtree. - /// - /// Assumes: - /// - The provided node is not the root. - /// - The provided `node` contain more than minimum number of keys. - fn remove_smallest_key<'a>( - &mut self, - mut node: StateRefMut<'a, state_btree_internals::Node, S>, - ) -> K - where - K: Ord + Serialize, - S: HasStateApi, { - while !node.is_leaf() { - // Node is not a leaf, so we move further down this subtree. - let child_index = 0; - node = self.prepare_child_for_key_removal(node, child_index); - } - // The node is a leaf, meaning we simply remove it. - // This will not violate the minimum keys invariant, since a node - // ensures a child can spare a key before iteration. - node.keys.remove(0) - } - - /// Internal function for key rotation, preparing a node for deleting a key. - /// Returns the now prepared child at `index`. - /// Assumes: - /// - The provided `node` is not a leaf and has a child at `index`. - /// - The minimum degree `M` is at least 2 or more. - fn prepare_child_for_key_removal<'b, 'c>( - &mut self, - mut node: StateRefMut<'b, state_btree_internals::Node, S>, - index: usize, - ) -> StateRefMut<'c, state_btree_internals::Node, S> - where - K: Ord + Serialize, - S: HasStateApi, { - let mut child = self.get_node_mut(node.children[index]); - if !child.is_at_min() { - return child; - } - // The child is at minimum keys, so first attempt to take a key from either - // sibling, otherwise merge with one of them. - let has_smaller_sibling = 0 < index; - let has_larger_sibling = index < node.children.len() - 1; - let smaller_sibling = if has_smaller_sibling { - let mut smaller_sibling = self.get_node_mut(node.children[index - 1]); - if !smaller_sibling.is_at_min() { - // The smaller sibling can spare a key, so we replace the largest - // key from the sibling, put it in - // the parent and take a key from - // the parent. - let largest_key_sibling = smaller_sibling.keys.pop().unwrap_abort(); - let swapped_node_key = mem::replace(&mut node.keys[index - 1], largest_key_sibling); - child.keys.insert(0, swapped_node_key); - if !child.is_leaf() { - child.children.insert(0, smaller_sibling.children.pop().unwrap_abort()); - } - return child; - } - Some(smaller_sibling) - } else { - None - }; - let larger_sibling = if has_larger_sibling { - let mut larger_sibling = self.get_node_mut(node.children[index + 1]); - if !larger_sibling.is_at_min() { - // The larger sibling can spare a key, so we replace the smallest - // key from the sibling, put it in - // the parent and take a key from - // the parent. - let first_key_sibling = larger_sibling.keys.remove(0); - let swapped_node_key = mem::replace(&mut node.keys[index], first_key_sibling); - child.keys.push(swapped_node_key); - - if !child.is_leaf() { - child.children.push(larger_sibling.children.remove(0)); - } - return child; - } - Some(larger_sibling) - } else { - None - }; - - if let Some(sibling) = larger_sibling { - self.merge(&mut node, index, &mut child, sibling); - child - } else if let Some(mut sibling) = smaller_sibling { - self.merge(&mut node, index - 1, &mut sibling, child); - sibling - } else { - // Unreachable code, since M must be 2 or larger (the minimum degree), a - // child node must have at least one sibling. - crate::trap(); - } - } - - /// Internal function for getting the highest key in a subtree. - /// Assumes the provided `node` is not a leaf and has a child at - /// `child_index`. - fn get_highest_key(&self, node: &state_btree_internals::Node, child_index: usize) -> K - where - K: Ord + Serialize, - S: HasStateApi, { - let mut node = self.get_node(node.children[child_index]); - while !node.is_leaf() { - let child_node_id = node.children.last().unwrap_abort(); - node = self.get_node(*child_node_id); - } - // This does not mutate the node in the end, just the representation in memory - // which is freed after the call. - node.keys.pop().unwrap_abort() - } + #[concordium_test] + fn high_level_nested_stateset() { + let inner_set_key = 0u8; + let value = 255u8; + let mut state_builder = StateBuilder::open(StateApi::open()); + let mut outer_map = state_builder.new_map::>(); + let mut inner_set = state_builder.new_set::(); - /// Internal function for getting the lowest key in a subtree. - /// Assumes the provided `node` is not a leaf and has a child at - /// `child_index`. - fn get_lowest_key(&self, node: &state_btree_internals::Node, child_index: usize) -> K - where - K: Ord + Serialize, - S: HasStateApi, { - let mut node = self.get_node(node.children[child_index]); - while !node.is_leaf() { - let child_node_id = node.children.first().unwrap_abort(); - node = self.get_node(*child_node_id); - } - // This does not mutate the node in the end, just the representation in memory - // which is freed after the call. - node.keys.swap_remove(0) - } + inner_set.insert(value); + outer_map.insert(inner_set_key, inner_set); - /// Moving key at `index` from the node to the lower child and then merges - /// this child with the content of its larger sibling, deleting the sibling. - /// - /// Assumes: - /// - `parent_node` has children `child` at `index` and `larger_child` at - /// `index + 1`. - /// - Both children are at minimum number of keys (`M - 1`). - fn merge( - &mut self, - parent_node: &mut state_btree_internals::Node, - index: usize, - child: &mut state_btree_internals::Node, - mut larger_child: StateRefMut, S>, - ) where - K: Ord + Serialize, - S: HasStateApi, { - let parent_key = parent_node.keys.remove(index); - parent_node.children.remove(index + 1); - child.keys.push(parent_key); - child.keys.append(&mut larger_child.keys); - child.children.append(&mut larger_child.children); - self.delete_node(larger_child); - } - - /// Internal function for constructing a node. It will increment the next - /// node ID and create an entry in the smart contract key-value store. - fn create_node<'b>( - &mut self, - keys: Vec, - children: Vec, - ) -> (state_btree_internals::NodeId, StateRefMut<'b, state_btree_internals::Node, S>) - where - K: Serialize, - S: HasStateApi, { - let node_id = self.next_node_id.copy_then_increment(); - let node = state_btree_internals::Node { - keys, - children, - }; - let entry = self.state_api.create_entry(&node_id.as_key(&self.prefix)).unwrap_abort(); - let mut ref_mut: StateRefMut<'_, state_btree_internals::Node, S> = - StateRefMut::new(entry, self.state_api.clone()); - ref_mut.set(node); - (node_id, ref_mut) + claim!(outer_map.get(&inner_set_key).unwrap().contains(&value)); } - /// Internal function for deleting a node, removing the entry in the smart - /// contract key-value store. Traps if no node was present. - fn delete_node(&mut self, node: StateRefMut, S>) - where - K: Serial, - S: HasStateApi, { - self.state_api.delete_entry(node.into_raw_parts().1).unwrap_abort() + #[concordium_test] + fn stateset_insert_remove() { + let mut state_builder = StateBuilder::open(StateApi::open()); + let mut set = state_builder.new_set(); + let _ = set.insert(42); + claim!(set.contains(&42)); + set.remove(&42); + claim!(!set.contains(&42)); } - /// Internal function for inserting into a subtree. - /// Assumes the given node is not full. - fn insert_non_full( - &mut self, - initial_node: StateRefMut, S>, - key: K, - ) -> bool - where - K: Serialize + Ord, - S: HasStateApi, { - let mut node = initial_node; - loop { - let Err(insert_index) = node.keys.binary_search(&key) else { - // We find the key in this node, so we do nothing. - return false; - }; - // The key is not in this node. - if node.is_leaf() { - // Since we can assume the node is not full and this is a leaf, we can just - // insert here. - node.keys.insert(insert_index, key); - return true; - } - - // The node is not a leaf, so we want to insert in the relevant child node. - let mut child = self.get_node_mut(node.children[insert_index]); - node = if !child.is_full() { - child - } else { - let larger_child = self.split_child(&mut node, insert_index, &mut child); - // Since the child is now split into two, we have to check which one to insert - // into. - if node.keys[insert_index] < key { - larger_child - } else { - child - } - }; - } + #[concordium_test] + fn stateset_clear() { + let mut state_builder = StateBuilder::open(StateApi::open()); + let mut set = state_builder.new_set(); + let _ = set.insert(1); + let _ = set.insert(2); + let _ = set.insert(3); + set.clear(); + claim!(set.is_empty()); } - /// Internal function for splitting the child node at a given index for a - /// given node. This will also mutate the given node adding a new key - /// and child after the provided child_index. - /// Returns the newly created node. - /// - /// Assumes: - /// - Node is not a leaf and has `child` as child at `child_index`. - /// - `child` is at maximum keys (`2 * M - 1`). - fn split_child<'b>( - &mut self, - node: &mut state_btree_internals::Node, - child_index: usize, - child: &mut state_btree_internals::Node, - ) -> StateRefMut<'b, state_btree_internals::Node, S> - where - K: Serialize + Ord, - S: HasStateApi, { - let split_index = state_btree_internals::Node::::MINIMUM_KEY_LEN + 1; - let (new_larger_sibling_id, new_larger_sibling) = self.create_node( - child.keys.split_off(split_index), - if child.is_leaf() { - Vec::new() - } else { - child.children.split_off(split_index) - }, - ); - let key = child.keys.pop().unwrap_abort(); - node.children.insert(child_index + 1, new_larger_sibling_id); - node.keys.insert(child_index, key); - new_larger_sibling + #[concordium_test] + fn stateset_iterator_unlocks_tree_once_dropped() { + let mut state_builder = StateBuilder::open(StateApi::open()); + let mut set = state_builder.new_set(); + set.insert(0u8); + set.insert(1); + { + let _iter = set.iter(); + // Uncommenting these two lines (and making iter mutable) should + // give a compile error: + // + // set.insert(2); + // let n = iter.next(); + } // iter is dropped here, unlocking the subtree. + set.insert(2); } - /// Internal function for looking up a node in the tree. - /// This assumes the node is present and traps if this is not the case. - fn get_node( - &self, - node_id: state_btree_internals::NodeId, - ) -> state_btree_internals::Node - where - Key: Deserial, - S: HasStateApi, { - let key = node_id.as_key(&self.prefix); - let mut entry = self.state_api.lookup_entry(&key).unwrap_abort(); - entry.get().unwrap_abort() + #[concordium_test] + fn allocate_and_get_statebox() { + let mut state_builder = StateBuilder::open(StateApi::open()); + let boxed_value = String::from("I'm boxed"); + let statebox = state_builder.new_box(boxed_value.clone()); + claim_eq!(*statebox.get(), boxed_value); } - /// Internal function for looking up a node, providing mutable access. - /// This assumes the node is present and traps if this is not the case. - fn get_node_mut<'b>( - &mut self, - node_id: state_btree_internals::NodeId, - ) -> StateRefMut<'b, state_btree_internals::Node, S> - where - K: Serial, - S: HasStateApi, { - let key = node_id.as_key(&self.prefix); - let entry = self.state_api.lookup_entry(&key).unwrap_abort(); - StateRefMut::new(entry, self.state_api.clone()) + #[concordium_test] + fn a_new_entry_can_not_be_created_under_a_locked_subtree() { + let expected_value: u64 = 123123123; + let key = to_bytes(b"ab"); + let sub_key = to_bytes(b"abc"); + let mut state = StateApi::open(); + state + .entry(&key[..]) + .or_insert_raw(&to_bytes(&expected_value)) + .expect("No iterators, so insertion should work."); + claim!(state.iterator(&key).is_ok(), "Iterator should be present"); + let entry = state.create_entry(&sub_key); + claim!(entry.is_err(), "Should not be able to create an entry under a locked subtree"); } -} - -/// Byte size of the key used to store a BTree internal node in the smart -/// contract key-value store. -const BTREE_NODE_KEY_SIZE: usize = - STATE_ITEM_PREFIX_SIZE + state_btree_internals::NodeId::SERIALIZED_BYTE_SIZE; - -impl state_btree_internals::Node { - /// The max length of the child list. - const MAXIMUM_CHILD_LEN: usize = 2 * M; - /// The max length of the key list. - const MAXIMUM_KEY_LEN: usize = Self::MAXIMUM_CHILD_LEN - 1; - /// The min length of the child list, when the node is not a leaf node. - const MINIMUM_CHILD_LEN: usize = M; - /// The min length of the key list, except when the node is root. - const MINIMUM_KEY_LEN: usize = Self::MINIMUM_CHILD_LEN - 1; - - /// The number of keys stored in this node. - fn len(&self) -> usize { self.keys.len() } - - /// Check if the node holds the maximum number of keys. - fn is_full(&self) -> bool { self.len() == Self::MAXIMUM_KEY_LEN } - - /// Check if the node is representing a leaf in the tree. - fn is_leaf(&self) -> bool { self.children.is_empty() } - /// Check if the node holds the minimum number of keys. - fn is_at_min(&self) -> bool { self.len() == Self::MINIMUM_KEY_LEN } -} - -impl state_btree_internals::NodeId { - /// Byte size of `NodeId` when serialized. - pub(crate) const SERIALIZED_BYTE_SIZE: usize = 4; - - /// Return a copy of the NodeId, then increments itself. - pub(crate) fn copy_then_increment(&mut self) -> Self { - let current = *self; - self.id += 1; - current - } - - /// Construct the key for the node in the key-value store from the node ID. - fn as_key(&self, prefix: &StateItemPrefix) -> [u8; BTREE_NODE_KEY_SIZE] { - // Create an uninitialized array of `MaybeUninit`. The `assume_init` is - // safe because the type we are claiming to have initialized here is a - // bunch of `MaybeUninit`s, which do not require initialization. - let mut prefixed: [MaybeUninit; BTREE_NODE_KEY_SIZE] = - unsafe { MaybeUninit::uninit().assume_init() }; - for i in 0..STATE_ITEM_PREFIX_SIZE { - prefixed[i].write(prefix[i]); - } - let id_bytes = self.id.to_le_bytes(); - for i in 0..id_bytes.len() { - prefixed[STATE_ITEM_PREFIX_SIZE + i].write(id_bytes[i]); - } - // Transmuting away the maybeuninit is safe since we have initialized all of - // them. - unsafe { mem::transmute(prefixed) } + #[concordium_test] + fn a_new_entry_can_be_created_under_a_different_subtree_in_same_super_tree() { + let expected_value: u64 = 123123123; + let key = to_bytes(b"abcd"); + let key2 = to_bytes(b"abe"); + let mut state = StateApi::open(); + state + .entry(&key[..]) + .or_insert_raw(&to_bytes(&expected_value)) + .expect("No iterators, so insertion should work."); + claim!(state.iterator(&key).is_ok(), "Iterator should be present"); + let entry = state.create_entry(&key2); + claim!(entry.is_ok(), "Failed to create a new entry under a different subtree"); } -} -impl Deserial for state_btree_internals::KeyWrapper { - fn deserial(source: &mut R) -> ParseResult { - let key = K::deserial(source)?; - Ok(Self { - key: Some(key), - }) + #[concordium_test] + fn an_existing_entry_can_not_be_deleted_under_a_locked_subtree() { + let expected_value: u64 = 123123123; + let key = to_bytes(b"ab"); + let sub_key = to_bytes(b"abc"); + let mut state = StateApi::open(); + state + .entry(&key[..]) + .or_insert_raw(&to_bytes(&expected_value)) + .expect("no iterators, so insertion should work."); + let sub_entry = state + .entry(sub_key) + .or_insert_raw(&to_bytes(&expected_value)) + .expect("Should be possible to create the entry."); + claim!(state.iterator(&key).is_ok(), "Iterator should be present"); + claim!( + state.delete_entry(sub_entry).is_err(), + "Should not be able to create an entry under a locked subtree" + ); } -} - -impl<'a, 'b, const M: usize, K, S> Iterator for StateBTreeSetIter<'a, 'b, K, S, M> -where - 'a: 'b, - K: Deserial, - S: HasStateApi, -{ - type Item = StateRef<'b, K>; - - fn next(&mut self) -> Option { - while let Some(id) = self.next_node.take() { - let node = self.tree.get_node(id); - if !node.is_leaf() { - self.next_node = Some(node.children[0]); - } - self.depth_first_stack.push((node, 0)); - } - if let Some((node, index)) = self.depth_first_stack.last_mut() { - let key = node.keys[*index].key.take().unwrap_abort(); - *index += 1; - let no_more_keys = index == &node.keys.len(); - if !node.is_leaf() { - let child_id = node.children[*index]; - self.next_node = Some(child_id); - } - if no_more_keys { - self.depth_first_stack.pop(); - } - self.length -= 1; - Some(StateRef::new(key)) - } else { - None - } + #[concordium_test] + fn an_existing_entry_can_be_deleted_from_a_different_subtree_in_same_super_tree() { + let expected_value: u64 = 123123123; + let key = to_bytes(b"abcd"); + let key2 = to_bytes(b"abe"); + let mut state = StateApi::open(); + state + .entry(&key[..]) + .or_insert_raw(&to_bytes(&expected_value)) + .expect("No iterators, so insertion should work."); + let entry2 = state + .entry(key2) + .or_insert_raw(&to_bytes(&expected_value)) + .expect("Should be possible to create the entry."); + claim!(state.iterator(&key).is_ok(), "Iterator should be present"); + claim!( + state.delete_entry(entry2).is_ok(), + "Failed to create a new entry under a different subtree" + ); } - fn size_hint(&self) -> (usize, Option) { (self.length, Some(self.length)) } -} - -impl<'a, 'b, const M: usize, K, V, S> Iterator for StateBTreeMapIter<'a, 'b, K, V, S, M> -where - 'a: 'b, - K: Serialize, - V: Serial + DeserialWithState + 'b, - S: HasStateApi, -{ - type Item = (StateRef<'b, K>, StateRef<'b, V>); - - fn next(&mut self) -> Option { - let next_key = self.key_iter.next()?; - let value = self.map.get(&next_key).unwrap_abort(); - // Unwrap is safe, otherwise the map and the set have inconsistencies. - Some((next_key, value)) + #[concordium_test] + fn deleting_nested_stateboxes_works() { + let mut state_builder = StateBuilder::open(StateApi::open()); + let inner_box = state_builder.new_box(99u8); + let middle_box = state_builder.new_box(inner_box); + let outer_box = state_builder.new_box(middle_box); + outer_box.delete(); + let mut iter = state_builder.state_api.iterator(&[]).expect("Could not get iterator"); + // The only remaining node should be the state_builder's next_item_prefix node. + claim!(iter.nth(1).is_none()); } - fn size_hint(&self) -> (usize, Option) { self.key_iter.size_hint() } -} - -impl Deletable for StateBTreeMap -where - S: HasStateApi, - K: Serialize, - V: Serial + DeserialWithState + Deletable, -{ - fn delete(mut self) { self.clear(); } -} - -#[cfg(test)] -mod tests { - - /// Check that you cannot have multiple active entries from a statemap at - /// the same time. See the test file for details. - #[test] - fn statemap_multiple_entries_not_allowed() { - let t = trybuild::TestCases::new(); - t.compile_fail("tests/state/map-multiple-entries.rs"); + #[concordium_test] + fn clearing_statemap_with_stateboxes_works() { + let mut state_builder = StateBuilder::open(StateApi::open()); + let box1 = state_builder.new_box(1u8); + let box2 = state_builder.new_box(2u8); + let box3 = state_builder.new_box(3u8); + let mut map = state_builder.new_map(); + map.insert(1u8, box1); + map.insert(2u8, box2); + map.insert(3u8, box3); + map.clear(); + let mut iter = state_builder.state_api.iterator(&[]).expect("Could not get iterator"); + // The only remaining node should be the state_builder's next_item_prefix node. + claim!(iter.nth(1).is_none()); } - #[test] - fn statemap_multiple_state_ref_mut_not_allowed() { - let t = trybuild::TestCases::new(); - t.compile_fail("tests/state/map-multiple-state-ref-mut.rs"); + #[concordium_test] + fn clearing_nested_statemaps_works() { + let mut state_builder = StateBuilder::open(StateApi::open()); + let mut inner_map_1 = state_builder.new_map(); + inner_map_1.insert(1u8, 2u8); + inner_map_1.insert(2u8, 3u8); + inner_map_1.insert(3u8, 4u8); + let mut inner_map_2 = state_builder.new_map(); + inner_map_2.insert(11u8, 12u8); + inner_map_2.insert(12u8, 13u8); + inner_map_2.insert(13u8, 14u8); + let mut outer_map = state_builder.new_map(); + outer_map.insert(0u8, inner_map_1); + outer_map.insert(1u8, inner_map_2); + outer_map.clear(); + let mut iter = state_builder.state_api.iterator(&[]).expect("Could not get iterator"); + // The only remaining node should be the state_builder's next_item_prefix node. + claim!(iter.nth(1).is_none()); } -} - -#[concordium_cfg_test] -mod wasm_test { - use concordium_contracts_common::concordium_test; - - use crate::{claim_eq, StateApi, StateBuilder}; #[concordium_test] - fn test() { + fn occupied_entry_truncates_leftover_data() { let mut state_builder = StateBuilder::open(StateApi::open()); let mut map = state_builder.new_map(); + map.insert(99u8, "A longer string that should be truncated".into()); + let a_short_string = "A short string".to_string(); + let expected_size = a_short_string.len() + 4; // 4 bytes for the length of the string. + map.entry(99u8).and_modify(|v| *v = a_short_string); + let actual_size = state_builder + .state_api + .lookup_entry(&[INITIAL_NEXT_ITEM_PREFIX[0], 0, 0, 0, 0, 0, 0, 0, 99]) + .expect("Lookup failed") + .size() + .expect("Getting size failed"); + claim_eq!(expected_size as u32, actual_size); + } - map.insert(0, "0".to_string()); - - claim_eq!(map.get(&0).as_deref(), Some(&"1".to_string())); + #[concordium_test] + fn occupied_entry_raw_truncates_leftover_data() { + let mut state = StateApi::open(); + state + .entry([]) + .or_insert_raw(&to_bytes(&"A longer string that should be truncated")) + .expect("No iterators, so insertion should work."); + + let a_short_string = "A short string"; + let expected_size = a_short_string.len() + 4; // 4 bytes for the length of the string. + + match state.entry([]) { + EntryRaw::Vacant(_) => panic!("Entry is vacant"), + EntryRaw::Occupied(mut occ) => occ.insert_raw(&to_bytes(&a_short_string)), + } + let actual_size = + state.lookup_entry(&[]).expect("Lookup failed").size().expect("Getting size failed"); + claim_eq!(expected_size as u32, actual_size); } } diff --git a/concordium-std/src/lib.rs b/concordium-std/src/lib.rs index 9622ff4b..1453e425 100644 --- a/concordium-std/src/lib.rs +++ b/concordium-std/src/lib.rs @@ -394,10 +394,12 @@ pub mod collections { pub mod constants; mod impls; pub mod prims; +mod state_btree; mod traits; mod types; pub use concordium_contracts_common::*; pub use impls::*; +pub use state_btree::*; pub use traits::*; pub use types::*; diff --git a/concordium-std/src/state_btree.rs b/concordium-std/src/state_btree.rs new file mode 100644 index 00000000..1d9d05aa --- /dev/null +++ b/concordium-std/src/state_btree.rs @@ -0,0 +1,1481 @@ +use crate::{ + marker::PhantomData, mem, Deletable, Deserial, DeserialWithState, Get, HasStateApi, + ParseResult, Read, Serial, Serialize, StateItemPrefix, StateMap, StateRef, StateRefMut, + UnwrapAbort, Write, STATE_ITEM_PREFIX_SIZE, +}; + +/// An ordered map based on [B-Tree](https://en.wikipedia.org/wiki/B-tree), where +/// each node is stored separately in the low-level key-value store. +/// +/// It can be seen as an extension adding tracking of the keys ordering on top +/// of [`StateMap`] to provide functions such as [`higher`](Self::higher) and +/// [`lower`](Self::lower). This results in some overhead when inserting and +/// deleting entries from the map compared to using [`StateMap`]. +/// +/// | Operation | Performance | +/// |-------------------------------------------------|---------------| +/// | [`get`](Self::get) / [`get_mut`](Self::get_mut) | O(k) | +/// | [`insert`](Self::insert) | O(k + log(n)) | +/// | [`remove`](Self::remove) | O(k + log(n)) | +/// | [`higher`](Self::higher)/[`lower`](Self::lower) | O(k + log(n)) | +/// +/// Where `k` is the byte size of the serialized keys and `n` is the number of +/// entries in the map. +/// +/// ## Type parameters +/// +/// The map `StateBTreeMap` is parametrized by the types: +/// - `K`: Keys used in the map. Most operations on the map require this to +/// implement [`Serialize`](crate::Serialize). Keys cannot contain references +/// to the low-level state, such as types containing [`StateBox`], +/// [`StateMap`] and [`StateSet`]. +/// - `V`: Values stored in the map. Most operations on the map require this to +/// implement [`Serial`](crate::Serial) and +/// [`DeserialWithState`](crate::DeserialWithState). +/// - `S`: The low-level state implementation used, this allows for mocking the +/// state API in unit tests, see +/// [`TestStateApi`](crate::test_infrastructure::TestStateApi). +/// - `M`: A `const usize` determining the _minimum degree_ of the B-tree. +/// _Must_ be a value of `2` or above for the tree to work. This can be used +/// to tweak the height of the tree vs size of each node in the tree. The +/// default is set based on benchmarks. +/// +/// ## Usage +/// +/// New maps can be constructed using the +/// [`new_btree_map`][StateBuilder::new_btree_map] method on the +/// [`StateBuilder`]. +/// +/// ``` +/// # use concordium_std::*; +/// # use concordium_std::test_infrastructure::*; +/// # let mut state_builder = TestStateBuilder::new(); +/// /// In an init method: +/// let mut map1 = state_builder.new_btree_map(); +/// # map1.insert(0u8, 1u8); // Specifies type of map. +/// +/// # let mut host = TestHost::new((), state_builder); +/// /// In a receive method: +/// let mut map2 = host.state_builder().new_btree_map(); +/// # map2.insert(0u16, 1u16); +/// ``` +/// +/// ### **Caution** +/// +/// `StateBTreeMap`s must be explicitly deleted when they are no longer needed, +/// otherwise they will remain in the contract's state, albeit unreachable. +/// +/// ```no_run +/// # use concordium_std::*; +/// struct MyState { +/// inner: StateBTreeMap, +/// } +/// fn incorrect_replace(state_builder: &mut StateBuilder, state: &mut MyState) { +/// // The following is incorrect. The old value of `inner` is not properly deleted. +/// // from the state. +/// state.inner = state_builder.new_btree_map(); // ⚠️ +/// } +/// ``` +/// Instead, either the map should be [cleared](StateBTreeMap::clear) or +/// explicitly deleted. +/// +/// ```no_run +/// # use concordium_std::*; +/// # struct MyState { +/// # inner: StateBTreeMap +/// # } +/// fn correct_replace(state_builder: &mut StateBuilder, state: &mut MyState) { +/// state.inner.clear_flat(); +/// } +/// ``` +/// Or alternatively +/// ```no_run +/// # use concordium_std::*; +/// # struct MyState { +/// # inner: StateBTreeMap +/// # } +/// fn correct_replace(state_builder: &mut StateBuilder, state: &mut MyState) { +/// let old_map = mem::replace(&mut state.inner, state_builder.new_btree_map()); +/// old_map.delete() +/// } +/// ``` +pub struct StateBTreeMap { + pub(crate) map: StateMap, + pub(crate) ordered_set: StateBTreeSet, +} + +impl StateBTreeMap { + /// Insert a key-value pair into the map. + /// Returns the previous value if the key was already in the map. + pub fn insert(&mut self, key: K, value: V) -> Option + where + S: HasStateApi, + K: Serialize + Ord, + V: Serial + DeserialWithState, { + let old_value_option = self.map.insert_borrowed(&key, value); + if old_value_option.is_none() && !self.ordered_set.insert(key) { + // Inconsistency between the map and ordered_set. + crate::trap(); + } + old_value_option + } + + /// Remove a key from the map, returning the value at the key if the key was + /// previously in the map. + /// + /// *Caution*: If `V` is a [StateBox], [StateMap], then it is + /// important to call [`Deletable::delete`] on the value returned when + /// you're finished with it. Otherwise, it will remain in the contract + /// state. + #[must_use] + pub fn remove_and_get(&mut self, key: &K) -> Option + where + S: HasStateApi, + K: Serialize + Ord, + V: Serial + DeserialWithState + Deletable, { + let v = self.map.remove_and_get(key); + if v.is_some() && !self.ordered_set.remove(key) { + // Inconsistency between the map and ordered_set. + crate::trap(); + } + v + } + + /// Remove a key from the map. + /// This also deletes the value in the state. + pub fn remove(&mut self, key: &K) + where + S: HasStateApi, + K: Serialize + Ord, + V: Serial + DeserialWithState + Deletable, { + if self.ordered_set.remove(key) { + self.map.remove(key); + } + } + + /// Get a reference to the value corresponding to the key. + pub fn get(&self, key: &K) -> Option> + where + K: Serialize, + S: HasStateApi, + V: Serial + DeserialWithState, { + if self.ordered_set.is_empty() { + None + } else { + self.map.get(key) + } + } + + /// Get a mutable reference to the value corresponding to the key. + pub fn get_mut(&mut self, key: &K) -> Option> + where + K: Serialize, + S: HasStateApi, + V: Serial + DeserialWithState, { + if self.ordered_set.is_empty() { + None + } else { + self.map.get_mut(key) + } + } + + /// Returns `true` if the map contains a value for the specified key. + pub fn contains_key(&self, key: &K) -> bool + where + K: Serialize + Ord, + S: HasStateApi, { + self.ordered_set.contains(key) + } + + /// Returns the smallest key in the map, which is strictly larger than the + /// provided key. `None` meaning no such key is present in the map. + pub fn higher(&self, key: &K) -> Option> + where + S: HasStateApi, + K: Serialize + Ord, { + self.ordered_set.higher(key) + } + + /// Returns the largest key in the map, which is strictly smaller than the + /// provided key. `None` meaning no such key is present in the map. + pub fn lower(&self, key: &K) -> Option> + where + S: HasStateApi, + K: Serialize + Ord, { + self.ordered_set.lower(key) + } + + /// Returns a reference to the first key in the map, if any. This key is + /// always the minimum of all keys in the map. + pub fn first_key(&self) -> Option> + where + S: HasStateApi, + K: Serialize + Ord, { + self.ordered_set.first() + } + + /// Returns a reference to the last key in the map, if any. This key is + /// always the maximum of all keys in the map. + pub fn last_key(&self) -> Option> + where + S: HasStateApi, + K: Serialize + Ord, { + self.ordered_set.last() + } + + /// Return the number of elements in the map. + pub fn len(&self) -> u32 { self.ordered_set.len() } + + /// Returns `true` is the map contains no elements. + pub fn is_empty(&self) -> bool { self.ordered_set.is_empty() } + + /// Create an iterator over the entries of [`StateBTreeMap`]. + /// Ordered by `K`. + pub fn iter(&self) -> StateBTreeMapIter + where + S: HasStateApi, { + StateBTreeMapIter { + key_iter: self.ordered_set.iter(), + map: &self.map, + } + } + + /// Clears the map, removing all key-value pairs. + /// This also includes values pointed at, if `V`, for example, is a + /// [StateBox]. **If applicable use [`clear_flat`](Self::clear_flat) + /// instead.** + pub fn clear(&mut self) + where + S: HasStateApi, + K: Serialize, + V: Serial + DeserialWithState + Deletable, { + self.map.clear(); + self.ordered_set.clear(); + } + + /// Clears the map, removing all key-value pairs. + /// **This should be used over [`clear`](Self::clear) if it is + /// applicable.** It avoids recursive deletion of values since the + /// values are required to be _flat_. + /// + /// Unfortunately it is not possible to automatically choose between these + /// implementations. Once Rust gets trait specialization then this might + /// be possible. + pub fn clear_flat(&mut self) + where + S: HasStateApi, + K: Serialize, + V: Serialize, { + self.map.clear_flat(); + self.ordered_set.clear(); + } +} + +/// An ordered set based on [B-Tree](https://en.wikipedia.org/wiki/B-tree), where +/// each node is stored separately in the low-level key-value store. +/// +/// | Operation | Performance | +/// |-------------------------------------------------|---------------| +/// | [`contains`](Self::contains) | O(k + log(n)) | +/// | [`insert`](Self::insert) | O(k + log(n)) | +/// | [`remove`](Self::remove) | O(k + log(n)) | +/// | [`higher`](Self::higher)/[`lower`](Self::lower) | O(k + log(n)) | +/// +/// Where `k` is the byte size of the serialized keys and `n` is the number of +/// entries in the map. +/// +/// ## Type parameters +/// +/// The map `StateBTreeSet` is parametrized by the types: +/// - `K`: Keys used in the set. Most operations on the set require this to +/// implement [`Serialize`](crate::Serialize). Keys cannot contain references +/// to the low-level state, such as types containing [`StateBox`], +/// [`StateMap`] and [`StateSet`]. +/// - `S`: The low-level state implementation used, this allows for mocking the +/// state API in unit tests, see +/// [`TestStateApi`](crate::test_infrastructure::TestStateApi). +/// - `M`: A `const usize` determining the _minimum degree_ of the B-tree. +/// _Must_ be a value of `2` or above for the tree to work. This can be used +/// to tweak the height of the tree vs size of each node in the tree. The +/// default is set based on benchmarks. +/// +/// ## Usage +/// +/// New sets can be constructed using the +/// [`new_btree_set`][StateBuilder::new_btree_set] method on the +/// [`StateBuilder`]. +/// +/// ``` +/// # use concordium_std::*; +/// # use concordium_std::test_infrastructure::*; +/// # let mut state_builder = TestStateBuilder::new(); +/// /// In an init method: +/// let mut map1 = state_builder.new_btree_set(); +/// # map1.insert(0u8); // Specifies type of map. +/// +/// # let mut host = TestHost::new((), state_builder); +/// /// In a receive method: +/// let mut map2 = host.state_builder().new_btree_set(); +/// # map2.insert(0u16); +/// ``` +/// +/// ### **Caution** +/// +/// `StateBTreeSet`s must be explicitly deleted when they are no longer needed, +/// otherwise they will remain in the contract's state, albeit unreachable. +/// +/// ```no_run +/// # use concordium_std::*; +/// struct MyState { +/// inner: StateBTreeSet, +/// } +/// fn incorrect_replace(state_builder: &mut StateBuilder, state: &mut MyState) { +/// // The following is incorrect. The old value of `inner` is not properly deleted. +/// // from the state. +/// state.inner = state_builder.new_btree_set(); // ⚠️ +/// } +/// ``` +/// Instead, the set should be [cleared](StateBTreeSet::clear): +/// +/// ```no_run +/// # use concordium_std::*; +/// # struct MyState { +/// # inner: StateBTreeSet +/// # } +/// fn correct_replace(state_builder: &mut StateBuilder, state: &mut MyState) { +/// state.inner.clear(); +/// } +/// ``` +pub struct StateBTreeSet { + /// Type marker for the key. + _marker_key: PhantomData, + /// The unique prefix to use for this map in the key-value store. + prefix: StateItemPrefix, + /// The API for interacting with the low-level state. + state_api: S, + /// The ID of the root node of the tree, where None represents the tree is + /// empty. + root: Option, + /// Tracking the number of items in the tree. + len: u32, + /// Tracking the next available ID for a new node. + next_node_id: NodeId, +} + +impl StateBTreeSet { + /// Construct a new [`StateBTreeSet`] given a unique prefix to use in the + /// key-value store. + pub(crate) fn new(state_api: S, prefix: StateItemPrefix) -> Self { + Self { + _marker_key: Default::default(), + prefix, + state_api, + root: None, + len: 0, + next_node_id: NodeId { + id: 0, + }, + } + } + + /// Insert a key into the set. + /// Returns true if the key is new in the collection. + pub fn insert(&mut self, key: K) -> bool + where + S: HasStateApi, + K: Serialize + Ord, { + let Some(root_id) = self.root else { + let node_id = { + let (node_id, _node) = self.create_node(crate::vec![key], Vec::new()); + node_id + }; + self.root = Some(node_id); + self.len = 1; + return true; + }; + + let root_node = self.get_node_mut(root_id); + if !root_node.is_full() { + let new = self.insert_non_full(root_node, key); + if new { + self.len += 1; + } + return new; + } else if root_node.keys.binary_search(&key).is_ok() { + return false; + } + // The root node is full, so we construct a new root node. + let (new_root_id, mut new_root) = self.create_node(Vec::new(), crate::vec![root_id]); + self.root = Some(new_root_id); + // The old root node is now a child node. + let mut child = root_node; + let new_larger_child = self.split_child(&mut new_root, 0, &mut child); + // new_root should now contain one key and two children, so we need to know + // which one to insert into. + let child = if new_root.keys[0] < key { + new_larger_child + } else { + child + }; + let new = self.insert_non_full(child, key); + if new { + self.len += 1; + } + new + } + + /// Returns `true` if the set contains an element equal to the key. + pub fn contains(&self, key: &K) -> bool + where + S: HasStateApi, + K: Serialize + Ord, { + let Some(root_node_id) = self.root else { + return false; + }; + let mut node = self.get_node(root_node_id); + loop { + let Err(child_index) = node.keys.binary_search(key) else { + return true; + }; + if node.is_leaf() { + return false; + } + let child_node_id = node.children[child_index]; + node = self.get_node(child_node_id); + } + } + + /// Return the number of elements in the set. + pub fn len(&self) -> u32 { self.len } + + /// Returns `true` is the set contains no elements. + pub fn is_empty(&self) -> bool { self.root.is_none() } + + /// Get an iterator over the elements in the `StateBTreeSet`. The iterator + /// returns elements in increasing order. + pub fn iter(&self) -> StateBTreeSetIter + where + S: HasStateApi, { + StateBTreeSetIter { + length: self.len.try_into().unwrap_abort(), + next_node: self.root, + depth_first_stack: Vec::new(), + tree: self, + _marker_lifetime: Default::default(), + } + } + + /// Clears the set, removing all elements. + pub fn clear(&mut self) + where + S: HasStateApi, { + // Reset the information. + self.root = None; + self.next_node_id = NodeId { + id: 0, + }; + self.len = 0; + // Then delete every node store in the state. + // Unwrapping is safe when only using the high-level API. + self.state_api.delete_prefix(&self.prefix).unwrap_abort(); + } + + /// Returns the smallest key in the set, which is strictly larger than the + /// provided key. `None` meaning no such key is present in the set. + pub fn higher(&self, key: &K) -> Option> + where + S: HasStateApi, + K: Serialize + Ord, { + let Some(root_node_id) = self.root else { + return None; + }; + + let mut node = self.get_node(root_node_id); + let mut higher_so_far = None; + loop { + let higher_key_index = match node.keys.binary_search(key) { + Ok(index) => index + 1, + Err(index) => index, + }; + + if node.is_leaf() { + return if higher_key_index < node.keys.len() { + // This does not mutate the node in the end, just the representation in memory + // which is freed after the call. + Some(StateRef::new(node.keys.swap_remove(higher_key_index))) + } else { + higher_so_far + }; + } else { + if higher_key_index < node.keys.len() { + // This does not mutate the node in the end, just the representation in memory + // which is freed after the call. + higher_so_far = Some(StateRef::new(node.keys.swap_remove(higher_key_index))) + } + + let child_node_id = node.children[higher_key_index]; + node = self.get_node(child_node_id); + } + } + } + + /// Returns the largest key in the set, which is strictly smaller than the + /// provided key. `None` meaning no such key is present in the set. + pub fn lower(&self, key: &K) -> Option> + where + S: HasStateApi, + K: Serialize + Ord, { + let Some(root_node_id) = self.root else { + return None; + }; + + let mut node = self.get_node(root_node_id); + let mut lower_so_far = None; + loop { + let lower_key_index = match node.keys.binary_search(key) { + Ok(index) => index, + Err(index) => index, + }; + + if node.is_leaf() { + return if lower_key_index == 0 { + lower_so_far + } else { + // lower_key_index cannot be 0 in this case, since the binary search will only + // return 0 in the true branch above. + Some(StateRef::new(node.keys.swap_remove(lower_key_index - 1))) + }; + } else { + if lower_key_index > 0 { + // This does not mutate the node in the end, just the representation in memory + // which is freed after the call. + lower_so_far = Some(StateRef::new(node.keys.swap_remove(lower_key_index - 1))); + } + let child_node_id = node.children[lower_key_index]; + node = self.get_node(child_node_id) + } + } + } + + /// Returns a reference to the first key in the set, if any. This key is + /// always the minimum of all keys in the set. + pub fn first(&self) -> Option> + where + S: HasStateApi, + K: Serialize + Ord, { + let Some(root_node_id) = self.root else { + return None; + }; + let mut root = self.get_node(root_node_id); + if root.is_leaf() { + Some(StateRef::new(root.keys.swap_remove(0))) + } else { + Some(StateRef::new(self.get_lowest_key(&root, 0))) + } + } + + /// Returns a reference to the last key in the set, if any. This key is + /// always the maximum of all keys in the set. + pub fn last(&self) -> Option> + where + S: HasStateApi, + K: Serialize + Ord, { + let Some(root_node_id) = self.root else { + return None; + }; + let mut root = self.get_node(root_node_id); + if root.is_leaf() { + Some(StateRef::new(root.keys.pop().unwrap_abort())) + } else { + Some(StateRef::new(self.get_highest_key(&root, root.children.len() - 1))) + } + } + + /// Remove a key from the set. + /// Returns whether such an element was present. + pub fn remove(&mut self, key: &K) -> bool + where + K: Ord + Serialize, + S: HasStateApi, { + let Some(root_node_id) = self.root else { + return false; + }; + + let deleted_something = { + let mut node = self.get_node_mut(root_node_id); + loop { + match node.keys.binary_search(key) { + Ok(index) => { + if node.is_leaf() { + // Found the key in this node and the node is a leaf, meaning we + // simply remove it. + // This will not violate the minimum keys invariant, since a node + // ensures a child can spare a key before iteration and the root + // node is not part of the + // invariant. + node.keys.remove(index); + break true; + } + // Found the key in this node, but the node is not a leaf. + let mut left_child = self.get_node_mut(node.children[index]); + if !left_child.is_at_min() { + // If the child with smaller keys can spare a key, we take the + // highest key from it. + node.keys[index] = self.remove_largest_key(left_child); + break true; + } + + let right_child = self.get_node_mut(node.children[index + 1]); + if !right_child.is_at_min() { + // If the child with larger keys can spare a key, we take the lowest + // key from it. + node.keys[index] = self.remove_smallest_key(right_child); + break true; + } + // No child on either side of the key can spare a key at this point, so + // we merge them into one child, moving the + // key into the merged child and try + // to remove from this. + self.merge(&mut node, index, &mut left_child, right_child); + node = left_child; + continue; + } + Err(index) => { + // Node did not contain the key. + if node.is_leaf() { + break false; + } + // Node did not contain the key and is not a leaf. + // Check and proactively prepare the child to be able to delete a key. + node = self.prepare_child_for_key_removal(node, index); + } + }; + } + }; + + // If something was deleted, we update the length and make sure to remove the + // root node if needed. + if deleted_something { + self.len -= 1; + let root = self.get_node_mut(root_node_id); + if self.len == 0 { + // Remote the root node if tree is empty. + self.root = None; + self.delete_node(root); + } else { + // If the root is empty but the tree is not, point to the only child of the root + // as the root. + if root.keys.is_empty() { + self.root = Some(root.children[0]); + self.delete_node(root); + } + } + } + deleted_something + } + + /// Internal function for taking the largest key in a subtree. + /// + /// Assumes: + /// - The provided node is not the root. + /// - The node contain more than minimum number of keys. + fn remove_largest_key(&mut self, mut node: StateRefMut<'_, Node, S>) -> K + where + K: Ord + Serialize, + S: HasStateApi, { + while !node.is_leaf() { + // Node is not a leaf, so we move further down this subtree. + let child_index = node.children.len() - 1; + node = self.prepare_child_for_key_removal(node, child_index); + } + // The node is a leaf, meaning we simply remove it. + // This will not violate the minimum keys invariant, since a node + // ensures a child can spare a key before iteration. + node.keys.pop().unwrap_abort() + } + + /// Internal function for taking the smallest key in a subtree. + /// + /// Assumes: + /// - The provided node is not the root. + /// - The provided `node` contain more than minimum number of keys. + fn remove_smallest_key(&mut self, mut node: StateRefMut<'_, Node, S>) -> K + where + K: Ord + Serialize, + S: HasStateApi, { + while !node.is_leaf() { + // Node is not a leaf, so we move further down this subtree. + let child_index = 0; + node = self.prepare_child_for_key_removal(node, child_index); + } + // The node is a leaf, meaning we simply remove it. + // This will not violate the minimum keys invariant, since a node + // ensures a child can spare a key before iteration. + node.keys.remove(0) + } + + /// Internal function for key rotation, preparing a node for deleting a key. + /// Returns the now prepared child at `index`. + /// Assumes: + /// - The provided `node` is not a leaf and has a child at `index`. + /// - The minimum degree `M` is at least 2 or more. + fn prepare_child_for_key_removal<'b, 'c>( + &mut self, + mut node: StateRefMut<'b, Node, S>, + index: usize, + ) -> StateRefMut<'c, Node, S> + where + K: Ord + Serialize, + S: HasStateApi, { + let mut child = self.get_node_mut(node.children[index]); + if !child.is_at_min() { + return child; + } + // The child is at minimum keys, so first attempt to take a key from either + // sibling, otherwise merge with one of them. + let has_smaller_sibling = 0 < index; + let has_larger_sibling = index < node.children.len() - 1; + let smaller_sibling = if has_smaller_sibling { + let mut smaller_sibling = self.get_node_mut(node.children[index - 1]); + if !smaller_sibling.is_at_min() { + // The smaller sibling can spare a key, so we replace the largest + // key from the sibling, put it in + // the parent and take a key from + // the parent. + let largest_key_sibling = smaller_sibling.keys.pop().unwrap_abort(); + let swapped_node_key = mem::replace(&mut node.keys[index - 1], largest_key_sibling); + child.keys.insert(0, swapped_node_key); + if !child.is_leaf() { + child.children.insert(0, smaller_sibling.children.pop().unwrap_abort()); + } + return child; + } + Some(smaller_sibling) + } else { + None + }; + let larger_sibling = if has_larger_sibling { + let mut larger_sibling = self.get_node_mut(node.children[index + 1]); + if !larger_sibling.is_at_min() { + // The larger sibling can spare a key, so we replace the smallest + // key from the sibling, put it in + // the parent and take a key from + // the parent. + let first_key_sibling = larger_sibling.keys.remove(0); + let swapped_node_key = mem::replace(&mut node.keys[index], first_key_sibling); + child.keys.push(swapped_node_key); + + if !child.is_leaf() { + child.children.push(larger_sibling.children.remove(0)); + } + return child; + } + Some(larger_sibling) + } else { + None + }; + + if let Some(sibling) = larger_sibling { + self.merge(&mut node, index, &mut child, sibling); + child + } else if let Some(mut sibling) = smaller_sibling { + self.merge(&mut node, index - 1, &mut sibling, child); + sibling + } else { + // Unreachable code, since M must be 2 or larger (the minimum degree), a + // child node must have at least one sibling. + crate::trap(); + } + } + + /// Internal function for getting the highest key in a subtree. + /// Assumes the provided `node` is not a leaf and has a child at + /// `child_index`. + fn get_highest_key(&self, node: &Node, child_index: usize) -> K + where + K: Ord + Serialize, + S: HasStateApi, { + let mut node = self.get_node(node.children[child_index]); + while !node.is_leaf() { + let child_node_id = node.children.last().unwrap_abort(); + node = self.get_node(*child_node_id); + } + // This does not mutate the node in the end, just the representation in memory + // which is freed after the call. + node.keys.pop().unwrap_abort() + } + + /// Internal function for getting the lowest key in a subtree. + /// Assumes the provided `node` is not a leaf and has a child at + /// `child_index`. + fn get_lowest_key(&self, node: &Node, child_index: usize) -> K + where + K: Ord + Serialize, + S: HasStateApi, { + let mut node = self.get_node(node.children[child_index]); + while !node.is_leaf() { + let child_node_id = node.children.first().unwrap_abort(); + node = self.get_node(*child_node_id); + } + // This does not mutate the node in the end, just the representation in memory + // which is freed after the call. + node.keys.swap_remove(0) + } + + /// Moving key at `index` from the node to the lower child and then merges + /// this child with the content of its larger sibling, deleting the sibling. + /// + /// Assumes: + /// - `parent_node` has children `child` at `index` and `larger_child` at + /// `index + 1`. + /// - Both children are at minimum number of keys (`M - 1`). + fn merge( + &mut self, + parent_node: &mut Node, + index: usize, + child: &mut Node, + mut larger_child: StateRefMut, S>, + ) where + K: Ord + Serialize, + S: HasStateApi, { + let parent_key = parent_node.keys.remove(index); + parent_node.children.remove(index + 1); + child.keys.push(parent_key); + child.keys.append(&mut larger_child.keys); + child.children.append(&mut larger_child.children); + self.delete_node(larger_child); + } + + /// Internal function for constructing a node. It will increment the next + /// node ID and create an entry in the smart contract key-value store. + fn create_node<'b>( + &mut self, + keys: Vec, + children: Vec, + ) -> (NodeId, StateRefMut<'b, Node, S>) + where + K: Serialize, + S: HasStateApi, { + let node_id = self.next_node_id.copy_then_increment(); + let node = Node { + keys, + children, + }; + let entry = self.state_api.create_entry(&node_id.as_key(&self.prefix)).unwrap_abort(); + let mut ref_mut: StateRefMut<'_, Node, S> = + StateRefMut::new(entry, self.state_api.clone()); + ref_mut.set(node); + (node_id, ref_mut) + } + + /// Internal function for deleting a node, removing the entry in the smart + /// contract key-value store. Traps if no node was present. + fn delete_node(&mut self, node: StateRefMut, S>) + where + K: Serial, + S: HasStateApi, { + self.state_api.delete_entry(node.into_raw_parts().1).unwrap_abort() + } + + /// Internal function for inserting into a subtree. + /// Assumes the given node is not full. + fn insert_non_full(&mut self, initial_node: StateRefMut, S>, key: K) -> bool + where + K: Serialize + Ord, + S: HasStateApi, { + let mut node = initial_node; + loop { + let Err(insert_index) = node.keys.binary_search(&key) else { + // We find the key in this node, so we do nothing. + return false; + }; + // The key is not in this node. + if node.is_leaf() { + // Since we can assume the node is not full and this is a leaf, we can just + // insert here. + node.keys.insert(insert_index, key); + return true; + } + + // The node is not a leaf, so we want to insert in the relevant child node. + let mut child = self.get_node_mut(node.children[insert_index]); + node = if !child.is_full() { + child + } else { + let larger_child = self.split_child(&mut node, insert_index, &mut child); + // Since the child is now split into two, we have to check which one to insert + // into. + if node.keys[insert_index] < key { + larger_child + } else { + child + } + }; + } + } + + /// Internal function for splitting the child node at a given index for a + /// given node. This will also mutate the given node adding a new key + /// and child after the provided child_index. + /// Returns the newly created node. + /// + /// Assumes: + /// - Node is not a leaf and has `child` as child at `child_index`. + /// - `child` is at maximum keys (`2 * M - 1`). + fn split_child<'b>( + &mut self, + node: &mut Node, + child_index: usize, + child: &mut Node, + ) -> StateRefMut<'b, Node, S> + where + K: Serialize + Ord, + S: HasStateApi, { + let split_index = Node::::MINIMUM_KEY_LEN + 1; + let (new_larger_sibling_id, new_larger_sibling) = self.create_node( + child.keys.split_off(split_index), + if child.is_leaf() { + Vec::new() + } else { + child.children.split_off(split_index) + }, + ); + let key = child.keys.pop().unwrap_abort(); + node.children.insert(child_index + 1, new_larger_sibling_id); + node.keys.insert(child_index, key); + new_larger_sibling + } + + /// Internal function for looking up a node in the tree. + /// This assumes the node is present and traps if this is not the case. + fn get_node(&self, node_id: NodeId) -> Node + where + Key: Deserial, + S: HasStateApi, { + let key = node_id.as_key(&self.prefix); + let mut entry = self.state_api.lookup_entry(&key).unwrap_abort(); + entry.get().unwrap_abort() + } + + /// Internal function for looking up a node, providing mutable access. + /// This assumes the node is present and traps if this is not the case. + fn get_node_mut<'b>(&mut self, node_id: NodeId) -> StateRefMut<'b, Node, S> + where + K: Serial, + S: HasStateApi, { + let key = node_id.as_key(&self.prefix); + let entry = self.state_api.lookup_entry(&key).unwrap_abort(); + StateRefMut::new(entry, self.state_api.clone()) + } +} + +/// An iterator over the entries of a [`StateBTreeSet`]. +/// +/// Ordered by `K`. +/// +/// This `struct` is created by the [`iter`][StateBTreeSet::iter] method on +/// [`StateBTreeSet`]. See its documentation for more. +pub struct StateBTreeSetIter<'a, 'b, K, S, const M: usize> { + /// The number of elements left to iterate. + length: usize, + /// Reference to a node in the tree to load and iterate before the current + /// node. + next_node: Option, + /// Tracking the nodes depth first, which are currently being iterated. + depth_first_stack: Vec<(Node>, usize)>, + /// Reference to the set, needed for looking up the nodes. + tree: &'a StateBTreeSet, + /// Marker for tracking the lifetime of the key. + _marker_lifetime: PhantomData<&'b K>, +} + +impl<'a, 'b, const M: usize, K, S> Iterator for StateBTreeSetIter<'a, 'b, K, S, M> +where + 'a: 'b, + K: Deserial, + S: HasStateApi, +{ + type Item = StateRef<'b, K>; + + fn next(&mut self) -> Option { + while let Some(id) = self.next_node.take() { + let node = self.tree.get_node(id); + if !node.is_leaf() { + self.next_node = Some(node.children[0]); + } + self.depth_first_stack.push((node, 0)); + } + + if let Some((node, index)) = self.depth_first_stack.last_mut() { + let key = node.keys[*index].key.take().unwrap_abort(); + *index += 1; + let no_more_keys = index == &node.keys.len(); + if !node.is_leaf() { + let child_id = node.children[*index]; + self.next_node = Some(child_id); + } + if no_more_keys { + self.depth_first_stack.pop(); + } + self.length -= 1; + Some(StateRef::new(key)) + } else { + None + } + } + + fn size_hint(&self) -> (usize, Option) { (self.length, Some(self.length)) } +} + +/// An iterator over the entries of a [`StateBTreeMap`]. +/// +/// Ordered by `K`. +/// +/// This `struct` is created by the [`iter`][StateBTreeMap::iter] method on +/// [`StateBTreeMap`]. See its documentation for more. +pub struct StateBTreeMapIter<'a, 'b, K, V, S, const M: usize> { + /// Iterator over the keys in the map. + key_iter: StateBTreeSetIter<'a, 'b, K, S, M>, + /// Reference to the map holding the values. + map: &'a StateMap, +} + +impl<'a, 'b, const M: usize, K, V, S> Iterator for StateBTreeMapIter<'a, 'b, K, V, S, M> +where + 'a: 'b, + K: Serialize, + V: Serial + DeserialWithState + 'b, + S: HasStateApi, +{ + type Item = (StateRef<'b, K>, StateRef<'b, V>); + + fn next(&mut self) -> Option { + let next_key = self.key_iter.next()?; + let value = self.map.get(&next_key).unwrap_abort(); + // Unwrap is safe, otherwise the map and the set have inconsistencies. + Some((next_key, value)) + } + + fn size_hint(&self) -> (usize, Option) { self.key_iter.size_hint() } +} + +/// Identifier for a node in the tree. Used to construct the key, where this +/// node is store in the smart contract key-value store. +#[derive(Debug, Copy, Clone)] +#[repr(transparent)] +struct NodeId { + id: u32, +} + +/// Type representing a node in the [`StateBTreeMap`]. +/// Each node is stored separately in the smart contract key-value store. +#[derive(Debug)] +struct Node { + /// List of sorted keys tracked by this node. + /// This list should never be empty and contain between `M - 1` and `2M + /// - 1` elements. + keys: Vec, + /// List of nodes which are children of this node in the tree. + /// + /// This list is empty when this node is representing a leaf. + /// When not a leaf, it will contain exactly `keys.len() + 1` elements. + /// + /// The elements are ordered such that for a key `keys[i]`: + /// - `children[i]` is a subtree containing strictly smaller keys. + /// - `children[i + 1]` is a subtree containing strictly larger keys. + children: Vec, +} + +/// Wrapper implement the exact same deserial as K, but wraps it in an +/// option in memory. This is used, to allow taking a key from a mutable +/// reference to a node, without cloning the key, during iteration of the +/// set. +#[repr(transparent)] +struct KeyWrapper { + key: Option, +} + +impl Deserial for KeyWrapper { + fn deserial(source: &mut R) -> ParseResult { + let key = K::deserial(source)?; + Ok(Self { + key: Some(key), + }) + } +} + +/// Byte size of the key used to store a BTree internal node in the smart +/// contract key-value store. +const BTREE_NODE_KEY_SIZE: usize = STATE_ITEM_PREFIX_SIZE + NodeId::SERIALIZED_BYTE_SIZE; + +impl Node { + /// The max length of the child list. + const MAXIMUM_CHILD_LEN: usize = 2 * M; + /// The max length of the key list. + const MAXIMUM_KEY_LEN: usize = Self::MAXIMUM_CHILD_LEN - 1; + /// The min length of the child list, when the node is not a leaf node. + const MINIMUM_CHILD_LEN: usize = M; + /// The min length of the key list, except when the node is root. + const MINIMUM_KEY_LEN: usize = Self::MINIMUM_CHILD_LEN - 1; + + /// The number of keys stored in this node. + fn len(&self) -> usize { self.keys.len() } + + /// Check if the node holds the maximum number of keys. + fn is_full(&self) -> bool { self.len() == Self::MAXIMUM_KEY_LEN } + + /// Check if the node is representing a leaf in the tree. + fn is_leaf(&self) -> bool { self.children.is_empty() } + + /// Check if the node holds the minimum number of keys. + fn is_at_min(&self) -> bool { self.len() == Self::MINIMUM_KEY_LEN } +} + +impl NodeId { + /// Byte size of `NodeId` when serialized. + const SERIALIZED_BYTE_SIZE: usize = 4; + + /// Return a copy of the NodeId, then increments itself. + fn copy_then_increment(&mut self) -> Self { + let current = *self; + self.id += 1; + current + } + + /// Construct the key for the node in the key-value store from the node ID. + fn as_key(&self, prefix: &StateItemPrefix) -> [u8; BTREE_NODE_KEY_SIZE] { + // Create an uninitialized array of `MaybeUninit`. The `assume_init` is + // safe because the type we are claiming to have initialized here is a + // bunch of `MaybeUninit`s, which do not require initialization. + let mut prefixed: [mem::MaybeUninit; BTREE_NODE_KEY_SIZE] = + unsafe { mem::MaybeUninit::uninit().assume_init() }; + for i in 0..STATE_ITEM_PREFIX_SIZE { + prefixed[i].write(prefix[i]); + } + let id_bytes = self.id.to_le_bytes(); + for i in 0..id_bytes.len() { + prefixed[STATE_ITEM_PREFIX_SIZE + i].write(id_bytes[i]); + } + // Transmuting away the maybeuninit is safe since we have initialized all of + // them. + unsafe { mem::transmute(prefixed) } + } +} + +impl Serial for StateBTreeMap { + fn serial(&self, out: &mut W) -> Result<(), W::Err> { + self.map.serial(out)?; + self.ordered_set.serial(out) + } +} + +impl DeserialWithState for StateBTreeMap { + fn deserial_with_state(state: &S, source: &mut R) -> ParseResult { + let map = DeserialWithState::deserial_with_state(state, source)?; + let ordered_set = DeserialWithState::deserial_with_state(state, source)?; + Ok(Self { + map, + ordered_set, + }) + } +} + +impl Serial for StateBTreeSet { + fn serial(&self, out: &mut W) -> Result<(), W::Err> { + self.prefix.serial(out)?; + self.root.serial(out)?; + self.len.serial(out)?; + self.next_node_id.serial(out) + } +} + +impl DeserialWithState for StateBTreeSet { + fn deserial_with_state(state: &S, source: &mut R) -> ParseResult { + let prefix = source.get()?; + let root = source.get()?; + let len = source.get()?; + let next_node_id = source.get()?; + + Ok(Self { + _marker_key: Default::default(), + prefix, + state_api: state.clone(), + root, + len, + next_node_id, + }) + } +} + +impl Serial for Node { + fn serial(&self, out: &mut W) -> Result<(), W::Err> { + self.keys.serial(out)?; + self.children.serial(out) + } +} + +impl Deserial for Node { + fn deserial(source: &mut R) -> ParseResult { + let keys = source.get()?; + let children = source.get()?; + Ok(Self { + keys, + children, + }) + } +} + +impl Serial for NodeId { + fn serial(&self, out: &mut W) -> Result<(), W::Err> { self.id.serial(out) } +} + +impl Deserial for NodeId { + fn deserial(source: &mut R) -> ParseResult { + Ok(Self { + id: source.get()?, + }) + } +} + +impl Deletable for StateBTreeMap +where + S: HasStateApi, + K: Serialize, + V: Serial + DeserialWithState + Deletable, +{ + fn delete(mut self) { self.clear(); } +} + +/// This test module rely on the runtime providing host functions and can only +/// be run using `cargo concordium test`. +#[cfg(feature = "wasm-test")] +mod wasm_test_btree { + use crate::{claim, claim_eq, concordium_test, StateApi, StateBuilder}; + + #[concordium_test] + fn test_btree_insert_6() { + let mut state_builder = StateBuilder::open(StateApi::open()); + let mut tree = state_builder.new_btree_set_degree::<5, _>(); + for n in 0..=5 { + tree.insert(n); + } + for n in 0..=5 { + claim!(tree.contains(&n)); + } + } + + #[concordium_test] + fn test_btree_insert_0_7() { + let mut state_builder = StateBuilder::open(StateApi::open()); + let mut tree = state_builder.new_btree_set_degree::<2, _>(); + for n in 0..=7 { + tree.insert(n); + } + for n in 0..=7 { + claim!(tree.contains(&n)); + } + } + + #[concordium_test] + fn test_btree_insert_7_no_order() { + let mut state_builder = StateBuilder::open(StateApi::open()); + let mut tree = state_builder.new_btree_set_degree::<2, _>(); + + tree.insert(0); + tree.insert(1); + tree.insert(2); + tree.insert(3); + tree.insert(7); + tree.insert(6); + tree.insert(5); + tree.insert(4); + + for n in 0..=7 { + claim!(tree.contains(&n)); + } + } + + #[concordium_test] + fn test_btree_higher() { + let mut state_builder = StateBuilder::open(StateApi::open()); + let mut tree = state_builder.new_btree_set_degree::<2, _>(); + tree.insert(1); + tree.insert(2); + tree.insert(3); + tree.insert(4); + tree.insert(5); + tree.insert(7); + claim_eq!(tree.higher(&3).as_deref(), Some(&4)); + claim_eq!(tree.higher(&5).as_deref(), Some(&7)); + claim_eq!(tree.higher(&6).as_deref(), Some(&7)); + claim_eq!(tree.higher(&7).as_deref(), None) + } + + #[concordium_test] + fn test_btree_lower() { + let mut state_builder = StateBuilder::open(StateApi::open()); + let mut tree = state_builder.new_btree_set_degree::<2, _>(); + tree.insert(1); + tree.insert(2); + tree.insert(3); + tree.insert(4); + tree.insert(5); + tree.insert(7); + claim_eq!(tree.lower(&3).as_deref(), Some(&2)); + claim_eq!(tree.lower(&7).as_deref(), Some(&5)); + claim_eq!(tree.lower(&6).as_deref(), Some(&5)); + claim_eq!(tree.lower(&1).as_deref(), None) + } + + #[concordium_test] + fn test_btree_insert_1000() { + let mut state_builder = StateBuilder::open(StateApi::open()); + let mut tree = state_builder.new_btree_set_degree::<2, _>(); + for n in 0..500 { + tree.insert(n); + } + + for n in (500..1000).into_iter().rev() { + tree.insert(n); + } + + for n in 0..1000 { + claim!(tree.contains(&n)) + } + + claim_eq!(tree.len(), 1000) + } + + #[concordium_test] + fn test_btree_7_get_8() { + let mut state_builder = StateBuilder::open(StateApi::open()); + let mut tree = state_builder.new_btree_set_degree::<2, _>(); + for n in 0..=7 { + tree.insert(n); + } + + claim!(!tree.contains(&8)); + } + + #[concordium_test] + fn test_btree_remove_from_one_node_tree() { + let mut state_builder = StateBuilder::open(StateApi::open()); + let mut tree = state_builder.new_btree_set_degree::<2, _>(); + for n in 0..3 { + tree.insert(n); + } + claim!(tree.contains(&1)); + claim!(tree.remove(&1)); + claim!(tree.contains(&0)); + claim!(!tree.contains(&1)); + claim!(tree.contains(&2)); + } + + #[concordium_test] + fn test_btree_remove_only_key_lower_leaf_in_three_node() { + let mut state_builder = StateBuilder::open(StateApi::open()); + let mut tree = state_builder.new_btree_set_degree::<2, _>(); + for n in 0..4 { + tree.insert(n); + } + tree.remove(&3); + + claim!(tree.contains(&0)); + claim!(tree.remove(&0)); + claim!(!tree.contains(&0)); + claim!(tree.contains(&1)); + claim!(tree.contains(&2)); + } + + #[concordium_test] + fn test_btree_remove_only_key_higher_leaf_in_three_node() { + let mut state_builder = StateBuilder::open(StateApi::open()); + let mut tree = state_builder.new_btree_set_degree::<2, _>(); + for n in (0..4).into_iter().rev() { + tree.insert(n); + } + tree.remove(&3); + claim!(tree.contains(&2)); + claim!(tree.remove(&2)); + claim!(tree.contains(&0)); + claim!(tree.contains(&1)); + claim!(!tree.contains(&2)); + } + + #[concordium_test] + fn test_btree_remove_from_higher_leaf_in_three_node_taking_from_sibling() { + let mut state_builder = StateBuilder::open(StateApi::open()); + let mut tree = state_builder.new_btree_set_degree::<2, _>(); + for n in (0..4).into_iter().rev() { + tree.insert(n); + } + claim!(tree.contains(&3)); + claim!(tree.remove(&3)); + claim!(!tree.contains(&3)); + } + + #[concordium_test] + fn test_btree_remove_from_lower_leaf_in_three_node_taking_from_sibling() { + let mut state_builder = StateBuilder::open(StateApi::open()); + let mut tree = state_builder.new_btree_set_degree::<2, _>(); + for n in 0..4 { + tree.insert(n); + } + claim!(tree.contains(&0)); + claim!(tree.remove(&0)); + claim!(!tree.contains(&0)); + claim!(tree.contains(&1)); + claim!(tree.contains(&2)); + claim!(tree.contains(&3)); + } + + #[concordium_test] + fn test_btree_remove_from_root_in_three_node_causing_merge() { + let mut state_builder = StateBuilder::open(StateApi::open()); + let mut tree = state_builder.new_btree_set_degree::<2, _>(); + for n in 0..4 { + tree.insert(n); + } + tree.remove(&3); + + claim!(tree.contains(&1)); + claim!(tree.remove(&1)); + claim!(!tree.contains(&1)); + } + + #[concordium_test] + fn test_btree_remove_from_root_in_three_node_taking_key_from_higher_child() { + let mut state_builder = StateBuilder::open(StateApi::open()); + let mut tree = state_builder.new_btree_set_degree::<2, _>(); + for n in 0..4 { + tree.insert(n); + } + claim!(tree.contains(&1)); + claim!(tree.remove(&1)); + claim!(!tree.contains(&1)); + } + + #[concordium_test] + fn test_btree_remove_from_root_in_three_node_taking_key_from_lower_child() { + let mut state_builder = StateBuilder::open(StateApi::open()); + let mut tree = state_builder.new_btree_set_degree::<2, _>(); + for n in (0..4).into_iter().rev() { + tree.insert(n); + } + claim!(tree.contains(&2)); + claim!(tree.remove(&2)); + claim!(!tree.contains(&2)); + } + + #[concordium_test] + fn test_btree_iter() { + let mut state_builder = StateBuilder::open(StateApi::open()); + let mut tree = state_builder.new_btree_set_degree::<2, _>(); + let keys: Vec = (0..15).into_iter().collect(); + for &k in &keys { + tree.insert(k); + } + let iter_keys: Vec = tree.iter().map(|k| k.clone()).collect(); + claim_eq!(keys, iter_keys); + } +} diff --git a/concordium-std/src/test_infrastructure.rs b/concordium-std/src/test_infrastructure.rs index 92762040..ff2805fd 100644 --- a/concordium-std/src/test_infrastructure.rs +++ b/concordium-std/src/test_infrastructure.rs @@ -2076,640 +2076,4 @@ mod test { "Correct data was written." ); } - - #[test] - fn high_level_insert_get() { - let expected_value: u64 = 123123123; - let mut state_builder = TestStateBuilder::new(); - state_builder.insert(0, expected_value).expect("Insert failed"); - let actual_value: u64 = state_builder.get(0).expect("Not found").expect("Not a valid u64"); - assert_eq!(expected_value, actual_value); - } - - #[test] - fn low_level_entry() { - let expected_value: u64 = 123123123; - let key = to_bytes(&42u64); - let mut state = TestStateApi::new(); - state - .entry(&key[..]) - .or_insert_raw(&to_bytes(&expected_value)) - .expect("No iterators, so insertion should work."); - - match state.entry(key) { - EntryRaw::Vacant(_) => panic!("Unexpected vacant entry."), - EntryRaw::Occupied(occ) => { - assert_eq!(u64::deserial(&mut occ.get()), Ok(expected_value)) - } - } - } - - #[test] - fn high_level_statemap() { - let my_map_key = "my_map"; - let mut state_builder = TestStateBuilder::new(); - - let map_to_insert = state_builder.new_map::(); - state_builder.insert(my_map_key, map_to_insert).expect("Insert failed"); - - let mut my_map: StateMap = state_builder - .get(my_map_key) - .expect("Could not get statemap") - .expect("Deserializing statemap failed"); - my_map.insert("abc".to_string(), "hello, world".to_string()); - my_map.insert("def".to_string(), "hallo, Weld".to_string()); - my_map.insert("ghi".to_string(), "hej, verden".to_string()); - assert_eq!(*my_map.get(&"abc".to_string()).unwrap(), "hello, world".to_string()); - - let mut iter = my_map.iter(); - let (k1, v1) = iter.next().unwrap(); - assert_eq!(*k1, "abc".to_string()); - assert_eq!(*v1, "hello, world".to_string()); - let (k2, v2) = iter.next().unwrap(); - assert_eq!(*k2, "def".to_string()); - assert_eq!(*v2, "hallo, Weld".to_string()); - let (k3, v3) = iter.next().unwrap(); - assert_eq!(*k3, "ghi".to_string()); - assert_eq!(*v3, "hej, verden".to_string()); - assert!(iter.next().is_none()); - } - - #[test] - fn statemap_insert_remove() { - let mut state_builder = TestStateBuilder::new(); - let mut map = state_builder.new_map(); - let value = String::from("hello"); - let _ = map.insert(42, value.clone()); - assert_eq!(*map.get(&42).unwrap(), value); - map.remove(&42); - assert!(map.get(&42).is_none()); - } - - #[test] - fn statemap_clear() { - let mut state_builder = TestStateBuilder::new(); - let mut map = state_builder.new_map(); - let _ = map.insert(1, 2); - let _ = map.insert(2, 3); - let _ = map.insert(3, 4); - map.clear(); - assert!(map.is_empty()); - } - - #[test] - fn high_level_nested_statemaps() { - let inner_map_key = 0u8; - let key_to_value = 77u8; - let value = 255u8; - let mut state_builder = TestStateBuilder::new(); - let mut outer_map = state_builder.new_map::>(); - let mut inner_map = state_builder.new_map::(); - - inner_map.insert(key_to_value, value); - outer_map.insert(inner_map_key, inner_map); - - assert_eq!(*outer_map.get(&inner_map_key).unwrap().get(&key_to_value).unwrap(), value); - } - - #[test] - fn statemap_iter_mut_works() { - let mut state_builder = TestStateBuilder::new(); - let mut map = state_builder.new_map(); - map.insert(0u8, 1u8); - map.insert(1u8, 2u8); - map.insert(2u8, 3u8); - for (_, mut v) in map.iter_mut() { - v.update(|old_value| *old_value += 10); - } - let mut iter = map.iter(); - let (k1, v1) = iter.next().unwrap(); - assert_eq!(*k1, 0); - assert_eq!(*v1, 11); - let (k2, v2) = iter.next().unwrap(); - assert_eq!(*k2, 1); - assert_eq!(*v2, 12); - let (k3, v3) = iter.next().unwrap(); - assert_eq!(*k3, 2); - assert_eq!(*v3, 13); - assert!(iter.next().is_none()); - } - - #[test] - fn iter_mut_works_on_nested_statemaps() { - let mut state_builder = TestStateBuilder::new(); - let mut outer_map = state_builder.new_map(); - let mut inner_map = state_builder.new_map(); - inner_map.insert(0u8, 1u8); - inner_map.insert(1u8, 2u8); - outer_map.insert(99u8, inner_map); - for (_, mut v_map) in outer_map.iter_mut() { - v_map.update(|v_map| { - for (_, mut inner_v) in v_map.iter_mut() { - inner_v.update(|inner_v| *inner_v += 10); - } - }); - } - - // Check the outer map. - let mut outer_iter = outer_map.iter(); - let (inner_map_key, inner_map) = outer_iter.next().unwrap(); - assert_eq!(*inner_map_key, 99); - assert!(outer_iter.next().is_none()); - - // Check the inner map. - let mut inner_iter = inner_map.iter(); - let (k1, v1) = inner_iter.next().unwrap(); - assert_eq!(*k1, 0); - assert_eq!(*v1, 11); - let (k2, v2) = inner_iter.next().unwrap(); - assert_eq!(*k2, 1); - assert_eq!(*v2, 12); - assert!(inner_iter.next().is_none()); - } - - #[test] - fn statemap_iterator_unlocks_tree_once_dropped() { - let mut state_builder = TestStateBuilder::new(); - let mut map = state_builder.new_map(); - map.insert(0u8, 1u8); - map.insert(1u8, 2u8); - { - let _iter = map.iter(); - // Uncommenting these two lines (and making iter mutable) should - // give a compile error: - // - // map.insert(2u8, 3u8); - // let n = iter.next(); - } // iter is dropped here, unlocking the subtree. - map.insert(2u8, 3u8); - } - - #[test] - fn high_level_stateset() { - let my_set_key = "my_set"; - let mut state_builder = TestStateBuilder::new(); - - let mut set = state_builder.new_set::(); - assert!(set.insert(0)); - assert!(set.insert(1)); - assert!(!set.insert(1)); - assert!(set.insert(2)); - assert!(set.remove(&2)); - state_builder.insert(my_set_key, set).expect("Insert failed"); - - assert!(state_builder.get::<_, StateSet>(my_set_key).unwrap().unwrap().contains(&0),); - assert!(!state_builder - .get::<_, StateSet>(my_set_key) - .unwrap() - .unwrap() - .contains(&2),); - - let set = state_builder.get::<_, StateSet>(my_set_key).unwrap().unwrap(); - let mut iter = set.iter(); - assert_eq!(*iter.next().unwrap(), 0); - assert_eq!(*iter.next().unwrap(), 1); - assert!(iter.next().is_none()); - } - - #[test] - fn high_level_nested_stateset() { - let inner_set_key = 0u8; - let value = 255u8; - let mut state_builder = TestStateBuilder::new(); - let mut outer_map = state_builder.new_map::>(); - let mut inner_set = state_builder.new_set::(); - - inner_set.insert(value); - outer_map.insert(inner_set_key, inner_set); - - assert!(outer_map.get(&inner_set_key).unwrap().contains(&value)); - } - - #[test] - fn stateset_insert_remove() { - let mut state_builder = TestStateBuilder::new(); - let mut set = state_builder.new_set(); - let _ = set.insert(42); - assert!(set.contains(&42)); - set.remove(&42); - assert!(!set.contains(&42)); - } - - #[test] - fn stateset_clear() { - let mut state_builder = TestStateBuilder::new(); - let mut set = state_builder.new_set(); - let _ = set.insert(1); - let _ = set.insert(2); - let _ = set.insert(3); - set.clear(); - assert!(set.is_empty()); - } - - #[test] - fn stateset_iterator_unlocks_tree_once_dropped() { - let mut state_builder = TestStateBuilder::new(); - let mut set = state_builder.new_set(); - set.insert(0u8); - set.insert(1); - { - let _iter = set.iter(); - // Uncommenting these two lines (and making iter mutable) should - // give a compile error: - // - // set.insert(2); - // let n = iter.next(); - } // iter is dropped here, unlocking the subtree. - set.insert(2); - } - - #[test] - fn allocate_and_get_statebox() { - let mut state_builder = TestStateBuilder::new(); - let boxed_value = String::from("I'm boxed"); - let statebox = state_builder.new_box(boxed_value.clone()); - assert_eq!(*statebox.get(), boxed_value); - } - - #[test] - fn a_new_entry_can_not_be_created_under_a_locked_subtree() { - let expected_value: u64 = 123123123; - let key = to_bytes(b"ab"); - let sub_key = to_bytes(b"abc"); - let mut state = TestStateApi::new(); - state - .entry(&key[..]) - .or_insert_raw(&to_bytes(&expected_value)) - .expect("No iterators, so insertion should work."); - assert!(state.iterator(&key).is_ok(), "Iterator should be present"); - let entry = state.create_entry(&sub_key); - assert!(entry.is_err(), "Should not be able to create an entry under a locked subtree"); - } - - #[test] - fn a_new_entry_can_be_created_under_a_different_subtree_in_same_super_tree() { - let expected_value: u64 = 123123123; - let key = to_bytes(b"abcd"); - let key2 = to_bytes(b"abe"); - let mut state = TestStateApi::new(); - state - .entry(&key[..]) - .or_insert_raw(&to_bytes(&expected_value)) - .expect("No iterators, so insertion should work."); - assert!(state.iterator(&key).is_ok(), "Iterator should be present"); - let entry = state.create_entry(&key2); - assert!(entry.is_ok(), "Failed to create a new entry under a different subtree"); - } - - #[test] - fn an_existing_entry_can_not_be_deleted_under_a_locked_subtree() { - let expected_value: u64 = 123123123; - let key = to_bytes(b"ab"); - let sub_key = to_bytes(b"abc"); - let mut state = TestStateApi::new(); - state - .entry(&key[..]) - .or_insert_raw(&to_bytes(&expected_value)) - .expect("no iterators, so insertion should work."); - let sub_entry = state - .entry(sub_key) - .or_insert_raw(&to_bytes(&expected_value)) - .expect("Should be possible to create the entry."); - assert!(state.iterator(&key).is_ok(), "Iterator should be present"); - assert!( - state.delete_entry(sub_entry).is_err(), - "Should not be able to create an entry under a locked subtree" - ); - } - - #[test] - fn an_existing_entry_can_be_deleted_from_a_different_subtree_in_same_super_tree() { - let expected_value: u64 = 123123123; - let key = to_bytes(b"abcd"); - let key2 = to_bytes(b"abe"); - let mut state = TestStateApi::new(); - state - .entry(&key[..]) - .or_insert_raw(&to_bytes(&expected_value)) - .expect("No iterators, so insertion should work."); - let entry2 = state - .entry(key2) - .or_insert_raw(&to_bytes(&expected_value)) - .expect("Should be possible to create the entry."); - assert!(state.iterator(&key).is_ok(), "Iterator should be present"); - assert!( - state.delete_entry(entry2).is_ok(), - "Failed to create a new entry under a different subtree" - ); - } - - #[test] - fn deleting_nested_stateboxes_works() { - let mut state_builder = TestStateBuilder::new(); - let inner_box = state_builder.new_box(99u8); - let middle_box = state_builder.new_box(inner_box); - let outer_box = state_builder.new_box(middle_box); - outer_box.delete(); - let mut iter = state_builder.state_api.iterator(&[]).expect("Could not get iterator"); - // The only remaining node should be the state_builder's next_item_prefix node. - assert!(iter.nth(1).is_none()); - } - - #[test] - fn clearing_statemap_with_stateboxes_works() { - let mut state_builder = TestStateBuilder::new(); - let box1 = state_builder.new_box(1u8); - let box2 = state_builder.new_box(2u8); - let box3 = state_builder.new_box(3u8); - let mut map = state_builder.new_map(); - map.insert(1u8, box1); - map.insert(2u8, box2); - map.insert(3u8, box3); - map.clear(); - let mut iter = state_builder.state_api.iterator(&[]).expect("Could not get iterator"); - // The only remaining node should be the state_builder's next_item_prefix node. - assert!(iter.nth(1).is_none()); - } - - #[test] - fn clearing_nested_statemaps_works() { - let mut state_builder = TestStateBuilder::new(); - let mut inner_map_1 = state_builder.new_map(); - inner_map_1.insert(1u8, 2u8); - inner_map_1.insert(2u8, 3u8); - inner_map_1.insert(3u8, 4u8); - let mut inner_map_2 = state_builder.new_map(); - inner_map_2.insert(11u8, 12u8); - inner_map_2.insert(12u8, 13u8); - inner_map_2.insert(13u8, 14u8); - let mut outer_map = state_builder.new_map(); - outer_map.insert(0u8, inner_map_1); - outer_map.insert(1u8, inner_map_2); - outer_map.clear(); - let mut iter = state_builder.state_api.iterator(&[]).expect("Could not get iterator"); - // The only remaining node should be the state_builder's next_item_prefix node. - assert!(iter.nth(1).is_none()); - } - - #[test] - fn occupied_entry_truncates_leftover_data() { - let mut state_builder = TestStateBuilder::new(); - let mut map = state_builder.new_map(); - map.insert(99u8, "A longer string that should be truncated".into()); - let a_short_string = "A short string".to_string(); - let expected_size = a_short_string.len() + 4; // 4 bytes for the length of the string. - map.entry(99u8).and_modify(|v| *v = a_short_string); - let actual_size = state_builder - .state_api - .lookup_entry(&[INITIAL_NEXT_ITEM_PREFIX[0], 0, 0, 0, 0, 0, 0, 0, 99]) - .expect("Lookup failed") - .size() - .expect("Getting size failed"); - assert_eq!(expected_size as u32, actual_size); - } - - #[test] - fn occupied_entry_raw_truncates_leftover_data() { - let mut state = TestStateApi::new(); - state - .entry([]) - .or_insert_raw(&to_bytes(&"A longer string that should be truncated")) - .expect("No iterators, so insertion should work."); - - let a_short_string = "A short string"; - let expected_size = a_short_string.len() + 4; // 4 bytes for the length of the string. - - match state.entry([]) { - EntryRaw::Vacant(_) => panic!("Entry is vacant"), - EntryRaw::Occupied(mut occ) => occ.insert_raw(&to_bytes(&a_short_string)), - } - let actual_size = - state.lookup_entry(&[]).expect("Lookup failed").size().expect("Getting size failed"); - assert_eq!(expected_size as u32, actual_size); - } - - #[test] - fn test_btree_insert_6() { - let mut state_builder = TestStateBuilder::new(); - let mut tree = state_builder.new_btree_set_degree::<5, _>(); - for n in 0..=5 { - tree.insert(n); - } - for n in 0..=5 { - assert!(tree.contains(&n)); - } - } - - #[test] - fn test_btree_insert_0_7() { - let mut state_builder = TestStateBuilder::new(); - let mut tree = state_builder.new_btree_set_degree::<2, _>(); - for n in 0..=7 { - tree.insert(n); - } - for n in 0..=7 { - assert!(tree.contains(&n)); - } - } - - #[test] - fn test_btree_insert_7_no_order() { - let mut state_builder = TestStateBuilder::new(); - let mut tree = state_builder.new_btree_set_degree::<2, _>(); - - tree.insert(0); - tree.insert(1); - tree.insert(2); - tree.insert(3); - tree.insert(7); - tree.insert(6); - tree.insert(5); - tree.insert(4); - - for n in 0..=7 { - assert!(tree.contains(&n)); - } - } - - #[test] - fn test_btree_higher() { - let mut state_builder = TestStateBuilder::new(); - let mut tree = state_builder.new_btree_set_degree::<2, _>(); - tree.insert(1); - tree.insert(2); - tree.insert(3); - tree.insert(4); - tree.insert(5); - tree.insert(7); - assert_eq!(tree.higher(&3).as_deref(), Some(&4)); - assert_eq!(tree.higher(&5).as_deref(), Some(&7)); - assert_eq!(tree.higher(&7).as_deref(), None) - } - - #[test] - fn test_btree_lower() { - let mut state_builder = TestStateBuilder::new(); - let mut tree = state_builder.new_btree_set_degree::<2, _>(); - tree.insert(1); - tree.insert(2); - tree.insert(3); - tree.insert(4); - tree.insert(5); - tree.insert(7); - assert_eq!(tree.lower(&3).as_deref(), Some(&2)); - assert_eq!(tree.lower(&7).as_deref(), Some(&5)); - assert_eq!(tree.lower(&1).as_deref(), None) - } - - #[test] - fn test_btree_insert_1000() { - let mut state_builder = TestStateBuilder::new(); - let mut tree = state_builder.new_btree_set_degree::<2, _>(); - for n in 0..500 { - tree.insert(n); - } - - for n in (500..1000).into_iter().rev() { - tree.insert(n); - } - - for n in 0..1000 { - assert!(tree.contains(&n)) - } - - assert_eq!(tree.len(), 1000) - } - - #[test] - fn test_btree_7_get_8() { - let mut state_builder = TestStateBuilder::new(); - let mut tree = state_builder.new_btree_set_degree::<2, _>(); - for n in 0..=7 { - tree.insert(n); - } - - assert!(!tree.contains(&8)); - } - - #[test] - fn test_btree_remove_from_one_node_tree() { - let mut state_builder = TestStateBuilder::new(); - let mut tree = state_builder.new_btree_set_degree::<2, _>(); - for n in 0..3 { - tree.insert(n); - } - assert!(tree.contains(&1)); - assert!(tree.remove(&1)); - assert!(tree.contains(&0)); - assert!(!tree.contains(&1)); - assert!(tree.contains(&2)); - } - - #[test] - fn test_btree_remove_only_key_lower_leaf_in_three_node() { - let mut state_builder = TestStateBuilder::new(); - let mut tree = state_builder.new_btree_set_degree::<2, _>(); - for n in 0..4 { - tree.insert(n); - } - tree.remove(&3); - - assert!(tree.contains(&0)); - assert!(tree.remove(&0)); - assert!(!tree.contains(&0)); - assert!(tree.contains(&1)); - assert!(tree.contains(&2)); - } - - #[test] - fn test_btree_remove_only_key_higher_leaf_in_three_node() { - let mut state_builder = TestStateBuilder::new(); - let mut tree = state_builder.new_btree_set_degree::<2, _>(); - for n in (0..4).into_iter().rev() { - tree.insert(n); - } - tree.remove(&3); - assert!(tree.contains(&2)); - assert!(tree.remove(&2)); - assert!(tree.contains(&0)); - assert!(tree.contains(&1)); - assert!(!tree.contains(&2)); - } - - #[test] - fn test_btree_remove_from_higher_leaf_in_three_node_taking_from_sibling() { - let mut state_builder = TestStateBuilder::new(); - let mut tree = state_builder.new_btree_set_degree::<2, _>(); - for n in (0..4).into_iter().rev() { - tree.insert(n); - } - assert!(tree.contains(&3)); - assert!(tree.remove(&3)); - assert!(!tree.contains(&3)); - } - - #[test] - fn test_btree_remove_from_lower_leaf_in_three_node_taking_from_sibling() { - let mut state_builder = TestStateBuilder::new(); - let mut tree = state_builder.new_btree_set_degree::<2, _>(); - for n in 0..4 { - tree.insert(n); - } - assert!(tree.contains(&0)); - assert!(tree.remove(&0)); - assert!(!tree.contains(&0)); - assert!(tree.contains(&1)); - assert!(tree.contains(&2)); - assert!(tree.contains(&3)); - } - - #[test] - fn test_btree_remove_from_root_in_three_node_causing_merge() { - let mut state_builder = TestStateBuilder::new(); - let mut tree = state_builder.new_btree_set_degree::<2, _>(); - for n in 0..4 { - tree.insert(n); - } - tree.remove(&3); - - assert!(tree.contains(&1)); - assert!(tree.remove(&1)); - assert!(!tree.contains(&1)); - } - - #[test] - fn test_btree_remove_from_root_in_three_node_taking_key_from_higher_child() { - let mut state_builder = TestStateBuilder::new(); - let mut tree = state_builder.new_btree_set_degree::<2, _>(); - for n in 0..4 { - tree.insert(n); - } - assert!(tree.contains(&1)); - assert!(tree.remove(&1)); - assert!(!tree.contains(&1)); - } - - #[test] - fn test_btree_remove_from_root_in_three_node_taking_key_from_lower_child() { - let mut state_builder = TestStateBuilder::new(); - let mut tree = state_builder.new_btree_set_degree::<2, _>(); - for n in (0..4).into_iter().rev() { - tree.insert(n); - } - assert!(tree.contains(&2)); - assert!(tree.remove(&2)); - assert!(!tree.contains(&2)); - } - - #[test] - fn test_btree_iter() { - let mut state_builder = TestStateBuilder::new(); - let mut tree = state_builder.new_btree_set_degree::<2, _>(); - let keys: Vec = (0..15).into_iter().collect(); - for &k in &keys { - tree.insert(k); - } - let iter_keys: Vec = tree.iter().map(|k| k.clone()).collect(); - assert_eq!(keys, iter_keys); - } } diff --git a/concordium-std/src/types.rs b/concordium-std/src/types.rs index c31906bb..7503b249 100644 --- a/concordium-std/src/types.rs +++ b/concordium-std/src/types.rs @@ -1264,269 +1264,3 @@ pub struct MetadataUrl { /// A optional hash of the content. pub hash: Option<[u8; 32]>, } - -/// An ordered map based on [B-Tree](https://en.wikipedia.org/wiki/B-tree), where -/// each node is stored separately in the low-level key-value store. -/// -/// It can be seen as an extension adding tracking of the keys ordering on top -/// of [`StateMap`] to provide functions such as [`higher`](Self::higher) and -/// [`lower`](Self::lower). This results in some overhead when inserting and -/// deleting entries from the map compared to using [`StateMap`]. -/// -/// | Operation | Performance | -/// |-------------------------------------------------|---------------| -/// | [`get`](Self::get) / [`get_mut`](Self::get_mut) | O(k) | -/// | [`insert`](Self::insert) | O(k + log(n)) | -/// | [`remove`](Self::remove) | O(k + log(n)) | -/// | [`higher`](Self::higher)/[`lower`](Self::lower) | O(k + log(n)) | -/// -/// Where `k` is the byte size of the serialized keys and `n` is the number of -/// entries in the map. -/// -/// ## Type parameters -/// -/// The map `StateBTreeMap` is parametrized by the types: -/// - `K`: Keys used in the map. Most operations on the map require this to -/// implement [`Serialize`](crate::Serialize). Keys cannot contain references -/// to the low-level state, such as types containing [`StateBox`], -/// [`StateMap`] and [`StateSet`]. -/// - `V`: Values stored in the map. Most operations on the map require this to -/// implement [`Serial`](crate::Serial) and -/// [`DeserialWithState`](crate::DeserialWithState). -/// - `S`: The low-level state implementation used, this allows for mocking the -/// state API in unit tests, see -/// [`TestStateApi`](crate::test_infrastructure::TestStateApi). -/// - `M`: A `const usize` determining the _minimum degree_ of the B-tree. -/// _Must_ be a value of `2` or above for the tree to work. This can be used -/// to tweak the height of the tree vs size of each node in the tree. The -/// default is set based on benchmarks. -/// -/// ## Usage -/// -/// New maps can be constructed using the -/// [`new_btree_map`][StateBuilder::new_btree_map] method on the -/// [`StateBuilder`]. -/// -/// ``` -/// # use concordium_std::*; -/// # use concordium_std::test_infrastructure::*; -/// # let mut state_builder = TestStateBuilder::new(); -/// /// In an init method: -/// let mut map1 = state_builder.new_btree_map(); -/// # map1.insert(0u8, 1u8); // Specifies type of map. -/// -/// # let mut host = TestHost::new((), state_builder); -/// /// In a receive method: -/// let mut map2 = host.state_builder().new_btree_map(); -/// # map2.insert(0u16, 1u16); -/// ``` -/// -/// ### **Caution** -/// -/// `StateBTreeMap`s must be explicitly deleted when they are no longer needed, -/// otherwise they will remain in the contract's state, albeit unreachable. -/// -/// ```no_run -/// # use concordium_std::*; -/// struct MyState { -/// inner: StateBTreeMap, -/// } -/// fn incorrect_replace(state_builder: &mut StateBuilder, state: &mut MyState) { -/// // The following is incorrect. The old value of `inner` is not properly deleted. -/// // from the state. -/// state.inner = state_builder.new_btree_map(); // ⚠️ -/// } -/// ``` -/// Instead, either the map should be [cleared](StateBTreeMap::clear) or -/// explicitly deleted. -/// -/// ```no_run -/// # use concordium_std::*; -/// # struct MyState { -/// # inner: StateBTreeMap -/// # } -/// fn correct_replace(state_builder: &mut StateBuilder, state: &mut MyState) { -/// state.inner.clear_flat(); -/// } -/// ``` -/// Or alternatively -/// ```no_run -/// # use concordium_std::*; -/// # struct MyState { -/// # inner: StateBTreeMap -/// # } -/// fn correct_replace(state_builder: &mut StateBuilder, state: &mut MyState) { -/// let old_map = mem::replace(&mut state.inner, state_builder.new_btree_map()); -/// old_map.delete() -/// } -/// ``` -pub struct StateBTreeMap { - pub(crate) map: StateMap, - pub(crate) ordered_set: StateBTreeSet, -} - -/// An ordered set based on [B-Tree](https://en.wikipedia.org/wiki/B-tree), where -/// each node is stored separately in the low-level key-value store. -/// -/// | Operation | Performance | -/// |-------------------------------------------------|---------------| -/// | [`contains`](Self::contains) | O(k + log(n)) | -/// | [`insert`](Self::insert) | O(k + log(n)) | -/// | [`remove`](Self::remove) | O(k + log(n)) | -/// | [`higher`](Self::higher)/[`lower`](Self::lower) | O(k + log(n)) | -/// -/// Where `k` is the byte size of the serialized keys and `n` is the number of -/// entries in the map. -/// -/// ## Type parameters -/// -/// The map `StateBTreeSet` is parametrized by the types: -/// - `K`: Keys used in the set. Most operations on the set require this to -/// implement [`Serialize`](crate::Serialize). Keys cannot contain references -/// to the low-level state, such as types containing [`StateBox`], -/// [`StateMap`] and [`StateSet`]. -/// - `S`: The low-level state implementation used, this allows for mocking the -/// state API in unit tests, see -/// [`TestStateApi`](crate::test_infrastructure::TestStateApi). -/// - `M`: A `const usize` determining the _minimum degree_ of the B-tree. -/// _Must_ be a value of `2` or above for the tree to work. This can be used -/// to tweak the height of the tree vs size of each node in the tree. The -/// default is set based on benchmarks. -/// -/// ## Usage -/// -/// New sets can be constructed using the -/// [`new_btree_set`][StateBuilder::new_btree_set] method on the -/// [`StateBuilder`]. -/// -/// ``` -/// # use concordium_std::*; -/// # use concordium_std::test_infrastructure::*; -/// # let mut state_builder = TestStateBuilder::new(); -/// /// In an init method: -/// let mut map1 = state_builder.new_btree_set(); -/// # map1.insert(0u8); // Specifies type of map. -/// -/// # let mut host = TestHost::new((), state_builder); -/// /// In a receive method: -/// let mut map2 = host.state_builder().new_btree_set(); -/// # map2.insert(0u16); -/// ``` -/// -/// ### **Caution** -/// -/// `StateBTreeSet`s must be explicitly deleted when they are no longer needed, -/// otherwise they will remain in the contract's state, albeit unreachable. -/// -/// ```no_run -/// # use concordium_std::*; -/// struct MyState { -/// inner: StateBTreeSet, -/// } -/// fn incorrect_replace(state_builder: &mut StateBuilder, state: &mut MyState) { -/// // The following is incorrect. The old value of `inner` is not properly deleted. -/// // from the state. -/// state.inner = state_builder.new_btree_set(); // ⚠️ -/// } -/// ``` -/// Instead, the set should be [cleared](StateBTreeSet::clear): -/// -/// ```no_run -/// # use concordium_std::*; -/// # struct MyState { -/// # inner: StateBTreeSet -/// # } -/// fn correct_replace(state_builder: &mut StateBuilder, state: &mut MyState) { -/// state.inner.clear(); -/// } -/// ``` -pub struct StateBTreeSet { - /// Type marker for the key. - pub(crate) _marker_key: PhantomData, - /// The unique prefix to use for this map in the key-value store. - pub(crate) prefix: StateItemPrefix, - /// The API for interacting with the low-level state. - pub(crate) state_api: S, - /// The ID of the root node of the tree, where None represents the tree is - /// empty. - pub(crate) root: Option, - /// Tracking the number of items in the tree. - pub(crate) len: u32, - /// Tracking the next available ID for a new node. - pub(crate) next_node_id: state_btree_internals::NodeId, -} - -/// Module with types used internally in [`StateBTreeMap`]. -pub(crate) mod state_btree_internals { - use super::*; - - /// Identifier for a node in the tree. Used to construct the key, where this - /// node is store in the smart contract key-value store. - #[derive(Debug, Copy, Clone)] - #[repr(transparent)] - pub(crate) struct NodeId { - pub(crate) id: u32, - } - - /// Type representing a node in the [`StateBTreeMap`]. - /// Each node is stored separately in the smart contract key-value store. - #[derive(Debug)] - pub(crate) struct Node { - /// List of sorted keys tracked by this node. - /// This list should never be empty and contain between `M - 1` and `2M - /// - 1` elements. - pub(crate) keys: Vec, - /// List of nodes which are children of this node in the tree. - /// - /// This list is empty when this node is representing a leaf. - /// When not a leaf, it will contain exactly `keys.len() + 1` elements. - /// - /// The elements are ordered such that for a key `keys[i]`: - /// - `children[i]` is a subtree containing strictly smaller keys. - /// - `children[i + 1]` is a subtree containing strictly larger keys. - pub(crate) children: Vec, - } - - /// Wrapper implement the exact same deserial as K, but wraps it in an - /// option in memory. This is used, to allow taking a key from a mutable - /// reference to a node, without cloning the key, during iteration of the - /// set. - #[repr(transparent)] - pub(crate) struct KeyWrapper { - pub(crate) key: Option, - } -} - -/// An iterator over the entries of a [`StateBTreeSet`]. -/// -/// Ordered by `K`. -/// -/// This `struct` is created by the [`iter`][StateBTreeSet::iter] method on -/// [`StateBTreeSet`]. See its documentation for more. -pub struct StateBTreeSetIter<'a, 'b, K, S, const M: usize> { - /// The number of elements left to iterate. - pub(crate) length: usize, - /// Reference to a node in the tree to load and iterate before the current - /// node. - pub(crate) next_node: Option, - /// Tracking the nodes depth first, which are currently being iterated. - pub(crate) depth_first_stack: - Vec<(state_btree_internals::Node>, usize)>, - /// Reference to the set, needed for looking up the nodes. - pub(crate) tree: &'a StateBTreeSet, - /// Marker for tracking the lifetime of the key. - pub(crate) _marker_lifetime: PhantomData<&'b K>, -} - -/// An iterator over the entries of a [`StateBTreeMap`]. -/// -/// Ordered by `K`. -/// -/// This `struct` is created by the [`iter`][StateBTreeMap::iter] method on -/// [`StateBTreeMap`]. See its documentation for more. -pub struct StateBTreeMapIter<'a, 'b, K, V, S, const M: usize> { - /// Iterator over the keys in the map. - pub(crate) key_iter: StateBTreeSetIter<'a, 'b, K, S, M>, - /// Reference to the map holding the values. - pub(crate) map: &'a StateMap, -} From 33e731be727efba5880c1638b8ebbcb2f7ac486c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Holm=20Gj=C3=B8rup?= Date: Tue, 12 Mar 2024 14:30:32 +0100 Subject: [PATCH 17/36] Fix related to empty root after removal, dublicate keys after split --- concordium-std/src/state_btree.rs | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/concordium-std/src/state_btree.rs b/concordium-std/src/state_btree.rs index 1d9d05aa..ecadf2de 100644 --- a/concordium-std/src/state_btree.rs +++ b/concordium-std/src/state_btree.rs @@ -385,10 +385,7 @@ impl StateBTreeSet { S: HasStateApi, K: Serialize + Ord, { let Some(root_id) = self.root else { - let node_id = { - let (node_id, _node) = self.create_node(crate::vec![key], Vec::new()); - node_id - }; + let (node_id, _) = self.create_node(crate::vec![key], Vec::new()); self.root = Some(node_id); self.len = 1; return true; @@ -655,22 +652,22 @@ impl StateBTreeSet { // If something was deleted, we update the length and make sure to remove the // root node if needed. + let root = self.get_node_mut(root_node_id); if deleted_something { self.len -= 1; - let root = self.get_node_mut(root_node_id); if self.len == 0 { // Remote the root node if tree is empty. self.root = None; self.delete_node(root); - } else { - // If the root is empty but the tree is not, point to the only child of the root - // as the root. - if root.keys.is_empty() { - self.root = Some(root.children[0]); - self.delete_node(root); - } + return true; } } + // If the root is empty but the tree is not, point to the only child of the root + // as the root. + if root.keys.is_empty() { + self.root = Some(root.children[0]); + self.delete_node(root); + } deleted_something } @@ -905,7 +902,11 @@ impl StateBTreeSet { let larger_child = self.split_child(&mut node, insert_index, &mut child); // Since the child is now split into two, we have to check which one to insert // into. - if node.keys[insert_index] < key { + let moved_up_key = &node.keys[insert_index]; + if moved_up_key == &key { + // If the key moved up during the split is the key we are inserting, we exit. + return false; + } else if moved_up_key < &key { larger_child } else { child From fa9c07f1f271010b83023f6fe65cdf08a15db8f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Holm=20Gj=C3=B8rup?= Date: Tue, 12 Mar 2024 15:23:13 +0100 Subject: [PATCH 18/36] Introduce unit-tests using quickcheck for the btree --- concordium-std/src/state_btree.rs | 412 ++++++++++++++++++++++++++---- 1 file changed, 363 insertions(+), 49 deletions(-) diff --git a/concordium-std/src/state_btree.rs b/concordium-std/src/state_btree.rs index ecadf2de..ac8a5e64 100644 --- a/concordium-std/src/state_btree.rs +++ b/concordium-std/src/state_btree.rs @@ -968,6 +968,73 @@ impl StateBTreeSet { let entry = self.state_api.lookup_entry(&key).unwrap_abort(); StateRefMut::new(entry, self.state_api.clone()) } + + /// Construct a string for displaying the btree and debug information. + /// Should only be used while debugging and testing the btree itself. + #[cfg(all(feature = "wasm-test", feature = "concordium-quickcheck"))] + pub(crate) fn debug(&self) -> String + where + K: Serialize + std::fmt::Debug + Ord, + S: HasStateApi, { + let Some(root_node_id) = self.root else { + return format!("no root"); + }; + let mut string = String::new(); + let root: Node = self.get_node(root_node_id); + string.push_str(format!("root: {:#?}", root).as_str()); + let mut stack = root.children; + + while let Some(node_id) = stack.pop() { + let node: Node = self.get_node(node_id); + string.push_str( + format!("node {} {:?}: {:#?},\n", node_id.id, node.check_invariants(), node) + .as_str(), + ); + + stack.extend(node.children); + } + string + } + + /// Check a number of invariants, producing an error if any of them are + /// violated. Should only be used while debugging and testing the btree + /// itself. + #[cfg(all(feature = "wasm-test", feature = "concordium-quickcheck"))] + pub(crate) fn check_invariants(&self) -> Result<(), InvariantViolation> + where + K: Serialize + Ord, + S: HasStateApi, { + use crate::ops::Deref; + let Some(root_node_id) = self.root else { + return if self.len == 0 { + Ok(()) + } else { + Err(InvariantViolation::NonZeroLenWithNoRoot) + }; + }; + let root: Node = self.get_node(root_node_id); + if root.keys.is_empty() { + return Err(InvariantViolation::ZeroKeysInRoot); + } + + let mut stack = root.children; + while let Some(node_id) = stack.pop() { + let node: Node = self.get_node(node_id); + node.check_invariants()?; + stack.extend(node.children); + } + + let mut prev = None; + for key in self.iter() { + if let Some(p) = prev.as_deref() { + if p >= key.deref() { + return Err(InvariantViolation::IterationOutOfOrder); + } + } + prev = Some(key); + } + Ok(()) + } } /// An iterator over the entries of a [`StateBTreeSet`]. @@ -1068,13 +1135,48 @@ struct NodeId { id: u32, } +/// Byte size of the key used to store a BTree internal node in the smart +/// contract key-value store. +const BTREE_NODE_KEY_SIZE: usize = STATE_ITEM_PREFIX_SIZE + NodeId::SERIALIZED_BYTE_SIZE; + +impl NodeId { + /// Byte size of `NodeId` when serialized. + const SERIALIZED_BYTE_SIZE: usize = 4; + + /// Return a copy of the NodeId, then increments itself. + fn copy_then_increment(&mut self) -> Self { + let current = *self; + self.id += 1; + current + } + + /// Construct the key for the node in the key-value store from the node ID. + fn as_key(&self, prefix: &StateItemPrefix) -> [u8; BTREE_NODE_KEY_SIZE] { + // Create an uninitialized array of `MaybeUninit`. The `assume_init` is + // safe because the type we are claiming to have initialized here is a + // bunch of `MaybeUninit`s, which do not require initialization. + let mut prefixed: [mem::MaybeUninit; BTREE_NODE_KEY_SIZE] = + unsafe { mem::MaybeUninit::uninit().assume_init() }; + for i in 0..STATE_ITEM_PREFIX_SIZE { + prefixed[i].write(prefix[i]); + } + let id_bytes = self.id.to_le_bytes(); + for i in 0..id_bytes.len() { + prefixed[STATE_ITEM_PREFIX_SIZE + i].write(id_bytes[i]); + } + // Transmuting away the maybeuninit is safe since we have initialized all of + // them. + unsafe { mem::transmute(prefixed) } + } +} + /// Type representing a node in the [`StateBTreeMap`]. /// Each node is stored separately in the smart contract key-value store. #[derive(Debug)] struct Node { /// List of sorted keys tracked by this node. /// This list should never be empty and contain between `M - 1` and `2M - /// - 1` elements. + /// - 1` elements. The root node being the only exception to this. keys: Vec, /// List of nodes which are children of this node in the tree. /// @@ -1087,28 +1189,6 @@ struct Node { children: Vec, } -/// Wrapper implement the exact same deserial as K, but wraps it in an -/// option in memory. This is used, to allow taking a key from a mutable -/// reference to a node, without cloning the key, during iteration of the -/// set. -#[repr(transparent)] -struct KeyWrapper { - key: Option, -} - -impl Deserial for KeyWrapper { - fn deserial(source: &mut R) -> ParseResult { - let key = K::deserial(source)?; - Ok(Self { - key: Some(key), - }) - } -} - -/// Byte size of the key used to store a BTree internal node in the smart -/// contract key-value store. -const BTREE_NODE_KEY_SIZE: usize = STATE_ITEM_PREFIX_SIZE + NodeId::SERIALIZED_BYTE_SIZE; - impl Node { /// The max length of the child list. const MAXIMUM_CHILD_LEN: usize = 2 * M; @@ -1130,36 +1210,75 @@ impl Node { /// Check if the node holds the minimum number of keys. fn is_at_min(&self) -> bool { self.len() == Self::MINIMUM_KEY_LEN } -} - -impl NodeId { - /// Byte size of `NodeId` when serialized. - const SERIALIZED_BYTE_SIZE: usize = 4; - /// Return a copy of the NodeId, then increments itself. - fn copy_then_increment(&mut self) -> Self { - let current = *self; - self.id += 1; - current - } + /// Check a number of invariants of a non-root node in a btree, producing an + /// error if any of them are violated. Should only be used while + /// debugging and testing the btree itself. + #[cfg(all(feature = "wasm-test", feature = "concordium-quickcheck"))] + pub(crate) fn check_invariants(&self) -> Result<(), InvariantViolation> + where + K: Ord, { + for i in 1..self.keys.len() { + if &self.keys[i - 1] >= &self.keys[i] { + return Err(InvariantViolation::NodeKeysOutOfOrder); + } + } - /// Construct the key for the node in the key-value store from the node ID. - fn as_key(&self, prefix: &StateItemPrefix) -> [u8; BTREE_NODE_KEY_SIZE] { - // Create an uninitialized array of `MaybeUninit`. The `assume_init` is - // safe because the type we are claiming to have initialized here is a - // bunch of `MaybeUninit`s, which do not require initialization. - let mut prefixed: [mem::MaybeUninit; BTREE_NODE_KEY_SIZE] = - unsafe { mem::MaybeUninit::uninit().assume_init() }; - for i in 0..STATE_ITEM_PREFIX_SIZE { - prefixed[i].write(prefix[i]); + if self.keys.len() < Self::MINIMUM_KEY_LEN { + return Err(InvariantViolation::KeysLenBelowMin); } - let id_bytes = self.id.to_le_bytes(); - for i in 0..id_bytes.len() { - prefixed[STATE_ITEM_PREFIX_SIZE + i].write(id_bytes[i]); + if self.keys.len() > Self::MAXIMUM_KEY_LEN { + return Err(InvariantViolation::KeysLenAboveMax); } - // Transmuting away the maybeuninit is safe since we have initialized all of - // them. - unsafe { mem::transmute(prefixed) } + + if self.is_leaf() { + if !self.children.is_empty() { + return Err(InvariantViolation::LeafWithChildren); + } + } else { + if self.children.len() < Self::MINIMUM_CHILD_LEN { + return Err(InvariantViolation::ChildrenLenBelowMin); + } + if self.children.len() > Self::MAXIMUM_CHILD_LEN { + return Err(InvariantViolation::ChildrenLenAboveMax); + } + } + + Ok(()) + } +} + +/// The invariants to check in a btree. +/// Should only be used while debugging and testing the btree itself. +#[cfg(all(feature = "wasm-test", feature = "concordium-quickcheck"))] +#[derive(Debug)] +pub(crate) enum InvariantViolation { + NonZeroLenWithNoRoot, + ZeroKeysInRoot, + IterationOutOfOrder, + NodeKeysOutOfOrder, + KeysLenBelowMin, + KeysLenAboveMax, + LeafWithChildren, + ChildrenLenBelowMin, + ChildrenLenAboveMax, +} + +/// Wrapper implement the exact same deserial as K, but wraps it in an +/// option in memory. This is used, to allow taking a key from a mutable +/// reference to a node, without cloning the key, during iteration of the +/// set. +#[repr(transparent)] +struct KeyWrapper { + key: Option, +} + +impl Deserial for KeyWrapper { + fn deserial(source: &mut R) -> ParseResult { + let key = K::deserial(source)?; + Ok(Self { + key: Some(key), + }) } } @@ -1479,4 +1598,199 @@ mod wasm_test_btree { let iter_keys: Vec = tree.iter().map(|k| k.clone()).collect(); claim_eq!(keys, iter_keys); } + + #[concordium_test] + fn test_btree_insert_present_key() { + let mut state_builder = StateBuilder::open(StateApi::open()); + let mut tree = state_builder.new_btree_set_degree::<2, _>(); + for n in [0, 3, 4, 1, 2].into_iter() { + tree.insert(n); + } + claim!(!tree.insert(1)); + } + + #[cfg(feature = "concordium-quickcheck")] + #[allow(deprecated)] + mod quickcheck { + use super::super::*; + use crate::{ + self as concordium_std, concordium_quickcheck, concordium_test, fail, StateApi, + StateBuilder, + }; + use ::quickcheck::{Arbitrary, Gen, TestResult}; + + #[concordium_quickcheck] + fn test_quickcheck_inserts(items: Vec) -> TestResult { + let mut state_builder = StateBuilder::open(StateApi::open()); + let mut tree = state_builder.new_btree_set_degree::<2, _>(); + for k in items.clone() { + tree.insert(k); + } + if let Err(violation) = tree.check_invariants() { + return TestResult::error(format!("Invariant violated: {:?}", violation)); + } + for k in items.iter() { + if !tree.contains(k) { + return TestResult::error(format!("Missing key: {}", k)); + } + } + TestResult::passed() + } + + #[concordium_quickcheck(num_tests = 500)] + fn test_quickcheck_inserts_removes(mutations: Mutations) -> TestResult { + let mut state_builder = StateBuilder::open(StateApi::open()); + let mut tree = state_builder.new_btree_set_degree::<2, _>(); + + if let Err(err) = run_mutations(&mut tree, &mutations.0) { + TestResult::error(format!("Error: {}, tree: {}", err, tree.debug())) + } else { + TestResult::passed() + } + } + + #[derive(Debug, Clone)] + struct Mutations(Vec<(K, Operation)>); + + #[derive(Debug, Clone, Copy)] + enum Operation { + InsertKeyNotPresent, + InsertKeyPresent, + RemoveKeyPresent, + RemoveKeyNotPresent, + } + + fn run_mutations( + tree: &mut StateBTreeSet, + mutations: &[(u32, Operation)], + ) -> Result<(), String> { + for (k, op) in mutations.into_iter() { + if let Err(violation) = tree.check_invariants() { + return Err(format!("Invariant violated: {:?}", violation)); + } + match op { + Operation::InsertKeyPresent => { + if tree.insert(*k) { + return Err(format!("InsertKeyPresent was not present: {}", k)); + } + } + Operation::InsertKeyNotPresent => { + if !tree.insert(*k) { + return Err(format!("InsertKeyNotPresent was present: {}", k)); + } + } + Operation::RemoveKeyNotPresent => { + if tree.remove(k) { + return Err(format!("RemoveKeyNotPresent was present: {}", k)); + } + } + Operation::RemoveKeyPresent => { + if !tree.remove(k) { + return Err(format!("RemoveKeyPresent was not present: {}", k)); + } + } + } + } + Ok(()) + } + + impl Arbitrary for Operation { + fn arbitrary(g: &mut Gen) -> Self { + g.choose(&[ + Self::InsertKeyNotPresent, + Self::InsertKeyPresent, + Self::RemoveKeyPresent, + Self::RemoveKeyNotPresent, + ]) + .unwrap() + .clone() + } + } + + impl Arbitrary for Mutations + where + K: Arbitrary + Ord, + { + fn arbitrary(g: &mut Gen) -> Self { + let mut inserted_keys: Vec = Vec::new(); + let mut mutations = Vec::new(); + + while mutations.len() < g.size() { + let op: Operation = Operation::arbitrary(g); + match op { + Operation::InsertKeyPresent if inserted_keys.len() > 0 => { + let indexes: Vec = + (0..inserted_keys.len()).into_iter().collect(); + let k_index = g.choose(&indexes).unwrap(); + let k = &inserted_keys[*k_index]; + mutations.push((k.clone(), op)); + } + Operation::InsertKeyNotPresent => { + let k = K::arbitrary(g); + if let Err(index) = inserted_keys.binary_search(&k) { + inserted_keys.insert(index, k.clone()); + mutations.push((k, op)); + } + } + Operation::RemoveKeyPresent if inserted_keys.len() > 0 => { + let indexes: Vec = + (0..inserted_keys.len()).into_iter().collect(); + let k_index = g.choose(&indexes).unwrap(); + let k = inserted_keys.remove(*k_index); + mutations.push((k, op)); + } + Operation::RemoveKeyNotPresent => { + let k = K::arbitrary(g); + if inserted_keys.binary_search(&k).is_err() { + mutations.push((k, op)); + } + } + _ => {} + } + } + + Self(mutations) + } + + fn shrink(&self) -> Box> { + let pop = { + let mut clone = self.0.clone(); + clone.pop(); + Self(clone) + }; + let mut v = vec![pop]; + for (i, (k, op)) in self.0.iter().enumerate() { + match op { + Operation::InsertKeyPresent | Operation::RemoveKeyNotPresent => { + let mut clone = self.0.clone(); + clone.remove(i); + v.push(Self(clone)); + } + Operation::RemoveKeyPresent => { + let mut clone = self.0.clone(); + let mut prev = clone[0..i].iter().enumerate().rev(); + let j = loop { + if let Some((j, (k2, op))) = prev.next() { + match op { + Operation::InsertKeyNotPresent if k == k2 => { + break j; + } + _ => {} + } + } else { + fail!("No insertion found before") + } + }; + clone.remove(i); + clone.remove(j); + v.push(Self(clone)); + } + _ => {} + } + } + + Box::new(v.into_iter()) + } + } + } } From e0cb873c01b67dbc7599f35b4a466c0902fbd910 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Holm=20Gj=C3=B8rup?= Date: Tue, 12 Mar 2024 15:39:37 +0100 Subject: [PATCH 19/36] Introduce concordium-std feature 'internal-wasm-test' --- concordium-std/Cargo.toml | 3 +++ concordium-std/src/impls.rs | 2 +- concordium-std/src/state_btree.rs | 11 +++++------ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/concordium-std/Cargo.toml b/concordium-std/Cargo.toml index 7804d777..535e4cb7 100644 --- a/concordium-std/Cargo.toml +++ b/concordium-std/Cargo.toml @@ -29,6 +29,9 @@ features = ["smart-contract"] default = ["std"] std = ["concordium-contracts-common/std"] wasm-test = ["concordium-contracts-common/wasm-test"] +# Own internal wasm-tests leaks out to the smart contracts using this library, +# so a separate feature 'internal-wasm-test' is introduced for these. +internal-wasm-test = ["wasm-test", "concordium-quickcheck"] build-schema = ["concordium-contracts-common/build-schema"] crypto-primitives = ["sha2", "sha3", "secp256k1", "ed25519-zebra"] concordium-quickcheck = ["getrandom", "quickcheck", "concordium-contracts-common/concordium-quickcheck", "std"] diff --git a/concordium-std/src/impls.rs b/concordium-std/src/impls.rs index c19145e1..b541d1af 100644 --- a/concordium-std/src/impls.rs +++ b/concordium-std/src/impls.rs @@ -3155,7 +3155,7 @@ mod tests { /// This test module rely on the runtime providing host functions and can only /// be run using `cargo concordium test`. -#[cfg(feature = "wasm-test")] +#[cfg(feature = "internal-wasm-test")] mod wasm_test { use crate::{ claim, claim_eq, concordium_test, to_bytes, Deletable, Deserial, DeserialWithState, diff --git a/concordium-std/src/state_btree.rs b/concordium-std/src/state_btree.rs index ac8a5e64..e1a977c3 100644 --- a/concordium-std/src/state_btree.rs +++ b/concordium-std/src/state_btree.rs @@ -971,7 +971,7 @@ impl StateBTreeSet { /// Construct a string for displaying the btree and debug information. /// Should only be used while debugging and testing the btree itself. - #[cfg(all(feature = "wasm-test", feature = "concordium-quickcheck"))] + #[cfg(feature = "internal-wasm-test")] pub(crate) fn debug(&self) -> String where K: Serialize + std::fmt::Debug + Ord, @@ -999,7 +999,7 @@ impl StateBTreeSet { /// Check a number of invariants, producing an error if any of them are /// violated. Should only be used while debugging and testing the btree /// itself. - #[cfg(all(feature = "wasm-test", feature = "concordium-quickcheck"))] + #[cfg(feature = "internal-wasm-test")] pub(crate) fn check_invariants(&self) -> Result<(), InvariantViolation> where K: Serialize + Ord, @@ -1214,7 +1214,7 @@ impl Node { /// Check a number of invariants of a non-root node in a btree, producing an /// error if any of them are violated. Should only be used while /// debugging and testing the btree itself. - #[cfg(all(feature = "wasm-test", feature = "concordium-quickcheck"))] + #[cfg(feature = "internal-wasm-test")] pub(crate) fn check_invariants(&self) -> Result<(), InvariantViolation> where K: Ord, { @@ -1250,7 +1250,7 @@ impl Node { /// The invariants to check in a btree. /// Should only be used while debugging and testing the btree itself. -#[cfg(all(feature = "wasm-test", feature = "concordium-quickcheck"))] +#[cfg(feature = "internal-wasm-test")] #[derive(Debug)] pub(crate) enum InvariantViolation { NonZeroLenWithNoRoot, @@ -1368,7 +1368,7 @@ where /// This test module rely on the runtime providing host functions and can only /// be run using `cargo concordium test`. -#[cfg(feature = "wasm-test")] +#[cfg(feature = "internal-wasm-test")] mod wasm_test_btree { use crate::{claim, claim_eq, concordium_test, StateApi, StateBuilder}; @@ -1609,7 +1609,6 @@ mod wasm_test_btree { claim!(!tree.insert(1)); } - #[cfg(feature = "concordium-quickcheck")] #[allow(deprecated)] mod quickcheck { use super::super::*; From 04460a731ccf70b71c7150135895317d1638d038 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Holm=20Gj=C3=B8rup?= Date: Tue, 12 Mar 2024 15:58:39 +0100 Subject: [PATCH 20/36] Fix clippy and doc complaints --- concordium-std/src/impls.rs | 12 ++++---- concordium-std/src/state_btree.rs | 48 +++++++++++++++---------------- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/concordium-std/src/impls.rs b/concordium-std/src/impls.rs index b541d1af..ccb5a782 100644 --- a/concordium-std/src/impls.rs +++ b/concordium-std/src/impls.rs @@ -2331,13 +2331,13 @@ where StateMap::open(state_api, prefix) } - /// Create a new empty [`StateBTreeSet`]. + /// Create a new empty [`StateBTreeSet`](crate::StateBTreeSet). pub fn new_btree_set(&mut self) -> state_btree::StateBTreeSet { let (state_api, prefix) = self.new_state_container(); state_btree::StateBTreeSet::new(state_api, prefix) } - /// Create a new empty [`StateBTreeMap`]. + /// Create a new empty [`StateBTreeMap`](crate::StateBTreeMap). pub fn new_btree_map(&mut self) -> state_btree::StateBTreeMap { state_btree::StateBTreeMap { map: self.new_map(), @@ -2345,8 +2345,8 @@ where } } - /// Create a new empty [`StateBTreeSet`], setting the minimum degree `M` of - /// the B-Tree explicitly. + /// Create a new empty [`StateBTreeSet`](crate::StateBTreeSet), setting the + /// minimum degree `M` of the B-Tree explicitly. pub fn new_btree_set_degree( &mut self, ) -> state_btree::StateBTreeSet { @@ -2354,8 +2354,8 @@ where state_btree::StateBTreeSet::new(state_api, prefix) } - /// Create a new empty [`StateBTreeMap`], setting the minimum degree `M` of - /// the B-Tree explicitly. + /// Create a new empty [`StateBTreeMap`](crate::StateBTreeMap), setting the + /// minimum degree `M` of the B-Tree explicitly. pub fn new_btree_map_degree( &mut self, ) -> state_btree::StateBTreeMap { diff --git a/concordium-std/src/state_btree.rs b/concordium-std/src/state_btree.rs index e1a977c3..80da1ea5 100644 --- a/concordium-std/src/state_btree.rs +++ b/concordium-std/src/state_btree.rs @@ -1,7 +1,7 @@ use crate::{ - marker::PhantomData, mem, Deletable, Deserial, DeserialWithState, Get, HasStateApi, - ParseResult, Read, Serial, Serialize, StateItemPrefix, StateMap, StateRef, StateRefMut, - UnwrapAbort, Write, STATE_ITEM_PREFIX_SIZE, + cmp::Ordering, marker::PhantomData, mem, Deletable, Deserial, DeserialWithState, Get, + HasStateApi, ParseResult, Read, Serial, Serialize, StateItemPrefix, StateMap, StateRef, + StateRefMut, UnwrapAbort, Write, STATE_ITEM_PREFIX_SIZE, }; /// An ordered map based on [B-Tree](https://en.wikipedia.org/wiki/B-tree), where @@ -27,8 +27,9 @@ use crate::{ /// The map `StateBTreeMap` is parametrized by the types: /// - `K`: Keys used in the map. Most operations on the map require this to /// implement [`Serialize`](crate::Serialize). Keys cannot contain references -/// to the low-level state, such as types containing [`StateBox`], -/// [`StateMap`] and [`StateSet`]. +/// to the low-level state, such as types containing +/// [`StateBox`](crate::StateBox), [`StateMap`](crate::StateMap) and +/// [`StateSet`](crate::StateSet). /// - `V`: Values stored in the map. Most operations on the map require this to /// implement [`Serial`](crate::Serial) and /// [`DeserialWithState`](crate::DeserialWithState). @@ -43,8 +44,8 @@ use crate::{ /// ## Usage /// /// New maps can be constructed using the -/// [`new_btree_map`][StateBuilder::new_btree_map] method on the -/// [`StateBuilder`]. +/// [`new_btree_map`](crate::StateBuilder::new_btree_map) method on the +/// [`StateBuilder`](crate::StateBuilder). /// /// ``` /// # use concordium_std::*; @@ -123,10 +124,11 @@ impl StateBTreeMap { /// Remove a key from the map, returning the value at the key if the key was /// previously in the map. /// - /// *Caution*: If `V` is a [StateBox], [StateMap], then it is - /// important to call [`Deletable::delete`] on the value returned when - /// you're finished with it. Otherwise, it will remain in the contract - /// state. + /// *Caution*: If `V` is a [StateBox](crate::StateBox), + /// [StateMap](crate::StateMap), then it is important to call + /// [`Deletable::delete`](crate::Deletable::delete) on the value returned + /// when you're finished with it. Otherwise, it will remain in the + /// contract state. #[must_use] pub fn remove_and_get(&mut self, key: &K) -> Option where @@ -242,8 +244,8 @@ impl StateBTreeMap { /// Clears the map, removing all key-value pairs. /// This also includes values pointed at, if `V`, for example, is a - /// [StateBox]. **If applicable use [`clear_flat`](Self::clear_flat) - /// instead.** + /// [StateBox](crate::StateBox). **If applicable use + /// [`clear_flat`](Self::clear_flat) instead.** pub fn clear(&mut self) where S: HasStateApi, @@ -289,8 +291,9 @@ impl StateBTreeMap { /// The map `StateBTreeSet` is parametrized by the types: /// - `K`: Keys used in the set. Most operations on the set require this to /// implement [`Serialize`](crate::Serialize). Keys cannot contain references -/// to the low-level state, such as types containing [`StateBox`], -/// [`StateMap`] and [`StateSet`]. +/// to the low-level state, such as types containing +/// [`StateBox`](crate::StateBox), [`StateMap`](crate::StateMap) and +/// [`StateSet`](crate::StateSet). /// - `S`: The low-level state implementation used, this allows for mocking the /// state API in unit tests, see /// [`TestStateApi`](crate::test_infrastructure::TestStateApi). @@ -302,8 +305,8 @@ impl StateBTreeMap { /// ## Usage /// /// New sets can be constructed using the -/// [`new_btree_set`][StateBuilder::new_btree_set] method on the -/// [`StateBuilder`]. +/// [`new_btree_set`](crate::StateBuilder::new_btree_set) method on the +/// [`StateBuilder`](crate::StateBuilder). /// /// ``` /// # use concordium_std::*; @@ -903,13 +906,10 @@ impl StateBTreeSet { // Since the child is now split into two, we have to check which one to insert // into. let moved_up_key = &node.keys[insert_index]; - if moved_up_key == &key { - // If the key moved up during the split is the key we are inserting, we exit. - return false; - } else if moved_up_key < &key { - larger_child - } else { - child + match moved_up_key.cmp(&key) { + Ordering::Equal => return false, + Ordering::Less => larger_child, + Ordering::Greater => child, } }; } From 01783cf67688d38b26027aef1b2efef1fd7269fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Holm=20Gj=C3=B8rup?= Date: Tue, 12 Mar 2024 16:08:46 +0100 Subject: [PATCH 21/36] Fix no_std issues in state_btree --- concordium-std/src/state_btree.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/concordium-std/src/state_btree.rs b/concordium-std/src/state_btree.rs index 80da1ea5..1aeccdb5 100644 --- a/concordium-std/src/state_btree.rs +++ b/concordium-std/src/state_btree.rs @@ -1,5 +1,5 @@ use crate::{ - cmp::Ordering, marker::PhantomData, mem, Deletable, Deserial, DeserialWithState, Get, + cmp::Ordering, marker::PhantomData, mem, vec::Vec, Deletable, Deserial, DeserialWithState, Get, HasStateApi, ParseResult, Read, Serial, Serialize, StateItemPrefix, StateMap, StateRef, StateRefMut, UnwrapAbort, Write, STATE_ITEM_PREFIX_SIZE, }; @@ -1199,17 +1199,14 @@ impl Node { /// The min length of the key list, except when the node is root. const MINIMUM_KEY_LEN: usize = Self::MINIMUM_CHILD_LEN - 1; - /// The number of keys stored in this node. - fn len(&self) -> usize { self.keys.len() } - /// Check if the node holds the maximum number of keys. - fn is_full(&self) -> bool { self.len() == Self::MAXIMUM_KEY_LEN } + fn is_full(&self) -> bool { self.keys.len() == Self::MAXIMUM_KEY_LEN } /// Check if the node is representing a leaf in the tree. fn is_leaf(&self) -> bool { self.children.is_empty() } /// Check if the node holds the minimum number of keys. - fn is_at_min(&self) -> bool { self.len() == Self::MINIMUM_KEY_LEN } + fn is_at_min(&self) -> bool { self.keys.len() == Self::MINIMUM_KEY_LEN } /// Check a number of invariants of a non-root node in a btree, producing an /// error if any of them are violated. Should only be used while From b69be0c8c17dbd253b585cf8f5b4d78d97e40a37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Holm=20Gj=C3=B8rup?= Date: Wed, 13 Mar 2024 16:08:45 +0100 Subject: [PATCH 22/36] Remove type parameter S from StateBTreeMap and StateBTreeSet --- concordium-std/CHANGELOG.md | 2 +- concordium-std/src/impls.rs | 90 ++--- concordium-std/src/state_btree.rs | 650 ++++++++++++++++++------------ concordium-std/src/traits.rs | 2 +- concordium-std/src/types.rs | 4 +- 5 files changed, 435 insertions(+), 313 deletions(-) diff --git a/concordium-std/CHANGELOG.md b/concordium-std/CHANGELOG.md index 6ea668de..fd456cf8 100644 --- a/concordium-std/CHANGELOG.md +++ b/concordium-std/CHANGELOG.md @@ -6,7 +6,7 @@ via the `HasHost::contract_module_reference` and `HasHost::contract_name` functions. These are only available from protocol version 7, and as such are guarded by the `p7` feature flag. -- Add two ordered collections: `StateBTreeMap` and `StateBTreeSet`. These are based on [B-Tree](https://en.wikipedia.org/wiki/B-tree), but where each node is stored in the low-level smart contract key-value store. +- Add two ordered collections: `StateBTreeMap` and `StateBTreeSet`. These are based on [B-Tree](https://en.wikipedia.org/wiki/B-tree), but where each node is stored in the low-level smart contract key-value store. Use one of these when needing operations related to the ordering of the keys, such as `higher(k)` providing the smallest key in collection which is stricly greater than `k`. ## concordium-std 10.0.0 (2024-02-22) diff --git a/concordium-std/src/impls.rs b/concordium-std/src/impls.rs index ccb5a782..341f5b2a 100644 --- a/concordium-std/src/impls.rs +++ b/concordium-std/src/impls.rs @@ -1201,7 +1201,7 @@ where V: DeserialWithState, { // Safe to unwrap below, since the entry can only be `None`, using methods which // are consuming self. - let entry = unsafe { &mut *self.entry.get() }.as_mut().unwrap_abort(); + let entry = unsafe { &mut *self.entry.get() }; entry.move_to_start(); V::deserial_with_state(&self.state_api, entry).unwrap_abort() } @@ -1210,7 +1210,7 @@ where pub fn set(&mut self, new_val: V) { // Safe to unwrap below, since the entry can only be `None`, using methods which // are consuming self. - let entry = self.entry.get_mut().as_mut().unwrap_abort(); + let entry = self.entry.get_mut(); entry.move_to_start(); new_val.serial(entry).unwrap_abort(); let _ = self.lazy_value.get_mut().insert(new_val); @@ -1224,7 +1224,7 @@ where let lv = self.lazy_value.get_mut(); // Safe to unwrap below, since the entry can only be `None`, using methods which // are consuming self. - let entry = self.entry.get_mut().as_mut().unwrap_abort(); + let entry = self.entry.get_mut(); let value = if let Some(v) = lv { v } else { @@ -1244,22 +1244,14 @@ where if let Some(value) = self.lazy_value.get_mut() { // Safe to unwrap below, since the entry can only be `None`, using methods which // are consuming self. - let entry = self.entry.get_mut().as_mut().unwrap_abort(); + let entry = self.entry.get_mut(); entry.move_to_start(); value.serial(entry).unwrap_abort(); } } - /// Get the inner entry and value if loaded, while consuming the - /// [`StateRefMut`]. Mutations will not be written to the smart contract - /// key-store when this is dropped. - /// Neither will they be written as part of this, so make sure to run - /// `store_mutations` first. - pub(crate) fn into_raw_parts(mut self) -> (Option, S::EntryType) { - let value = self.lazy_value.get_mut().take(); - let entry = self.entry.get_mut().take().unwrap_abort(); - (value, entry) - } + /// Drop the ref without storing mutations to the state entry. + pub(crate) fn drop_without_storing(mut self) { *self.lazy_value.get_mut() = None; } } impl Serial for StateMap { @@ -2331,40 +2323,6 @@ where StateMap::open(state_api, prefix) } - /// Create a new empty [`StateBTreeSet`](crate::StateBTreeSet). - pub fn new_btree_set(&mut self) -> state_btree::StateBTreeSet { - let (state_api, prefix) = self.new_state_container(); - state_btree::StateBTreeSet::new(state_api, prefix) - } - - /// Create a new empty [`StateBTreeMap`](crate::StateBTreeMap). - pub fn new_btree_map(&mut self) -> state_btree::StateBTreeMap { - state_btree::StateBTreeMap { - map: self.new_map(), - ordered_set: self.new_btree_set(), - } - } - - /// Create a new empty [`StateBTreeSet`](crate::StateBTreeSet), setting the - /// minimum degree `M` of the B-Tree explicitly. - pub fn new_btree_set_degree( - &mut self, - ) -> state_btree::StateBTreeSet { - let (state_api, prefix) = self.new_state_container(); - state_btree::StateBTreeSet::new(state_api, prefix) - } - - /// Create a new empty [`StateBTreeMap`](crate::StateBTreeMap), setting the - /// minimum degree `M` of the B-Tree explicitly. - pub fn new_btree_map_degree( - &mut self, - ) -> state_btree::StateBTreeMap { - state_btree::StateBTreeMap { - map: self.new_map(), - ordered_set: self.new_btree_set_degree(), - } - } - /// Create a new empty [`StateSet`]. pub fn new_set(&mut self) -> StateSet { let (state_api, prefix) = self.new_state_container(); @@ -2440,6 +2398,40 @@ where } } +impl StateBuilder { + /// Create a new empty [`StateBTreeSet`](crate::StateBTreeSet). + pub fn new_btree_set(&mut self) -> state_btree::StateBTreeSet { + let (state_api, prefix) = self.new_state_container(); + state_btree::StateBTreeSet::new(state_api, prefix) + } + + /// Create a new empty [`StateBTreeMap`](crate::StateBTreeMap). + pub fn new_btree_map(&mut self) -> state_btree::StateBTreeMap { + state_btree::StateBTreeMap { + key_value: self.new_map(), + key_order: self.new_btree_set(), + } + } + + /// Create a new empty [`StateBTreeSet`](crate::StateBTreeSet), setting the + /// minimum degree `M` of the B-Tree explicitly. + pub fn new_btree_set_degree(&mut self) -> state_btree::StateBTreeSet { + let (state_api, prefix) = self.new_state_container(); + state_btree::StateBTreeSet::new(state_api, prefix) + } + + /// Create a new empty [`StateBTreeMap`](crate::StateBTreeMap), setting the + /// minimum degree `M` of the B-Tree explicitly. + pub fn new_btree_map_degree( + &mut self, + ) -> state_btree::StateBTreeMap { + state_btree::StateBTreeMap { + key_value: self.new_map(), + key_order: self.new_btree_set_degree(), + } + } +} + impl HasHost for ExternHost where S: Serial + DeserialWithState, @@ -3153,7 +3145,7 @@ mod tests { } } -/// This test module rely on the runtime providing host functions and can only +/// This test module relies on the runtime providing host functions and can only /// be run using `cargo concordium test`. #[cfg(feature = "internal-wasm-test")] mod wasm_test { diff --git a/concordium-std/src/state_btree.rs b/concordium-std/src/state_btree.rs index 1aeccdb5..b1a16952 100644 --- a/concordium-std/src/state_btree.rs +++ b/concordium-std/src/state_btree.rs @@ -1,7 +1,7 @@ use crate::{ - cmp::Ordering, marker::PhantomData, mem, vec::Vec, Deletable, Deserial, DeserialWithState, Get, - HasStateApi, ParseResult, Read, Serial, Serialize, StateItemPrefix, StateMap, StateRef, - StateRefMut, UnwrapAbort, Write, STATE_ITEM_PREFIX_SIZE, + self as concordium_std, cmp::Ordering, marker::PhantomData, mem, prims, vec::Vec, Deletable, + Deserial, DeserialWithState, Get, HasStateApi, ParseResult, Read, Serial, Serialize, StateApi, + StateItemPrefix, StateMap, StateRef, StateRefMut, UnwrapAbort, Write, STATE_ITEM_PREFIX_SIZE, }; /// An ordered map based on [B-Tree](https://en.wikipedia.org/wiki/B-tree), where @@ -24,7 +24,7 @@ use crate::{ /// /// ## Type parameters /// -/// The map `StateBTreeMap` is parametrized by the types: +/// The map `StateBTreeMap` is parametrized by the types: /// - `K`: Keys used in the map. Most operations on the map require this to /// implement [`Serialize`](crate::Serialize). Keys cannot contain references /// to the low-level state, such as types containing @@ -32,7 +32,7 @@ use crate::{ /// [`StateSet`](crate::StateSet). /// - `V`: Values stored in the map. Most operations on the map require this to /// implement [`Serial`](crate::Serial) and -/// [`DeserialWithState`](crate::DeserialWithState). +/// [`DeserialWithState`](crate::DeserialWithState). /// - `S`: The low-level state implementation used, this allows for mocking the /// state API in unit tests, see /// [`TestStateApi`](crate::test_infrastructure::TestStateApi). @@ -68,8 +68,8 @@ use crate::{ /// /// ```no_run /// # use concordium_std::*; -/// struct MyState { -/// inner: StateBTreeMap, +/// struct MyState { +/// inner: StateBTreeMap, /// } /// fn incorrect_replace(state_builder: &mut StateBuilder, state: &mut MyState) { /// // The following is incorrect. The old value of `inner` is not properly deleted. @@ -82,8 +82,8 @@ use crate::{ /// /// ```no_run /// # use concordium_std::*; -/// # struct MyState { -/// # inner: StateBTreeMap +/// # struct MyState { +/// # inner: StateBTreeMap /// # } /// fn correct_replace(state_builder: &mut StateBuilder, state: &mut MyState) { /// state.inner.clear_flat(); @@ -92,29 +92,42 @@ use crate::{ /// Or alternatively /// ```no_run /// # use concordium_std::*; -/// # struct MyState { -/// # inner: StateBTreeMap +/// # struct MyState { +/// # inner: StateBTreeMap /// # } /// fn correct_replace(state_builder: &mut StateBuilder, state: &mut MyState) { /// let old_map = mem::replace(&mut state.inner, state_builder.new_btree_map()); /// old_map.delete() /// } /// ``` -pub struct StateBTreeMap { - pub(crate) map: StateMap, - pub(crate) ordered_set: StateBTreeSet, +#[derive(Serial)] +pub struct StateBTreeMap { + /// Mapping from key to value. + /// Each key in this map must also be in the `key_order` set. + pub(crate) key_value: StateMap, + /// A set for tracking the order of the inserted keys. + /// Each key in this set mush also have an associated value in the + /// `key_value` map. + pub(crate) key_order: StateBTreeSet, } -impl StateBTreeMap { +impl StateBTreeMap { /// Insert a key-value pair into the map. /// Returns the previous value if the key was already in the map. + /// + /// *Caution*: If `Option` is to be deleted and contains a data structure + /// prefixed with `State` (such as [StateBox](crate::StateBox) or + /// [StateMap](crate::StateMap)), then it is important to call + /// [`Deletable::delete`](crate::Deletable::delete) on the value returned + /// when you're finished with it. Otherwise, it will remain in the + /// contract state. + #[must_use] pub fn insert(&mut self, key: K, value: V) -> Option where - S: HasStateApi, K: Serialize + Ord, - V: Serial + DeserialWithState, { - let old_value_option = self.map.insert_borrowed(&key, value); - if old_value_option.is_none() && !self.ordered_set.insert(key) { + V: Serial + DeserialWithState, { + let old_value_option = self.key_value.insert_borrowed(&key, value); + if old_value_option.is_none() && !self.key_order.insert(key) { // Inconsistency between the map and ordered_set. crate::trap(); } @@ -132,11 +145,10 @@ impl StateBTreeMap { #[must_use] pub fn remove_and_get(&mut self, key: &K) -> Option where - S: HasStateApi, K: Serialize + Ord, - V: Serial + DeserialWithState + Deletable, { - let v = self.map.remove_and_get(key); - if v.is_some() && !self.ordered_set.remove(key) { + V: Serial + DeserialWithState + Deletable, { + let v = self.key_value.remove_and_get(key); + if v.is_some() && !self.key_order.remove(key) { // Inconsistency between the map and ordered_set. crate::trap(); } @@ -147,11 +159,10 @@ impl StateBTreeMap { /// This also deletes the value in the state. pub fn remove(&mut self, key: &K) where - S: HasStateApi, K: Serialize + Ord, - V: Serial + DeserialWithState + Deletable, { - if self.ordered_set.remove(key) { - self.map.remove(key); + V: Serial + DeserialWithState + Deletable, { + if self.key_order.remove(key) { + self.key_value.remove(key); } } @@ -159,86 +170,101 @@ impl StateBTreeMap { pub fn get(&self, key: &K) -> Option> where K: Serialize, - S: HasStateApi, - V: Serial + DeserialWithState, { - if self.ordered_set.is_empty() { + V: Serial + DeserialWithState, { + // Minor optimization by first checking whether the collection is empty, since + // this information is kept at the root of the ordered set it is already loaded + // into memory. In the case of the empty collection we can then save a key + // lookup. + if self.key_order.is_empty() { None } else { - self.map.get(key) + self.key_value.get(key) } } /// Get a mutable reference to the value corresponding to the key. - pub fn get_mut(&mut self, key: &K) -> Option> + pub fn get_mut(&mut self, key: &K) -> Option> where K: Serialize, - S: HasStateApi, - V: Serial + DeserialWithState, { - if self.ordered_set.is_empty() { + V: Serial + DeserialWithState, { + // Minor optimization by first checking whether the collection is empty, since + // this information is kept at the root of the ordered set it is already loaded + // into memory. In the case of the empty collection we can then save a key + // lookup. + if self.key_order.is_empty() { None } else { - self.map.get_mut(key) + self.key_value.get_mut(key) } } /// Returns `true` if the map contains a value for the specified key. pub fn contains_key(&self, key: &K) -> bool where - K: Serialize + Ord, - S: HasStateApi, { - self.ordered_set.contains(key) + K: Serialize + Ord, { + self.key_order.contains(key) } /// Returns the smallest key in the map, which is strictly larger than the /// provided key. `None` meaning no such key is present in the map. pub fn higher(&self, key: &K) -> Option> where - S: HasStateApi, K: Serialize + Ord, { - self.ordered_set.higher(key) + self.key_order.higher(key) + } + + /// Returns the smallest key in the map, which is equal or larger than the + /// provided key. `None` meaning no such key is present in the map. + pub fn eq_or_higher(&self, key: &K) -> Option> + where + K: Serialize + Ord, { + self.key_order.eq_or_higher(key) } /// Returns the largest key in the map, which is strictly smaller than the /// provided key. `None` meaning no such key is present in the map. pub fn lower(&self, key: &K) -> Option> where - S: HasStateApi, K: Serialize + Ord, { - self.ordered_set.lower(key) + self.key_order.lower(key) + } + + /// Returns the largest key in the map, which is equal or smaller than the + /// provided key. `None` meaning no such key is present in the map. + pub fn eq_or_lower(&self, key: &K) -> Option> + where + K: Serialize + Ord, { + self.key_order.eq_or_lower(key) } /// Returns a reference to the first key in the map, if any. This key is /// always the minimum of all keys in the map. pub fn first_key(&self) -> Option> where - S: HasStateApi, K: Serialize + Ord, { - self.ordered_set.first() + self.key_order.first() } /// Returns a reference to the last key in the map, if any. This key is /// always the maximum of all keys in the map. pub fn last_key(&self) -> Option> where - S: HasStateApi, K: Serialize + Ord, { - self.ordered_set.last() + self.key_order.last() } /// Return the number of elements in the map. - pub fn len(&self) -> u32 { self.ordered_set.len() } + pub fn len(&self) -> u32 { self.key_order.len() } /// Returns `true` is the map contains no elements. - pub fn is_empty(&self) -> bool { self.ordered_set.is_empty() } + pub fn is_empty(&self) -> bool { self.key_order.is_empty() } /// Create an iterator over the entries of [`StateBTreeMap`]. - /// Ordered by `K`. - pub fn iter(&self) -> StateBTreeMapIter - where - S: HasStateApi, { + /// Ordered by `K` ascending. + pub fn iter(&self) -> StateBTreeMapIter { StateBTreeMapIter { - key_iter: self.ordered_set.iter(), - map: &self.map, + key_iter: self.key_order.iter(), + map: &self.key_value, } } @@ -248,11 +274,10 @@ impl StateBTreeMap { /// [`clear_flat`](Self::clear_flat) instead.** pub fn clear(&mut self) where - S: HasStateApi, K: Serialize, - V: Serial + DeserialWithState + Deletable, { - self.map.clear(); - self.ordered_set.clear(); + V: Serial + DeserialWithState + Deletable, { + self.key_value.clear(); + self.key_order.clear(); } /// Clears the map, removing all key-value pairs. @@ -265,11 +290,10 @@ impl StateBTreeMap { /// be possible. pub fn clear_flat(&mut self) where - S: HasStateApi, K: Serialize, V: Serialize, { - self.map.clear_flat(); - self.ordered_set.clear(); + self.key_value.clear_flat(); + self.key_order.clear(); } } @@ -288,7 +312,7 @@ impl StateBTreeMap { /// /// ## Type parameters /// -/// The map `StateBTreeSet` is parametrized by the types: +/// The map `StateBTreeSet` is parametrized by the types: /// - `K`: Keys used in the set. Most operations on the set require this to /// implement [`Serialize`](crate::Serialize). Keys cannot contain references /// to the low-level state, such as types containing @@ -349,13 +373,13 @@ impl StateBTreeMap { /// state.inner.clear(); /// } /// ``` -pub struct StateBTreeSet { +pub struct StateBTreeSet { /// Type marker for the key. _marker_key: PhantomData, /// The unique prefix to use for this map in the key-value store. prefix: StateItemPrefix, /// The API for interacting with the low-level state. - state_api: S, + state_api: StateApi, /// The ID of the root node of the tree, where None represents the tree is /// empty. root: Option, @@ -365,10 +389,10 @@ pub struct StateBTreeSet { next_node_id: NodeId, } -impl StateBTreeSet { +impl StateBTreeSet { /// Construct a new [`StateBTreeSet`] given a unique prefix to use in the /// key-value store. - pub(crate) fn new(state_api: S, prefix: StateItemPrefix) -> Self { + pub(crate) fn new(state_api: StateApi, prefix: StateItemPrefix) -> Self { Self { _marker_key: Default::default(), prefix, @@ -385,7 +409,6 @@ impl StateBTreeSet { /// Returns true if the key is new in the collection. pub fn insert(&mut self, key: K) -> bool where - S: HasStateApi, K: Serialize + Ord, { let Some(root_id) = self.root else { let (node_id, _) = self.create_node(crate::vec![key], Vec::new()); @@ -410,9 +433,10 @@ impl StateBTreeSet { // The old root node is now a child node. let mut child = root_node; let new_larger_child = self.split_child(&mut new_root, 0, &mut child); - // new_root should now contain one key and two children, so we need to know + // new_root now contains one key and two children, so we need to know // which one to insert into. - let child = if new_root.keys[0] < key { + let key_in_root = unsafe { new_root.keys.get_unchecked(0) }; + let child = if key_in_root < &key { new_larger_child } else { child @@ -427,7 +451,6 @@ impl StateBTreeSet { /// Returns `true` if the set contains an element equal to the key. pub fn contains(&self, key: &K) -> bool where - S: HasStateApi, K: Serialize + Ord, { let Some(root_node_id) = self.root else { return false; @@ -453,9 +476,7 @@ impl StateBTreeSet { /// Get an iterator over the elements in the `StateBTreeSet`. The iterator /// returns elements in increasing order. - pub fn iter(&self) -> StateBTreeSetIter - where - S: HasStateApi, { + pub fn iter(&self) -> StateBTreeSetIter { StateBTreeSetIter { length: self.len.try_into().unwrap_abort(), next_node: self.root, @@ -466,9 +487,7 @@ impl StateBTreeSet { } /// Clears the set, removing all elements. - pub fn clear(&mut self) - where - S: HasStateApi, { + pub fn clear(&mut self) { // Reset the information. self.root = None; self.next_node_id = NodeId { @@ -484,7 +503,6 @@ impl StateBTreeSet { /// provided key. `None` meaning no such key is present in the set. pub fn higher(&self, key: &K) -> Option> where - S: HasStateApi, K: Serialize + Ord, { let Some(root_node_id) = self.root else { return None; @@ -519,11 +537,48 @@ impl StateBTreeSet { } } + /// Returns the smallest key in the set, which is equal or larger than the + /// provided key. `None` meaning no such key is present in the set. + pub fn eq_or_higher(&self, key: &K) -> Option> + where + K: Serialize + Ord, { + let Some(root_node_id) = self.root else { + return None; + }; + + let mut node = self.get_node(root_node_id); + let mut higher_so_far = None; + loop { + let higher_key_index = match node.keys.binary_search(key) { + Ok(index) => { + // This does not mutate the node in the end, just the representation in memory + // which is freed after the call. + return Some(StateRef::new(node.keys.swap_remove(index))); + } + Err(index) => index, + }; + + if node.is_leaf() { + return if higher_key_index < node.keys.len() { + Some(StateRef::new(node.keys.swap_remove(higher_key_index))) + } else { + higher_so_far + }; + } else { + if higher_key_index < node.keys.len() { + higher_so_far = Some(StateRef::new(node.keys.swap_remove(higher_key_index))) + } + + let child_node_id = node.children[higher_key_index]; + node = self.get_node(child_node_id); + } + } + } + /// Returns the largest key in the set, which is strictly smaller than the /// provided key. `None` meaning no such key is present in the set. pub fn lower(&self, key: &K) -> Option> where - S: HasStateApi, K: Serialize + Ord, { let Some(root_node_id) = self.root else { return None; @@ -541,18 +596,51 @@ impl StateBTreeSet { return if lower_key_index == 0 { lower_so_far } else { - // lower_key_index cannot be 0 in this case, since the binary search will only - // return 0 in the true branch above. Some(StateRef::new(node.keys.swap_remove(lower_key_index - 1))) }; } else { if lower_key_index > 0 { + lower_so_far = Some(StateRef::new(node.keys.swap_remove(lower_key_index - 1))); + } + let child_node_id = unsafe { node.children.get_unchecked(lower_key_index) }; + node = self.get_node(*child_node_id) + } + } + } + + /// Returns the largest key in the set, which is equal or smaller than the + /// provided key. `None` meaning no such key is present in the set. + pub fn eq_or_lower(&self, key: &K) -> Option> + where + K: Serialize + Ord, { + let Some(root_node_id) = self.root else { + return None; + }; + + let mut node = self.get_node(root_node_id); + let mut lower_so_far = None; + loop { + let lower_key_index = match node.keys.binary_search(key) { + Ok(index) => { // This does not mutate the node in the end, just the representation in memory // which is freed after the call. + return Some(StateRef::new(node.keys.swap_remove(index))); + } + Err(index) => index, + }; + + if node.is_leaf() { + return if lower_key_index == 0 { + lower_so_far + } else { + Some(StateRef::new(node.keys.swap_remove(lower_key_index - 1))) + }; + } else { + if lower_key_index > 0 { lower_so_far = Some(StateRef::new(node.keys.swap_remove(lower_key_index - 1))); } - let child_node_id = node.children[lower_key_index]; - node = self.get_node(child_node_id) + let child_node_id = unsafe { node.children.get_unchecked(lower_key_index) }; + node = self.get_node(*child_node_id) } } } @@ -561,7 +649,6 @@ impl StateBTreeSet { /// always the minimum of all keys in the set. pub fn first(&self) -> Option> where - S: HasStateApi, K: Serialize + Ord, { let Some(root_node_id) = self.root else { return None; @@ -578,7 +665,6 @@ impl StateBTreeSet { /// always the maximum of all keys in the set. pub fn last(&self) -> Option> where - S: HasStateApi, K: Serialize + Ord, { let Some(root_node_id) = self.root else { return None; @@ -595,8 +681,7 @@ impl StateBTreeSet { /// Returns whether such an element was present. pub fn remove(&mut self, key: &K) -> bool where - K: Ord + Serialize, - S: HasStateApi, { + K: Ord + Serialize, { let Some(root_node_id) = self.root else { return false; }; @@ -661,7 +746,7 @@ impl StateBTreeSet { if self.len == 0 { // Remote the root node if tree is empty. self.root = None; - self.delete_node(root); + self.delete_node(root_node_id, root); return true; } } @@ -669,7 +754,7 @@ impl StateBTreeSet { // as the root. if root.keys.is_empty() { self.root = Some(root.children[0]); - self.delete_node(root); + self.delete_node(root_node_id, root); } deleted_something } @@ -679,10 +764,9 @@ impl StateBTreeSet { /// Assumes: /// - The provided node is not the root. /// - The node contain more than minimum number of keys. - fn remove_largest_key(&mut self, mut node: StateRefMut<'_, Node, S>) -> K + fn remove_largest_key(&mut self, mut node: StateRefMut<'_, Node, StateApi>) -> K where - K: Ord + Serialize, - S: HasStateApi, { + K: Ord + Serialize, { while !node.is_leaf() { // Node is not a leaf, so we move further down this subtree. let child_index = node.children.len() - 1; @@ -699,10 +783,9 @@ impl StateBTreeSet { /// Assumes: /// - The provided node is not the root. /// - The provided `node` contain more than minimum number of keys. - fn remove_smallest_key(&mut self, mut node: StateRefMut<'_, Node, S>) -> K + fn remove_smallest_key(&mut self, mut node: StateRefMut<'_, Node, StateApi>) -> K where - K: Ord + Serialize, - S: HasStateApi, { + K: Ord + Serialize, { while !node.is_leaf() { // Node is not a leaf, so we move further down this subtree. let child_index = 0; @@ -721,12 +804,11 @@ impl StateBTreeSet { /// - The minimum degree `M` is at least 2 or more. fn prepare_child_for_key_removal<'b, 'c>( &mut self, - mut node: StateRefMut<'b, Node, S>, + mut node: StateRefMut<'b, Node, StateApi>, index: usize, - ) -> StateRefMut<'c, Node, S> + ) -> StateRefMut<'c, Node, StateApi> where - K: Ord + Serialize, - S: HasStateApi, { + K: Ord + Serialize, { let mut child = self.get_node_mut(node.children[index]); if !child.is_at_min() { return child; @@ -793,8 +875,7 @@ impl StateBTreeSet { /// `child_index`. fn get_highest_key(&self, node: &Node, child_index: usize) -> K where - K: Ord + Serialize, - S: HasStateApi, { + K: Ord + Serialize, { let mut node = self.get_node(node.children[child_index]); while !node.is_leaf() { let child_node_id = node.children.last().unwrap_abort(); @@ -810,15 +891,12 @@ impl StateBTreeSet { /// `child_index`. fn get_lowest_key(&self, node: &Node, child_index: usize) -> K where - K: Ord + Serialize, - S: HasStateApi, { + K: Ord + Serialize, { let mut node = self.get_node(node.children[child_index]); while !node.is_leaf() { let child_node_id = node.children.first().unwrap_abort(); node = self.get_node(*child_node_id); } - // This does not mutate the node in the end, just the representation in memory - // which is freed after the call. node.keys.swap_remove(0) } @@ -834,16 +912,15 @@ impl StateBTreeSet { parent_node: &mut Node, index: usize, child: &mut Node, - mut larger_child: StateRefMut, S>, + mut larger_child: StateRefMut, StateApi>, ) where - K: Ord + Serialize, - S: HasStateApi, { + K: Ord + Serialize, { let parent_key = parent_node.keys.remove(index); - parent_node.children.remove(index + 1); + let larger_child_id = parent_node.children.remove(index + 1); child.keys.push(parent_key); child.keys.append(&mut larger_child.keys); child.children.append(&mut larger_child.children); - self.delete_node(larger_child); + self.delete_node(larger_child_id, larger_child); } /// Internal function for constructing a node. It will increment the next @@ -852,17 +929,16 @@ impl StateBTreeSet { &mut self, keys: Vec, children: Vec, - ) -> (NodeId, StateRefMut<'b, Node, S>) + ) -> (NodeId, StateRefMut<'b, Node, StateApi>) where - K: Serialize, - S: HasStateApi, { + K: Serialize, { let node_id = self.next_node_id.copy_then_increment(); let node = Node { keys, children, }; let entry = self.state_api.create_entry(&node_id.as_key(&self.prefix)).unwrap_abort(); - let mut ref_mut: StateRefMut<'_, Node, S> = + let mut ref_mut: StateRefMut<'_, Node, StateApi> = StateRefMut::new(entry, self.state_api.clone()); ref_mut.set(node); (node_id, ref_mut) @@ -870,19 +946,19 @@ impl StateBTreeSet { /// Internal function for deleting a node, removing the entry in the smart /// contract key-value store. Traps if no node was present. - fn delete_node(&mut self, node: StateRefMut, S>) + fn delete_node(&mut self, node_id: NodeId, node: StateRefMut, StateApi>) where - K: Serial, - S: HasStateApi, { - self.state_api.delete_entry(node.into_raw_parts().1).unwrap_abort() + K: Serial, { + let key = node_id.as_key(&self.prefix); + node.drop_without_storing(); + unsafe { prims::state_delete_entry(key.as_ptr(), key.len() as u32) }; } /// Internal function for inserting into a subtree. /// Assumes the given node is not full. - fn insert_non_full(&mut self, initial_node: StateRefMut, S>, key: K) -> bool + fn insert_non_full(&mut self, initial_node: StateRefMut, StateApi>, key: K) -> bool where - K: Serialize + Ord, - S: HasStateApi, { + K: Serialize + Ord, { let mut node = initial_node; loop { let Err(insert_index) = node.keys.binary_search(&key) else { @@ -898,7 +974,8 @@ impl StateBTreeSet { } // The node is not a leaf, so we want to insert in the relevant child node. - let mut child = self.get_node_mut(node.children[insert_index]); + let child_id = unsafe { node.children.get_unchecked(insert_index) }; + let mut child = self.get_node_mut(*child_id); node = if !child.is_full() { child } else { @@ -928,10 +1005,9 @@ impl StateBTreeSet { node: &mut Node, child_index: usize, child: &mut Node, - ) -> StateRefMut<'b, Node, S> + ) -> StateRefMut<'b, Node, StateApi> where - K: Serialize + Ord, - S: HasStateApi, { + K: Serialize + Ord, { let split_index = Node::::MINIMUM_KEY_LEN + 1; let (new_larger_sibling_id, new_larger_sibling) = self.create_node( child.keys.split_off(split_index), @@ -951,8 +1027,7 @@ impl StateBTreeSet { /// This assumes the node is present and traps if this is not the case. fn get_node(&self, node_id: NodeId) -> Node where - Key: Deserial, - S: HasStateApi, { + Key: Deserial, { let key = node_id.as_key(&self.prefix); let mut entry = self.state_api.lookup_entry(&key).unwrap_abort(); entry.get().unwrap_abort() @@ -960,10 +1035,9 @@ impl StateBTreeSet { /// Internal function for looking up a node, providing mutable access. /// This assumes the node is present and traps if this is not the case. - fn get_node_mut<'b>(&mut self, node_id: NodeId) -> StateRefMut<'b, Node, S> + fn get_node_mut<'b>(&mut self, node_id: NodeId) -> StateRefMut<'b, Node, StateApi> where - K: Serial, - S: HasStateApi, { + K: Serial, { let key = node_id.as_key(&self.prefix); let entry = self.state_api.lookup_entry(&key).unwrap_abort(); StateRefMut::new(entry, self.state_api.clone()) @@ -974,8 +1048,7 @@ impl StateBTreeSet { #[cfg(feature = "internal-wasm-test")] pub(crate) fn debug(&self) -> String where - K: Serialize + std::fmt::Debug + Ord, - S: HasStateApi, { + K: Serialize + std::fmt::Debug + Ord, { let Some(root_node_id) = self.root else { return format!("no root"); }; @@ -1002,8 +1075,7 @@ impl StateBTreeSet { #[cfg(feature = "internal-wasm-test")] pub(crate) fn check_invariants(&self) -> Result<(), InvariantViolation> where - K: Serialize + Ord, - S: HasStateApi, { + K: Serialize + Ord, { use crate::ops::Deref; let Some(root_node_id) = self.root else { return if self.len == 0 { @@ -1043,7 +1115,7 @@ impl StateBTreeSet { /// /// This `struct` is created by the [`iter`][StateBTreeSet::iter] method on /// [`StateBTreeSet`]. See its documentation for more. -pub struct StateBTreeSetIter<'a, 'b, K, S, const M: usize> { +pub struct StateBTreeSetIter<'a, 'b, K, const M: usize> { /// The number of elements left to iterate. length: usize, /// Reference to a node in the tree to load and iterate before the current @@ -1052,16 +1124,15 @@ pub struct StateBTreeSetIter<'a, 'b, K, S, const M: usize> { /// Tracking the nodes depth first, which are currently being iterated. depth_first_stack: Vec<(Node>, usize)>, /// Reference to the set, needed for looking up the nodes. - tree: &'a StateBTreeSet, + tree: &'a StateBTreeSet, /// Marker for tracking the lifetime of the key. _marker_lifetime: PhantomData<&'b K>, } -impl<'a, 'b, const M: usize, K, S> Iterator for StateBTreeSetIter<'a, 'b, K, S, M> +impl<'a, 'b, const M: usize, K> Iterator for StateBTreeSetIter<'a, 'b, K, M> where 'a: 'b, K: Deserial, - S: HasStateApi, { type Item = StateRef<'b, K>; @@ -1074,22 +1145,19 @@ where self.depth_first_stack.push((node, 0)); } - if let Some((node, index)) = self.depth_first_stack.last_mut() { - let key = node.keys[*index].key.take().unwrap_abort(); - *index += 1; - let no_more_keys = index == &node.keys.len(); - if !node.is_leaf() { - let child_id = node.children[*index]; - self.next_node = Some(child_id); - } - if no_more_keys { - self.depth_first_stack.pop(); - } - self.length -= 1; - Some(StateRef::new(key)) - } else { - None + let (node, index) = self.depth_first_stack.last_mut()?; + let key = node.keys[*index].key.take().unwrap_abort(); + *index += 1; + let no_more_keys = index == &node.keys.len(); + if !node.is_leaf() { + let child_id = node.children[*index]; + self.next_node = Some(child_id); + } + if no_more_keys { + self.depth_first_stack.pop(); } + self.length -= 1; + Some(StateRef::new(key)) } fn size_hint(&self) -> (usize, Option) { (self.length, Some(self.length)) } @@ -1101,19 +1169,18 @@ where /// /// This `struct` is created by the [`iter`][StateBTreeMap::iter] method on /// [`StateBTreeMap`]. See its documentation for more. -pub struct StateBTreeMapIter<'a, 'b, K, V, S, const M: usize> { +pub struct StateBTreeMapIter<'a, 'b, K, V, const M: usize> { /// Iterator over the keys in the map. - key_iter: StateBTreeSetIter<'a, 'b, K, S, M>, + key_iter: StateBTreeSetIter<'a, 'b, K, M>, /// Reference to the map holding the values. - map: &'a StateMap, + map: &'a StateMap, } -impl<'a, 'b, const M: usize, K, V, S> Iterator for StateBTreeMapIter<'a, 'b, K, V, S, M> +impl<'a, 'b, const M: usize, K, V> Iterator for StateBTreeMapIter<'a, 'b, K, V, M> where 'a: 'b, K: Serialize, - V: Serial + DeserialWithState + 'b, - S: HasStateApi, + V: Serial + DeserialWithState + 'b, { type Item = (StateRef<'b, K>, StateRef<'b, V>); @@ -1128,8 +1195,8 @@ where } /// Identifier for a node in the tree. Used to construct the key, where this -/// node is store in the smart contract key-value store. -#[derive(Debug, Copy, Clone)] +/// node is stored in the smart contract key-value store. +#[derive(Debug, Copy, Clone, Serialize)] #[repr(transparent)] struct NodeId { id: u32, @@ -1157,12 +1224,12 @@ impl NodeId { // bunch of `MaybeUninit`s, which do not require initialization. let mut prefixed: [mem::MaybeUninit; BTREE_NODE_KEY_SIZE] = unsafe { mem::MaybeUninit::uninit().assume_init() }; - for i in 0..STATE_ITEM_PREFIX_SIZE { - prefixed[i].write(prefix[i]); + for (place, value) in prefixed.iter_mut().zip(prefix) { + place.write(*value); } let id_bytes = self.id.to_le_bytes(); - for i in 0..id_bytes.len() { - prefixed[STATE_ITEM_PREFIX_SIZE + i].write(id_bytes[i]); + for (place, value) in prefixed[STATE_ITEM_PREFIX_SIZE..].iter_mut().zip(id_bytes) { + place.write(value); } // Transmuting away the maybeuninit is safe since we have initialized all of // them. @@ -1172,7 +1239,7 @@ impl NodeId { /// Type representing a node in the [`StateBTreeMap`]. /// Each node is stored separately in the smart contract key-value store. -#[derive(Debug)] +#[derive(Debug, Serialize)] struct Node { /// List of sorted keys tracked by this node. /// This list should never be empty and contain between `M - 1` and `2M @@ -1200,12 +1267,15 @@ impl Node { const MINIMUM_KEY_LEN: usize = Self::MINIMUM_CHILD_LEN - 1; /// Check if the node holds the maximum number of keys. + #[inline(always)] fn is_full(&self) -> bool { self.keys.len() == Self::MAXIMUM_KEY_LEN } /// Check if the node is representing a leaf in the tree. + #[inline(always)] fn is_leaf(&self) -> bool { self.children.is_empty() } /// Check if the node holds the minimum number of keys. + #[inline(always)] fn is_at_min(&self) -> bool { self.keys.len() == Self::MINIMUM_KEY_LEN } /// Check a number of invariants of a non-root node in a btree, producing an @@ -1262,7 +1332,7 @@ pub(crate) enum InvariantViolation { } /// Wrapper implement the exact same deserial as K, but wraps it in an -/// option in memory. This is used, to allow taking a key from a mutable +/// option in memory. This is used to allow taking a key from a mutable /// reference to a node, without cloning the key, during iteration of the /// set. #[repr(transparent)] @@ -1279,25 +1349,7 @@ impl Deserial for KeyWrapper { } } -impl Serial for StateBTreeMap { - fn serial(&self, out: &mut W) -> Result<(), W::Err> { - self.map.serial(out)?; - self.ordered_set.serial(out) - } -} - -impl DeserialWithState for StateBTreeMap { - fn deserial_with_state(state: &S, source: &mut R) -> ParseResult { - let map = DeserialWithState::deserial_with_state(state, source)?; - let ordered_set = DeserialWithState::deserial_with_state(state, source)?; - Ok(Self { - map, - ordered_set, - }) - } -} - -impl Serial for StateBTreeSet { +impl Serial for StateBTreeSet { fn serial(&self, out: &mut W) -> Result<(), W::Err> { self.prefix.serial(out)?; self.root.serial(out)?; @@ -1306,8 +1358,8 @@ impl Serial for StateBTreeSet { } } -impl DeserialWithState for StateBTreeSet { - fn deserial_with_state(state: &S, source: &mut R) -> ParseResult { +impl DeserialWithState for StateBTreeSet { + fn deserial_with_state(state: &StateApi, source: &mut R) -> ParseResult { let prefix = source.get()?; let root = source.get()?; let len = source.get()?; @@ -1324,94 +1376,90 @@ impl DeserialWithState for StateBTreeSet Serial for Node { - fn serial(&self, out: &mut W) -> Result<(), W::Err> { - self.keys.serial(out)?; - self.children.serial(out) - } -} - -impl Deserial for Node { - fn deserial(source: &mut R) -> ParseResult { - let keys = source.get()?; - let children = source.get()?; - Ok(Self { - keys, - children, - }) - } -} - -impl Serial for NodeId { - fn serial(&self, out: &mut W) -> Result<(), W::Err> { self.id.serial(out) } -} - -impl Deserial for NodeId { - fn deserial(source: &mut R) -> ParseResult { +impl DeserialWithState for StateBTreeMap { + fn deserial_with_state(state: &StateApi, source: &mut R) -> ParseResult { + let key_value = StateMap::deserial_with_state(state, source)?; + let key_order = StateBTreeSet::deserial_with_state(state, source)?; Ok(Self { - id: source.get()?, + key_value, + key_order, }) } } -impl Deletable for StateBTreeMap +impl Deletable for StateBTreeMap where - S: HasStateApi, K: Serialize, - V: Serial + DeserialWithState + Deletable, + V: Serial + DeserialWithState + Deletable, { fn delete(mut self) { self.clear(); } } -/// This test module rely on the runtime providing host functions and can only +/// This test module relies on the runtime providing host functions and can only /// be run using `cargo concordium test`. #[cfg(feature = "internal-wasm-test")] mod wasm_test_btree { use crate::{claim, claim_eq, concordium_test, StateApi, StateBuilder}; + /// Insert `2 * M` items such that the btree contains more than the root + /// node. Checking that every item is contained in the collection. #[concordium_test] - fn test_btree_insert_6() { + fn test_btree_insert_asc_above_max_branching_degree() { let mut state_builder = StateBuilder::open(StateApi::open()); - let mut tree = state_builder.new_btree_set_degree::<5, _>(); - for n in 0..=5 { - tree.insert(n); + const M: usize = 5; + let mut tree = state_builder.new_btree_set_degree::(); + let items = (2 * M) as u32; + for n in 0..items { + claim!(tree.insert(n)); } - for n in 0..=5 { + for n in 0..items { claim!(tree.contains(&n)); } + claim_eq!(tree.len(), items) } + /// Insert items such that the btree must be at least height 3 to contain + /// all of them. With a minimum degree of 2, each node can contain up to + /// 3 items and have 4 children, meaning 16 items is needed. + /// Then checks that every item is contained in the collection. #[concordium_test] - fn test_btree_insert_0_7() { + fn test_btree_insert_asc_height_3() { let mut state_builder = StateBuilder::open(StateApi::open()); let mut tree = state_builder.new_btree_set_degree::<2, _>(); - for n in 0..=7 { - tree.insert(n); + for n in 0..16 { + claim!(tree.insert(n)); } - for n in 0..=7 { + for n in 0..16 { claim!(tree.contains(&n)); } + claim_eq!(tree.len(), 16); + claim!(!tree.contains(&17)); } + /// Insert items in a random order. + /// Then checks that every item is contained in the collection. #[concordium_test] - fn test_btree_insert_7_no_order() { + fn test_btree_insert_random_order() { let mut state_builder = StateBuilder::open(StateApi::open()); let mut tree = state_builder.new_btree_set_degree::<2, _>(); tree.insert(0); - tree.insert(1); - tree.insert(2); tree.insert(3); + tree.insert(2); + tree.insert(1); + tree.insert(5); tree.insert(7); tree.insert(6); - tree.insert(5); tree.insert(4); for n in 0..=7 { claim!(tree.contains(&n)); } + claim_eq!(tree.len(), 8) } + /// Build a set and query `higher` on each key plus some keys outside of the + /// set. #[concordium_test] fn test_btree_higher() { let mut state_builder = StateBuilder::open(StateApi::open()); @@ -1422,12 +1470,19 @@ mod wasm_test_btree { tree.insert(4); tree.insert(5); tree.insert(7); + claim_eq!(tree.higher(&0).as_deref(), Some(&1)); + claim_eq!(tree.higher(&1).as_deref(), Some(&2)); + claim_eq!(tree.higher(&2).as_deref(), Some(&3)); claim_eq!(tree.higher(&3).as_deref(), Some(&4)); + claim_eq!(tree.higher(&4).as_deref(), Some(&5)); claim_eq!(tree.higher(&5).as_deref(), Some(&7)); claim_eq!(tree.higher(&6).as_deref(), Some(&7)); - claim_eq!(tree.higher(&7).as_deref(), None) + claim_eq!(tree.higher(&7).as_deref(), None); + claim_eq!(tree.higher(&8).as_deref(), None); } + /// Build a set and query `lower` on each key plus some keys outside of the + /// set. #[concordium_test] fn test_btree_lower() { let mut state_builder = StateBuilder::open(StateApi::open()); @@ -1438,42 +1493,44 @@ mod wasm_test_btree { tree.insert(4); tree.insert(5); tree.insert(7); + claim_eq!(tree.lower(&0).as_deref(), None); + claim_eq!(tree.lower(&1).as_deref(), None); + claim_eq!(tree.lower(&2).as_deref(), Some(&1)); claim_eq!(tree.lower(&3).as_deref(), Some(&2)); - claim_eq!(tree.lower(&7).as_deref(), Some(&5)); + claim_eq!(tree.lower(&4).as_deref(), Some(&3)); + claim_eq!(tree.lower(&5).as_deref(), Some(&4)); claim_eq!(tree.lower(&6).as_deref(), Some(&5)); - claim_eq!(tree.lower(&1).as_deref(), None) + claim_eq!(tree.lower(&7).as_deref(), Some(&5)); + claim_eq!(tree.lower(&8).as_deref(), Some(&7)); } + /// Insert a large number of items. + /// Check the set contains each item. + /// Insert the same items again, checking the set is the same size + /// afterwards. #[concordium_test] - fn test_btree_insert_1000() { + fn test_btree_insert_a_lot_then_reinsert() { let mut state_builder = StateBuilder::open(StateApi::open()); let mut tree = state_builder.new_btree_set_degree::<2, _>(); for n in 0..500 { - tree.insert(n); + claim!(tree.insert(n)); } - for n in (500..1000).into_iter().rev() { - tree.insert(n); + claim!(tree.insert(n)); } for n in 0..1000 { claim!(tree.contains(&n)) } + claim_eq!(tree.len(), 1000); - claim_eq!(tree.len(), 1000) - } - - #[concordium_test] - fn test_btree_7_get_8() { - let mut state_builder = StateBuilder::open(StateApi::open()); - let mut tree = state_builder.new_btree_set_degree::<2, _>(); - for n in 0..=7 { - tree.insert(n); + for n in 0..1000 { + claim!(!tree.insert(n)) } - - claim!(!tree.contains(&8)); + claim_eq!(tree.len(), 1000) } + /// Remove from a btree with only the root node. #[concordium_test] fn test_btree_remove_from_one_node_tree() { let mut state_builder = StateBuilder::open(StateApi::open()); @@ -1488,6 +1545,12 @@ mod wasm_test_btree { claim!(tree.contains(&2)); } + /// Removing from a the lower child node which is at minimum keys, causing a + /// merge. + /// + /// - Builds a tree of the form: [[0], 1, [2]] + /// - Remove 0 + /// - Expecting a tree of the form: [1, 2] #[concordium_test] fn test_btree_remove_only_key_lower_leaf_in_three_node() { let mut state_builder = StateBuilder::open(StateApi::open()); @@ -1497,13 +1560,19 @@ mod wasm_test_btree { } tree.remove(&3); - claim!(tree.contains(&0)); claim!(tree.remove(&0)); + claim!(!tree.contains(&0)); claim!(tree.contains(&1)); claim!(tree.contains(&2)); } + /// Removing from a the higher child node which is at minimum keys, causing + /// a merge. + /// + /// - Builds a tree of the form: [[0], 1, [2]] + /// - Remove 2 + /// - Expecting a tree of the form: [0, 1] #[concordium_test] fn test_btree_remove_only_key_higher_leaf_in_three_node() { let mut state_builder = StateBuilder::open(StateApi::open()); @@ -1513,12 +1582,20 @@ mod wasm_test_btree { } tree.remove(&3); claim!(tree.contains(&2)); + claim!(tree.remove(&2)); + claim!(tree.contains(&0)); claim!(tree.contains(&1)); claim!(!tree.contains(&2)); } + /// Removing from a the higher child node which is at minimum keys, causing + /// it to move a key from its higher sibling. + /// + /// - Builds a tree of the form: [[0, 1], 2, [3]] + /// - Remove 3 + /// - Expecting a tree of the form: [[0], 1, [2]] #[concordium_test] fn test_btree_remove_from_higher_leaf_in_three_node_taking_from_sibling() { let mut state_builder = StateBuilder::open(StateApi::open()); @@ -1531,6 +1608,12 @@ mod wasm_test_btree { claim!(!tree.contains(&3)); } + /// Removing from a the higher child node which is at minimum keys, causing + /// it to move a key from its lower sibling. + /// + /// - Builds a tree of the form: [[0], 1, [2, 3]] + /// - Remove 0 + /// - Expecting a tree of the form: [[1], 2, [3]] #[concordium_test] fn test_btree_remove_from_lower_leaf_in_three_node_taking_from_sibling() { let mut state_builder = StateBuilder::open(StateApi::open()); @@ -1538,14 +1621,20 @@ mod wasm_test_btree { for n in 0..4 { tree.insert(n); } - claim!(tree.contains(&0)); claim!(tree.remove(&0)); + claim!(!tree.contains(&0)); claim!(tree.contains(&1)); claim!(tree.contains(&2)); claim!(tree.contains(&3)); } + /// Removing from a the root node which is at minimum keys, likewise are the + /// children, causing a merge. + /// + /// - Builds a tree of the form: [[0], 1, [2]] + /// - Remove 1 + /// - Expecting a tree of the form: [0, 2] #[concordium_test] fn test_btree_remove_from_root_in_three_node_causing_merge() { let mut state_builder = StateBuilder::open(StateApi::open()); @@ -1554,12 +1643,19 @@ mod wasm_test_btree { tree.insert(n); } tree.remove(&3); - - claim!(tree.contains(&1)); claim!(tree.remove(&1)); + + claim!(tree.contains(&0)); claim!(!tree.contains(&1)); + claim!(tree.contains(&2)); } + /// Removing from a the root node which is at minimum keys, taking a child + /// from its higher child. + /// + /// - Builds a tree of the form: [[0], 1, [2, 3]] + /// - Remove 1 + /// - Expecting a tree of the form: [[0], 2, [3]] #[concordium_test] fn test_btree_remove_from_root_in_three_node_taking_key_from_higher_child() { let mut state_builder = StateBuilder::open(StateApi::open()); @@ -1572,6 +1668,12 @@ mod wasm_test_btree { claim!(!tree.contains(&1)); } + /// Removing from a the root node which is at minimum keys, taking a child + /// from its lower child. + /// + /// - Builds a tree of the form: [[0, 1], 2, [3]] + /// - Remove 1 + /// - Expecting a tree of the form: [[0], 1, [3]] #[concordium_test] fn test_btree_remove_from_root_in_three_node_taking_key_from_lower_child() { let mut state_builder = StateBuilder::open(StateApi::open()); @@ -1584,6 +1686,7 @@ mod wasm_test_btree { claim!(!tree.contains(&2)); } + /// Test iteration of the set. #[concordium_test] fn test_btree_iter() { let mut state_builder = StateBuilder::open(StateApi::open()); @@ -1596,6 +1699,14 @@ mod wasm_test_btree { claim_eq!(keys, iter_keys); } + /// Testcase for dublicate keys in the set bug. Due to an edgecase where the + /// key moved up as part of splitting a child node, is equal to the + /// inserted key. + /// + /// - Builds a tree of the form: [[0, 1, 2], 3, [4]] + /// - Insert 1 (again) causing the [0,1,2] to split, moving 1 up to the + /// root. + /// - Expecting a tree of the form: [[0], 1, [2], 3, [4]] #[concordium_test] fn test_btree_insert_present_key() { let mut state_builder = StateBuilder::open(StateApi::open()); @@ -1615,6 +1726,7 @@ mod wasm_test_btree { }; use ::quickcheck::{Arbitrary, Gen, TestResult}; + /// Quickcheck inserting random items. #[concordium_quickcheck] fn test_quickcheck_inserts(items: Vec) -> TestResult { let mut state_builder = StateBuilder::open(StateApi::open()); @@ -1633,6 +1745,8 @@ mod wasm_test_btree { TestResult::passed() } + /// Quickcheck random mutations see `Mutations` and `run_mutations` for + /// the details. #[concordium_quickcheck(num_tests = 500)] fn test_quickcheck_inserts_removes(mutations: Mutations) -> TestResult { let mut state_builder = StateBuilder::open(StateApi::open()); @@ -1645,19 +1759,28 @@ mod wasm_test_btree { } } + /// Newtype wrapper for Vec to implement Arbitrary. #[derive(Debug, Clone)] struct Mutations(Vec<(K, Operation)>); + /// The different mutating operations to generate for the btree. #[derive(Debug, Clone, Copy)] enum Operation { + /// Insert a new key in the set. InsertKeyNotPresent, + /// Insert a key already in the set. InsertKeyPresent, + /// Remove a key in the set. RemoveKeyPresent, + /// Remove a key not already in the set. RemoveKeyNotPresent, } - fn run_mutations( - tree: &mut StateBTreeSet, + /// Run a list of mutations on a btree, checking the return value and + /// tree invariants using `StateBTreeSet::check_invariants` + /// between each mutation, returning an error string if violated. + fn run_mutations( + tree: &mut StateBTreeSet, mutations: &[(u32, Operation)], ) -> Result<(), String> { for (k, op) in mutations.into_iter() { @@ -1708,7 +1831,11 @@ mod wasm_test_btree { K: Arbitrary + Ord, { fn arbitrary(g: &mut Gen) -> Self { + // Tracking the keys expected in the set at the point of each mutation. + // This is used to ensure operations such as inserting and removing a key which + // is present are actually valid. let mut inserted_keys: Vec = Vec::new(); + // The generated mutations to return. let mut mutations = Vec::new(); while mutations.len() < g.size() { @@ -1764,21 +1891,24 @@ mod wasm_test_btree { } Operation::RemoveKeyPresent => { let mut clone = self.0.clone(); - let mut prev = clone[0..i].iter().enumerate().rev(); - let j = loop { + let mut prev = self.0[0..i].iter().enumerate().rev(); + clone.remove(i); + loop { if let Some((j, (k2, op))) = prev.next() { match op { + Operation::InsertKeyPresent if k == k2 => { + clone.remove(j); + } Operation::InsertKeyNotPresent if k == k2 => { - break j; + clone.remove(j); + break; } _ => {} } } else { fail!("No insertion found before") } - }; - clone.remove(i); - clone.remove(j); + } v.push(Self(clone)); } _ => {} diff --git a/concordium-std/src/traits.rs b/concordium-std/src/traits.rs index 1dd54d03..89303fc3 100644 --- a/concordium-std/src/traits.rs +++ b/concordium-std/src/traits.rs @@ -294,7 +294,7 @@ pub trait HasStateApi: Clone { /// Delete an entry. /// Returns an error if the entry did not exist, or if it is part of a /// locked subtree. - fn delete_entry(&mut self, key: Self::EntryType) -> Result<(), StateError>; + fn delete_entry(&mut self, entry: Self::EntryType) -> Result<(), StateError>; /// Delete the entire subtree. /// Returns whether any values were deleted, or an error if the given prefix diff --git a/concordium-std/src/types.rs b/concordium-std/src/types.rs index 7503b249..5a498f0c 100644 --- a/concordium-std/src/types.rs +++ b/concordium-std/src/types.rs @@ -389,7 +389,7 @@ pub struct StateRefMut<'a, V: Serial, S: HasStateApi> { /// The `Option` allows for having an internal method destroying the /// `StateRefMut` into its raw parts without `Drop` causing a write to the /// contract state. - pub(crate) entry: UnsafeCell>, + pub(crate) entry: UnsafeCell, pub(crate) state_api: S, pub(crate) lazy_value: UnsafeCell>, pub(crate) _marker_lifetime: PhantomData<&'a mut V>, @@ -399,7 +399,7 @@ impl<'a, V: Serial, S: HasStateApi> StateRefMut<'a, V, S> { #[inline(always)] pub(crate) fn new(entry: S::EntryType, state_api: S) -> Self { Self { - entry: UnsafeCell::new(Some(entry)), + entry: UnsafeCell::new(entry), state_api, lazy_value: UnsafeCell::new(None), _marker_lifetime: PhantomData, From 99b461bb3ef6c7e71de2dcc9348cf940a74ddc7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Holm=20Gj=C3=B8rup?= Date: Thu, 14 Mar 2024 09:26:41 +0100 Subject: [PATCH 23/36] Add quickcheck tests for iter, higher and lower --- concordium-std/src/state_btree.rs | 130 +++++++++++++++++++++++++----- 1 file changed, 111 insertions(+), 19 deletions(-) diff --git a/concordium-std/src/state_btree.rs b/concordium-std/src/state_btree.rs index b1a16952..2baec6b1 100644 --- a/concordium-std/src/state_btree.rs +++ b/concordium-std/src/state_btree.rs @@ -1726,9 +1726,10 @@ mod wasm_test_btree { }; use ::quickcheck::{Arbitrary, Gen, TestResult}; - /// Quickcheck inserting random items. + /// Quickcheck inserting random items, check invariants on the tree and + /// query every item ensuring the tree contains it. #[concordium_quickcheck] - fn test_quickcheck_inserts(items: Vec) -> TestResult { + fn quickcheck_btree_inserts(items: Vec) -> TestResult { let mut state_builder = StateBuilder::open(StateApi::open()); let mut tree = state_builder.new_btree_set_degree::<2, _>(); for k in items.clone() { @@ -1745,14 +1746,96 @@ mod wasm_test_btree { TestResult::passed() } + /// Quickcheck inserting random items, then we call query the tree for + /// higher and lower of every item validating the outcome. + #[concordium_quickcheck(num_tests = 500)] + fn quickcheck_btree_iter(mut items: Vec) -> TestResult { + let mut state_builder = StateBuilder::open(StateApi::open()); + let mut tree = state_builder.new_btree_set_degree::<2, _>(); + for k in items.clone() { + tree.insert(k); + } + if let Err(violation) = tree.check_invariants() { + return TestResult::error(format!("Invariant violated: {:?}", violation)); + } + + items.sort(); + items.dedup(); + + for (value, expected) in tree.iter().zip(items.into_iter()) { + if *value != expected { + return TestResult::error(format!("Got {} but expected {expected}", *value)); + } + } + + TestResult::passed() + } + + /// Quickcheck inserting random items, then we call query the tree for + /// higher and lower of every item validating the outcome. + #[concordium_quickcheck(num_tests = 500)] + fn quickcheck_btree_higher_lower(mut items: Vec) -> TestResult { + let mut state_builder = StateBuilder::open(StateApi::open()); + let mut tree = state_builder.new_btree_set_degree::<2, _>(); + for k in items.clone() { + tree.insert(k); + } + if let Err(violation) = tree.check_invariants() { + return TestResult::error(format!("Invariant violated: {:?}", violation)); + } + + items.sort(); + items.dedup(); + for window in items.windows(2) { + let l = &window[0]; + let r = &window[1]; + let l_higher = tree.higher(l); + if l_higher.as_deref() != Some(r) { + return TestResult::error(format!( + "higher({l}) gave {:?} instead of the expected Some({r})", + l_higher.as_deref() + )); + } + let r_lower = tree.lower(r); + if r_lower.as_deref() != Some(l) { + return TestResult::error(format!( + "lower({r}) gave {:?} instead of the expected Some({l})", + r_lower.as_deref() + )); + } + } + + if let Some(first) = items.first() { + let lower = tree.lower(first); + if lower.is_some() { + return TestResult::error(format!( + "lower({first}) gave {:?} instead of the expected None", + lower.as_deref() + )); + } + } + + if let Some(last) = items.last() { + let higher = tree.higher(last); + if higher.is_some() { + return TestResult::error(format!( + "higher({last}) gave {:?} instead of the expected None", + higher.as_deref() + )); + } + } + + TestResult::passed() + } + /// Quickcheck random mutations see `Mutations` and `run_mutations` for /// the details. #[concordium_quickcheck(num_tests = 500)] - fn test_quickcheck_inserts_removes(mutations: Mutations) -> TestResult { + fn quickcheck_btree_inserts_removes(mutations: Mutations) -> TestResult { let mut state_builder = StateBuilder::open(StateApi::open()); let mut tree = state_builder.new_btree_set_degree::<2, _>(); - if let Err(err) = run_mutations(&mut tree, &mutations.0) { + if let Err(err) = run_mutations(&mut tree, &mutations.mutations) { TestResult::error(format!("Error: {}, tree: {}", err, tree.debug())) } else { TestResult::passed() @@ -1761,7 +1844,10 @@ mod wasm_test_btree { /// Newtype wrapper for Vec to implement Arbitrary. #[derive(Debug, Clone)] - struct Mutations(Vec<(K, Operation)>); + struct Mutations { + expected_keys: crate::collections::BTreeSet, + mutations: Vec<(K, Operation)>, + } /// The different mutating operations to generate for the btree. #[derive(Debug, Clone, Copy)] @@ -1872,35 +1958,41 @@ mod wasm_test_btree { } } - Self(mutations) + Self { + expected_keys: crate::collections::BTreeSet::from_iter( + inserted_keys.into_iter(), + ), + mutations, + } } fn shrink(&self) -> Box> { let pop = { - let mut clone = self.0.clone(); - clone.pop(); - Self(clone) + let mut clone = self.clone(); + clone.mutations.pop(); + clone }; let mut v = vec![pop]; - for (i, (k, op)) in self.0.iter().enumerate() { + for (i, (k, op)) in self.mutations.iter().enumerate() { match op { Operation::InsertKeyPresent | Operation::RemoveKeyNotPresent => { - let mut clone = self.0.clone(); - clone.remove(i); - v.push(Self(clone)); + let mut clone = self.clone(); + clone.mutations.remove(i); + v.push(clone); } Operation::RemoveKeyPresent => { - let mut clone = self.0.clone(); - let mut prev = self.0[0..i].iter().enumerate().rev(); - clone.remove(i); + let mut clone = self.clone(); + let mut prev = self.mutations[0..i].iter().enumerate().rev(); + clone.mutations.remove(i); + clone.expected_keys.remove(k); loop { if let Some((j, (k2, op))) = prev.next() { match op { Operation::InsertKeyPresent if k == k2 => { - clone.remove(j); + clone.mutations.remove(j); } Operation::InsertKeyNotPresent if k == k2 => { - clone.remove(j); + clone.mutations.remove(j); break; } _ => {} @@ -1909,7 +2001,7 @@ mod wasm_test_btree { fail!("No insertion found before") } } - v.push(Self(clone)); + v.push(clone); } _ => {} } From 4a919349fa0b2d57a2ddcb6787e50f082ff131a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Holm=20Gj=C3=B8rup?= Date: Thu, 14 Mar 2024 09:42:43 +0100 Subject: [PATCH 24/36] Add unit tests for eq_or_lower and eq_or_higher --- concordium-std/src/state_btree.rs | 69 +++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/concordium-std/src/state_btree.rs b/concordium-std/src/state_btree.rs index 2baec6b1..ceb42fdb 100644 --- a/concordium-std/src/state_btree.rs +++ b/concordium-std/src/state_btree.rs @@ -1504,6 +1504,52 @@ mod wasm_test_btree { claim_eq!(tree.lower(&8).as_deref(), Some(&7)); } + /// Build a set and query `eq_or_higher` on each key plus some keys outside + /// of the set. + #[concordium_test] + fn test_btree_eq_or_higher() { + let mut state_builder = StateBuilder::open(StateApi::open()); + let mut tree = state_builder.new_btree_set_degree::<2, _>(); + tree.insert(1); + tree.insert(2); + tree.insert(3); + tree.insert(4); + tree.insert(5); + tree.insert(7); + claim_eq!(tree.eq_or_higher(&0).as_deref(), Some(&1)); + claim_eq!(tree.eq_or_higher(&1).as_deref(), Some(&1)); + claim_eq!(tree.eq_or_higher(&2).as_deref(), Some(&2)); + claim_eq!(tree.eq_or_higher(&3).as_deref(), Some(&3)); + claim_eq!(tree.eq_or_higher(&4).as_deref(), Some(&4)); + claim_eq!(tree.eq_or_higher(&5).as_deref(), Some(&5)); + claim_eq!(tree.eq_or_higher(&6).as_deref(), Some(&7)); + claim_eq!(tree.eq_or_higher(&7).as_deref(), Some(&7)); + claim_eq!(tree.eq_or_higher(&8).as_deref(), None); + } + + /// Build a set and query `eq_or_lower` on each key plus some keys outside + /// of the set. + #[concordium_test] + fn test_btree_eq_or_lower() { + let mut state_builder = StateBuilder::open(StateApi::open()); + let mut tree = state_builder.new_btree_set_degree::<2, _>(); + tree.insert(1); + tree.insert(2); + tree.insert(3); + tree.insert(4); + tree.insert(5); + tree.insert(7); + claim_eq!(tree.eq_or_lower(&0).as_deref(), None); + claim_eq!(tree.eq_or_lower(&1).as_deref(), Some(&1)); + claim_eq!(tree.eq_or_lower(&2).as_deref(), Some(&2)); + claim_eq!(tree.eq_or_lower(&3).as_deref(), Some(&3)); + claim_eq!(tree.eq_or_lower(&4).as_deref(), Some(&4)); + claim_eq!(tree.eq_or_lower(&5).as_deref(), Some(&5)); + claim_eq!(tree.eq_or_lower(&6).as_deref(), Some(&5)); + claim_eq!(tree.eq_or_lower(&7).as_deref(), Some(&7)); + claim_eq!(tree.eq_or_lower(&8).as_deref(), Some(&7)); + } + /// Insert a large number of items. /// Check the set contains each item. /// Insert the same items again, checking the set is the same size @@ -1803,6 +1849,29 @@ mod wasm_test_btree { r_lower.as_deref() )); } + + let space_between = r - l > 1; + if space_between { + let l_eq_or_higher = tree.eq_or_higher(&(l + 1)); + if l_eq_or_higher.as_deref() != Some(r) { + return TestResult::error(format!( + "eq_or_higher({}) gave {:?} instead of the expected Some({r})", + l + 1, + l_higher.as_deref() + )); + } + } + + if space_between { + let r_eq_or_lower = tree.eq_or_lower(&(r - 1)); + if r_eq_or_lower.as_deref() != Some(l) { + return TestResult::error(format!( + "eq_or_lower({}) gave {:?} instead of the expected Some({l})", + r - 1, + l_higher.as_deref() + )); + } + } } if let Some(first) = items.first() { From c73d37fc18dd2ad32ef0724b859fa17be4a23762 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Holm=20Gj=C3=B8rup?= Date: Thu, 14 Mar 2024 09:46:58 +0100 Subject: [PATCH 25/36] Add must_use and warning for insert on StateMap --- concordium-std/src/impls.rs | 62 +++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 27 deletions(-) diff --git a/concordium-std/src/impls.rs b/concordium-std/src/impls.rs index 341f5b2a..2d11fe08 100644 --- a/concordium-std/src/impls.rs +++ b/concordium-std/src/impls.rs @@ -939,6 +939,14 @@ where /// Inserts the value with the given key. If a value already exists at the /// given key it is replaced and the old value is returned. + /// + /// *Caution*: If `Option` is to be deleted and contains a data structure + /// prefixed with `State` (such as [StateBox](crate::StateBox) or + /// [StateMap](crate::StateMap)), then it is important to call + /// [`Deletable::delete`](crate::Deletable::delete) on the value returned + /// when you're finished with it. Otherwise, it will remain in the + /// contract state. + #[must_use] pub fn insert(&mut self, key: K, value: V) -> Option { self.insert_borrowed(&key, value) } /// Get an entry for the given key. @@ -3243,9 +3251,9 @@ mod wasm_test { .get(my_map_key) .expect("Could not get statemap") .expect("Deserializing statemap failed"); - my_map.insert("abc".to_string(), "hello, world".to_string()); - my_map.insert("def".to_string(), "hallo, Weld".to_string()); - my_map.insert("ghi".to_string(), "hej, verden".to_string()); + let _ = my_map.insert("abc".to_string(), "hello, world".to_string()); + let _ = my_map.insert("def".to_string(), "hallo, Weld".to_string()); + let _ = my_map.insert("ghi".to_string(), "hej, verden".to_string()); claim_eq!(*my_map.get(&"abc".to_string()).unwrap(), "hello, world".to_string()); let mut iter = my_map.iter(); @@ -3292,8 +3300,8 @@ mod wasm_test { let mut outer_map = state_builder.new_map::>(); let mut inner_map = state_builder.new_map::(); - inner_map.insert(key_to_value, value); - outer_map.insert(inner_map_key, inner_map); + let _ = inner_map.insert(key_to_value, value); + let _ = outer_map.insert(inner_map_key, inner_map); claim_eq!(*outer_map.get(&inner_map_key).unwrap().get(&key_to_value).unwrap(), value); } @@ -3302,9 +3310,9 @@ mod wasm_test { fn statemap_iter_mut_works() { let mut state_builder = StateBuilder::open(StateApi::open()); let mut map = state_builder.new_map(); - map.insert(0u8, 1u8); - map.insert(1u8, 2u8); - map.insert(2u8, 3u8); + let _ = map.insert(0u8, 1u8); + let _ = map.insert(1u8, 2u8); + let _ = map.insert(2u8, 3u8); for (_, mut v) in map.iter_mut() { v.update(|old_value| *old_value += 10); } @@ -3326,9 +3334,9 @@ mod wasm_test { let mut state_builder = StateBuilder::open(StateApi::open()); let mut outer_map = state_builder.new_map(); let mut inner_map = state_builder.new_map(); - inner_map.insert(0u8, 1u8); - inner_map.insert(1u8, 2u8); - outer_map.insert(99u8, inner_map); + let _ = inner_map.insert(0u8, 1u8); + let _ = inner_map.insert(1u8, 2u8); + let _ = outer_map.insert(99u8, inner_map); for (_, mut v_map) in outer_map.iter_mut() { v_map.update(|v_map| { for (_, mut inner_v) in v_map.iter_mut() { @@ -3358,8 +3366,8 @@ mod wasm_test { fn statemap_iterator_unlocks_tree_once_dropped() { let mut state_builder = StateBuilder::open(StateApi::open()); let mut map = state_builder.new_map(); - map.insert(0u8, 1u8); - map.insert(1u8, 2u8); + let _ = map.insert(0u8, 1u8); + let _ = map.insert(1u8, 2u8); { let _iter = map.iter(); // Uncommenting these two lines (and making iter mutable) should @@ -3368,7 +3376,7 @@ mod wasm_test { // map.insert(2u8, 3u8); // let n = iter.next(); } // iter is dropped here, unlocking the subtree. - map.insert(2u8, 3u8); + let _ = map.insert(2u8, 3u8); } #[concordium_test] @@ -3403,7 +3411,7 @@ mod wasm_test { let mut inner_set = state_builder.new_set::(); inner_set.insert(value); - outer_map.insert(inner_set_key, inner_set); + let _ = outer_map.insert(inner_set_key, inner_set); claim!(outer_map.get(&inner_set_key).unwrap().contains(&value)); } @@ -3545,9 +3553,9 @@ mod wasm_test { let box2 = state_builder.new_box(2u8); let box3 = state_builder.new_box(3u8); let mut map = state_builder.new_map(); - map.insert(1u8, box1); - map.insert(2u8, box2); - map.insert(3u8, box3); + let _ = map.insert(1u8, box1); + let _ = map.insert(2u8, box2); + let _ = map.insert(3u8, box3); map.clear(); let mut iter = state_builder.state_api.iterator(&[]).expect("Could not get iterator"); // The only remaining node should be the state_builder's next_item_prefix node. @@ -3558,16 +3566,16 @@ mod wasm_test { fn clearing_nested_statemaps_works() { let mut state_builder = StateBuilder::open(StateApi::open()); let mut inner_map_1 = state_builder.new_map(); - inner_map_1.insert(1u8, 2u8); - inner_map_1.insert(2u8, 3u8); - inner_map_1.insert(3u8, 4u8); + let _ = inner_map_1.insert(1u8, 2u8); + let _ = inner_map_1.insert(2u8, 3u8); + let _ = inner_map_1.insert(3u8, 4u8); let mut inner_map_2 = state_builder.new_map(); - inner_map_2.insert(11u8, 12u8); - inner_map_2.insert(12u8, 13u8); - inner_map_2.insert(13u8, 14u8); + let _ = inner_map_2.insert(11u8, 12u8); + let _ = inner_map_2.insert(12u8, 13u8); + let _ = inner_map_2.insert(13u8, 14u8); let mut outer_map = state_builder.new_map(); - outer_map.insert(0u8, inner_map_1); - outer_map.insert(1u8, inner_map_2); + let _ = outer_map.insert(0u8, inner_map_1); + let _ = outer_map.insert(1u8, inner_map_2); outer_map.clear(); let mut iter = state_builder.state_api.iterator(&[]).expect("Could not get iterator"); // The only remaining node should be the state_builder's next_item_prefix node. @@ -3578,7 +3586,7 @@ mod wasm_test { fn occupied_entry_truncates_leftover_data() { let mut state_builder = StateBuilder::open(StateApi::open()); let mut map = state_builder.new_map(); - map.insert(99u8, "A longer string that should be truncated".into()); + let _ = map.insert(99u8, "A longer string that should be truncated".into()); let a_short_string = "A short string".to_string(); let expected_size = a_short_string.len() + 4; // 4 bytes for the length of the string. map.entry(99u8).and_modify(|v| *v = a_short_string); From fd57aee824d008c843f7fd8438da1263ebcd9c86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Holm=20Gj=C3=B8rup?= Date: Thu, 14 Mar 2024 10:21:17 +0100 Subject: [PATCH 26/36] Add quickcheck test for clearing btree, checking state --- concordium-std/src/state_btree.rs | 39 ++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/concordium-std/src/state_btree.rs b/concordium-std/src/state_btree.rs index ceb42fdb..3ece3f0b 100644 --- a/concordium-std/src/state_btree.rs +++ b/concordium-std/src/state_btree.rs @@ -1768,7 +1768,7 @@ mod wasm_test_btree { use super::super::*; use crate::{ self as concordium_std, concordium_quickcheck, concordium_test, fail, StateApi, - StateBuilder, + StateBuilder, StateError, }; use ::quickcheck::{Arbitrary, Gen, TestResult}; @@ -1792,9 +1792,42 @@ mod wasm_test_btree { TestResult::passed() } + /// Quickcheck inserting random items and then clear the entire tree + /// again. Use state api to ensure the btree nodes are no longer + /// stored in the state. + #[concordium_quickcheck] + fn quickcheck_btree_clear(items: Vec) -> TestResult { + let mut state_builder = StateBuilder::open(StateApi::open()); + let mut tree = state_builder.new_btree_set_degree::<2, _>(); + for k in items.clone() { + tree.insert(k); + } + tree.clear(); + for k in items.iter() { + if tree.contains(k) { + return TestResult::error(format!("Found {k} in a cleared btree")); + } + } + + let state_api = StateApi::open(); + match state_api.iterator(&tree.prefix) { + Ok(node_iter) => { + let nodes_in_state = node_iter.count(); + TestResult::error(format!( + "Found {} nodes still stored in the state", + nodes_in_state + )) + } + Err(StateError::SubtreeWithPrefixNotFound) => TestResult::passed(), + Err(err) => { + TestResult::error(format!("Failed to get iterator for btree nodes: {err:?}")) + } + } + } + /// Quickcheck inserting random items, then we call query the tree for /// higher and lower of every item validating the outcome. - #[concordium_quickcheck(num_tests = 500)] + #[concordium_quickcheck(num_tests = 100)] fn quickcheck_btree_iter(mut items: Vec) -> TestResult { let mut state_builder = StateBuilder::open(StateApi::open()); let mut tree = state_builder.new_btree_set_degree::<2, _>(); @@ -1819,7 +1852,7 @@ mod wasm_test_btree { /// Quickcheck inserting random items, then we call query the tree for /// higher and lower of every item validating the outcome. - #[concordium_quickcheck(num_tests = 500)] + #[concordium_quickcheck(num_tests = 100)] fn quickcheck_btree_higher_lower(mut items: Vec) -> TestResult { let mut state_builder = StateBuilder::open(StateApi::open()); let mut tree = state_builder.new_btree_set_degree::<2, _>(); From 166687b31652b194c87e6131408dc55380452fcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Holm=20Gj=C3=B8rup?= Date: Thu, 21 Mar 2024 13:39:55 +0100 Subject: [PATCH 27/36] Update btree documentation related to benchmarks and update rust sdk --- concordium-rust-sdk | 2 +- concordium-std/src/state_btree.rs | 18 ++++++++---------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/concordium-rust-sdk b/concordium-rust-sdk index 6eb5f3b7..62f2a44f 160000 --- a/concordium-rust-sdk +++ b/concordium-rust-sdk @@ -1 +1 @@ -Subproject commit 6eb5f3b768256b1e77d6d4340e121ead4a40aa20 +Subproject commit 62f2a44f14685c29957119ec4f6a5736d9f54ffd diff --git a/concordium-std/src/state_btree.rs b/concordium-std/src/state_btree.rs index 3ece3f0b..3fb71ee5 100644 --- a/concordium-std/src/state_btree.rs +++ b/concordium-std/src/state_btree.rs @@ -33,13 +33,12 @@ use crate::{ /// - `V`: Values stored in the map. Most operations on the map require this to /// implement [`Serial`](crate::Serial) and /// [`DeserialWithState`](crate::DeserialWithState). -/// - `S`: The low-level state implementation used, this allows for mocking the -/// state API in unit tests, see -/// [`TestStateApi`](crate::test_infrastructure::TestStateApi). /// - `M`: A `const usize` determining the _minimum degree_ of the B-tree. /// _Must_ be a value of `2` or above for the tree to work. This can be used /// to tweak the height of the tree vs size of each node in the tree. The -/// default is set based on benchmarks. +/// default is set to 8, which seems to perform well on benchmarks. These +/// benchmarks ran operations on a collection of 1000 elements, some using +/// keys of 4 bytes others 16 bytes. /// /// ## Usage /// @@ -318,13 +317,12 @@ impl StateBTreeMap { /// to the low-level state, such as types containing /// [`StateBox`](crate::StateBox), [`StateMap`](crate::StateMap) and /// [`StateSet`](crate::StateSet). -/// - `S`: The low-level state implementation used, this allows for mocking the -/// state API in unit tests, see -/// [`TestStateApi`](crate::test_infrastructure::TestStateApi). /// - `M`: A `const usize` determining the _minimum degree_ of the B-tree. /// _Must_ be a value of `2` or above for the tree to work. This can be used /// to tweak the height of the tree vs size of each node in the tree. The -/// default is set based on benchmarks. +/// default is set to 8, which seems to perform well on benchmarks. These +/// benchmarks ran operations on a collection of 1000 elements, some using +/// keys of 4 bytes others 16 bytes. /// /// ## Usage /// @@ -1199,7 +1197,7 @@ where #[derive(Debug, Copy, Clone, Serialize)] #[repr(transparent)] struct NodeId { - id: u32, + id: u64, } /// Byte size of the key used to store a BTree internal node in the smart @@ -1208,7 +1206,7 @@ const BTREE_NODE_KEY_SIZE: usize = STATE_ITEM_PREFIX_SIZE + NodeId::SERIALIZED_B impl NodeId { /// Byte size of `NodeId` when serialized. - const SERIALIZED_BYTE_SIZE: usize = 4; + const SERIALIZED_BYTE_SIZE: usize = 8; /// Return a copy of the NodeId, then increments itself. fn copy_then_increment(&mut self) -> Self { From 29de383f39caa14333356ffd6145bda3a34e6531 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Holm=20Gj=C3=B8rup?= Date: Fri, 22 Mar 2024 13:33:27 +0100 Subject: [PATCH 28/36] Add guide for choosing the right collection (#409) * Document when to use StateMap and StateBTreeMap * Address review comments --- concordium-std/src/lib.rs | 71 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 69 insertions(+), 2 deletions(-) diff --git a/concordium-std/src/lib.rs b/concordium-std/src/lib.rs index 1453e425..efcb45d9 100644 --- a/concordium-std/src/lib.rs +++ b/concordium-std/src/lib.rs @@ -7,6 +7,7 @@ //! libraries. //! //! # Versions +//! //! The concordium blockchain at present supports two variants of smart //! contracts. The original V0 contracts that use message-passing for //! communication and have limited state, and V1 contracts which use synchronous @@ -24,6 +25,7 @@ //! `test_infrastructure`](#deprecating-the-test_infrastructure) section. //! //! # Panic handler +//! //! When compiled without the `std` feature this crate sets the panic handler //! so that it terminates the process immediately, without any unwinding or //! prints. @@ -45,6 +47,7 @@ //! #crypto-primitives-for-testing-crypto-with-actual-implementations //! //! ## `std`: Build with the Rust standard library +//! //! By default this library will be linked with the //! [std](https://doc.rust-lang.org/std/) crate, the rust standard library, //! however to minimize code size this library supports toggling compilation @@ -61,6 +64,7 @@ //! In your project's `Cargo.toml` file. //! //! ## `build-schema`: Build for generating a module schema +//! //! **WARNING** Building with this feature enabled is meant for tooling, and the //! result is not intended to be deployed on chain. //! @@ -77,6 +81,7 @@ //! schema and for most cases this feature should not be set manually. //! //! ## `wasm-test`: Build for testing in Wasm +//! //! **WARNING** Building with this feature enabled is meant for tooling, and the //! result is not intended to be deployed on chain. //! @@ -99,6 +104,7 @@ //! testing and for most cases this feature should not be set manually. //! //! ## `crypto-primitives`: For testing crypto with actual implementations +//! //! This features is only relevant when using the **deprecated** //! [test_infrastructure]. //! @@ -137,7 +143,7 @@ //! whether `bump_alloc` is the best option. See the Rust [allocator](https://doc.rust-lang.org/std/alloc/index.html#the-global_allocator-attribute) //! documentation for more context and details on using custom allocators. //! -//! Emit debug information +//! # Emit debug information //! //! During testing and debugging it is often useful to emit debug information to //! narrow down the source of the problem. `concordium-std` supports this using @@ -154,6 +160,7 @@ //! ignore its arguments when the `debug` feature is not enabled. //! //! # Essential types +//! //! This crate has a number of essential types that are used when writing smart //! contracts. The structure of these are, at present, a bit odd without the //! historic context, which is explained below. @@ -187,6 +194,7 @@ //! simplify the names with aliases. //! //! # Signalling errors +//! //! On the Wasm level contracts can signal errors by returning a negative i32 //! value as a result of either initialization or invocation of the receive //! method. If the error is a logic error and the contract executes successfully @@ -242,7 +250,65 @@ //! Other error codes may be added in the future and custom error codes should //! not use the range `i32::MIN` to `i32::MIN + 100`. //! +//! # Collections +//! +//! Several collections are available for use in a smart contract and choosing +//! the right one can result in significant cost savings. +//! +//! First off, Rust's own standard library provides [several efficient data structures](https://doc.rust-lang.org/std/collections/) +//! which can be used in a smart contract. +//! +//! However, these can become costly for collections holding a large number of +//! elements, which needs to be persisted in the smart contract state across +//! contract updates. +//! The reason being these data structures are designed to be kept in-memory and +//! persisting them means reading and writing the entire collection to a single +//! entry in the smart contract key-value store on every contract update. +//! This is wasteful when only part of the collection is relevant on each +//! update. +//! +//! In the above mentioned scenarios, it is instead recommended to use one of +//! the smart contract tailored collections, as these partisions the collection +//! into multiple key-value stores, resulting in cheaper read and writes to the +//! state. +//! +//! The collections can be grouped as: +//! - Maps: [`StateMap`], [`StateBTreeMap`] +//! - Sets: [`StateSet`], [`StateBTreeSet`] +//! +//! ## When should you use which collection? +//! +//! This section presents a rough guideline for when to reach for each of the +//! collections. +//! +//! ### Use `StateMap` when: +//! +//! - You want to track which keys you have seen. +//! - Arbitrary values are associated with each of the keys. +//! +//! ### Use `StateBTreeMap` when: +//! +//! - You want to track which keys you have seen. +//! - Arbitrary values are associated with each of the keys. +//! - The keys have some _ordering_ which is relevant e.g. if you need the key +//! which is located right above/below another key using +//! [`higher`](StateBTreeMap::higher)/[`lower`](StateBTreeMap::lower). +//! +//! ### Use `StateSet` when: +//! +//! - You want to track which keys you have seen. +//! - There is no meaningful value to associate with your keys. +//! +//! ### Use `StateBTreeSet` when: +//! +//! - You want to track which keys you have seen. +//! - There is no meaningful value to associate with your keys. +//! - The keys have some _ordering_ which is relevant e.g. if you need the key +//! which is located right above/below another key using +//! [`higher`](StateBTreeMap::higher)/[`lower`](StateBTreeMap::lower). +//! //! # Deprecating the `test_infrastructure` +//! //! Version 8.1 deprecates the [test_infrastructure] in favor of the library //! [concordium_smart_contract_testing]. A number of traits are also //! deprecated at the same time since they only exist to support the @@ -295,7 +361,7 @@ //! ctx: &impl HasReceiveContext, //! host: &impl HasHost, //! ) -> ReceiveResult { todo!() } -//! +//! //! /// After //! #[receive(contract = "my_contract", name = "my_receive")] //! fn receive_after( // `` removed @@ -325,6 +391,7 @@ //! [1]: https://doc.rust-lang.org/std/primitive.unit.html //! [test_infrastructure]: ./test_infrastructure/index.html //! [concordium_smart_contract_testing]: https://docs.rs/concordium-smart-contract-testing + #![cfg_attr(not(feature = "std"), no_std, feature(core_intrinsics))] #[cfg(not(feature = "std"))] From b4f08f82cc62495a6a161e02d118064e47e84823 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Holm=20Gj=C3=B8rup?= Date: Fri, 22 Mar 2024 19:28:02 +0100 Subject: [PATCH 29/36] Update rust version and fix clippy --- .github/workflows/linter.yml | 33 +++++++++++++++++++++++++++++-- concordium-std/Cargo.toml | 2 +- concordium-std/src/impls.rs | 12 ++++------- concordium-std/src/state_btree.rs | 4 ++-- 4 files changed, 38 insertions(+), 13 deletions(-) diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index a56d246c..da960035 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -12,9 +12,9 @@ name: Clippy & fmt env: RUST_FMT: nightly-2023-04-01 - RUST_VERSION: "1.66" + RUST_VERSION: "1.73" RUST_VERSION_TESTING_LIBRARY: "1.72" - CARGO_CONCORDIUM_VERSION: "3.2.0" + CARGO_CONCORDIUM_VERSION: "3.3.0-internal" jobs: rustfmt: @@ -103,6 +103,35 @@ jobs: run: | RUSTDOCFLAGS="-D warnings" cargo doc --no-deps --all-features --color=always + # All templates are generated with the `cargo-generate` command and it is checked that the 'cargo test' command + # can be executed without errors on the generated smart contracts. + std-internal-wasm-test: + name: concordium-std internal wasm tests + runs-on: ubuntu-latest + + steps: + - name: Checkout sources + uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Install toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: ${{ env.RUST_VERSION }} + + - name: Install Wasm target + run: rustup target install wasm32-unknown-unknown + + - name: Run internal wasm unit test + run: | + CARGO_CCD=cargo-concordium_${{ env.CARGO_CONCORDIUM_VERSION }} + wget https://distribution.concordium.software/tools/linux/$CARGO_CCD + chmod +x $CARGO_CCD + sudo mv $CARGO_CCD /usr/bin/cargo-concordium + cd concordium-std + cargo concordium test --only-unit-tests -- --features internal-wasm-test + # All templates are generated with the `cargo-generate` command and it is checked that the 'cargo test' command # can be executed without errors on the generated smart contracts. cargo-generate-templates: diff --git a/concordium-std/Cargo.toml b/concordium-std/Cargo.toml index 535e4cb7..8e97dbfd 100644 --- a/concordium-std/Cargo.toml +++ b/concordium-std/Cargo.toml @@ -3,7 +3,7 @@ name = "concordium-std" version = "10.0.0" authors = ["Concordium "] edition = "2021" -rust-version = "1.66" +rust-version = "1.73" license = "MPL-2.0" description = "A standard library for writing smart contracts for the Concordium blockchain in Rust." homepage = "https://github.com/Concordium/concordium-rust-smart-contracts/" diff --git a/concordium-std/src/impls.rs b/concordium-std/src/impls.rs index 2d11fe08..696898bf 100644 --- a/concordium-std/src/impls.rs +++ b/concordium-std/src/impls.rs @@ -765,6 +765,7 @@ where StateApi: HasStateApi, { /// Ensures a value is in the entry by inserting the default value if empty. + #[allow(clippy::unwrap_or_default)] pub fn or_default(self) -> OccupiedEntry<'a, K, V, StateApi> { self.or_insert_with(Default::default) } @@ -2228,9 +2229,8 @@ fn query_exchange_rates_worker() -> ExchangeRates { /// two extern hosts below. fn query_account_public_keys_worker(address: AccountAddress) -> QueryAccountPublicKeysResult { let data: &[u8] = address.as_ref(); - let response = unsafe { - prims::invoke(INVOKE_QUERY_ACCOUNT_PUBLIC_KEYS_TAG, data.as_ptr() as *const u8, 32) - }; + let response = + unsafe { prims::invoke(INVOKE_QUERY_ACCOUNT_PUBLIC_KEYS_TAG, data.as_ptr(), 32) }; let mut return_value = parse_query_account_public_keys_response_code(response)?; Ok(crate::AccountPublicKeys::deserial(&mut return_value).unwrap_abort()) } @@ -2246,11 +2246,7 @@ fn check_account_signature_worker( signatures.serial(&mut buffer).unwrap_abort(); let response = unsafe { - prims::invoke( - INVOKE_CHECK_ACCOUNT_SIGNATURE_TAG, - buffer.as_ptr() as *const u8, - buffer.len() as u32, - ) + prims::invoke(INVOKE_CHECK_ACCOUNT_SIGNATURE_TAG, buffer.as_ptr(), buffer.len() as u32) }; // Be explicit that the buffer must survive up to here. drop(buffer); diff --git a/concordium-std/src/state_btree.rs b/concordium-std/src/state_btree.rs index 3fb71ee5..4be9b02e 100644 --- a/concordium-std/src/state_btree.rs +++ b/concordium-std/src/state_btree.rs @@ -800,9 +800,9 @@ impl StateBTreeSet { /// Assumes: /// - The provided `node` is not a leaf and has a child at `index`. /// - The minimum degree `M` is at least 2 or more. - fn prepare_child_for_key_removal<'b, 'c>( + fn prepare_child_for_key_removal<'c>( &mut self, - mut node: StateRefMut<'b, Node, StateApi>, + mut node: StateRefMut, StateApi>, index: usize, ) -> StateRefMut<'c, Node, StateApi> where From aeaae93c4b498990289a325c6ac8819dcae87039 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Holm=20Gj=C3=B8rup?= Date: Fri, 22 Mar 2024 19:49:52 +0100 Subject: [PATCH 30/36] Fix warnings --- concordium-std/src/state_btree.rs | 23 ++++++++----------- .../tests/state/map-multiple-entries.stderr | 4 ++-- .../state/map-multiple-state-ref-mut.stderr | 4 ++-- examples/cis2-dynamic-nft/src/lib.rs | 4 ++-- examples/cis2-multi-royalties/src/lib.rs | 4 ++-- examples/cis2-multi/src/lib.rs | 4 ++-- examples/cis2-nft/src/lib.rs | 2 +- examples/cis2-wccd/src/lib.rs | 2 +- examples/cis3-nft-sponsored-txs/src/lib.rs | 2 +- examples/credential-registry/src/lib.rs | 2 +- examples/eSealing/src/lib.rs | 2 +- examples/factory/src/lib.rs | 2 +- examples/nametoken/src/lib.rs | 2 +- examples/offchain-transfers/src/lib.rs | 20 ++++++++-------- .../sponsored-tx-enabled-auction/src/lib.rs | 2 +- examples/two-step-transfer/src/lib.rs | 14 +++++------ templates/cis2-nft/src/lib.rs | 2 +- templates/credential-registry/src/lib.rs | 2 +- 18 files changed, 47 insertions(+), 50 deletions(-) diff --git a/concordium-std/src/state_btree.rs b/concordium-std/src/state_btree.rs index 4be9b02e..c4096fd2 100644 --- a/concordium-std/src/state_btree.rs +++ b/concordium-std/src/state_btree.rs @@ -46,15 +46,14 @@ use crate::{ /// [`new_btree_map`](crate::StateBuilder::new_btree_map) method on the /// [`StateBuilder`](crate::StateBuilder). /// -/// ``` +/// ```no_run /// # use concordium_std::*; -/// # use concordium_std::test_infrastructure::*; -/// # let mut state_builder = TestStateBuilder::new(); +/// # let mut state_builder = StateBuilder::open(StateApi::open()); /// /// In an init method: /// let mut map1 = state_builder.new_btree_map(); /// # map1.insert(0u8, 1u8); // Specifies type of map. /// -/// # let mut host = TestHost::new((), state_builder); +/// # let mut host = ExternHost { state: (), state_builder }; /// /// In a receive method: /// let mut map2 = host.state_builder().new_btree_map(); /// # map2.insert(0u16, 1u16); @@ -330,15 +329,13 @@ impl StateBTreeMap { /// [`new_btree_set`](crate::StateBuilder::new_btree_set) method on the /// [`StateBuilder`](crate::StateBuilder). /// -/// ``` +/// ```no_run /// # use concordium_std::*; -/// # use concordium_std::test_infrastructure::*; -/// # let mut state_builder = TestStateBuilder::new(); +/// # let mut state_builder = StateBuilder::open(StateApi::open()); /// /// In an init method: /// let mut map1 = state_builder.new_btree_set(); /// # map1.insert(0u8); // Specifies type of map. -/// -/// # let mut host = TestHost::new((), state_builder); +/// # let mut host = ExternHost { state: (), state_builder }; /// /// In a receive method: /// let mut map2 = host.state_builder().new_btree_set(); /// # map2.insert(0u16); @@ -351,8 +348,8 @@ impl StateBTreeMap { /// /// ```no_run /// # use concordium_std::*; -/// struct MyState { -/// inner: StateBTreeSet, +/// struct MyState { +/// inner: StateBTreeSet, /// } /// fn incorrect_replace(state_builder: &mut StateBuilder, state: &mut MyState) { /// // The following is incorrect. The old value of `inner` is not properly deleted. @@ -364,8 +361,8 @@ impl StateBTreeMap { /// /// ```no_run /// # use concordium_std::*; -/// # struct MyState { -/// # inner: StateBTreeSet +/// # struct MyState { +/// # inner: StateBTreeSet /// # } /// fn correct_replace(state_builder: &mut StateBuilder, state: &mut MyState) { /// state.inner.clear(); diff --git a/concordium-std/tests/state/map-multiple-entries.stderr b/concordium-std/tests/state/map-multiple-entries.stderr index 9fa1d65a..94fba8ed 100644 --- a/concordium-std/tests/state/map-multiple-entries.stderr +++ b/concordium-std/tests/state/map-multiple-entries.stderr @@ -2,9 +2,9 @@ error[E0499]: cannot borrow `map` as mutable more than once at a time --> tests/state/map-multiple-entries.rs:12:14 | 11 | let e1 = map.entry(0u8); - | -------------- first mutable borrow occurs here + | --- first mutable borrow occurs here 12 | let e2 = map.entry(1u8); - | ^^^^^^^^^^^^^^ second mutable borrow occurs here + | ^^^ second mutable borrow occurs here 13 | // Use them, so we are certain that their lifetimes overlap. 14 | e1.or_insert(1); | -- first borrow later used here diff --git a/concordium-std/tests/state/map-multiple-state-ref-mut.stderr b/concordium-std/tests/state/map-multiple-state-ref-mut.stderr index 444cfbf3..eb996224 100644 --- a/concordium-std/tests/state/map-multiple-state-ref-mut.stderr +++ b/concordium-std/tests/state/map-multiple-state-ref-mut.stderr @@ -2,9 +2,9 @@ error[E0499]: cannot borrow `map` as mutable more than once at a time --> tests/state/map-multiple-state-ref-mut.rs:14:14 | 13 | let e1 = map.get_mut(&0u8).unwrap(); - | ----------------- first mutable borrow occurs here + | --- first mutable borrow occurs here 14 | let e2 = map.get_mut(&1u8).unwrap(); - | ^^^^^^^^^^^^^^^^^ second mutable borrow occurs here + | ^^^ second mutable borrow occurs here 15 | // Use them, so we are certain that their lifetimes overlap. 16 | assert_eq!(*e1, *e2); | -- first borrow later used here diff --git a/examples/cis2-dynamic-nft/src/lib.rs b/examples/cis2-dynamic-nft/src/lib.rs index f19ebded..cd7ad00a 100644 --- a/examples/cis2-dynamic-nft/src/lib.rs +++ b/examples/cis2-dynamic-nft/src/lib.rs @@ -196,7 +196,7 @@ impl State { owner: &Address, state_builder: &mut StateBuilder, ) { - self.tokens.insert(*token_id, TokenMetadataState { + let _ = self.tokens.insert(*token_id, TokenMetadataState { token_metadata_current_state_counter: 0, token_metadata_list: mint_param.metadata_url.clone(), }); @@ -368,7 +368,7 @@ impl State { std_id: StandardIdentifierOwned, implementors: Vec, ) { - self.implementors.insert(std_id, implementors); + let _ = self.implementors.insert(std_id, implementors); } } diff --git a/examples/cis2-multi-royalties/src/lib.rs b/examples/cis2-multi-royalties/src/lib.rs index 025515c1..70bc5df9 100644 --- a/examples/cis2-multi-royalties/src/lib.rs +++ b/examples/cis2-multi-royalties/src/lib.rs @@ -229,7 +229,7 @@ impl State { royalty: u8, ) { self.tokens.insert(*token_id); - self.token_details.insert(*token_id, TokenDetails { + let _ = self.token_details.insert(*token_id, TokenDetails { minter: *owner, royalty, }); @@ -355,7 +355,7 @@ impl State { std_id: StandardIdentifierOwned, implementors: Vec, ) { - self.implementors.insert(std_id, implementors); + let _ = self.implementors.insert(std_id, implementors); } fn calculate_royalty_payments( diff --git a/examples/cis2-multi/src/lib.rs b/examples/cis2-multi/src/lib.rs index 62ac6b66..1dc49ce7 100644 --- a/examples/cis2-multi/src/lib.rs +++ b/examples/cis2-multi/src/lib.rs @@ -642,7 +642,7 @@ impl State { ) -> MetadataUrl { let token_metadata = self.tokens.get(token_id).map(|x| x.to_owned()); if token_metadata.is_none() { - self.tokens.insert(*token_id, metadata_url.to_owned()); + let _ = self.tokens.insert(*token_id, metadata_url.to_owned()); } let mut owner_state = @@ -788,7 +788,7 @@ impl State { std_id: StandardIdentifierOwned, implementors: Vec, ) { - self.implementors.insert(std_id, implementors); + let _ = self.implementors.insert(std_id, implementors); } /// Grant role to an address. diff --git a/examples/cis2-nft/src/lib.rs b/examples/cis2-nft/src/lib.rs index 4b352ef3..e2157418 100644 --- a/examples/cis2-nft/src/lib.rs +++ b/examples/cis2-nft/src/lib.rs @@ -275,7 +275,7 @@ impl State { std_id: StandardIdentifierOwned, implementors: Vec, ) { - self.implementors.insert(std_id, implementors); + let _ = self.implementors.insert(std_id, implementors); } } diff --git a/examples/cis2-wccd/src/lib.rs b/examples/cis2-wccd/src/lib.rs index 19fb8110..4c7b3e86 100644 --- a/examples/cis2-wccd/src/lib.rs +++ b/examples/cis2-wccd/src/lib.rs @@ -412,7 +412,7 @@ impl State { std_id: StandardIdentifierOwned, implementors: Vec, ) { - self.implementors.insert(std_id, implementors); + let _ = self.implementors.insert(std_id, implementors); } } diff --git a/examples/cis3-nft-sponsored-txs/src/lib.rs b/examples/cis3-nft-sponsored-txs/src/lib.rs index 6c07bb68..e702e3d5 100644 --- a/examples/cis3-nft-sponsored-txs/src/lib.rs +++ b/examples/cis3-nft-sponsored-txs/src/lib.rs @@ -501,7 +501,7 @@ impl State { std_id: StandardIdentifierOwned, implementors: Vec, ) { - self.implementors.insert(std_id, implementors); + let _ = self.implementors.insert(std_id, implementors); } } diff --git a/examples/credential-registry/src/lib.rs b/examples/credential-registry/src/lib.rs index d7fcbe6a..965b9e96 100644 --- a/examples/credential-registry/src/lib.rs +++ b/examples/credential-registry/src/lib.rs @@ -314,7 +314,7 @@ impl State { std_id: StandardIdentifierOwned, implementors: Vec, ) { - self.implementors.insert(std_id, implementors); + let _ = self.implementors.insert(std_id, implementors); } } diff --git a/examples/eSealing/src/lib.rs b/examples/eSealing/src/lib.rs index 2b30ef37..b565a35e 100644 --- a/examples/eSealing/src/lib.rs +++ b/examples/eSealing/src/lib.rs @@ -84,7 +84,7 @@ impl State { /// Add a new file hash (replaces existing file if present). fn add_file(&mut self, file_hash: HashSha2256, timestamp: Timestamp, witness: AccountAddress) { - self.files.insert(file_hash, FileState { + let _ = self.files.insert(file_hash, FileState { timestamp, witness, }); diff --git a/examples/factory/src/lib.rs b/examples/factory/src/lib.rs index 2573f770..8740dfd7 100644 --- a/examples/factory/src/lib.rs +++ b/examples/factory/src/lib.rs @@ -269,7 +269,7 @@ pub mod factory { let state = host.state_mut(); let next_product = state.next_product; state.next_product = next_product + 1; - state.products.insert(next_product, product_address); + let _ = state.products.insert(next_product, product_address); // Invoke the initialize entrypoint on the product passing in the index for this // product. host.invoke_contract( diff --git a/examples/nametoken/src/lib.rs b/examples/nametoken/src/lib.rs index 20b6441f..2ba737a8 100644 --- a/examples/nametoken/src/lib.rs +++ b/examples/nametoken/src/lib.rs @@ -487,7 +487,7 @@ impl State { std_id: StandardIdentifierOwned, implementors: Vec, ) { - self.implementors.insert(std_id, implementors); + let _ = self.implementors.insert(std_id, implementors); } } diff --git a/examples/offchain-transfers/src/lib.rs b/examples/offchain-transfers/src/lib.rs index f6a8aa69..35045ca5 100644 --- a/examples/offchain-transfers/src/lib.rs +++ b/examples/offchain-transfers/src/lib.rs @@ -701,7 +701,7 @@ mod tests { balance, ); //Set account3 balance - host.state_mut().balance_sheet.insert(account3, balance); + let _ = host.state_mut().balance_sheet.insert(account3, balance); //Test 1: Try to withdraw too much money from Account 3 let mut ctx = TestReceiveContext::empty(); @@ -782,9 +782,9 @@ mod tests { alice_balance + bob_balance + charlie_balance, ); //Set balance sheet - host.state_mut().balance_sheet.insert(alice, alice_balance); - host.state_mut().balance_sheet.insert(bob, bob_balance); - host.state_mut().balance_sheet.insert(charlie, charlie_balance); + let _ = host.state_mut().balance_sheet.insert(alice, alice_balance); + let _ = host.state_mut().balance_sheet.insert(bob, bob_balance); + let _ = host.state_mut().balance_sheet.insert(charlie, charlie_balance); //Define settlements let settlement1 = Settlement { @@ -1084,9 +1084,9 @@ mod tests { ); //Set balance sheet - host.state_mut().balance_sheet.insert(alice, alice_balance); - host.state_mut().balance_sheet.insert(bob, bob_balance); - host.state_mut().balance_sheet.insert(charlie, charlie_balance); + let _ = host.state_mut().balance_sheet.insert(alice, alice_balance); + let _ = host.state_mut().balance_sheet.insert(bob, bob_balance); + let _ = host.state_mut().balance_sheet.insert(charlie, charlie_balance); // First settlement is fine and with past finality let settlement1 = Settlement { @@ -1266,9 +1266,9 @@ mod tests { alice_balance + bob_balance + charlie_balance, ); //Set balance sheet - host.state_mut().balance_sheet.insert(alice, alice_balance); - host.state_mut().balance_sheet.insert(bob, bob_balance); - host.state_mut().balance_sheet.insert(charlie, charlie_balance); + let _ = host.state_mut().balance_sheet.insert(alice, alice_balance); + let _ = host.state_mut().balance_sheet.insert(bob, bob_balance); + let _ = host.state_mut().balance_sheet.insert(charlie, charlie_balance); //Define settlements let settlement1 = Settlement { diff --git a/examples/sponsored-tx-enabled-auction/src/lib.rs b/examples/sponsored-tx-enabled-auction/src/lib.rs index 8fe900d9..1e1669ca 100644 --- a/examples/sponsored-tx-enabled-auction/src/lib.rs +++ b/examples/sponsored-tx-enabled-auction/src/lib.rs @@ -337,7 +337,7 @@ fn add_item( host.state_mut().counter = item_index; // Insert the item into the state. - host.state_mut().items.insert(item_index, ItemState { + let _ = host.state_mut().items.insert(item_index, ItemState { auction_state: AuctionState::NotSoldYet, highest_bidder: None, name: item.name, diff --git a/examples/two-step-transfer/src/lib.rs b/examples/two-step-transfer/src/lib.rs index 9e6f91d9..bf81dbbd 100644 --- a/examples/two-step-transfer/src/lib.rs +++ b/examples/two-step-transfer/src/lib.rs @@ -232,7 +232,7 @@ fn contract_receive_message( // Persist the active requests host.state_mut().requests = host.state_builder().new_map(); for (key, req) in active_requests.iter() { - host.state_mut().requests.insert(*key, req.clone()); + let _ = host.state_mut().requests.insert(*key, req.clone()); } // Check if a request already exists @@ -264,7 +264,7 @@ fn contract_receive_message( supporters, }; - host.state_mut().requests.insert(req_id, new_request); + let _ = host.state_mut().requests.insert(req_id, new_request); Ok(()) } @@ -467,7 +467,7 @@ mod tests { let mut state_builder = TestStateBuilder::new(); let mut requests = state_builder.new_map(); - requests.insert(request_id, request); + let _ = requests.insert(request_id, request); let state = State { init_params, requests, @@ -534,7 +534,7 @@ mod tests { let mut state_builder = TestStateBuilder::new(); let mut requests = state_builder.new_map(); - requests.insert(request_id, request); + let _ = requests.insert(request_id, request); let state = State { init_params, requests, @@ -601,7 +601,7 @@ mod tests { supporters.insert(account1); // Outdated request - requests.insert(request_id_outdated, TransferRequest { + let _ = requests.insert(request_id_outdated, TransferRequest { transfer_amount, target_account, times_out_at: ctx @@ -613,7 +613,7 @@ mod tests { }); // Active request - requests.insert(request_id_active, TransferRequest { + let _ = requests.insert(request_id_active, TransferRequest { transfer_amount, target_account, times_out_at: ctx @@ -696,7 +696,7 @@ mod tests { let mut state_builder = TestStateBuilder::new(); let mut requests = state_builder.new_map(); - requests.insert(request_id, request); + let _ = requests.insert(request_id, request); let state = State { init_params, requests, diff --git a/templates/cis2-nft/src/lib.rs b/templates/cis2-nft/src/lib.rs index c055fea4..ce0365ec 100644 --- a/templates/cis2-nft/src/lib.rs +++ b/templates/cis2-nft/src/lib.rs @@ -275,7 +275,7 @@ impl State { std_id: StandardIdentifierOwned, implementors: Vec, ) { - self.implementors.insert(std_id, implementors); + let _ = self.implementors.insert(std_id, implementors); } } diff --git a/templates/credential-registry/src/lib.rs b/templates/credential-registry/src/lib.rs index 1b206cdb..bcb6d4c7 100644 --- a/templates/credential-registry/src/lib.rs +++ b/templates/credential-registry/src/lib.rs @@ -316,7 +316,7 @@ impl State { std_id: StandardIdentifierOwned, implementors: Vec, ) { - self.implementors.insert(std_id, implementors); + let _ = self.implementors.insert(std_id, implementors); } } From c20981abec59ce57e98cff3e27fa91aa619bcbe3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Holm=20Gj=C3=B8rup?= Date: Mon, 25 Mar 2024 13:26:51 +0100 Subject: [PATCH 31/36] Address review comments --- .github/workflows/linter.yml | 2 +- concordium-std/Cargo.toml | 1 + concordium-std/src/impls.rs | 27 ++- concordium-std/src/state_btree.rs | 285 ++++++++++++++++-------------- concordium-std/src/types.rs | 3 - 5 files changed, 176 insertions(+), 142 deletions(-) diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index da960035..fed42a59 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -619,7 +619,7 @@ jobs: uses: actions-rs/cargo@v1 with: command: build - args: --manifest-path ${{ matrix.crates }} --target=${{ matrix.target }} --no-default-features + args: --manifest-path ${{ matrix.crates }} --target=${{ matrix.target }} --no-default-features --features bump_alloc check-no-std-examples: name: Build on nightly, diff --git a/concordium-std/Cargo.toml b/concordium-std/Cargo.toml index 8e97dbfd..da7760c8 100644 --- a/concordium-std/Cargo.toml +++ b/concordium-std/Cargo.toml @@ -41,6 +41,7 @@ bump_alloc = [] p7 = [] [lib] +# cdylib is needed below to compile into a wasm module with internal unit tests. crate-type = ["cdylib", "rlib"] [profile.release] diff --git a/concordium-std/src/impls.rs b/concordium-std/src/impls.rs index 696898bf..aaff802d 100644 --- a/concordium-std/src/impls.rs +++ b/concordium-std/src/impls.rs @@ -2418,10 +2418,18 @@ impl StateBuilder { } /// Create a new empty [`StateBTreeSet`](crate::StateBTreeSet), setting the - /// minimum degree `M` of the B-Tree explicitly. + /// minimum degree `M` of the B-Tree explicitly. `M` must be 2 or higher + /// otherwise constructing the B-Tree results in aborting. pub fn new_btree_set_degree(&mut self) -> state_btree::StateBTreeSet { - let (state_api, prefix) = self.new_state_container(); - state_btree::StateBTreeSet::new(state_api, prefix) + if M >= 2 { + let (state_api, prefix) = self.new_state_container(); + state_btree::StateBTreeSet::new(state_api, prefix) + } else { + crate::fail!( + "Invalid minimum degree used for StateBTreeSet, must be >=2 instead got {}", + M + ) + } } /// Create a new empty [`StateBTreeMap`](crate::StateBTreeMap), setting the @@ -2429,9 +2437,16 @@ impl StateBuilder { pub fn new_btree_map_degree( &mut self, ) -> state_btree::StateBTreeMap { - state_btree::StateBTreeMap { - key_value: self.new_map(), - key_order: self.new_btree_set_degree(), + if M >= 2 { + state_btree::StateBTreeMap { + key_value: self.new_map(), + key_order: self.new_btree_set_degree(), + } + } else { + crate::fail!( + "Invalid minimum degree used for StateBTreeMap, must be >=2 instead got {}", + M + ) } } } diff --git a/concordium-std/src/state_btree.rs b/concordium-std/src/state_btree.rs index c4096fd2..aad156e0 100644 --- a/concordium-std/src/state_btree.rs +++ b/concordium-std/src/state_btree.rs @@ -169,10 +169,8 @@ impl StateBTreeMap { where K: Serialize, V: Serial + DeserialWithState, { - // Minor optimization by first checking whether the collection is empty, since - // this information is kept at the root of the ordered set it is already loaded - // into memory. In the case of the empty collection we can then save a key - // lookup. + // Minor optimization in the case of the empty collection. Since the length is + // tracked by the ordered set, we can return early, saving a key lookup. if self.key_order.is_empty() { None } else { @@ -185,10 +183,8 @@ impl StateBTreeMap { where K: Serialize, V: Serial + DeserialWithState, { - // Minor optimization by first checking whether the collection is empty, since - // this information is kept at the root of the ordered set it is already loaded - // into memory. In the case of the empty collection we can then save a key - // lookup. + // Minor optimization in the case of the empty collection. Since the length is + // tracked by the ordered set, we can return early, saving a key lookup. if self.key_order.is_empty() { None } else { @@ -458,7 +454,7 @@ impl StateBTreeSet { if node.is_leaf() { return false; } - let child_node_id = node.children[child_index]; + let child_node_id = unsafe { *node.children.get_unchecked(child_index) }; node = self.get_node(child_node_id); } } @@ -526,7 +522,7 @@ impl StateBTreeSet { higher_so_far = Some(StateRef::new(node.keys.swap_remove(higher_key_index))) } - let child_node_id = node.children[higher_key_index]; + let child_node_id = unsafe { *node.children.get_unchecked(higher_key_index) }; node = self.get_node(child_node_id); } } @@ -564,7 +560,7 @@ impl StateBTreeSet { higher_so_far = Some(StateRef::new(node.keys.swap_remove(higher_key_index))) } - let child_node_id = node.children[higher_key_index]; + let child_node_id = unsafe { *node.children.get_unchecked(higher_key_index) }; node = self.get_node(child_node_id); } } @@ -895,7 +891,7 @@ impl StateBTreeSet { node.keys.swap_remove(0) } - /// Moving key at `index` from the node to the lower child and then merges + /// Move key at `index` from the node to the lower child and then merges /// this child with the content of its larger sibling, deleting the sibling. /// /// Assumes: @@ -927,7 +923,7 @@ impl StateBTreeSet { ) -> (NodeId, StateRefMut<'b, Node, StateApi>) where K: Serialize, { - let node_id = self.next_node_id.copy_then_increment(); + let node_id = self.next_node_id.fetch_and_add(); let node = Node { keys, children, @@ -1037,71 +1033,6 @@ impl StateBTreeSet { let entry = self.state_api.lookup_entry(&key).unwrap_abort(); StateRefMut::new(entry, self.state_api.clone()) } - - /// Construct a string for displaying the btree and debug information. - /// Should only be used while debugging and testing the btree itself. - #[cfg(feature = "internal-wasm-test")] - pub(crate) fn debug(&self) -> String - where - K: Serialize + std::fmt::Debug + Ord, { - let Some(root_node_id) = self.root else { - return format!("no root"); - }; - let mut string = String::new(); - let root: Node = self.get_node(root_node_id); - string.push_str(format!("root: {:#?}", root).as_str()); - let mut stack = root.children; - - while let Some(node_id) = stack.pop() { - let node: Node = self.get_node(node_id); - string.push_str( - format!("node {} {:?}: {:#?},\n", node_id.id, node.check_invariants(), node) - .as_str(), - ); - - stack.extend(node.children); - } - string - } - - /// Check a number of invariants, producing an error if any of them are - /// violated. Should only be used while debugging and testing the btree - /// itself. - #[cfg(feature = "internal-wasm-test")] - pub(crate) fn check_invariants(&self) -> Result<(), InvariantViolation> - where - K: Serialize + Ord, { - use crate::ops::Deref; - let Some(root_node_id) = self.root else { - return if self.len == 0 { - Ok(()) - } else { - Err(InvariantViolation::NonZeroLenWithNoRoot) - }; - }; - let root: Node = self.get_node(root_node_id); - if root.keys.is_empty() { - return Err(InvariantViolation::ZeroKeysInRoot); - } - - let mut stack = root.children; - while let Some(node_id) = stack.pop() { - let node: Node = self.get_node(node_id); - node.check_invariants()?; - stack.extend(node.children); - } - - let mut prev = None; - for key in self.iter() { - if let Some(p) = prev.as_deref() { - if p >= key.deref() { - return Err(InvariantViolation::IterationOutOfOrder); - } - } - prev = Some(key); - } - Ok(()) - } } /// An iterator over the entries of a [`StateBTreeSet`]. @@ -1149,7 +1080,8 @@ where self.next_node = Some(child_id); } if no_more_keys { - self.depth_first_stack.pop(); + // This was the last key in the node, so remove the node from the stack. + let _ = self.depth_first_stack.pop(); } self.length -= 1; Some(StateRef::new(key)) @@ -1206,7 +1138,7 @@ impl NodeId { const SERIALIZED_BYTE_SIZE: usize = 8; /// Return a copy of the NodeId, then increments itself. - fn copy_then_increment(&mut self) -> Self { + fn fetch_and_add(&mut self) -> Self { let current = *self; self.id += 1; current @@ -1272,58 +1204,6 @@ impl Node { /// Check if the node holds the minimum number of keys. #[inline(always)] fn is_at_min(&self) -> bool { self.keys.len() == Self::MINIMUM_KEY_LEN } - - /// Check a number of invariants of a non-root node in a btree, producing an - /// error if any of them are violated. Should only be used while - /// debugging and testing the btree itself. - #[cfg(feature = "internal-wasm-test")] - pub(crate) fn check_invariants(&self) -> Result<(), InvariantViolation> - where - K: Ord, { - for i in 1..self.keys.len() { - if &self.keys[i - 1] >= &self.keys[i] { - return Err(InvariantViolation::NodeKeysOutOfOrder); - } - } - - if self.keys.len() < Self::MINIMUM_KEY_LEN { - return Err(InvariantViolation::KeysLenBelowMin); - } - if self.keys.len() > Self::MAXIMUM_KEY_LEN { - return Err(InvariantViolation::KeysLenAboveMax); - } - - if self.is_leaf() { - if !self.children.is_empty() { - return Err(InvariantViolation::LeafWithChildren); - } - } else { - if self.children.len() < Self::MINIMUM_CHILD_LEN { - return Err(InvariantViolation::ChildrenLenBelowMin); - } - if self.children.len() > Self::MAXIMUM_CHILD_LEN { - return Err(InvariantViolation::ChildrenLenAboveMax); - } - } - - Ok(()) - } -} - -/// The invariants to check in a btree. -/// Should only be used while debugging and testing the btree itself. -#[cfg(feature = "internal-wasm-test")] -#[derive(Debug)] -pub(crate) enum InvariantViolation { - NonZeroLenWithNoRoot, - ZeroKeysInRoot, - IterationOutOfOrder, - NodeKeysOutOfOrder, - KeysLenBelowMin, - KeysLenAboveMax, - LeafWithChildren, - ChildrenLenBelowMin, - ChildrenLenAboveMax, } /// Wrapper implement the exact same deserial as K, but wraps it in an @@ -1394,8 +1274,138 @@ where /// be run using `cargo concordium test`. #[cfg(feature = "internal-wasm-test")] mod wasm_test_btree { + use super::*; use crate::{claim, claim_eq, concordium_test, StateApi, StateBuilder}; + /// The invariants to check in a btree. + /// Should only be used while debugging and testing the btree itself. + #[derive(Debug)] + pub(crate) enum InvariantViolation { + /// The collection have length above 0, but no root. + NonZeroLenWithNoRoot, + /// The collection contain a root node, but this has no keys. + ZeroKeysInRoot, + /// Iterating the keys in the entire collection, is not in strictly + /// ascending order. + IterationOutOfOrder, + /// The keys in a node is not in strictly ascending order. + NodeKeysOutOfOrder, + /// The non-root node contains fewer keys than the minimum. + KeysLenBelowMin, + /// The non-root node contains more keys than the maximum. + KeysLenAboveMax, + /// The leaf node contains children nodes. + LeafWithChildren, + /// The non-root non-leaf node contains fewer children than the minimum. + ChildrenLenBelowMin, + /// The non-root non-leaf node contains more children than the maximum. + ChildrenLenAboveMax, + } + + impl StateBTreeSet { + /// Check invariants, producing an error if any of them are + /// violated. + /// See [`InvariantViolation`] for the of list invariants being checked. + /// Should only be used while debugging and testing the btree itself. + fn check_invariants(&self) -> Result<(), InvariantViolation> + where + K: Serialize + Ord, { + use crate::ops::Deref; + let Some(root_node_id) = self.root else { + return if self.len == 0 { + Ok(()) + } else { + Err(InvariantViolation::NonZeroLenWithNoRoot) + }; + }; + let root: Node = self.get_node(root_node_id); + if root.keys.is_empty() { + return Err(InvariantViolation::ZeroKeysInRoot); + } + + let mut stack = root.children; + while let Some(node_id) = stack.pop() { + let node: Node = self.get_node(node_id); + node.check_invariants()?; + stack.extend(node.children); + } + + let mut prev = None; + for key in self.iter() { + if let Some(p) = prev.as_deref() { + if p > key.deref() { + return Err(InvariantViolation::IterationOutOfOrder); + } + } + prev = Some(key); + } + Ok(()) + } + + /// Construct a string for displaying the btree and debug information. + /// Should only be used while debugging and testing the btree itself. + pub(crate) fn debug(&self) -> String + where + K: Serialize + std::fmt::Debug + Ord, { + let Some(root_node_id) = self.root else { + return format!("no root"); + }; + let mut string = String::new(); + let root: Node = self.get_node(root_node_id); + string.push_str(format!("root: {:#?}", root).as_str()); + let mut stack = root.children; + + while let Some(node_id) = stack.pop() { + let node: Node = self.get_node(node_id); + string.push_str( + format!("node {} {:?}: {:#?},\n", node_id.id, node.check_invariants(), node) + .as_str(), + ); + + stack.extend(node.children); + } + string + } + } + + impl Node { + /// Check invariants of a non-root node in a btree, producing an error + /// if any of them are violated. + /// See [`InvariantViolation`] for the of list invariants being checked. + /// Should only be used while debugging and testing the btree itself. + pub(crate) fn check_invariants(&self) -> Result<(), InvariantViolation> + where + K: Ord, { + for i in 1..self.keys.len() { + if &self.keys[i - 1] >= &self.keys[i] { + return Err(InvariantViolation::NodeKeysOutOfOrder); + } + } + + if self.keys.len() < Self::MINIMUM_KEY_LEN { + return Err(InvariantViolation::KeysLenBelowMin); + } + if self.keys.len() > Self::MAXIMUM_KEY_LEN { + return Err(InvariantViolation::KeysLenAboveMax); + } + + if self.is_leaf() { + if !self.children.is_empty() { + return Err(InvariantViolation::LeafWithChildren); + } + } else { + if self.children.len() < Self::MINIMUM_CHILD_LEN { + return Err(InvariantViolation::ChildrenLenBelowMin); + } + if self.children.len() > Self::MAXIMUM_CHILD_LEN { + return Err(InvariantViolation::ChildrenLenAboveMax); + } + } + + Ok(()) + } + } + /// Insert `2 * M` items such that the btree contains more than the root /// node. Checking that every item is contained in the collection. #[concordium_test] @@ -1758,6 +1768,8 @@ mod wasm_test_btree { claim!(!tree.insert(1)); } + // The module is using `concordium_quickcheck` which is located in a deprecated + // module. #[allow(deprecated)] mod quickcheck { use super::super::*; @@ -2063,6 +2075,15 @@ mod wasm_test_btree { } } + /// We attempt to produce several shrinked versions: + /// + /// - Simply remove the last mutation from the list of mutations. + /// - Remove mutations, which are not adding or removing keys + /// (Remove non-present keys and inserting present keys), note + /// these might still mutate the internal structure. + /// - Iterate the mutations and when a key is removed, we traverse + /// back in the mutations and remove other mutations of the same + /// key back to when it was inserted. fn shrink(&self) -> Box> { let pop = { let mut clone = self.clone(); diff --git a/concordium-std/src/types.rs b/concordium-std/src/types.rs index 5a498f0c..e64a88e8 100644 --- a/concordium-std/src/types.rs +++ b/concordium-std/src/types.rs @@ -386,9 +386,6 @@ impl<'a, V> crate::ops::Deref for StateRef<'a, V> { pub struct StateRefMut<'a, V: Serial, S: HasStateApi> { /// This is set as an `UnsafeCell`, to be able to get a mutable reference to /// the entry without `StateRefMut` being mutable. - /// The `Option` allows for having an internal method destroying the - /// `StateRefMut` into its raw parts without `Drop` causing a write to the - /// contract state. pub(crate) entry: UnsafeCell, pub(crate) state_api: S, pub(crate) lazy_value: UnsafeCell>, From 5e7fdce5f3c890de9508547dad61e5da1ca5f5c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Holm=20Gj=C3=B8rup?= Date: Mon, 25 Mar 2024 13:39:25 +0100 Subject: [PATCH 32/36] Update rust-sdk --- concordium-rust-sdk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/concordium-rust-sdk b/concordium-rust-sdk index 62f2a44f..5e4b93c0 160000 --- a/concordium-rust-sdk +++ b/concordium-rust-sdk @@ -1 +1 @@ -Subproject commit 62f2a44f14685c29957119ec4f6a5736d9f54ffd +Subproject commit 5e4b93c0a0ae7aef16d3bb6c0e62c30424d900bc From 3a9b1c6f6224afa85155d8664aac61d652a298e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Holm=20Gj=C3=B8rup?= Date: Mon, 25 Mar 2024 13:44:01 +0100 Subject: [PATCH 33/36] Update rust version in contract-testing CI --- .github/workflows/linter-testing-lib.yaml | 2 +- .github/workflows/linter.yml | 25 ++++++++--------------- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/.github/workflows/linter-testing-lib.yaml b/.github/workflows/linter-testing-lib.yaml index dc89950f..e21c53a5 100644 --- a/.github/workflows/linter-testing-lib.yaml +++ b/.github/workflows/linter-testing-lib.yaml @@ -28,7 +28,7 @@ on: env: RUST_FMT: nightly-2023-04-01-x86_64-unknown-linux-gnu - RUST_CLIPPY: 1.72 + RUST_CLIPPY: 1.73 jobs: "lint_fmt": diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index fed42a59..09265002 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -13,7 +13,7 @@ name: Clippy & fmt env: RUST_FMT: nightly-2023-04-01 RUST_VERSION: "1.73" - RUST_VERSION_TESTING_LIBRARY: "1.72" + RUST_VERSION_TESTING_LIBRARY: "1.73" CARGO_CONCORDIUM_VERSION: "3.3.0-internal" jobs: @@ -589,18 +589,9 @@ jobs: args: --manifest-path ${{ matrix.lib-crates }} --target=${{ matrix.target }} --features=${{ matrix.features }} -- -D warnings check-std-no-std: - name: Build on nightly, + name: Build no-std on nightly, runs-on: ubuntu-latest needs: rustfmt - strategy: - matrix: - target: - - wasm32-unknown-unknown - - crates: - - concordium-std/Cargo.toml - - concordium-cis2/Cargo.toml - steps: - name: Checkout sources uses: actions/checkout@v2 @@ -615,11 +606,13 @@ jobs: target: ${{ matrix.target }} override: true - - name: Run cargo check with no-std - uses: actions-rs/cargo@v1 - with: - command: build - args: --manifest-path ${{ matrix.crates }} --target=${{ matrix.target }} --no-default-features --features bump_alloc + - name: Run no-std build: concordium-std + working-directory: concordium-std + run: cargo build --target wasm32-unknown-unknown --no-default-features --features bump_alloc + + - name: Run no-std build: concordium-cis2 + working-directory: concordium-cis2 + run: cargo build --target wasm32-unknown-unknown --no-default-features --features concordium-std/bump_alloc check-no-std-examples: name: Build on nightly, From e3b3d6f530adf5b180934685b44e7ebd8615d6f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Holm=20Gj=C3=B8rup?= Date: Mon, 25 Mar 2024 14:46:13 +0100 Subject: [PATCH 34/36] Address yet another round of comments --- .github/workflows/linter.yml | 41 +++++++++++------------- concordium-std/src/impls.rs | 3 +- concordium-std/src/state_btree.rs | 10 ++++++ templates/credential-registry/Cargo.toml | 2 +- 4 files changed, 31 insertions(+), 25 deletions(-) diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 09265002..96ed2395 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -14,7 +14,7 @@ env: RUST_FMT: nightly-2023-04-01 RUST_VERSION: "1.73" RUST_VERSION_TESTING_LIBRARY: "1.73" - CARGO_CONCORDIUM_VERSION: "3.3.0-internal" + CARGO_CONCORDIUM_VERSION: "3.3.0" jobs: rustfmt: @@ -124,12 +124,11 @@ jobs: run: rustup target install wasm32-unknown-unknown - name: Run internal wasm unit test + working-directory: concordium-std run: | - CARGO_CCD=cargo-concordium_${{ env.CARGO_CONCORDIUM_VERSION }} - wget https://distribution.concordium.software/tools/linux/$CARGO_CCD - chmod +x $CARGO_CCD - sudo mv $CARGO_CCD /usr/bin/cargo-concordium - cd concordium-std + curl -L https://github.com/Concordium/concordium-smart-contract-tools/releases/download/releases/cargo-concordium/${CARGO_CONCORDIUM_VERSION}/cargo-concordium-linux-amd64 -o /tmp/cargo-concordium + chmod +x /tmp/cargo-concordium + sudo mv /tmp/cargo-concordium /usr/bin/cargo-concordium cargo concordium test --only-unit-tests -- --features internal-wasm-test # All templates are generated with the `cargo-generate` command and it is checked that the 'cargo test' command @@ -174,10 +173,9 @@ jobs: # Run all tests, including doc tests. - name: Run cargo test run: | - CARGO_CCD=cargo-concordium_${{ env.CARGO_CONCORDIUM_VERSION }} - wget https://distribution.concordium.software/tools/linux/$CARGO_CCD - chmod +x $CARGO_CCD - sudo mv $CARGO_CCD /usr/bin/cargo-concordium + curl -L https://github.com/Concordium/concordium-smart-contract-tools/releases/download/releases/cargo-concordium/${CARGO_CONCORDIUM_VERSION}/cargo-concordium-linux-amd64 -o /tmp/cargo-concordium + chmod +x /tmp/cargo-concordium + sudo mv /tmp/cargo-concordium /usr/bin/cargo-concordium mv $PROJECT_NAME ${{ runner.temp }}/ cd ${{ runner.temp }}/$PROJECT_NAME cargo concordium test --out "./concordium-out/module.wasm.v1" @@ -224,10 +222,9 @@ jobs: # Run all tests, including doc tests. - name: Run cargo test run: | - CARGO_CCD=cargo-concordium_${{ env.CARGO_CONCORDIUM_VERSION }} - wget https://distribution.concordium.software/tools/linux/$CARGO_CCD - chmod +x $CARGO_CCD - sudo mv $CARGO_CCD /usr/bin/cargo-concordium + curl -L https://github.com/Concordium/concordium-smart-contract-tools/releases/download/releases/cargo-concordium/${CARGO_CONCORDIUM_VERSION}/cargo-concordium-linux-amd64 -o /tmp/cargo-concordium + chmod +x /tmp/cargo-concordium + sudo mv /tmp/cargo-concordium /usr/bin/cargo-concordium mv $PROJECT_NAME ${{ runner.temp }}/ cd ${{ runner.temp }}/$PROJECT_NAME cargo concordium test --out "./concordium-out/module.wasm.v1" @@ -428,7 +425,7 @@ jobs: sed -i "s/root/Concordium /g" ${{ runner.temp }}/$PROJECT_NAME/Cargo.toml sed -i "s/version = \"10.0\", default-features = false/path = \"..\/..\/concordium-std\", version = \"10.0\", default-features = false/g" ${{ runner.temp }}/$PROJECT_NAME/Cargo.toml sed -i "s/version = \"6.1\", default-features = false/path = \"..\/..\/concordium-cis2\", version = \"6.1\", default-features = false/g" ${{ runner.temp }}/$PROJECT_NAME/Cargo.toml - sed -i "s/concordium-smart-contract-testing = \"4.0\"/concordium-smart-contract-testing = {path = \"..\/..\/contract-testing\"}/g" ${{ runner.temp }}/$PROJECT_NAME/Cargo.toml + sed -i "s/concordium-smart-contract-testing = \"4.1\"/concordium-smart-contract-testing = {path = \"..\/..\/contract-testing\"}/g" ${{ runner.temp }}/$PROJECT_NAME/Cargo.toml diff ${{ runner.temp }}/$PROJECT_NAME/Cargo.toml examples/credential-registry/Cargo.toml diff ${{ runner.temp }}/$PROJECT_NAME/src/lib.rs examples/credential-registry/src/lib.rs diff ${{ runner.temp }}/$PROJECT_NAME/tests/tests.rs examples/credential-registry/tests/tests.rs @@ -603,14 +600,14 @@ jobs: with: profile: minimal toolchain: nightly - target: ${{ matrix.target }} + target: wasm32-unknown-unknown override: true - - name: Run no-std build: concordium-std + - name: Run no-std build concordium-std working-directory: concordium-std run: cargo build --target wasm32-unknown-unknown --no-default-features --features bump_alloc - - name: Run no-std build: concordium-cis2 + - name: Run no-std build concordium-cis2 working-directory: concordium-cis2 run: cargo build --target wasm32-unknown-unknown --no-default-features --features concordium-std/bump_alloc @@ -886,11 +883,9 @@ jobs: - name: Download and install Cargo Concordium run: | - CARGO_CCD=cargo-concordium_${{ env.CARGO_CONCORDIUM_VERSION }} - wget https://distribution.concordium.software/tools/linux/$CARGO_CCD - chmod +x $CARGO_CCD - sudo mv $CARGO_CCD /usr/bin/cargo-concordium - + curl -L https://github.com/Concordium/concordium-smart-contract-tools/releases/download/releases/cargo-concordium/${CARGO_CONCORDIUM_VERSION}/cargo-concordium-linux-amd64 -o /tmp/cargo-concordium + chmod +x /tmp/cargo-concordium + sudo mv /tmp/cargo-concordium /usr/bin/cargo-concordium # The 'smart-contract-upgrade' example has a v1 and v2 contract and the tests in v1 needs the wasm module from v2 for upgrading. - name: Build contract-upgrade version 2 module if needed if: ${{ matrix.crates == 'examples/smart-contract-upgrade/contract-version1' }} diff --git a/concordium-std/src/impls.rs b/concordium-std/src/impls.rs index aaff802d..7282d4ed 100644 --- a/concordium-std/src/impls.rs +++ b/concordium-std/src/impls.rs @@ -2433,7 +2433,8 @@ impl StateBuilder { } /// Create a new empty [`StateBTreeMap`](crate::StateBTreeMap), setting the - /// minimum degree `M` of the B-Tree explicitly. + /// minimum degree `M` of the B-Tree explicitly. `M` must be 2 or higher + /// otherwise constructing the B-Tree results in aborting. pub fn new_btree_map_degree( &mut self, ) -> state_btree::StateBTreeMap { diff --git a/concordium-std/src/state_btree.rs b/concordium-std/src/state_btree.rs index aad156e0..263ce4fd 100644 --- a/concordium-std/src/state_btree.rs +++ b/concordium-std/src/state_btree.rs @@ -193,6 +193,7 @@ impl StateBTreeMap { } /// Returns `true` if the map contains a value for the specified key. + #[inline(always)] pub fn contains_key(&self, key: &K) -> bool where K: Serialize + Ord, { @@ -201,6 +202,7 @@ impl StateBTreeMap { /// Returns the smallest key in the map, which is strictly larger than the /// provided key. `None` meaning no such key is present in the map. + #[inline(always)] pub fn higher(&self, key: &K) -> Option> where K: Serialize + Ord, { @@ -209,6 +211,7 @@ impl StateBTreeMap { /// Returns the smallest key in the map, which is equal or larger than the /// provided key. `None` meaning no such key is present in the map. + #[inline(always)] pub fn eq_or_higher(&self, key: &K) -> Option> where K: Serialize + Ord, { @@ -217,6 +220,7 @@ impl StateBTreeMap { /// Returns the largest key in the map, which is strictly smaller than the /// provided key. `None` meaning no such key is present in the map. + #[inline(always)] pub fn lower(&self, key: &K) -> Option> where K: Serialize + Ord, { @@ -225,6 +229,7 @@ impl StateBTreeMap { /// Returns the largest key in the map, which is equal or smaller than the /// provided key. `None` meaning no such key is present in the map. + #[inline(always)] pub fn eq_or_lower(&self, key: &K) -> Option> where K: Serialize + Ord, { @@ -233,6 +238,7 @@ impl StateBTreeMap { /// Returns a reference to the first key in the map, if any. This key is /// always the minimum of all keys in the map. + #[inline(always)] pub fn first_key(&self) -> Option> where K: Serialize + Ord, { @@ -241,6 +247,7 @@ impl StateBTreeMap { /// Returns a reference to the last key in the map, if any. This key is /// always the maximum of all keys in the map. + #[inline(always)] pub fn last_key(&self) -> Option> where K: Serialize + Ord, { @@ -248,13 +255,16 @@ impl StateBTreeMap { } /// Return the number of elements in the map. + #[inline(always)] pub fn len(&self) -> u32 { self.key_order.len() } /// Returns `true` is the map contains no elements. + #[inline(always)] pub fn is_empty(&self) -> bool { self.key_order.is_empty() } /// Create an iterator over the entries of [`StateBTreeMap`]. /// Ordered by `K` ascending. + #[inline(always)] pub fn iter(&self) -> StateBTreeMapIter { StateBTreeMapIter { key_iter: self.key_order.iter(), diff --git a/templates/credential-registry/Cargo.toml b/templates/credential-registry/Cargo.toml index ec535bae..5c4e8fec 100644 --- a/templates/credential-registry/Cargo.toml +++ b/templates/credential-registry/Cargo.toml @@ -20,7 +20,7 @@ concordium-cis2 = {version = "6.1", default-features = false} quickcheck = {version = "1"} [dev-dependencies] -concordium-smart-contract-testing = "4.0" +concordium-smart-contract-testing = "4.1" [lib] crate-type=["cdylib", "rlib"] From 1702ccf62960be66a21a1d996cf58221fdd37517 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Holm=20Gj=C3=B8rup?= Date: Wed, 27 Mar 2024 11:04:08 +0100 Subject: [PATCH 35/36] Address review comments --- .github/workflows/linter.yml | 3 +-- concordium-std/Cargo.toml | 2 +- concordium-std/src/impls.rs | 4 ++-- concordium-std/src/state_btree.rs | 40 +++++++++++++++++++++++-------- 4 files changed, 34 insertions(+), 15 deletions(-) diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 1465a2ad..21498ab8 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -103,8 +103,7 @@ jobs: run: | RUSTDOCFLAGS="-D warnings" cargo doc --no-deps --all-features --color=always - # All templates are generated with the `cargo-generate` command and it is checked that the 'cargo test' command - # can be executed without errors on the generated smart contracts. + # Run unit-tests for concordium-std compiled to wasm using cargo concordium test. std-internal-wasm-test: name: concordium-std internal wasm tests runs-on: ubuntu-latest diff --git a/concordium-std/Cargo.toml b/concordium-std/Cargo.toml index da7760c8..67a812e0 100644 --- a/concordium-std/Cargo.toml +++ b/concordium-std/Cargo.toml @@ -29,7 +29,7 @@ features = ["smart-contract"] default = ["std"] std = ["concordium-contracts-common/std"] wasm-test = ["concordium-contracts-common/wasm-test"] -# Own internal wasm-tests leaks out to the smart contracts using this library, +# Own internal wasm-tests leak out to the smart contracts using this library, # so a separate feature 'internal-wasm-test' is introduced for these. internal-wasm-test = ["wasm-test", "concordium-quickcheck"] build-schema = ["concordium-contracts-common/build-schema"] diff --git a/concordium-std/src/impls.rs b/concordium-std/src/impls.rs index 7282d4ed..ab878269 100644 --- a/concordium-std/src/impls.rs +++ b/concordium-std/src/impls.rs @@ -3521,7 +3521,7 @@ mod wasm_test { claim!(state.iterator(&key).is_ok(), "Iterator should be present"); claim!( state.delete_entry(sub_entry).is_err(), - "Should not be able to create an entry under a locked subtree" + "Should not be able to delete entry under a locked subtree" ); } @@ -3542,7 +3542,7 @@ mod wasm_test { claim!(state.iterator(&key).is_ok(), "Iterator should be present"); claim!( state.delete_entry(entry2).is_ok(), - "Failed to create a new entry under a different subtree" + "Should be able to delete entry under a different subtree" ); } diff --git a/concordium-std/src/state_btree.rs b/concordium-std/src/state_btree.rs index 263ce4fd..ae532add 100644 --- a/concordium-std/src/state_btree.rs +++ b/concordium-std/src/state_btree.rs @@ -104,7 +104,7 @@ pub struct StateBTreeMap { /// Each key in this map must also be in the `key_order` set. pub(crate) key_value: StateMap, /// A set for tracking the order of the inserted keys. - /// Each key in this set mush also have an associated value in the + /// Each key in this set must also have an associated value in the /// `key_value` map. pub(crate) key_order: StateBTreeSet, } @@ -1298,8 +1298,12 @@ mod wasm_test_btree { /// Iterating the keys in the entire collection, is not in strictly /// ascending order. IterationOutOfOrder, - /// The keys in a node is not in strictly ascending order. + /// Leaf node found at different depths. + LeafAtDifferentDepth, + /// The keys in a node are not in strictly ascending order. NodeKeysOutOfOrder, + /// The non-leaf node does not contain `keys.len() + 1` children. + MismatchingChildrenLenKeyLen, /// The non-root node contains fewer keys than the minimum. KeysLenBelowMin, /// The non-root node contains more keys than the maximum. @@ -1333,11 +1337,24 @@ mod wasm_test_btree { return Err(InvariantViolation::ZeroKeysInRoot); } - let mut stack = root.children; - while let Some(node_id) = stack.pop() { - let node: Node = self.get_node(node_id); - node.check_invariants()?; - stack.extend(node.children); + if !root.is_leaf() && root.children.len() != root.keys.len() + 1 { + return Err(InvariantViolation::MismatchingChildrenLenKeyLen); + } + let mut stack = vec![(0usize, root.children)]; + let mut leaf_depth = None; + while let Some((node_level, mut nodes)) = stack.pop() { + while let Some(node_id) = nodes.pop() { + let node: Node = self.get_node(node_id); + node.check_invariants()?; + if node.is_leaf() { + let depth = leaf_depth.get_or_insert(node_level); + if *depth != node_level { + return Err(InvariantViolation::LeafAtDifferentDepth); + } + } else { + stack.push((node_level + 1, node.children)); + } + } } let mut prev = None; @@ -1404,6 +1421,9 @@ mod wasm_test_btree { return Err(InvariantViolation::LeafWithChildren); } } else { + if self.children.len() != self.keys.len() + 1 { + return Err(InvariantViolation::MismatchingChildrenLenKeyLen); + } if self.children.len() < Self::MINIMUM_CHILD_LEN { return Err(InvariantViolation::ChildrenLenBelowMin); } @@ -1733,7 +1753,7 @@ mod wasm_test_btree { /// from its lower child. /// /// - Builds a tree of the form: [[0, 1], 2, [3]] - /// - Remove 1 + /// - Remove 2 /// - Expecting a tree of the form: [[0], 1, [3]] #[concordium_test] fn test_btree_remove_from_root_in_three_node_taking_key_from_lower_child() { @@ -1760,12 +1780,12 @@ mod wasm_test_btree { claim_eq!(keys, iter_keys); } - /// Testcase for dublicate keys in the set bug. Due to an edgecase where the + /// Testcase for duplicate keys in the set. Due to an edge case where the /// key moved up as part of splitting a child node, is equal to the /// inserted key. /// /// - Builds a tree of the form: [[0, 1, 2], 3, [4]] - /// - Insert 1 (again) causing the [0,1,2] to split, moving 1 up to the + /// - Insert 1 (again) causing the [0, 1, 2] to split, moving 1 up to the /// root. /// - Expecting a tree of the form: [[0], 1, [2], 3, [4]] #[concordium_test] From 66a283e528f16655561dd7c4eedc40a1f3ba2208 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20Holm=20Gj=C3=B8rup?= Date: Wed, 27 Mar 2024 20:37:02 +0100 Subject: [PATCH 36/36] Address last round of comments --- concordium-std/src/state_btree.rs | 41 ++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/concordium-std/src/state_btree.rs b/concordium-std/src/state_btree.rs index ae532add..6b0f60c0 100644 --- a/concordium-std/src/state_btree.rs +++ b/concordium-std/src/state_btree.rs @@ -200,7 +200,7 @@ impl StateBTreeMap { self.key_order.contains(key) } - /// Returns the smallest key in the map, which is strictly larger than the + /// Returns the smallest key in the map that is strictly larger than the /// provided key. `None` meaning no such key is present in the map. #[inline(always)] pub fn higher(&self, key: &K) -> Option> @@ -209,7 +209,7 @@ impl StateBTreeMap { self.key_order.higher(key) } - /// Returns the smallest key in the map, which is equal or larger than the + /// Returns the smallest key in the map that is equal or larger than the /// provided key. `None` meaning no such key is present in the map. #[inline(always)] pub fn eq_or_higher(&self, key: &K) -> Option> @@ -218,7 +218,7 @@ impl StateBTreeMap { self.key_order.eq_or_higher(key) } - /// Returns the largest key in the map, which is strictly smaller than the + /// Returns the largest key in the map that is strictly smaller than the /// provided key. `None` meaning no such key is present in the map. #[inline(always)] pub fn lower(&self, key: &K) -> Option> @@ -227,7 +227,7 @@ impl StateBTreeMap { self.key_order.lower(key) } - /// Returns the largest key in the map, which is equal or smaller than the + /// Returns the largest key in the map that is equal or smaller than the /// provided key. `None` meaning no such key is present in the map. #[inline(always)] pub fn eq_or_lower(&self, key: &K) -> Option> @@ -500,7 +500,7 @@ impl StateBTreeSet { self.state_api.delete_prefix(&self.prefix).unwrap_abort(); } - /// Returns the smallest key in the set, which is strictly larger than the + /// Returns the smallest key in the set that is strictly larger than the /// provided key. `None` meaning no such key is present in the set. pub fn higher(&self, key: &K) -> Option> where @@ -538,7 +538,7 @@ impl StateBTreeSet { } } - /// Returns the smallest key in the set, which is equal or larger than the + /// Returns the smallest key in the set that is equal or larger than the /// provided key. `None` meaning no such key is present in the set. pub fn eq_or_higher(&self, key: &K) -> Option> where @@ -576,7 +576,7 @@ impl StateBTreeSet { } } - /// Returns the largest key in the set, which is strictly smaller than the + /// Returns the largest key in the set that is strictly smaller than the /// provided key. `None` meaning no such key is present in the set. pub fn lower(&self, key: &K) -> Option> where @@ -609,7 +609,7 @@ impl StateBTreeSet { } } - /// Returns the largest key in the set, which is equal or smaller than the + /// Returns the largest key in the set that is equal or smaller than the /// provided key. `None` meaning no such key is present in the set. pub fn eq_or_lower(&self, key: &K) -> Option> where @@ -1291,7 +1291,7 @@ mod wasm_test_btree { /// Should only be used while debugging and testing the btree itself. #[derive(Debug)] pub(crate) enum InvariantViolation { - /// The collection have length above 0, but no root. + /// The collection has length above 0, but no root. NonZeroLenWithNoRoot, /// The collection contain a root node, but this has no keys. ZeroKeysInRoot, @@ -1337,9 +1337,28 @@ mod wasm_test_btree { return Err(InvariantViolation::ZeroKeysInRoot); } - if !root.is_leaf() && root.children.len() != root.keys.len() + 1 { - return Err(InvariantViolation::MismatchingChildrenLenKeyLen); + for i in 1..root.keys.len() { + if &root.keys[i - 1] >= &root.keys[i] { + return Err(InvariantViolation::NodeKeysOutOfOrder); + } + } + if root.keys.len() > Node::::MAXIMUM_KEY_LEN { + return Err(InvariantViolation::KeysLenAboveMax); + } + + if root.is_leaf() { + if !root.children.is_empty() { + return Err(InvariantViolation::LeafWithChildren); + } + } else { + if root.children.len() != root.keys.len() + 1 { + return Err(InvariantViolation::MismatchingChildrenLenKeyLen); + } + if root.children.len() > Node::::MAXIMUM_CHILD_LEN { + return Err(InvariantViolation::ChildrenLenAboveMax); + } } + let mut stack = vec![(0usize, root.children)]; let mut leaf_depth = None; while let Some((node_level, mut nodes)) = stack.pop() {