diff --git a/src/engine.rs b/src/engine.rs index 27e5f8f..6e4c0d4 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -1,11 +1,11 @@ pub mod block; -pub mod port_buffer; -pub mod regressor; +pub mod buffer_handler; pub mod graph; -pub mod optimizer; pub mod hogwild; +pub mod multithread_helpers; +pub mod optimizer; pub mod persistence; +pub mod port_buffer; pub mod quantization; -pub mod buffer_handler; -pub mod multithread_helpers; +pub mod regressor; pub mod serving; diff --git a/src/engine/block.rs b/src/engine/block.rs index 172264e..d589bf0 100644 --- a/src/engine/block.rs +++ b/src/engine/block.rs @@ -1,4 +1,5 @@ pub mod ffm; +pub mod file; pub mod iterators; pub mod loss_functions; pub mod lr; @@ -8,4 +9,3 @@ pub mod normalize; pub mod relu; #[cfg(test)] pub mod test; -pub mod file; \ No newline at end of file diff --git a/src/engine/block/ffm.rs b/src/engine/block/ffm.rs index 6e40f96..f430059 100644 --- a/src/engine/block/ffm.rs +++ b/src/engine/block/ffm.rs @@ -12,18 +12,18 @@ use merand48::*; use optimizer::OptimizerTrait; -use crate::engine::block::{file, iterators}; use crate::engine::block::iterators::OptimizerData; -use crate::namespace::feature_buffer; -use crate::namespace::feature_buffer::{FeatureBuffer, HashAndValueAndSeq}; +use crate::engine::block::{file, iterators}; use crate::engine::graph; -use crate::model_instance; use crate::engine::optimizer; use crate::engine::port_buffer; use crate::engine::port_buffer::PortBuffer; use crate::engine::quantization; use crate::engine::regressor; use crate::engine::regressor::{BlockCache, BlockTrait, FFM_CONTRA_BUF_LEN}; +use crate::model_instance; +use crate::namespace::feature_buffer; +use crate::namespace::feature_buffer::{FeatureBuffer, HashAndValueAndSeq}; const FFM_STACK_BUF_LEN: usize = 170393; const STEP: usize = 4; @@ -294,7 +294,7 @@ impl BlockTrait for BlockFFM { fb.ffm_buffer.len() * (self.ffm_k * self.ffm_num_fields) as usize; if local_data_ffm_len < FFM_STACK_BUF_LEN { // Fast-path - using on-stack data structures - let local_data_ffm_values: [f32; FFM_STACK_BUF_LEN ] = + let local_data_ffm_values: [f32; FFM_STACK_BUF_LEN] = MaybeUninit::uninit().assume_init(); core_macro!(local_data_ffm_values); } else { @@ -1203,11 +1203,11 @@ impl BlockFFM { #[cfg(test)] mod tests { use crate::assert_epsilon; - use crate::engine::block::test::{slearn2, spredict2, spredict2_with_cache, ssetup_cache2}; use crate::engine::block::loss_functions; - use crate::namespace::feature_buffer::HashAndValueAndSeq; + use crate::engine::block::test::{slearn2, spredict2, spredict2_with_cache, ssetup_cache2}; use crate::engine::graph::BlockGraph; use crate::model_instance::Optimizer; + use crate::namespace::feature_buffer::HashAndValueAndSeq; // Note this useful idiom: importing names from outer (for mod tests) scope. use super::*; diff --git a/src/engine/block/file.rs b/src/engine/block/file.rs index 3669ff3..ebdda64 100644 --- a/src/engine/block/file.rs +++ b/src/engine/block/file.rs @@ -1,15 +1,14 @@ -use std::error::Error; -use std::{io, mem, slice}; -use std::cmp::min; -use std::io::{Read, Write}; use crate::engine::block::iterators::WeightAndOptimizerData; use crate::engine::optimizer::{OptimizerSGD, OptimizerTrait}; - +use std::cmp::min; +use std::error::Error; +use std::io::{Read, Write}; +use std::{io, mem, slice}; pub fn read_weights_from_buf( weights: &mut Vec, input_bufreader: &mut dyn Read, - _use_quantization: bool + _use_quantization: bool, ) -> Result<(), Box> { if weights.is_empty() { return Err("Loading weights to unallocated weighs buffer".to_string())?; @@ -39,7 +38,7 @@ pub fn skip_weights_from_buf( pub fn write_weights_to_buf( weights: &Vec, output_bufwriter: &mut dyn Write, - _use_quantization: bool + _use_quantization: bool, ) -> Result<(), Box> { if weights.is_empty() { assert!(false); @@ -85,4 +84,4 @@ pub fn read_weights_only_from_buf2( } } Ok(()) -} \ No newline at end of file +} diff --git a/src/engine/block/iterators.rs b/src/engine/block/iterators.rs index 11cb457..b6e3c5e 100644 --- a/src/engine/block/iterators.rs +++ b/src/engine/block/iterators.rs @@ -1,9 +1,8 @@ use crate::engine::optimizer::OptimizerTrait; -use crate::namespace::feature_buffer; use crate::engine::port_buffer; use crate::engine::regressor::{BlockCache, BlockTrait}; - +use crate::namespace::feature_buffer; #[derive(Clone, Debug)] #[repr(C)] diff --git a/src/engine/block/loss_functions.rs b/src/engine/block/loss_functions.rs index 4c1af03..cbeec16 100644 --- a/src/engine/block/loss_functions.rs +++ b/src/engine/block/loss_functions.rs @@ -2,14 +2,14 @@ use std::any::Any; use std::error::Error; use crate::engine::block::iterators; -use crate::namespace::feature_buffer; -use crate::namespace::feature_buffer::FeatureBuffer; use crate::engine::graph; use crate::engine::port_buffer; use crate::engine::port_buffer::PortBuffer; use crate::engine::regressor; use crate::engine::regressor::BlockCache; use crate::engine::regressor::BlockTrait; +use crate::namespace::feature_buffer; +use crate::namespace::feature_buffer::FeatureBuffer; #[inline(always)] pub fn logistic(t: f32) -> f32 { @@ -42,11 +42,7 @@ pub fn new_logloss_block( impl BlockSigmoid { #[inline(always)] - fn internal_forward( - &self, - fb: &FeatureBuffer, - pb: &mut PortBuffer, - ) { + fn internal_forward(&self, fb: &FeatureBuffer, pb: &mut PortBuffer) { unsafe { debug_assert!(self.input_offset != usize::MAX); debug_assert!(self.output_offset != usize::MAX); diff --git a/src/engine/block/lr.rs b/src/engine/block/lr.rs index 0bc7317..f727c91 100644 --- a/src/engine/block/lr.rs +++ b/src/engine/block/lr.rs @@ -1,9 +1,9 @@ use std::any::Any; use crate::engine::graph; -use crate::model_instance; use crate::engine::optimizer; use crate::engine::regressor; +use crate::model_instance; use crate::namespace::{feature_buffer, parser}; use std::error::Error; @@ -12,9 +12,9 @@ use std::io; use crate::engine::block::{file, iterators}; use crate::engine::port_buffer; use crate::engine::regressor::BlockCache; +use crate::engine::regressor::BlockTrait; use iterators::WeightAndOptimizerData; use optimizer::OptimizerTrait; -use crate::engine::regressor::BlockTrait; pub struct BlockLR { pub weights: Vec>, diff --git a/src/engine/block/misc.rs b/src/engine/block/misc.rs index f48ed2a..420ccdd 100644 --- a/src/engine/block/misc.rs +++ b/src/engine/block/misc.rs @@ -3,10 +3,10 @@ use std::error::Error; use crate::engine::block::iterators; use crate::engine::graph; -use crate::namespace::feature_buffer::FeatureBuffer; use crate::engine::port_buffer::PortBuffer; use crate::engine::regressor::BlockCache; use crate::engine::regressor::BlockTrait; +use crate::namespace::feature_buffer::FeatureBuffer; #[derive(PartialEq)] pub enum Observe { @@ -941,12 +941,7 @@ mod tests { let triangle_block = new_triangle_block(&mut bg, observe_block_backward).unwrap(); let observe_block_forward = new_observe_block(&mut bg, triangle_block, Observe::Forward, None).unwrap(); - new_sink_block( - &mut bg, - observe_block_forward, - SinkType::Untouched, - ) - .unwrap(); + new_sink_block(&mut bg, observe_block_forward, SinkType::Untouched).unwrap(); bg.finalize(); bg.allocate_and_init_weights(&mi); @@ -972,11 +967,9 @@ mod tests { let (copy_block_1, copy_block_2) = new_copy_block_2(&mut bg, observe_block_backward).unwrap(); let _observe_block_1_forward = - new_observe_block(&mut bg, copy_block_1, Observe::Forward, Some(5.0)) - .unwrap(); + new_observe_block(&mut bg, copy_block_1, Observe::Forward, Some(5.0)).unwrap(); let _observe_block_2_forward = - new_observe_block(&mut bg, copy_block_2, Observe::Forward, Some(6.0)) - .unwrap(); + new_observe_block(&mut bg, copy_block_2, Observe::Forward, Some(6.0)).unwrap(); bg.finalize(); bg.allocate_and_init_weights(&mi); @@ -1015,14 +1008,11 @@ mod tests { let (copy_block_3, copy_block_4) = new_copy_block_2(&mut bg, copy_block_1).unwrap(); let _observe_block_1_forward = - new_observe_block(&mut bg, copy_block_2, Observe::Forward, Some(5.0)) - .unwrap(); + new_observe_block(&mut bg, copy_block_2, Observe::Forward, Some(5.0)).unwrap(); let _observe_block_2_forward = - new_observe_block(&mut bg, copy_block_3, Observe::Forward, Some(6.0)) - .unwrap(); + new_observe_block(&mut bg, copy_block_3, Observe::Forward, Some(6.0)).unwrap(); let _observe_block_3_forward = - new_observe_block(&mut bg, copy_block_4, Observe::Forward, Some(7.0)) - .unwrap(); + new_observe_block(&mut bg, copy_block_4, Observe::Forward, Some(7.0)).unwrap(); bg.finalize(); bg.allocate_and_init_weights(&mi); @@ -1057,8 +1047,7 @@ mod tests { let input_block_1 = new_const_block(&mut bg, vec![2.0, 3.0]).unwrap(); let join_block = new_join_block(&mut bg, vec![input_block_1]).unwrap(); let _observe_block = - new_observe_block(&mut bg, join_block, Observe::Forward, Some(6.0)) - .unwrap(); + new_observe_block(&mut bg, join_block, Observe::Forward, Some(6.0)).unwrap(); bg.finalize(); bg.allocate_and_init_weights(&mi); @@ -1079,8 +1068,7 @@ mod tests { let input_block_2 = new_const_block(&mut bg, vec![4.0, 5.0, 6.0]).unwrap(); let join_block = new_join_block(&mut bg, vec![input_block_1, input_block_2]).unwrap(); let _observe_block = - new_observe_block(&mut bg, join_block, Observe::Forward, Some(6.0)) - .unwrap(); + new_observe_block(&mut bg, join_block, Observe::Forward, Some(6.0)).unwrap(); bg.finalize(); bg.allocate_and_init_weights(&mi); @@ -1103,8 +1091,7 @@ mod tests { let join_block = new_join_block(&mut bg, vec![input_block_1, input_block_2, input_block_3]).unwrap(); let _observe_block = - new_observe_block(&mut bg, join_block, Observe::Forward, Some(6.0)) - .unwrap(); + new_observe_block(&mut bg, join_block, Observe::Forward, Some(6.0)).unwrap(); bg.finalize(); bg.allocate_and_init_weights(&mi); @@ -1128,8 +1115,7 @@ mod tests { let input_block_3 = new_const_block(&mut bg, vec![6.0, 7.0]).unwrap(); let join_block_2 = new_join_block(&mut bg, vec![input_block_3, join_block_1]).unwrap(); let _observe_block = - new_observe_block(&mut bg, join_block_2, Observe::Forward, Some(6.0)) - .unwrap(); + new_observe_block(&mut bg, join_block_2, Observe::Forward, Some(6.0)).unwrap(); bg.finalize(); bg.allocate_and_init_weights(&mi); @@ -1151,12 +1137,10 @@ mod tests { let input_block_1 = new_const_block(&mut bg, vec![2.0, 3.0]).unwrap(); let observe_block_backward = new_observe_block(&mut bg, input_block_1, Observe::Backward, None).unwrap(); - let (copy_1, copy_2) = - new_copy_block_2(&mut bg, observe_block_backward).unwrap(); + let (copy_1, copy_2) = new_copy_block_2(&mut bg, observe_block_backward).unwrap(); let join_block = new_join_block(&mut bg, vec![copy_1, copy_2]).unwrap(); let _observe_block = - new_observe_block(&mut bg, join_block, Observe::Forward, Some(6.0)) - .unwrap(); + new_observe_block(&mut bg, join_block, Observe::Forward, Some(6.0)).unwrap(); bg.finalize(); bg.allocate_and_init_weights(&mi); diff --git a/src/engine/block/neural.rs b/src/engine/block/neural.rs index fd71239..766fbf5 100644 --- a/src/engine/block/neural.rs +++ b/src/engine/block/neural.rs @@ -7,18 +7,18 @@ use std::io; use std::io::Error as IOError; use std::io::ErrorKind; -use crate::engine::block::{file, iterators}; use crate::engine::block::misc; +use crate::engine::block::{file, iterators}; use crate::engine::graph; -use crate::model_instance; use crate::engine::optimizer; use crate::engine::regressor::BlockCache; +use crate::engine::regressor::BlockTrait; +use crate::model_instance; use iterators::OptimizerData; use optimizer::OptimizerTrait; -use crate::engine::regressor::BlockTrait; -use crate::namespace::feature_buffer::FeatureBuffer; use crate::engine::port_buffer::PortBuffer; +use crate::namespace::feature_buffer::FeatureBuffer; use blas::*; const MAX_NUM_INPUTS: usize = 16000; @@ -483,11 +483,11 @@ mod tests { // Note this useful idiom: importing names from outer (for mod tests) scope. use super::*; use crate::assert_epsilon; - use crate::engine:: block::misc; + use crate::engine::block::misc; use crate::engine::block::misc::Observe; + use crate::engine::block::test::slearn2; use crate::engine::graph::BlockGraph; use crate::model_instance::Optimizer; - use crate::engine::block::test::slearn2; fn fb_vec() -> FeatureBuffer { FeatureBuffer { @@ -521,8 +521,7 @@ mod tests { ) .unwrap(); let _observe_block = - misc::new_observe_block(&mut bg, neuron_block, Observe::Forward, Some(1.0)) - .unwrap(); + misc::new_observe_block(&mut bg, neuron_block, Observe::Forward, Some(1.0)).unwrap(); bg.finalize(); bg.allocate_and_init_weights(&mi); @@ -556,8 +555,7 @@ mod tests { ) .unwrap(); let _observe_block = - misc::new_observe_block(&mut bg, neuron_block, Observe::Forward, Some(1.0)) - .unwrap(); + misc::new_observe_block(&mut bg, neuron_block, Observe::Forward, Some(1.0)).unwrap(); bg.finalize(); bg.allocate_and_init_weights(&mi); diff --git a/src/engine/block/normalize.rs b/src/engine/block/normalize.rs index 4fcd527..830e35b 100644 --- a/src/engine/block/normalize.rs +++ b/src/engine/block/normalize.rs @@ -2,12 +2,12 @@ use std::any::Any; use std::error::Error; use crate::engine::block::iterators; -use crate::namespace::feature_buffer::FeatureBuffer; use crate::engine::graph; -use crate::model_instance; use crate::engine::port_buffer::PortBuffer; use crate::engine::regressor::BlockCache; use crate::engine::regressor::BlockTrait; +use crate::model_instance; +use crate::namespace::feature_buffer::FeatureBuffer; const EPS: f32 = 1e-2; diff --git a/src/engine/block/relu.rs b/src/engine/block/relu.rs index e69a0f8..02d79ab 100644 --- a/src/engine/block/relu.rs +++ b/src/engine/block/relu.rs @@ -2,12 +2,12 @@ use std::any::Any; use std::error::Error; use crate::engine::block::iterators; -use crate::namespace::feature_buffer::FeatureBuffer; use crate::engine::graph::{BlockGraph, BlockPtrOutput, InputSlot, OutputSlot}; -use crate::model_instance; use crate::engine::port_buffer::PortBuffer; use crate::engine::regressor::BlockCache; use crate::engine::regressor::BlockTrait; +use crate::model_instance; +use crate::namespace::feature_buffer::FeatureBuffer; pub struct BlockRELU { pub num_inputs: usize, @@ -136,8 +136,8 @@ mod tests { use crate::assert_epsilon; use crate::engine::block::misc; use crate::engine::block::test; - use test::slearn2; use misc::Observe; + use test::slearn2; fn fb_vec() -> FeatureBuffer { FeatureBuffer { @@ -156,8 +156,7 @@ mod tests { let input_block = misc::new_const_block(&mut bg, vec![2.0]).unwrap(); let relu_block = new_relu_block(&mut bg, &mi, input_block).unwrap(); let _observe_block = - misc::new_observe_block(&mut bg, relu_block, Observe::Forward, Some(1.0)) - .unwrap(); + misc::new_observe_block(&mut bg, relu_block, Observe::Forward, Some(1.0)).unwrap(); bg.finalize(); bg.allocate_and_init_weights(&mi); diff --git a/src/engine/block/test.rs b/src/engine/block/test.rs index e182286..e6a8f4c 100644 --- a/src/engine/block/test.rs +++ b/src/engine/block/test.rs @@ -1,5 +1,8 @@ #[cfg(test)] -use crate::{namespace::feature_buffer, engine:: {graph, port_buffer, regressor::BlockCache}}; +use crate::{ + engine::{graph, port_buffer, regressor::BlockCache}, + namespace::feature_buffer, +}; #[cfg(test)] #[macro_export] @@ -45,7 +48,6 @@ pub fn ssetup_cache2( ); } - #[cfg(test)] pub fn spredict2_with_cache( bg: &mut graph::BlockGraph, @@ -70,4 +72,4 @@ pub fn spredict2( let (block_run, further_blocks) = bg.blocks_final.split_at(1); block_run[0].forward(further_blocks, fb, pb); pb.observations[0] -} \ No newline at end of file +} diff --git a/src/engine/graph.rs b/src/engine/graph.rs index 1876396..8409186 100644 --- a/src/engine/graph.rs +++ b/src/engine/graph.rs @@ -1,7 +1,7 @@ use crate::engine::block::misc; -use crate::{engine::block, model_instance}; use crate::engine::port_buffer; use crate::engine::regressor::BlockTrait; +use crate::{engine::block, model_instance}; use std::error::Error; use std::mem; @@ -429,8 +429,7 @@ mod tests { // Using the join block, we merge two outputs into one single output (copy-less implementation) let union_output = - misc::new_join_block(&mut bg, vec![const_block_output1, const_block_output2]) - .unwrap(); + misc::new_join_block(&mut bg, vec![const_block_output1, const_block_output2]).unwrap(); assert_eq!(union_output, BlockPtrOutput(BlockPtr(2), OutputSlot(0))); assert_eq!(bg.nodes[0].edges_in, vec![]); assert_eq!( @@ -501,32 +500,24 @@ mod tests { let const_2 = misc::new_const_block(&mut bg, vec![3.0]).unwrap(); // 1 let const_3 = misc::new_const_block(&mut bg, vec![4.0]).unwrap(); // 2 let const_4 = misc::new_const_block(&mut bg, vec![4.0]).unwrap(); // 3 - let (copy_output_1, copy_output_2) = - misc::new_copy_block_2(&mut bg, const_1).unwrap(); // 4 - let (_copy_output_3, _copy_output_4) = - misc::new_copy_block_2(&mut bg, const_2).unwrap(); // 5 + let (copy_output_1, copy_output_2) = misc::new_copy_block_2(&mut bg, const_1).unwrap(); // 4 + let (_copy_output_3, _copy_output_4) = misc::new_copy_block_2(&mut bg, const_2).unwrap(); // 5 // this is not zero copy let _join_1 = misc::new_join_block(&mut bg, vec![copy_output_1, const_3]).unwrap(); // 6 - // this is zero copy + // this is zero copy let _join_2 = misc::new_join_block(&mut bg, vec![const_4, copy_output_2]); // 7 bg.finalize(); let mut list = bg.take_blocks(); { // first one goes to copy again, so it cannot use input as an output - let copy_block_1 = list[4] - .as_any() - .downcast_mut::() - .unwrap(); + let copy_block_1 = list[4].as_any().downcast_mut::().unwrap(); assert_ne!(copy_block_1.input_offset, copy_block_1.output_offsets[0]); } { // But second one can re-use the input as output - let copy_block_2 = list[5] - .as_any() - .downcast_mut::() - .unwrap(); + let copy_block_2 = list[5].as_any().downcast_mut::().unwrap(); assert_eq!(copy_block_2.input_offset, copy_block_2.output_offsets[0]); } } diff --git a/src/engine/hogwild.rs b/src/engine/hogwild.rs index aae1b3d..3c7fef3 100644 --- a/src/engine/hogwild.rs +++ b/src/engine/hogwild.rs @@ -3,10 +3,10 @@ use std::sync::{mpsc, Arc, Mutex}; use std::thread; use std::thread::JoinHandle; -use crate::namespace::feature_buffer::FeatureBufferTranslator; -use crate::model_instance::ModelInstance; use crate::engine::multithread_helpers::BoxedRegressorTrait; use crate::engine::port_buffer::PortBuffer; +use crate::model_instance::ModelInstance; +use crate::namespace::feature_buffer::FeatureBufferTranslator; static CHANNEL_CAPACITY: usize = 100_000; diff --git a/src/engine/persistence.rs b/src/engine/persistence.rs index 502e873..a344838 100644 --- a/src/engine/persistence.rs +++ b/src/engine/persistence.rs @@ -7,8 +7,8 @@ use std::fs::File; use std::io; use std::io::Read; -use crate::model_instance; use crate::engine::regressor; +use crate::model_instance; use crate::namespace::vwmap; use crate::engine::multithread_helpers::BoxedRegressorTrait; @@ -207,14 +207,14 @@ mod tests { // Note this useful idiom: importing names from outer (for mod tests) scope. use super::*; use crate::assert_epsilon; - use crate::namespace::feature_buffer; - use crate::namespace::feature_buffer::{HashAndValue, HashAndValueAndSeq}; - use crate::model_instance::Optimizer; + use crate::engine::block::ffm; use crate::engine::optimizer; use crate::engine::optimizer::OptimizerTrait; - use crate::engine::block::ffm; use crate::engine::regressor::BlockTrait; use crate::engine::regressor::Regressor; + use crate::model_instance::Optimizer; + use crate::namespace::feature_buffer; + use crate::namespace::feature_buffer::{HashAndValue, HashAndValueAndSeq}; use tempfile::tempdir; diff --git a/src/engine/regressor.rs b/src/engine/regressor.rs index 6c5d8ca..2faf35b 100644 --- a/src/engine/regressor.rs +++ b/src/engine/regressor.rs @@ -5,6 +5,7 @@ use std::error::Error; use std::io; use std::io::Cursor; +use crate::engine::block; use crate::engine::block::ffm; use crate::engine::block::iterators; use crate::engine::block::loss_functions; @@ -14,12 +15,11 @@ use crate::engine::block::neural; use crate::engine::block::neural::InitType; use crate::engine::block::normalize; use crate::engine::block::relu; -use crate::engine::block; -use crate::namespace::feature_buffer; -use crate::namespace::feature_buffer::HashAndValueAndSeq; use crate::engine::graph; -use crate::model_instance; use crate::engine::port_buffer; +use crate::model_instance; +use crate::namespace::feature_buffer; +use crate::namespace::feature_buffer::HashAndValueAndSeq; pub const FFM_CONTRA_BUF_LEN: usize = 41472; @@ -294,21 +294,18 @@ impl Regressor { .unwrap(); if layernorm == NNLayerNorm::BeforeRelu { - output = - normalize::new_normalize_layer_block(&mut bg, mi, output).unwrap(); + output = normalize::new_normalize_layer_block(&mut bg, mi, output).unwrap(); } if activation == NNActivation::Relu { output = relu::new_relu_block(&mut bg, mi, output).unwrap(); } if layernorm == NNLayerNorm::AfterRelu { - output = - normalize::new_normalize_layer_block(&mut bg, mi, output).unwrap(); + output = normalize::new_normalize_layer_block(&mut bg, mi, output).unwrap(); } } // If we have split if join_block.is_some() { - output = - misc::new_join_block(&mut bg, vec![output, join_block.unwrap()]).unwrap(); + output = misc::new_join_block(&mut bg, vec![output, join_block.unwrap()]).unwrap(); } output = neural::new_neuron_block( &mut bg, @@ -539,8 +536,8 @@ impl Regressor { mod tests { // Note this useful idiom: importing names from outer (for mod tests) scope. use super::*; - use crate::namespace::feature_buffer::HashAndValue; use crate::engine::optimizer; + use crate::namespace::feature_buffer::HashAndValue; /* LR TESTS */ fn lr_vec(v: Vec) -> feature_buffer::FeatureBuffer { diff --git a/src/engine/serving.rs b/src/engine/serving.rs index 5332786..c206eb1 100644 --- a/src/engine/serving.rs +++ b/src/engine/serving.rs @@ -9,14 +9,14 @@ use std::sync::Arc; use std::sync::Mutex; use std::thread; -use crate::namespace::feature_buffer; -use crate::model_instance; use crate::engine::multithread_helpers::BoxedRegressorTrait; -use crate::namespace::parser; -use crate::persistence; use crate::engine::port_buffer; use crate::engine::regressor; +use crate::model_instance; +use crate::namespace::feature_buffer; +use crate::namespace::parser; use crate::namespace::vwmap; +use crate::persistence; pub struct Serving { listening_interface: String, @@ -257,8 +257,8 @@ impl Serving { mod tests { // Note this useful idiom: importing names from outer (for mod tests) scope. use super::*; - use crate::namespace::feature_buffer; use crate::engine::regressor; + use crate::namespace::feature_buffer; use mockstream::{FailingMockStream, SharedMockStream}; use std::io::ErrorKind; use std::str; diff --git a/src/lib.rs b/src/lib.rs index 25673c4..6d7d7f3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,5 @@ -pub mod engine; pub mod cmdline; +pub mod engine; pub mod logging; pub mod model_instance; pub mod namespace; @@ -8,17 +8,17 @@ extern crate blas; extern crate half; extern crate intel_mkl_src; -use crate::namespace::feature_buffer::FeatureBufferTranslator; use crate::engine::multithread_helpers::BoxedRegressorTrait; -use crate::namespace::parser::VowpalParser; use crate::engine::port_buffer::PortBuffer; use crate::engine::regressor::BlockCache; +use crate::namespace::feature_buffer::FeatureBufferTranslator; +use crate::namespace::parser::VowpalParser; use crate::namespace::vwmap::NamespaceType; +use engine::persistence; use shellwords; use std::ffi::CStr; use std::io::Cursor; use std::os::raw::c_char; -use engine::persistence; const EOF_ERROR_CODE: f32 = -1.0; const EXCEPTION_ERROR_CODE: f32 = -1.0; diff --git a/src/logging.rs b/src/logging.rs index 6676349..dcf3064 100644 --- a/src/logging.rs +++ b/src/logging.rs @@ -44,4 +44,4 @@ fn log_detected_x86_features() { } else { log::info!("Detected CPU features: {:?}", features.join(", ")); } -} \ No newline at end of file +} diff --git a/src/main.rs b/src/main.rs index 7aff2d5..cadfdbb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -23,20 +23,20 @@ extern crate intel_mkl_src; extern crate nom; extern crate core; -use fw::namespace::cache::RecordCache; -use fw::namespace::feature_buffer::FeatureBufferTranslator; +use fw::engine::buffer_handler::create_buffered_input; use fw::engine::hogwild::HogwildTrainer; -use fw::model_instance::{ModelInstance, Optimizer}; use fw::engine::multithread_helpers::BoxedRegressorTrait; -use fw::namespace::parser::VowpalParser; -use fw::engine::buffer_handler::create_buffered_input; -use fw::persistence::{ - new_regressor_from_filename, save_regressor_to_filename, save_sharable_regressor_to_filename, -}; use fw::engine::regressor; use fw::engine::regressor::{get_regressor_with_weights, Regressor}; use fw::engine::serving::Serving; +use fw::model_instance::{ModelInstance, Optimizer}; +use fw::namespace::cache::RecordCache; +use fw::namespace::feature_buffer::FeatureBufferTranslator; +use fw::namespace::parser::VowpalParser; use fw::namespace::vwmap::VwNamespaceMap; +use fw::persistence::{ + new_regressor_from_filename, save_regressor_to_filename, save_sharable_regressor_to_filename, +}; use fw::{cmdline, logging}; fn main() { @@ -138,8 +138,7 @@ fn main_fw_loop() -> Result<(), Box> { let filename = cl .value_of("initial_regressor") .expect("Convert mode requires --initial regressor"); - let (mut mi2, vw2, re_fixed) = - new_regressor_from_filename(filename, true, Some(&cl))?; + let (mut mi2, vw2, re_fixed) = new_regressor_from_filename(filename, true, Some(&cl))?; mi2.optimizer = Optimizer::SGD; if cl.is_present("weight_quantization") { mi2.dequantize_weights = Some(true); diff --git a/src/model_instance.rs b/src/model_instance.rs index a8549cc..ac13177 100644 --- a/src/model_instance.rs +++ b/src/model_instance.rs @@ -174,11 +174,8 @@ impl ModelInstance { let namespaces_str = vsplit[0]; let mut namespace_descriptors: Vec = Vec::new(); for char in namespaces_str.chars() { - let namespace_descriptor = parser::get_namespace_descriptor( - &self.transform_namespaces, - vw, - char, - )?; + let namespace_descriptor = + parser::get_namespace_descriptor(&self.transform_namespaces, vw, char)?; namespace_descriptors.push(namespace_descriptor); } Ok(FeatureComboDesc { @@ -390,11 +387,8 @@ impl ModelInstance { for namespaces_str in in_v { let mut field: Vec = Vec::new(); for char in namespaces_str.chars() { - let namespace_descriptor = parser::get_namespace_descriptor( - &mi.transform_namespaces, - vw, - char, - )?; + let namespace_descriptor = + parser::get_namespace_descriptor(&mi.transform_namespaces, vw, char)?; field.push(namespace_descriptor); } mi.ffm_fields.push(field); @@ -555,7 +549,9 @@ impl ModelInstance { mod tests { // Note this useful idiom: importing names from outer (for mod tests) scope. use super::*; - use crate::namespace::vwmap::{NamespaceDescriptor, NamespaceFormat, NamespaceType, VwNamespaceMap}; + use crate::namespace::vwmap::{ + NamespaceDescriptor, NamespaceFormat, NamespaceType, VwNamespaceMap, + }; fn ns_desc(i: u16) -> NamespaceDescriptor { NamespaceDescriptor { diff --git a/src/namespace.rs b/src/namespace.rs index 923e385..221e3c6 100644 --- a/src/namespace.rs +++ b/src/namespace.rs @@ -1,6 +1,6 @@ +pub mod cache; pub mod feature; +pub mod feature_buffer; pub mod parser; pub mod radix_tree; pub mod vwmap; -pub mod feature_buffer; -pub mod cache; diff --git a/src/namespace/feature.rs b/src/namespace/feature.rs index 2ad0dfe..f6ae86e 100644 --- a/src/namespace/feature.rs +++ b/src/namespace/feature.rs @@ -1,3 +1,3 @@ pub mod executors; -pub mod transformers; pub mod parser; +pub mod transformers; diff --git a/src/namespace/feature/executors.rs b/src/namespace/feature/executors.rs index 1eb41a6..2bebc42 100644 --- a/src/namespace/feature/executors.rs +++ b/src/namespace/feature/executors.rs @@ -86,8 +86,8 @@ impl ExecutorToNamespace { let hash_index = murmur3::hash32_with_seed(to_data1.to_le_bytes(), unsafe { *self.namespace_seeds.get_unchecked(SEED_ID) }); - let hash_index = - murmur3::hash32_with_seed(to_data2.to_le_bytes(), hash_index) & transformers::parser::MASK31; + let hash_index = murmur3::hash32_with_seed(to_data2.to_le_bytes(), hash_index) + & transformers::parser::MASK31; self.tmp_data.push((hash_index, hash_value)); } } diff --git a/src/namespace/feature/parser.rs b/src/namespace/feature/parser.rs index aca4948..bc93e3f 100644 --- a/src/namespace/feature/parser.rs +++ b/src/namespace/feature/parser.rs @@ -1,9 +1,7 @@ -use serde::{Deserialize, Serialize}; -use std::cell::Cell; -use std::collections::HashMap; -use std::error::Error; -use std::io::Error as IOError; -use std::io::ErrorKind; +use crate::namespace::feature; +use crate::namespace::vwmap::{ + NamespaceDescriptor, NamespaceFormat, NamespaceType, VwNamespaceMap, +}; use nom::bytes::complete::take_while; use nom::character; use nom::character::complete; @@ -11,8 +9,12 @@ use nom::number; use nom::sequence::tuple; use nom::AsChar; use nom::IResult; -use crate::namespace::feature; -use crate::namespace::vwmap::{NamespaceDescriptor, NamespaceFormat, NamespaceType, VwNamespaceMap}; +use serde::{Deserialize, Serialize}; +use std::cell::Cell; +use std::collections::HashMap; +use std::error::Error; +use std::io::Error as IOError; +use std::io::ErrorKind; #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] pub struct Namespace { @@ -370,7 +372,9 @@ pub fn parse_namespace_statement( mod tests { // Note this useful idiom: importing names from outer (for mod tests) scope. use super::*; - use crate::namespace::vwmap::{NamespaceDescriptor, NamespaceFormat, NamespaceType, VwNamespaceMap}; + use crate::namespace::vwmap::{ + NamespaceDescriptor, NamespaceFormat, NamespaceType, VwNamespaceMap, + }; fn ns_desc(i: u16) -> NamespaceDescriptor { NamespaceDescriptor { diff --git a/src/namespace/feature_buffer.rs b/src/namespace/feature_buffer.rs index 6c29cec..a22905b 100644 --- a/src/namespace/feature_buffer.rs +++ b/src/namespace/feature_buffer.rs @@ -1,5 +1,5 @@ -use crate::namespace::feature::executors; use crate::model_instance; +use crate::namespace::feature::executors; use crate::namespace::parser; use crate::namespace::vwmap::{NamespaceFormat, NamespaceType}; @@ -164,10 +164,9 @@ impl FeatureBufferTranslator { feature_buffer: fb, lr_hash_mask, ffm_hash_mask, - transform_executors: - executors::TransformExecutors::from_namespace_transforms( - &mi.transform_namespaces, - ), + transform_executors: executors::TransformExecutors::from_namespace_transforms( + &mi.transform_namespaces, + ), } } @@ -517,11 +516,7 @@ mod tests { fbt.translate(&rb, 0); assert_eq!(fbt.feature_buffer.lr_buffer, vec![]); // since the other feature is missing - VW compatibility says no feature is here - let rb = add_header(vec![ - 2988156968 & MASK31, - 2422381320 & MASK31, - NO_FEATURES, - ]); + let rb = add_header(vec![2988156968 & MASK31, 2422381320 & MASK31, NO_FEATURES]); fbt.translate(&rb, 0); assert_eq!(