make maxMinorTokens constructor parametrized (#13)

* make maxMinorTokens constructor parametrized * constructor args are now final again
teragrep · Oct 12, 2023 · ae2935e · ae2935e
1 parent 949223a
commit ae2935e
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/src/main/scala/com/teragrep/functions/dpf_03/TokenAggregator.scala b/src/main/scala/com/teragrep/functions/dpf_03/TokenAggregator.scala
@@ -56,13 +56,13 @@ import org.apache.spark.unsafe.types.UTF8String
 import java.nio.charset.StandardCharsets
 import scala.reflect.ClassTag
 
-class TokenAggregator(private final val columnName: String) extends Aggregator[Row, TokenBuffer, Set[String]]
+class TokenAggregator(final val columnName: String, final val maxMinorTokens: Long) extends Aggregator[Row, TokenBuffer, Set[String]]
   with Serializable {
 
   var tokenizer: Option[Tokenizer] = None
 
   override def zero(): TokenBuffer = {
-    tokenizer = Some(new Tokenizer(32))
+    tokenizer = Some(new Tokenizer(maxMinorTokens))
     new TokenBuffer()
   }