Skip to content

Commit

Permalink
[facets] remove faceted search functionality
Browse files Browse the repository at this point in the history
  • Loading branch information
Anastasios Zouzias authored and zouzias committed Apr 13, 2019
1 parent a3413dc commit 4b84ad3
Show file tree
Hide file tree
Showing 14 changed files with 19 additions and 631 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,7 @@ import com.twitter.algebird.TopK
import com.twitter.chill.Kryo
import org.apache.spark.SparkConf
import org.apache.spark.serializer.{KryoRegistrator, KryoSerializer}
import org.zouzias.spark.lucenerdd.facets.FacetedLuceneRDD
import org.zouzias.spark.lucenerdd.models.{SparkDoc, SparkFacetResult, SparkScoreDoc}
import org.zouzias.spark.lucenerdd.models.{SparkDoc, SparkScoreDoc}
import org.zouzias.spark.lucenerdd.partition.LuceneRDDPartition
import org.zouzias.spark.lucenerdd.response.{LuceneRDDResponse, LuceneRDDResponsePartition}
import org.zouzias.spark.lucenerdd.testing.{FavoriteCaseClass, Person}
Expand All @@ -30,7 +29,6 @@ class LuceneRDDKryoRegistrator extends KryoRegistrator {
def registerClasses(kryo: Kryo): Unit = {
kryo.register(classOf[LuceneRDD[_]])
kryo.register(classOf[LuceneRDDPartition[_]])
kryo.register(classOf[FacetedLuceneRDD[_]])
kryo.register(classOf[SparkDoc])
kryo.register(classOf[Number])
kryo.register(classOf[java.lang.Double])
Expand All @@ -57,7 +55,6 @@ class LuceneRDDKryoRegistrator extends KryoRegistrator {
kryo.register(classOf[scala.collection.immutable.Set$EmptySet$])
kryo.register(classOf[scala.collection.immutable.Map[_, _]])
kryo.register(classOf[Array[scala.collection.immutable.Map[_, _]]])
kryo.register(classOf[SparkFacetResult])
kryo.register(classOf[SparkScoreDoc])
kryo.register(classOf[LuceneRDDResponse])
kryo.register(classOf[LuceneRDDResponsePartition])
Expand Down

This file was deleted.

165 changes: 0 additions & 165 deletions src/main/scala/org/zouzias/spark/lucenerdd/facets/package.scala

This file was deleted.

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ package org.zouzias.spark.lucenerdd.partition

import org.apache.lucene.search.{BooleanClause, Query}
import org.zouzias.spark.lucenerdd.models.indexstats.IndexStatistics
import org.zouzias.spark.lucenerdd.models.{SparkFacetResult, TermVectorEntry}
import org.zouzias.spark.lucenerdd.models.TermVectorEntry
import org.zouzias.spark.lucenerdd.response.LuceneRDDResponsePartition

import scala.reflect.ClassTag
Expand Down Expand Up @@ -81,15 +81,6 @@ private[lucenerdd] abstract class AbstractLuceneRDDPartition[T] extends Serializ
def queries(searchString: Iterable[String], topK: Int)
: Iterable[(String, LuceneRDDResponsePartition)]

/**
* Generic Lucene faceted Query using QueryParser
* @param searchString Lucene query string, i.e., textField:hello*
* @param topK Number of facets to return
* @return
*/
def facetQuery(searchString: String, facetField: String, topK: Int)
: SparkFacetResult

/**
* Term Query
* @param fieldName Name of field
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,18 +19,15 @@ package org.zouzias.spark.lucenerdd.partition
import org.apache.lucene.analysis.Analyzer
import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper
import org.apache.lucene.document._
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader
import org.apache.lucene.index.{DirectoryReader, IndexReader}
import org.apache.lucene.search._
import org.joda.time.DateTime
import org.zouzias.spark.lucenerdd.facets.FacetedLuceneRDD
import org.zouzias.spark.lucenerdd.models.indexstats.{FieldStatistics, IndexStatistics}
import org.zouzias.spark.lucenerdd.models.{SparkFacetResult, TermVectorEntry}
import org.zouzias.spark.lucenerdd.models.TermVectorEntry
import org.zouzias.spark.lucenerdd.query.{LuceneQueryHelpers, SimilarityConfigurable}
import org.zouzias.spark.lucenerdd.response.LuceneRDDResponsePartition
import org.zouzias.spark.lucenerdd.store.IndexWithTaxonomyWriter
import org.zouzias.spark.lucenerdd.LuceneRDD
import scala.collection.JavaConverters._
import org.zouzias.spark.lucenerdd.store.IndexWritable

import scala.reflect.{ClassTag, _}
import scala.collection.mutable.ArrayBuffer
Expand Down Expand Up @@ -64,7 +61,7 @@ private[lucenerdd] class LuceneRDDPartition[T]
(implicit docConversion: T => Document,
override implicit val kTag: ClassTag[T])
extends AbstractLuceneRDDPartition[T]
with IndexWithTaxonomyWriter
with IndexWritable
with SimilarityConfigurable {

logInfo(s"[partId=${partitionId}] Partition is created...")
Expand Down Expand Up @@ -92,7 +89,7 @@ private[lucenerdd] class LuceneRDDPartition[T]
iterIndex.foreach { case elem =>
// (implicitly) convert type T to Lucene document
val doc = docConversion(elem)
indexWriter.addDocument(FacetsConfig.build(taxoWriter, doc))
indexWriter.addDocument(doc)
}
private val endTime = new DateTime(System.currentTimeMillis())
logInfo(s"[partId=${partitionId}]Indexing process completed at ${endTime}...")
Expand All @@ -106,7 +103,6 @@ private[lucenerdd] class LuceneRDDPartition[T]
logDebug(s"[partId=${partitionId}]Instantiating index/facet readers")
private val indexReader = DirectoryReader.open(IndexDir)
private lazy val indexSearcher = initializeIndexSearcher(indexReader)
private val taxoReader = new DirectoryTaxonomyReader(TaxonomyDir)
logDebug(s"[partId=${partitionId}]Index readers instantiated successfully")
logInfo(s"[partId=${partitionId}]Indexed ${size} documents")

Expand Down Expand Up @@ -199,15 +195,6 @@ private[lucenerdd] class LuceneRDDPartition[T]
LuceneRDDResponsePartition(results.toIterator)
}

override def facetQuery(searchString: String,
facetField: String,
topK: Int): SparkFacetResult = {
LuceneQueryHelpers.facetedTextSearch(indexSearcher, taxoReader, FacetsConfig,
searchString,
facetField + FacetedLuceneRDD.FacetTextFieldSuffix,
topK, QueryAnalyzer)
}

override def moreLikeThis(fieldName: String, query: String,
minTermFreq: Int, minDocFreq: Int, topK: Int)
: LuceneRDDResponsePartition = {
Expand Down
Loading

0 comments on commit 4b84ad3

Please sign in to comment.