[facets] remove faceted search functionality

.travis.yml

@@ -1,4 +1,6 @@
 language: scala
+scala:
+  - 2.11.12
 sudo: false
 dist: trusty
 cache:
@@ -15,22 +17,16 @@ before_cache:
 matrix:
   include:
     - jdk: oraclejdk8
-      scala: 2.11.12
-      env: TEST_SPARK_VERSION="2.4.0" LUCENE_ANALYZER="en" LINKER_METHOD="cartesian"
+      env: TEST_SPARK_VERSION="2.4.1" LUCENERDD_ANALYZER_NAME="en" LUCENERDD_LINKER_METHOD="cartesian"
     - jdk: openjdk8
-      scala: 2.11.12
-      env: TEST_SPARK_VERSION="2.4.0" LUCENE_ANALYZER="en" LINKER_METHOD="collectbroadcast"
+      env: TEST_SPARK_VERSION="2.4.1" LUCENERDD_ANALYZER_NAME="en" LUCENERDD_LINKER_METHOD="collectbroadcast"
     - jdk: openjdk8
-      scala: 2.11.12
-      env: TEST_SPARK_VERSION="2.4.0" LUCENE_ANALYZER="whitespace" LINKER_METHOD="cartesian"
+      env: TEST_SPARK_VERSION="2.4.1" LUCENERDD_ANALYZER_NAME="whitespace" LUCENERDD_LINKER_METHOD="cartesian"
     - jdk: oraclejdk8
-      scala: 2.11.12
-      env: TEST_SPARK_VERSION="2.4.0" LUCENE_ANALYZER="whitespace" LINKER_METHOD="collectbroadcast"
+      env: TEST_SPARK_VERSION="2.4.1" LUCENERDD_ANALYZER_NAME="whitespace" LUCENERDD_LINKER_METHOD="collectbroadcast"
 script:
   - sbt ++$TRAVIS_SCALA_VERSION clean update
-      -Dlucenerdd.linker.method=${LINKER_METHOD}
-      -Dlucenerdd.spatial.linker.method=${LINKER_METHOD}
-      -Dlucenerdd.analyzer.name=${LUCENE_ANALYZER} -test
+      -Dlucenerdd.spatial.linker.method=${LUCENE_SPATIAL_LINKER_METHOD} -test
   - sbt ++$TRAVIS_SCALA_VERSION scalastyle
   - sbt ++$TRAVIS_SCALA_VERSION assembly
   - travis_wait 30 sbt ++$TRAVIS_SCALA_VERSION clean coverage test coverageReport

README.md

@@ -9,7 +9,7 @@
 
 [Spark](http://spark.apache.org) RDD with Apache [Lucene](https://lucene.apache.org)'s query capabilities.
 
-The main abstractions are special types of `RDD` called `LuceneRDD`, `FacetedLuceneRDD` and `ShapeLuceneRDD`, which instantiate a Lucene index on each Spark executor. These `RDD`s distribute search queries and aggregate search results between the Spark driver and its executors. Currently, the following queries are supported:
+The main abstractions are special types of `RDD` called `LuceneRDD` and `ShapeLuceneRDD`, which instantiate a Lucene index on each Spark executor. These `RDD`s distribute search queries and aggregate search results between the Spark driver and its executors. Currently, the following queries are supported:
 
 |Operation| Syntax| Description |
 |-------|---------------------|----------|
@@ -18,7 +18,6 @@ The main abstractions are special types of `RDD` called `LuceneRDD`, `FacetedLuc
 |Phrase Query | `LuceneRDD.phraseQuery(field, query, topK)` | Phrase search |
 |Prefix Query | `LuceneRDD.prefixSearch(field, prefix, topK)` | Prefix search |
 |Query Parser | `LuceneRDD.query(queryString, topK)` | Query parser search|
-|Faceted Search| `FacetedLuceneRDD.facetQuery(queryString, field, topK)` | Faceted Search |
 |[Record Linkage](https://github.com/zouzias/spark-lucenerdd/wiki/Record-Linkage)| `LuceneRDD.link(otherEntity: RDD[T], linkageFct: T => searchQuery, topK)`| Record linkage via Lucene queries|
 |Circle Search| `ShapeLuceneRDD.circleSearch((x,y), radius, topK)` | Search within radius |
 |Bbox Search| `ShapeLuceneRDD.bboxSearch(lowerLeft, upperLeft, topK)` | Bounding box |

build.sbt

@@ -77,10 +77,12 @@ pomExtra := <scm>
     </developer>
   </developers>
 
-val luceneV = "7.7.1"
+credentials += Credentials(Path.userHome / ".sbt" / ".credentials")
+
+val luceneV = "8.0.0"
 
 spName := "zouzias/spark-lucenerdd"
-sparkVersion := "2.4.0"
+sparkVersion := "2.4.1"
 spShortDescription := "Spark RDD with Lucene's query capabilities"
 sparkComponents ++= Seq("core", "sql", "mllib")
 spAppendScalaVersion := true
@@ -94,8 +96,8 @@ testSparkVersion := sys.props.get("spark.testVersion").getOrElse(sparkVersion.va
 
 
 // scalastyle:off
-val scalactic                 = "org.scalactic"                  %% "scalactic"                % "3.0.6"
-val scalatest                 = "org.scalatest"                  %% "scalatest"                % "3.0.6" % "test"
+val scalactic                 = "org.scalactic"                  %% "scalactic"                % "3.0.7"
+val scalatest                 = "org.scalatest"                  %% "scalatest"                % "3.0.7" % "test"
 
 val joda_time                 = "joda-time"                      % "joda-time"                 % "2.10.1"
 val algebird                  = "com.twitter"                    %% "algebird-core"            % "0.13.5"
@@ -104,7 +106,6 @@ val spatial4j                 = "org.locationtech.spatial4j"     % "spatial4j"
 
 val typesafe_config           = "com.typesafe"                   % "config"                    % "1.3.3"
 
-val lucene_facet              = "org.apache.lucene"              % "lucene-facet"              % luceneV
 val lucene_analyzers          = "org.apache.lucene"              % "lucene-analyzers-common"   % luceneV
 val lucene_query_parsers      = "org.apache.lucene"              % "lucene-queryparser"        % luceneV
 val lucene_expressions        = "org.apache.lucene"              % "lucene-expressions"        % luceneV
@@ -117,7 +118,6 @@ val jts                       = "org.locationtech.jts"           % "jts-core"
 
 libraryDependencies ++= Seq(
   algebird,
-  lucene_facet,
   lucene_analyzers,
   lucene_expressions,
   lucene_query_parsers,

project/build.properties

@@ -1 +1 @@
-sbt.version=0.13.17
+sbt.version=0.13.18

project/plugins.sbt

@@ -17,11 +17,13 @@
 
 resolvers += "bintray-spark-packages" at "https://dl.bintray.com/spark-packages/maven/"
 
-addSbtPlugin("com.timushev.sbt" % "sbt-updates" % "0.3.0")
+addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.7.0")
 
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.7")
+addSbtPlugin("com.timushev.sbt" % "sbt-updates" % "0.4.0")
 
-addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.9")
+addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.9")
+
+addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.11")
 
 addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "1.0.0")
 

src/main/resources/reference.conf

@@ -2,15 +2,12 @@ lucenerdd {
 
   // Name of analyzer as it is under Lucene's package org.apache.lucene.analysis.XX
   analyzer.name = "en"
+  analyzer.name=${?LUCENERDD_ANALYZER_NAME}
 
-  // Analyzer name must be "ngram"
-  analyzer {
-    ngram.mingram = 2
-    ngram.maxgram = 5
-  }
-
+
   // Similarity scoring for Lucenes
   similarity.name = "bm25" // anything else will default to Lucene classic similarity
+  similarity.name = ${?LUCENERDD_SIMILARITY_NAME}
 
   // Supported linkage methods
   // "collectbroadcast" : Collects the RDD that contains the queries (to be used only if query RDD
@@ -20,6 +17,7 @@ lucenerdd {
   // of LuceneRDD. Note it duplicates each partition of LuceneRDD n times where n is the number of
   // partitions of the queries RDD.
   linker.method = "collectbroadcast"
+  linker.method = ${?LUCENERDD_LINKER_METHOD}
 
   index {
 
@@ -28,17 +26,20 @@ lucenerdd {
     // Otherwise the index will be stored in memory
     // Do not use memory, see http://lucene.apache.org/core/7_5_0/core/org/apache/lucene/store/RAMDirectory.html
     store.mode = "disk"
+    store.mode = ${?LUCENERDD_INDEX_STORE_MODE}
 
     stringfields{
 
       // Analyze string fields by default or not
       // Implicit fields, like _1, _2, etc will use this option
       analyzed = true
+      analyzed = ${?LUCENERDD_INDEX_STRINGFIELDS_ANALYZED}
 
       // Select a subset of string fields that you do not wish to be analyzed
       // Due to serialization issues this list should be set before starting a Spark Session
       // Moreover, all text/string fields that end with '_notanalyzed' are not analyzed
       not_analyzed_list = []
+      not_analyzed_list = ${?LUCENERDD_INDEX_STRINGFIELDS_NOT_ANALYZED_LIST}
 
       // Text fields options as in org.apache.lucene.index.IndexOptions
       //
@@ -49,36 +50,55 @@ lucenerdd {
       // "DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS"
       // "NONE"
       options = "docs_and_freqs_and_positions_and_offsets"
+      options = ${?LUCENERDD_INDEX_STRINGFIELDS_OPTIONS}
 
-      // Omit terms norms
-      terms.omitnorms = false
+      terms {
+        // Omit terms norms
+        omitnorms = false
+        omitnorms = ${?LUCENERDD_INDEX_STRINGFIELDS_TERMS_OMITNORMS}
 
-      // Store term positions
-      terms.positions = false
+        // Store term positions
+        positions = false
+        positions = ${?LUCENERDD_INDEX_STRINGFIELDS_TERMS_POSITIONS}
 
-      // Store Term vectors (set true, otherwise LuceneRDD.termVectors(fieldName) will fail)
-      terms.vectors = true
+        // Store Term vectors (set true, otherwise LuceneRDD.termVectors(fieldName) will fail)
+        vectors = true
+        vectors = ${?LUCENERDD_INDEX_STRINGFIELDS_TERMS_VECTORS}
+      }
     }
   }
 
-  // Maximum value on topK queries
-  query.topk.maxvalue = 100
-  // Default value of number of returned results
-  query.topk.default = 10
+  query {
+    // Maximum value on topK queries
+    topk.maxvalue = 100
+    topk.maxvalue = ${?LUCENERDD_QUERY_TOPK_MAXVALUE}
 
-  // Default value of number of faceted results
-  query.facets.number.default = 10
+    // Default value of number of returned results
+    topk.default = 10
+    topk.default = ${?LUCENERDD_QUERY_TOPK_DEFAULT}
+
+  }
 
   // Spatial related configurations used by ShapeLuceneRDD
   spatial {
     prefixtree {
+
+      // Spatial tree data structure
       name = "quad"  // "geohash" or "quad"
+      name = ${?LUCENE_SPATIAL_PREFIXTREE_NAME}
+
       maxlevel = 9 // 11 results in sub-meter precision for geohash
+      maxlevel = ${?LUCENE_SPATIAL_PREFIXTREE_MAXLEVEL}
+
+
       maxDistErr = 5.0 // in kilometers
+      maxDistErr = ${?LUCENE_SPATIAL_PREFIXTREE_MAXDISTERR}
+
     }
 
     // Shape format can be one of ShapeIO.GeoJSON, ShapeIO.LEGACY, ShapeIO.POLY, ShapeIO.WKT
     shape.io.format = "WKT"
+    shape.io.format = ${?LUCENE_SPATIAL_SHAPE_IO_FORMAT}
 
     // Supported linkage methods
     // "collectbroadcast" : Collects the RDD that contains the queries (to be used only if query RDD
@@ -88,5 +108,6 @@ lucenerdd {
     // of LuceneRDD. Note it duplicates each partition of LuceneRDD n times where n is the number of
     // partitions of the queries RDD.
     linker.method = "collectbroadcast"
+    linker.method = ${?LUCENE_SPATIAL_LINKER_METHOD}
   }
 }