From ef5b42405600a8296b673761b6ecbf167e2c8626 Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Fri, 4 Aug 2017 21:58:13 +0200
Subject: [PATCH 01/47] sbt: update test library

---
 build.sbt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.sbt b/build.sbt
index fa8ab90d..d60d5e21 100644
--- a/build.sbt
+++ b/build.sbt
@@ -137,7 +137,7 @@ libraryDependencies ++= Seq(
 libraryDependencies ++= Seq(
   "org.apache.spark" %% "spark-core" % testSparkVersion.value % "test" force(),
   "org.apache.spark" %% "spark-sql" % testSparkVersion.value % "test" force(),
-  "com.holdenkarau"  %% "spark-testing-base" % s"${sparkVersion.value}_0.7.2" % "test" intransitive(),
+  "com.holdenkarau"  %% "spark-testing-base" % s"${sparkVersion.value}_0.7.3" % "test" intransitive(),
   "org.scala-lang"    % "scala-library" % scalaVersion.value % "compile"
 )
 

From 576e29e5b4bb9ec06e9972147d71f0a4b99dacc8 Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Sat, 26 Aug 2017 09:20:07 +0200
Subject: [PATCH 02/47] minor version updates

---
 build.sbt | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/build.sbt b/build.sbt
index d60d5e21..4cb77a78 100644
--- a/build.sbt
+++ b/build.sbt
@@ -96,11 +96,11 @@ testSparkVersion := sys.props.get("spark.testVersion").getOrElse(sparkVersion.va
 
 
 // scalastyle:off
-val scalactic                 = "org.scalactic"                  %% "scalactic" % "3.0.3"
+val scalactic                 = "org.scalactic"                  %% "scalactic" % "3.0.4"
 val scalatest                 = "org.scalatest"                  %% "scalatest"                % "3.0.3" % "test"
 
 val joda_time                 = "joda-time"                      % "joda-time"                 % "2.9.9"
-val joda_convert              = "org.joda"                       % "joda-convert"              % "1.8.2"
+val joda_convert              = "org.joda"                       % "joda-convert"              % "1.8.3"
 val algebird                  = "com.twitter"                    %% "algebird-core"            % "0.13.0"
 val spatial4j                 = "org.locationtech.spatial4j"     % "spatial4j"                 % "0.6"
 
@@ -137,7 +137,7 @@ libraryDependencies ++= Seq(
 libraryDependencies ++= Seq(
   "org.apache.spark" %% "spark-core" % testSparkVersion.value % "test" force(),
   "org.apache.spark" %% "spark-sql" % testSparkVersion.value % "test" force(),
-  "com.holdenkarau"  %% "spark-testing-base" % s"${sparkVersion.value}_0.7.3" % "test" intransitive(),
+  "com.holdenkarau"  %% "spark-testing-base" % s"${sparkVersion.value}_0.7.4" % "test" intransitive(),
   "org.scala-lang"    % "scala-library" % scalaVersion.value % "compile"
 )
 

From 3b3110e3a18b7ee9a780ecd12f11b3daf3100f73 Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Tue, 20 Dec 2016 08:12:26 +0100
Subject: [PATCH 03/47] cart_link: init commit

---
 .../spatial/shape/ShapeLuceneRDD.scala        | 37 +++++++++++--------
 1 file changed, 21 insertions(+), 16 deletions(-)

diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala
index c80c3ddc..6b5981a3 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala
@@ -24,9 +24,9 @@ import org.apache.spark.rdd.RDD
 import org.apache.spark.storage.StorageLevel
 import org.apache.spark._
 import org.apache.spark.sql.Dataset
-import org.apache.spark.sql.{DataFrame, Row}
 import org.locationtech.spatial4j.shape.Shape
 import org.zouzias.spark.lucenerdd.analyzers.AnalyzerConfigurable
+import org.apache.spark.sql.{DataFrame, Dataset, Row}
 import org.zouzias.spark.lucenerdd.config.LuceneRDDConfigurable
 import org.zouzias.spark.lucenerdd.models.SparkScoreDoc
 import org.zouzias.spark.lucenerdd.query.{LuceneQueryHelpers, SimilarityConfigurable}
@@ -94,28 +94,33 @@ class ShapeLuceneRDD[K: ClassTag, V: ClassTag]
   }
 
   private def linker[T: ClassTag](that: RDD[T], pointFunctor: T => PointType,
-    mapper: ( PointType, AbstractShapeLuceneRDDPartition[K, V]) =>
-                            Iterable[SparkScoreDoc]): RDD[(T, Array[SparkScoreDoc])] = {
+                                  mapper: ( PointType, AbstractShapeLuceneRDDPartition[K, V]) =>
+                                    Iterable[SparkScoreDoc])
+  : RDD[(T, Array[SparkScoreDoc])] = {
     logDebug("Linker requested")
 
     val topKMonoid = new TopKMonoid[SparkScoreDoc](MaxDefaultTopKValue)(SparkScoreDoc.ascending)
-    val thatWithIndex = that.zipWithIndex().map(_.swap)
-    val queries = thatWithIndex.mapValues(pointFunctor).collect()
-    val queriesB = partitionsRDD.context.broadcast(queries)
-
-    val resultsByPart: RDD[(Long, TopK[SparkScoreDoc])] = partitionsRDD.mapPartitions {
-      case partitions =>
-        partitions.flatMap { case partition =>
-          queriesB.value.par
-            .map{ case (index, (x, y)) =>
-            (index, topKMonoid.build(mapper((x, y), partition)))
-          }
-        }
+    val queries = that.map(pointFunctor)
+
+    val concated: RDD[String] = queries.zipWithIndex().map(_.swap).mapPartitions { case iter =>
+      val all = iter.map { case (ind, (x, y)) => s"${ind}#${x}#${y}"}
+        .reduce( (a, b) => s"${a}|${b}")
+      Iterator(all)
     }
+    val resultsByPart = concated.cartesian(partitionsRDD)
+      .flatMap { case (qs, lucene) =>
+        qs.split('|').filter(_.nonEmpty).par.map { case x =>
+          val arr = x.split('#')
+          (arr(0).toLong,
+            topKMonoid.build(mapper((arr(1).toDouble, arr(2).toDouble), lucene)))
+        }.toIterator
+      }
 
     logDebug("Merge topK linkage results")
     val results = resultsByPart.reduceByKey(topKMonoid.plus)
-    thatWithIndex.join(results).values.map(joined => (joined._1, joined._2.items.toArray))
+
+    that.zipWithIndex.map(_.swap).join(results).values
+      .map(joined => (joined._1, joined._2.items.toArray))
   }
 
   /**

From b09fdfcdb4724a6f09fa08d7d1304294a827d77f Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Sat, 26 Aug 2017 09:35:42 +0200
Subject: [PATCH 04/47] cart_linker: remove unused imports

---
 .../zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala
index 6b5981a3..2897a353 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala
@@ -17,13 +17,12 @@
 package org.zouzias.spark.lucenerdd.spatial.shape
 
 
-import com.twitter.algebird.{TopK, TopKMonoid}
+import com.twitter.algebird.TopKMonoid
 import org.apache.lucene.document.Document
 import org.apache.lucene.spatial.query.SpatialOperation
 import org.apache.spark.rdd.RDD
 import org.apache.spark.storage.StorageLevel
 import org.apache.spark._
-import org.apache.spark.sql.Dataset
 import org.locationtech.spatial4j.shape.Shape
 import org.zouzias.spark.lucenerdd.analyzers.AnalyzerConfigurable
 import org.apache.spark.sql.{DataFrame, Dataset, Row}

From 87c7e211ca458090607bda4635d95f6d135a829b Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Sat, 26 Aug 2017 09:37:36 +0200
Subject: [PATCH 05/47] cart_linker: avoid _.swap

---
 .../spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala      | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala
index 2897a353..801542aa 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala
@@ -101,12 +101,12 @@ class ShapeLuceneRDD[K: ClassTag, V: ClassTag]
     val topKMonoid = new TopKMonoid[SparkScoreDoc](MaxDefaultTopKValue)(SparkScoreDoc.ascending)
     val queries = that.map(pointFunctor)
 
-    val concated: RDD[String] = queries.zipWithIndex().map(_.swap).mapPartitions { case iter =>
-      val all = iter.map { case (ind, (x, y)) => s"${ind}#${x}#${y}"}
+    val concatenated: RDD[String] = queries.zipWithIndex().mapPartitions { case iter =>
+      val all = iter.map { case ((x, y), ind) => s"${ind}#${x}#${y}"}
         .reduce( (a, b) => s"${a}|${b}")
       Iterator(all)
     }
-    val resultsByPart = concated.cartesian(partitionsRDD)
+    val resultsByPart = concatenated.cartesian(partitionsRDD)
       .flatMap { case (qs, lucene) =>
         qs.split('|').filter(_.nonEmpty).par.map { case x =>
           val arr = x.split('#')

From de07126008860a2d512a9b22156a53672771ab02 Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Sat, 26 Aug 2017 18:22:48 +0200
Subject: [PATCH 06/47] fix reduce Option issue

---
 .../lucenerdd/spatial/shape/ShapeLuceneRDD.scala     | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala
index 801542aa..c26ad153 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala
@@ -102,9 +102,15 @@ class ShapeLuceneRDD[K: ClassTag, V: ClassTag]
     val queries = that.map(pointFunctor)
 
     val concatenated: RDD[String] = queries.zipWithIndex().mapPartitions { case iter =>
-      val all = iter.map { case ((x, y), ind) => s"${ind}#${x}#${y}"}
-        .reduce( (a, b) => s"${a}|${b}")
-      Iterator(all)
+      if (iter.nonEmpty) {
+        val all = iter.map { case ((x, y), ind) => s"${ind}#${x}#${y}" }
+          .reduceLeft((a, b) => s"${a}|${b}")
+
+        Iterator(all)
+      }
+      else {
+        Iterator[String]()
+      }
     }
     val resultsByPart = concatenated.cartesian(partitionsRDD)
       .flatMap { case (qs, lucene) =>

From f25b49368fa8bfb3f54eb43dfd737b8936a963d0 Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Sat, 26 Aug 2017 21:58:23 +0200
Subject: [PATCH 07/47] use glom on cartesian linker

---
 .../spatial/shape/ShapeLuceneRDD.scala        | 21 +++++--------------
 1 file changed, 5 insertions(+), 16 deletions(-)

diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala
index c26ad153..b563ec21 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala
@@ -101,24 +101,13 @@ class ShapeLuceneRDD[K: ClassTag, V: ClassTag]
     val topKMonoid = new TopKMonoid[SparkScoreDoc](MaxDefaultTopKValue)(SparkScoreDoc.ascending)
     val queries = that.map(pointFunctor)
 
-    val concatenated: RDD[String] = queries.zipWithIndex().mapPartitions { case iter =>
-      if (iter.nonEmpty) {
-        val all = iter.map { case ((x, y), ind) => s"${ind}#${x}#${y}" }
-          .reduceLeft((a, b) => s"${a}|${b}")
+    val concatenated: RDD[Array[((Double, Double), Long)]] = queries.zipWithIndex().glom()
 
-        Iterator(all)
-      }
-      else {
-        Iterator[String]()
-      }
-    }
     val resultsByPart = concatenated.cartesian(partitionsRDD)
-      .flatMap { case (qs, lucene) =>
-        qs.split('|').filter(_.nonEmpty).par.map { case x =>
-          val arr = x.split('#')
-          (arr(0).toLong,
-            topKMonoid.build(mapper((arr(1).toDouble, arr(2).toDouble), lucene)))
-        }.toIterator
+      .flatMap { case (queries, lucene) =>
+        queries.map{ case (query, ind) =>
+          (ind, topKMonoid.build(mapper((query._1, query._2), lucene)))
+        }
       }
 
     logDebug("Merge topK linkage results")

From de51245ea654175ceee8c9bb495b2e961f23725a Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Sun, 27 Aug 2017 16:10:32 +0200
Subject: [PATCH 08/47] lucenerdd: use cartesian linker

---
 .../org/zouzias/spark/lucenerdd/LuceneRDD.scala    | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/LuceneRDD.scala b/src/main/scala/org/zouzias/spark/lucenerdd/LuceneRDD.scala
index 75a9749c..b187867d 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/LuceneRDD.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/LuceneRDD.scala
@@ -175,18 +175,16 @@ class LuceneRDD[T: ClassTag](protected val partitionsRDD: RDD[AbstractLuceneRDDP
     val topKMonoid = new TopKMonoid[SparkScoreDoc](topK)(SparkScoreDoc.descending)
     logInfo("Collecting query points to driver")
     val otherWithIndex = other.zipWithIndex().map(_.swap)
-    val queries = otherWithIndex.mapValues(searchQueryGen).collect()
-    val queriesB = partitionsRDD.context.broadcast(queries)
+    val queries = otherWithIndex.mapValues(searchQueryGen).glom()
 
-    val resultsByPart: RDD[(Long, TopK[SparkScoreDoc])] = partitionsRDD.mapPartitions(partitions =>
-      partitions.flatMap { case partition =>
-        queriesB.value.par.map { case (index, qr) =>
-          (index, topKMonoid.build(partition.query(qr, topK)))
-        }
-      })
+    val resultsByPart = queries.cartesian(partitionsRDD)
+      .flatMap { case (qs, part) =>
+        qs.map(q => (q._1, topKMonoid.build(part.query(q._2, topK))))
+      }
 
     logInfo("Compute topK linkage per partition")
     val results = resultsByPart.reduceByKey(topKMonoid.plus)
+
     otherWithIndex.join(results).values
       .map(joined => (joined._1, joined._2.items.toArray))
   }

From 9fbfee1ffc86c0a5aeb7c5ab88cd6f3fa77b57c8 Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Sat, 2 Sep 2017 10:28:21 +0200
Subject: [PATCH 09/47] shape: configurable linker "cartesian" and
 "collectbroadcast"

Two strategies for shape linkage are implemented.
---
 src/main/resources/reference.conf             |  8 ++++
 .../config/ShapeLuceneRDDConfigurable.scala   |  7 ++++
 .../spatial/shape/ShapeLuceneRDD.scala        | 37 +++++++++++++------
 3 files changed, 41 insertions(+), 11 deletions(-)

diff --git a/src/main/resources/reference.conf b/src/main/resources/reference.conf
index 36ff63fb..b4e365e7 100644
--- a/src/main/resources/reference.conf
+++ b/src/main/resources/reference.conf
@@ -65,5 +65,13 @@ lucenerdd {
     // Shape format can be one of ShapeIO.GeoJSON, ShapeIO.LEGACY, ShapeIO.POLY, ShapeIO.WKT
     shape.io.format = "WKT"
 
+    // Supported linkage methods
+    // "collectbroadcast" : Collects the RDD that contains the queries (to be used only if query RDD
+    // fits in spark driver's memory)
+    //
+    // "cartesian" : Uses cartesian product between the partitions of the queries RDD and the partitions
+    // of LuceneRDD. Note it duplicates each partition of LuceneRDD n times where n is the number of
+    // partitions of the queries RDD.
+    linker.method = "collectbroadcast"
   }
 }
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/config/ShapeLuceneRDDConfigurable.scala b/src/main/scala/org/zouzias/spark/lucenerdd/config/ShapeLuceneRDDConfigurable.scala
index 18361b75..b7df7d6f 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/config/ShapeLuceneRDDConfigurable.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/config/ShapeLuceneRDDConfigurable.scala
@@ -56,5 +56,12 @@ trait ShapeLuceneRDDConfigurable extends LuceneRDDConfigurable {
     }
     else ShapeIO.WKT
   }
+
+  protected val getLinkerMethod: String = {
+    if (Config.hasPath("lucenerdd.spatial.linker.method ")) {
+      Config.getString("lucenerdd.spatial.linker.method ")
+    }
+    else "collectbroadcast"  // collectbroadcast by default
+  }
 }
 
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala
index b563ec21..0059bc39 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala
@@ -26,7 +26,7 @@ import org.apache.spark._
 import org.locationtech.spatial4j.shape.Shape
 import org.zouzias.spark.lucenerdd.analyzers.AnalyzerConfigurable
 import org.apache.spark.sql.{DataFrame, Dataset, Row}
-import org.zouzias.spark.lucenerdd.config.LuceneRDDConfigurable
+import org.zouzias.spark.lucenerdd.config.ShapeLuceneRDDConfigurable
 import org.zouzias.spark.lucenerdd.models.SparkScoreDoc
 import org.zouzias.spark.lucenerdd.query.{LuceneQueryHelpers, SimilarityConfigurable}
 import org.zouzias.spark.lucenerdd.response.{LuceneRDDResponse, LuceneRDDResponsePartition}
@@ -49,7 +49,7 @@ class ShapeLuceneRDD[K: ClassTag, V: ClassTag]
    val queryAnalyzerName: String,
    val similarity: String)
   extends RDD[(K, V)](partitionsRDD.context, List(new OneToOneDependency(partitionsRDD)))
-    with LuceneRDDConfigurable {
+    with ShapeLuceneRDDConfigurable {
 
   logInfo("Instance is created...")
   logInfo(s"Number of partitions: ${partitionsRDD.count()}")
@@ -99,16 +99,31 @@ class ShapeLuceneRDD[K: ClassTag, V: ClassTag]
     logDebug("Linker requested")
 
     val topKMonoid = new TopKMonoid[SparkScoreDoc](MaxDefaultTopKValue)(SparkScoreDoc.ascending)
-    val queries = that.map(pointFunctor)
-
-    val concatenated: RDD[Array[((Double, Double), Long)]] = queries.zipWithIndex().glom()
-
-    val resultsByPart = concatenated.cartesian(partitionsRDD)
-      .flatMap { case (queries, lucene) =>
-        queries.map{ case (query, ind) =>
-          (ind, topKMonoid.build(mapper((query._1, query._2), lucene)))
+    val queries = that.map(pointFunctor).zipWithIndex()
+
+    val resultsByPart = getLinkerMethod match {
+      case "cartesian" =>
+        val concatenated = queries.glom()
+
+        concatenated.cartesian(partitionsRDD)
+          .flatMap { case (qs, lucene) =>
+            qs.map { case (query, ind) =>
+              (ind, topKMonoid.build(mapper((query._1, query._2), lucene)))
+            }
+          }
+      case _ =>
+        val thatWithIndex = queries.map(_.swap)
+        val collectedQueries = thatWithIndex.collect()
+        val queriesB = partitionsRDD.context.broadcast(collectedQueries)
+
+        partitionsRDD.mapPartitions { partitions =>
+            partitions.flatMap { partition =>
+              queriesB.value.map { case (index, (x, y)) =>
+                  (index, topKMonoid.build(mapper((x, y), partition)))
+                }
+            }
         }
-      }
+    }
 
     logDebug("Merge topK linkage results")
     val results = resultsByPart.reduceByKey(topKMonoid.plus)

From a878919c27479db52da68fd994eac6144e3942db Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Sat, 2 Sep 2017 10:47:54 +0200
Subject: [PATCH 10/47] cartesian linker for LuceneRDD

---
 .travis.yml                                   | 13 ++++---
 src/main/resources/reference.conf             |  8 ++++
 .../zouzias/spark/lucenerdd/LuceneRDD.scala   | 37 ++++++++++++++-----
 .../config/LuceneRDDConfigurable.scala        |  7 ++++
 .../config/ShapeLuceneRDDConfigurable.scala   |  2 +-
 .../spatial/shape/ShapeLuceneRDD.scala        |  7 ++--
 6 files changed, 55 insertions(+), 19 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 0c00aff3..3412e81b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,18 +16,21 @@ matrix:
   include:
     - jdk: oraclejdk8
       scala: 2.11.11
-      env: TEST_SPARK_VERSION="2.2.0" LUCENE_ANALYZER="en"
+      env: TEST_SPARK_VERSION="2.2.0" LUCENE_ANALYZER="en" LINKER_METHOD="cartesian"
     - jdk: openjdk8
       scala: 2.11.11
-      env: TEST_SPARK_VERSION="2.2.0" LUCENE_ANALYZER="en"
+      env: TEST_SPARK_VERSION="2.2.0" LUCENE_ANALYZER="en" LINKER_METHOD="collectbroadcast"
     - jdk: openjdk8
       scala: 2.11.11
-      env: TEST_SPARK_VERSION="2.2.0" LUCENE_ANALYZER="whitespace"
+      env: TEST_SPARK_VERSION="2.2.0" LUCENE_ANALYZER="whitespace" LINKER_METHOD="cartesian"
     - jdk: oraclejdk8
       scala: 2.11.11
-      env: TEST_SPARK_VERSION="2.2.0" LUCENE_ANALYZER="whitespace"
+      env: TEST_SPARK_VERSION="2.2.0" LUCENE_ANALYZER="whitespace" LINKER_METHOD="collectbroadcast"
 script:
-  - sbt ++$TRAVIS_SCALA_VERSION clean update -Dlucenerdd.analyzer.name=${LUCENE_ANALYZER} -test
+  - sbt ++$TRAVIS_SCALA_VERSION clean update
+      -Dlucenerdd.linker.method=${LINKER_METHOD}
+      -Dlucenerdd.spatial.linker.method=${LINKER_METHOD}
+      -Dlucenerdd.analyzer.name=${LUCENE_ANALYZER} -test
   - sbt ++$TRAVIS_SCALA_VERSION scalastyle
   - sbt ++$TRAVIS_SCALA_VERSION assembly
   - travis_wait 30 sbt ++$TRAVIS_SCALA_VERSION clean coverage test coverageReport
diff --git a/src/main/resources/reference.conf b/src/main/resources/reference.conf
index b4e365e7..17f72bbc 100644
--- a/src/main/resources/reference.conf
+++ b/src/main/resources/reference.conf
@@ -12,6 +12,14 @@ lucenerdd {
   // Similarity scoring for Lucenes
   similarity.name = "bm25" // anything else will default to Lucene classic similarity
 
+  // Supported linkage methods
+  // "collectbroadcast" : Collects the RDD that contains the queries (to be used only if query RDD
+  // fits in spark driver's memory)
+  //
+  // "cartesian" : Uses cartesian product between the partitions of the queries RDD and the partitions
+  // of LuceneRDD. Note it duplicates each partition of LuceneRDD n times where n is the number of
+  // partitions of the queries RDD.
+  linker.method = "collectbroadcast"
 
   index {
 
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/LuceneRDD.scala b/src/main/scala/org/zouzias/spark/lucenerdd/LuceneRDD.scala
index b187867d..1dc312cf 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/LuceneRDD.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/LuceneRDD.scala
@@ -173,19 +173,36 @@ class LuceneRDD[T: ClassTag](protected val partitionsRDD: RDD[AbstractLuceneRDDP
     logInfo("Linkage requested")
 
     val topKMonoid = new TopKMonoid[SparkScoreDoc](topK)(SparkScoreDoc.descending)
-    logInfo("Collecting query points to driver")
-    val otherWithIndex = other.zipWithIndex().map(_.swap)
-    val queries = otherWithIndex.mapValues(searchQueryGen).glom()
-
-    val resultsByPart = queries.cartesian(partitionsRDD)
-      .flatMap { case (qs, part) =>
-        qs.map(q => (q._1, topKMonoid.build(part.query(q._2, topK))))
-      }
+    val queriesWithIndex = other.zipWithIndex().map(_.swap)
+
+    val resultsByPart = getLinkerMethod match {
+      case "cartesian" =>
+        val concatenated = queriesWithIndex.mapValues(searchQueryGen).glom()
+
+        concatenated.cartesian(partitionsRDD)
+          .flatMap { case (qs, part) =>
+            qs.map(q => (q._1, topKMonoid.build(part.query(q._2, topK))))
+          }
+      case _ =>
+        logInfo("Collecting query points to driver")
+        val collectedQueries = queriesWithIndex.mapValues(searchQueryGen).collect()
+        val queriesB = partitionsRDD.context.broadcast(collectedQueries)
+
+        val resultsByPart: RDD[(Long, TopK[SparkScoreDoc])] =
+          partitionsRDD.mapPartitions(partitions =>
+          partitions.flatMap { partition =>
+            queriesB.value.map { case (index, qr) =>
+              (index, topKMonoid.build(partition.query(qr, topK)))
+            }
+          })
+
+        resultsByPart
+    }
 
-    logInfo("Compute topK linkage per partition")
+    logInfo("Computing top-k linkage per partition")
     val results = resultsByPart.reduceByKey(topKMonoid.plus)
 
-    otherWithIndex.join(results).values
+    queriesWithIndex.join(results).values
       .map(joined => (joined._1, joined._2.items.toArray))
   }
 
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/config/LuceneRDDConfigurable.scala b/src/main/scala/org/zouzias/spark/lucenerdd/config/LuceneRDDConfigurable.scala
index 4e791850..6633cda0 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/config/LuceneRDDConfigurable.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/config/LuceneRDDConfigurable.scala
@@ -88,4 +88,11 @@ trait LuceneRDDConfigurable extends Configurable {
     }
     else IndexOptions.DOCS_AND_FREQS_AND_POSITIONS  // Default
   }
+
+  protected val getLinkerMethod: String = {
+    if (Config.hasPath("lucenerdd.linker.method ")) {
+      Config.getString("lucenerdd.linker.method ")
+    }
+    else "collectbroadcast"  // collectbroadcast by default
+  }
 }
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/config/ShapeLuceneRDDConfigurable.scala b/src/main/scala/org/zouzias/spark/lucenerdd/config/ShapeLuceneRDDConfigurable.scala
index b7df7d6f..f4a38d2a 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/config/ShapeLuceneRDDConfigurable.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/config/ShapeLuceneRDDConfigurable.scala
@@ -57,7 +57,7 @@ trait ShapeLuceneRDDConfigurable extends LuceneRDDConfigurable {
     else ShapeIO.WKT
   }
 
-  protected val getLinkerMethod: String = {
+  protected val getShapeLinkerMethod: String = {
     if (Config.hasPath("lucenerdd.spatial.linker.method ")) {
       Config.getString("lucenerdd.spatial.linker.method ")
     }
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala
index 0059bc39..01872050 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala
@@ -96,12 +96,12 @@ class ShapeLuceneRDD[K: ClassTag, V: ClassTag]
                                   mapper: ( PointType, AbstractShapeLuceneRDDPartition[K, V]) =>
                                     Iterable[SparkScoreDoc])
   : RDD[(T, Array[SparkScoreDoc])] = {
-    logDebug("Linker requested")
+    logInfo("Shape Linkage requested")
 
     val topKMonoid = new TopKMonoid[SparkScoreDoc](MaxDefaultTopKValue)(SparkScoreDoc.ascending)
     val queries = that.map(pointFunctor).zipWithIndex()
 
-    val resultsByPart = getLinkerMethod match {
+    val resultsByPart = getShapeLinkerMethod match {
       case "cartesian" =>
         val concatenated = queries.glom()
 
@@ -113,6 +113,7 @@ class ShapeLuceneRDD[K: ClassTag, V: ClassTag]
           }
       case _ =>
         val thatWithIndex = queries.map(_.swap)
+        logInfo("Collecting query points to driver")
         val collectedQueries = thatWithIndex.collect()
         val queriesB = partitionsRDD.context.broadcast(collectedQueries)
 
@@ -125,7 +126,7 @@ class ShapeLuceneRDD[K: ClassTag, V: ClassTag]
         }
     }
 
-    logDebug("Merge topK linkage results")
+    logInfo("Computing top-k linkage per partition")
     val results = resultsByPart.reduceByKey(topKMonoid.plus)
 
     that.zipWithIndex.map(_.swap).join(results).values

From 87d989d443c9286bd3e6474cbca8abfcadd5ab91 Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Sat, 2 Sep 2017 10:50:27 +0200
Subject: [PATCH 11/47] sbt: minor update scalatest

---
 build.sbt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.sbt b/build.sbt
index 4cb77a78..1b35f301 100644
--- a/build.sbt
+++ b/build.sbt
@@ -97,7 +97,7 @@ testSparkVersion := sys.props.get("spark.testVersion").getOrElse(sparkVersion.va
 
 // scalastyle:off
 val scalactic                 = "org.scalactic"                  %% "scalactic" % "3.0.4"
-val scalatest                 = "org.scalatest"                  %% "scalatest"                % "3.0.3" % "test"
+val scalatest                 = "org.scalatest"                  %% "scalatest"                % "3.0.4" % "test"
 
 val joda_time                 = "joda-time"                      % "joda-time"                 % "2.9.9"
 val joda_convert              = "org.joda"                       % "joda-convert"              % "1.8.3"

From e505d1abc8f627fc62e08a6ea5912d3b3dc6e8fe Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Sat, 2 Sep 2017 11:04:17 +0200
Subject: [PATCH 12/47] fix ShapeLuceneRDD linker issues

---
 .../lucenerdd/spatial/shape/ShapeLuceneRDD.scala      | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala
index 01872050..6d25cdb3 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala
@@ -99,22 +99,21 @@ class ShapeLuceneRDD[K: ClassTag, V: ClassTag]
     logInfo("Shape Linkage requested")
 
     val topKMonoid = new TopKMonoid[SparkScoreDoc](MaxDefaultTopKValue)(SparkScoreDoc.ascending)
-    val queries = that.map(pointFunctor).zipWithIndex()
+    val queries = that.zipWithIndex().map(_.swap)
 
     val resultsByPart = getShapeLinkerMethod match {
       case "cartesian" =>
-        val concatenated = queries.glom()
+        val concatenated = queries.mapValues(pointFunctor).glom()
 
         concatenated.cartesian(partitionsRDD)
           .flatMap { case (qs, lucene) =>
-            qs.map { case (query, ind) =>
+            qs.map { case (ind, query) =>
               (ind, topKMonoid.build(mapper((query._1, query._2), lucene)))
             }
           }
       case _ =>
-        val thatWithIndex = queries.map(_.swap)
         logInfo("Collecting query points to driver")
-        val collectedQueries = thatWithIndex.collect()
+        val collectedQueries = queries.mapValues(pointFunctor).collect()
         val queriesB = partitionsRDD.context.broadcast(collectedQueries)
 
         partitionsRDD.mapPartitions { partitions =>
@@ -129,7 +128,7 @@ class ShapeLuceneRDD[K: ClassTag, V: ClassTag]
     logInfo("Computing top-k linkage per partition")
     val results = resultsByPart.reduceByKey(topKMonoid.plus)
 
-    that.zipWithIndex.map(_.swap).join(results).values
+    queries.join(results).values
       .map(joined => (joined._1, joined._2.items.toArray))
   }
 

From 35f0f316d40a3bc371efdef327a0c5cfd327802c Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Wed, 13 Sep 2017 14:08:16 +0200
Subject: [PATCH 13/47] LuceneRDD: linkerMethod input in linkage

---
 .../zouzias/spark/lucenerdd/LuceneRDD.scala   | 67 ++++++++++++-------
 1 file changed, 42 insertions(+), 25 deletions(-)

diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/LuceneRDD.scala b/src/main/scala/org/zouzias/spark/lucenerdd/LuceneRDD.scala
index 1dc312cf..ff280f67 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/LuceneRDD.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/LuceneRDD.scala
@@ -135,12 +135,16 @@ class LuceneRDD[T: ClassTag](protected val partitionsRDD: RDD[AbstractLuceneRDDP
     *
     * @param searchQueryGen Search query mapper function
     * @param topK Number of results to deduplication
+    * @param linkerMethod Method to perform linkage
+    *
     * @return
     */
-  def dedup[T1: ClassTag](searchQueryGen: T1 => String, topK: Int = DefaultTopK)
+  def dedup[T1: ClassTag](searchQueryGen: T1 => String,
+                          topK: Int = DefaultTopK,
+                          linkerMethod: String = getLinkerMethod)
   : RDD[(T1, Array[SparkScoreDoc])] = {
     // FIXME: is this asInstanceOf necessary?
-    link[T1](this.asInstanceOf[RDD[T1]], searchQueryGen, topK)
+    link[T1](this.asInstanceOf[RDD[T1]], searchQueryGen, topK, linkerMethod)
   }
 
   /**
@@ -149,12 +153,39 @@ class LuceneRDD[T: ClassTag](protected val partitionsRDD: RDD[AbstractLuceneRDDP
    * @param other DataFrame to be linked
    * @param searchQueryGen Function that generates a search query for each element of other
    * @param topK
+   * @param linkerMethod Method to perform linkage
    * @return an RDD of Tuple2 that contains the linked search Lucene documents in the second
    */
-  def linkDataFrame(other: DataFrame, searchQueryGen: Row => String, topK: Int = DefaultTopK)
+  def linkDataFrame(other: DataFrame,
+                    searchQueryGen: Row => String,
+                    topK: Int = DefaultTopK,
+                    linkerMethod: String = getLinkerMethod)
   : RDD[(Row, Array[SparkScoreDoc])] = {
     logInfo("LinkDataFrame requested")
-    link[Row](other.rdd, searchQueryGen, topK)
+    link[Row](other.rdd, searchQueryGen, topK, linkerMethod)
+  }
+
+  /**
+    * Entity linkage via Lucene query over all elements of an RDD.
+    *
+    * @param other RDD to be linked
+    * @param searchQueryGen Function that generates a Lucene Query object for each element of other
+    * @param linkerMethod Method to perform linkage
+    * @tparam T1 A type
+    * @return an RDD of Tuple2 that contains the linked search Lucene Document
+    *         in the second position
+    */
+  def linkByQuery[T1: ClassTag](other: RDD[T1],
+                                searchQueryGen: T1 => Query,
+                                topK: Int = DefaultTopK,
+                                linkerMethod: String = getLinkerMethod)
+  : RDD[(T1, Array[SparkScoreDoc])] = {
+    logInfo("LinkByQuery requested")
+    def typeToQueryString = (input: T1) => {
+      searchQueryGen(input).toString
+    }
+
+    link[T1](other, typeToQueryString, topK, linkerMethod)
   }
 
   /**
@@ -162,20 +193,25 @@ class LuceneRDD[T: ClassTag](protected val partitionsRDD: RDD[AbstractLuceneRDDP
    *
    * @param other RDD to be linked
    * @param searchQueryGen Function that generates a search query for each element of other
+   * @param linkerMethod Method to perform linkage
    * @tparam T1 A type
    * @return an RDD of Tuple2 that contains the linked search Lucene documents in the second
    *
    * Note: Currently the query strings of the other RDD are collected to the driver and
    * broadcast to the workers.
    */
-  def link[T1: ClassTag](other: RDD[T1], searchQueryGen: T1 => String, topK: Int = DefaultTopK)
+  def link[T1: ClassTag](other: RDD[T1],
+                         searchQueryGen: T1 => String,
+                         topK: Int = DefaultTopK,
+                         linkerMethod: String)
     : RDD[(T1, Array[SparkScoreDoc])] = {
     logInfo("Linkage requested")
 
     val topKMonoid = new TopKMonoid[SparkScoreDoc](topK)(SparkScoreDoc.descending)
     val queriesWithIndex = other.zipWithIndex().map(_.swap)
 
-    val resultsByPart = getLinkerMethod match {
+    logInfo(s"Linker method is ${linkerMethod}")
+    val resultsByPart = linkerMethod match {
       case "cartesian" =>
         val concatenated = queriesWithIndex.mapValues(searchQueryGen).glom()
 
@@ -206,25 +242,6 @@ class LuceneRDD[T: ClassTag](protected val partitionsRDD: RDD[AbstractLuceneRDDP
       .map(joined => (joined._1, joined._2.items.toArray))
   }
 
-  /**
-   * Entity linkage via Lucene query over all elements of an RDD.
-   *
-   * @param other RDD to be linked
-   * @param searchQueryGen Function that generates a Lucene Query object for each element of other
-   * @tparam T1 A type
-   * @return an RDD of Tuple2 that contains the linked search Lucene Document in the second position
-   */
-  def linkByQuery[T1: ClassTag](other: RDD[T1],
-                                searchQueryGen: T1 => Query, topK: Int = DefaultTopK)
-  : RDD[(T1, Array[SparkScoreDoc])] = {
-    logInfo("LinkByQuery requested")
-    def typeToQueryString = (input: T1) => {
-      searchQueryGen(input).toString
-    }
-
-    link[T1](other, typeToQueryString, topK)
-  }
-
   /**
    * Lucene term query
    *

From 118df889ce053c9f58c4e2147275f42a8f4825e0 Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Wed, 13 Sep 2017 14:20:22 +0200
Subject: [PATCH 14/47] sbt: Lucene version update to 6.6.1

---
 build.sbt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.sbt b/build.sbt
index 1b35f301..220b2ab3 100644
--- a/build.sbt
+++ b/build.sbt
@@ -79,7 +79,7 @@ pomExtra := (
   </developers>
 )
 
-val luceneV = "6.6.0"
+val luceneV = "6.6.1"
 
 spName := "zouzias/spark-lucenerdd"
 sparkVersion := "2.2.0"

From 7bf947b566d5ccd3168cda2a9e0f0e5619859daa Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Wed, 13 Sep 2017 14:35:41 +0200
Subject: [PATCH 15/47] ShapeLuceneRDD: linkerMethod in ShapeLuceneRDD

---
 .../spatial/shape/ShapeLuceneRDD.scala        | 57 ++++++++++------
 src/test/resources/reference.conf             | 68 +++++++++++++++++--
 .../LuceneRDDRecordLinkageSpec.scala          | 24 +++++++
 3 files changed, 123 insertions(+), 26 deletions(-)

diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala
index 6d25cdb3..e07b3870 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala
@@ -92,16 +92,18 @@ class ShapeLuceneRDD[K: ClassTag, V: ClassTag]
     new LuceneRDDResponse(partitionsRDD.map(f), SparkScoreDoc.ascending)
   }
 
-  private def linker[T: ClassTag](that: RDD[T], pointFunctor: T => PointType,
+  private def linker[T: ClassTag](that: RDD[T],
+                                  pointFunctor: T => PointType,
                                   mapper: ( PointType, AbstractShapeLuceneRDDPartition[K, V]) =>
-                                    Iterable[SparkScoreDoc])
+                                    Iterable[SparkScoreDoc],
+                                  linkerMethod: String)
   : RDD[(T, Array[SparkScoreDoc])] = {
     logInfo("Shape Linkage requested")
 
     val topKMonoid = new TopKMonoid[SparkScoreDoc](MaxDefaultTopKValue)(SparkScoreDoc.ascending)
     val queries = that.zipWithIndex().map(_.swap)
 
-    val resultsByPart = getShapeLinkerMethod match {
+    val resultsByPart = linkerMethod match {
       case "cartesian" =>
         val concatenated = queries.mapValues(pointFunctor).glom()
 
@@ -140,18 +142,22 @@ class ShapeLuceneRDD[K: ClassTag, V: ClassTag]
    *
    * @param that An RDD of entities to be linked
    * @param pointFunctor Function that generates a point from each element of other
+   * @param linkerMethod Method to perform linkage
    * @tparam T A type
    * @return an RDD of Tuple2 that contains the linked results
    *
    * Note: Currently the query coordinates of the other RDD are collected to the driver and
    * broadcast to the workers.
    */
-  def linkByKnn[T: ClassTag](that: RDD[T], pointFunctor: T => PointType,
-                           topK: Int = DefaultTopK)
+  def linkByKnn[T: ClassTag](that: RDD[T],
+                             pointFunctor: T => PointType,
+                             topK: Int = DefaultTopK,
+                             linkerMethod: String = getShapeLinkerMethod)
   : RDD[(T, Array[SparkScoreDoc])] = {
     logInfo("linkByKnn requested")
     linker[T](that, pointFunctor, (queryPoint, part) =>
-      part.knnSearch(queryPoint, topK, LuceneQueryHelpers.MatchAllDocsString))
+      part.knnSearch(queryPoint, topK, LuceneQueryHelpers.MatchAllDocsString),
+      linkerMethod)
   }
 
   /**
@@ -161,18 +167,24 @@ class ShapeLuceneRDD[K: ClassTag, V: ClassTag]
    *
    * @param that An RDD of entities to be linked
    * @param pointFunctor Function that generates a point from each element of other
+   * @param linkerMethod Method to perform linkage
    * @tparam T A type
    * @return an RDD of Tuple2 that contains the linked results
    *
    * Note: Currently the query coordinates of the other RDD are collected to the driver and
    * broadcast to the workers.
    */
-  def linkByRadius[T: ClassTag](that: RDD[T], pointFunctor: T => PointType, radius: Double,
-    topK: Int = DefaultTopK, spatialOp: String = SpatialOperation.Intersects.getName)
+  def linkByRadius[T: ClassTag](that: RDD[T],
+                                pointFunctor: T => PointType,
+                                radius: Double,
+                                topK: Int = DefaultTopK,
+                                spatialOp: String = SpatialOperation.Intersects.getName,
+                                linkerMethod: String = getShapeLinkerMethod)
   : RDD[(T, Array[SparkScoreDoc])] = {
     logInfo("linkByRadius requested")
     linker[T](that, pointFunctor, (queryPoint, part) =>
-      part.circleSearch(queryPoint, radius, topK, spatialOp))
+      part.circleSearch(queryPoint, radius, topK, spatialOp),
+      linkerMethod)
   }
 
 
@@ -182,16 +194,19 @@ class ShapeLuceneRDD[K: ClassTag, V: ClassTag]
    * Links this and that based on nearest neighbors, returns Knn
    *
    *
-   * @param other
-   * @param searchQueryGen
-   * @param topK
+   * @param other DataFrame of entities to be linked
+   * @param searchQueryGen Function that generates a point from each element of other
+    * @param topK Number of linked items
+   * @param linkerMethod Method to perform linkage
    * @return
    */
-  def linkDataFrameByKnn(other: DataFrame, searchQueryGen: Row => PointType,
-                         topK: Int = DefaultTopK)
+  def linkDataFrameByKnn(other: DataFrame,
+                         searchQueryGen: Row => PointType,
+                         topK: Int = DefaultTopK,
+                         linkerMethod: String = getShapeLinkerMethod)
   : RDD[(Row, Array[SparkScoreDoc])] = {
     logInfo("linkDataFrameByKnn requested")
-    linkByKnn[Row](other.rdd, searchQueryGen, topK)
+    linkByKnn[Row](other.rdd, searchQueryGen, topK, linkerMethod)
   }
 
   /**
@@ -201,17 +216,21 @@ class ShapeLuceneRDD[K: ClassTag, V: ClassTag]
    *
    * @param other DataFrame of entities to be linked
    * @param pointFunctor Function that generates a point from each element of other
+   * @param linkerMethod Method to perform linkage
    * @return an RDD of Tuple2 that contains the linked results
    *
    * Note: Currently the query coordinates of the other RDD are collected to the driver and
    * broadcast to the workers.
    */
-  def linkDataFrameByRadius(other: DataFrame, pointFunctor: Row => PointType,
-                            radius: Double, topK: Int = DefaultTopK,
-                            spatialOp: String = SpatialOperation.Intersects.getName)
+  def linkDataFrameByRadius(other: DataFrame,
+                            pointFunctor: Row => PointType,
+                            radius: Double,
+                            topK: Int = DefaultTopK,
+                            spatialOp: String = SpatialOperation.Intersects.getName,
+                            linkerMethod: String = getShapeLinkerMethod)
   : RDD[(Row, Array[SparkScoreDoc])] = {
     logInfo("linkDataFrameByRadius requested")
-    linkByRadius[Row](other.rdd, pointFunctor, radius, topK, spatialOp)
+    linkByRadius[Row](other.rdd, pointFunctor, radius, topK, spatialOp, linkerMethod)
   }
 
   /**
diff --git a/src/test/resources/reference.conf b/src/test/resources/reference.conf
index effb01e9..17f72bbc 100644
--- a/src/test/resources/reference.conf
+++ b/src/test/resources/reference.conf
@@ -3,10 +3,56 @@ lucenerdd {
   // Name of analyzer as it is under Lucene's package org.apache.lucene.analysis.XX
   analyzer.name = "en"
 
-  // Lucene index storage
-  // Use 'disk' to store the index in Java's temp directory
-  // Otherwise the index will be stored in memory
-  index.store.mode = "memory"
+  // Analyzer name must be "ngram"
+  analyzer {
+    ngram.mingram = 2
+    ngram.maxgram = 5
+  }
+
+  // Similarity scoring for Lucenes
+  similarity.name = "bm25" // anything else will default to Lucene classic similarity
+
+  // Supported linkage methods
+  // "collectbroadcast" : Collects the RDD that contains the queries (to be used only if query RDD
+  // fits in spark driver's memory)
+  //
+  // "cartesian" : Uses cartesian product between the partitions of the queries RDD and the partitions
+  // of LuceneRDD. Note it duplicates each partition of LuceneRDD n times where n is the number of
+  // partitions of the queries RDD.
+  linker.method = "collectbroadcast"
+
+  index {
+
+    // Lucene index storage
+    // Use 'disk' to store the index in Java's temp directory
+    // Otherwise the index will be stored in memory
+    store.mode = "memory"
+
+    stringfields{
+
+      // Analyze text fields or not
+      analyzed = true
+
+      // Text fields options as in org.apache.lucene.index.IndexOptions
+      //
+      // Other options are:
+      // "DOCS"
+      // "DOCS_AND_FREQS"
+      // "DOCS_AND_FREQS_AND_POSITIONS"
+      // "DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS"
+      // "NONE"
+      options = "docs_and_freqs_and_positions_and_offsets"
+
+      // Omit terms norms
+      terms.omitnorms = false
+
+      // Store term positions
+      terms.positions = false
+
+      // Store Term vectors (set true, otherwise LuceneRDD.termVectors(fieldName) will fail)
+      terms.vectors = true
+    }
+  }
 
   // Maximum value on topK queries
   query.topk.maxvalue = 100
@@ -19,13 +65,21 @@ lucenerdd {
   // Spatial related configurations used by ShapeLuceneRDD
   spatial {
     prefixtree {
-      name = "geohash"
-      maxlevel = 11 // 11 results in sub-meter precision for geohash
-      maxDistErr = 1.0 // in kilometers
+      name = "quad"  // "geohash" or "quad"
+      maxlevel = 9 // 11 results in sub-meter precision for geohash
+      maxDistErr = 5.0 // in kilometers
     }
 
     // Shape format can be one of ShapeIO.GeoJSON, ShapeIO.LEGACY, ShapeIO.POLY, ShapeIO.WKT
     shape.io.format = "WKT"
 
+    // Supported linkage methods
+    // "collectbroadcast" : Collects the RDD that contains the queries (to be used only if query RDD
+    // fits in spark driver's memory)
+    //
+    // "cartesian" : Uses cartesian product between the partitions of the queries RDD and the partitions
+    // of LuceneRDD. Note it duplicates each partition of LuceneRDD n times where n is the number of
+    // partitions of the queries RDD.
+    linker.method = "collectbroadcast"
   }
 }
diff --git a/src/test/scala/org/zouzias/spark/lucenerdd/LuceneRDDRecordLinkageSpec.scala b/src/test/scala/org/zouzias/spark/lucenerdd/LuceneRDDRecordLinkageSpec.scala
index e13f5640..b7c99f55 100644
--- a/src/test/scala/org/zouzias/spark/lucenerdd/LuceneRDDRecordLinkageSpec.scala
+++ b/src/test/scala/org/zouzias/spark/lucenerdd/LuceneRDDRecordLinkageSpec.scala
@@ -64,6 +64,30 @@ class LuceneRDDRecordLinkageSpec extends FlatSpec
     linked.collect().exists(link => link._1 == "ita" && link._2.length == 1) should equal(true)
   }
 
+  "LuceneRDD.linkByQuery" should "correctly link with prefix query (query / )" in {
+    val leftCountries = Array("gree", "germa", "spa", "ita")
+    implicit val mySC = sc
+    val leftCountriesRDD = sc.parallelize(leftCountries)
+
+    val countries = sc.parallelize(Source.fromFile("src/test/resources/countries.txt").getLines()
+      .map(_.toLowerCase()).toSeq)
+
+    luceneRDD = LuceneRDD(countries)
+
+    val linker = (country: String) => {
+      val term = new Term("_1", country)
+      new PrefixQuery(term)
+    }
+
+    val linked = luceneRDD.linkByQuery(leftCountriesRDD, linker, 10, "cartesian")
+
+    linked.count() should equal(leftCountries.length)
+    // Greece and Greenland should appear
+    linked.collect().exists(link => link._1 == "gree" && link._2.length == 2) should equal(true)
+    // Italy should appear
+    linked.collect().exists(link => link._1 == "ita" && link._2.length == 1) should equal(true)
+  }
+
   "LuceneRDD.linkByQuery" should "correctly link with query parser (fuzzy)" in {
     val leftCountries = Array("gree", "germa", "spa", "ita")
     implicit val mySC = sc

From b9d3b8f77aae05cfb1153fa589ac649081e3d758 Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Wed, 13 Sep 2017 14:38:44 +0200
Subject: [PATCH 16/47] sbt: update to 0.13.16

---
 project/build.properties | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/project/build.properties b/project/build.properties
index 64317fda..c091b86c 100644
--- a/project/build.properties
+++ b/project/build.properties
@@ -1 +1 @@
-sbt.version=0.13.15
+sbt.version=0.13.16

From 630bf01fcbaa08289d6eedf5a36a5a97cb7bb9b3 Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Wed, 13 Sep 2017 14:39:58 +0200
Subject: [PATCH 17/47] LuceneRDD: link: default linker method

---
 src/main/scala/org/zouzias/spark/lucenerdd/LuceneRDD.scala | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/LuceneRDD.scala b/src/main/scala/org/zouzias/spark/lucenerdd/LuceneRDD.scala
index ff280f67..0e356111 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/LuceneRDD.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/LuceneRDD.scala
@@ -193,7 +193,7 @@ class LuceneRDD[T: ClassTag](protected val partitionsRDD: RDD[AbstractLuceneRDDP
    *
    * @param other RDD to be linked
    * @param searchQueryGen Function that generates a search query for each element of other
-   * @param linkerMethod Method to perform linkage
+   * @param linkerMethod Method to perform linkage, default value from configuration
    * @tparam T1 A type
    * @return an RDD of Tuple2 that contains the linked search Lucene documents in the second
    *
@@ -203,7 +203,7 @@ class LuceneRDD[T: ClassTag](protected val partitionsRDD: RDD[AbstractLuceneRDDP
   def link[T1: ClassTag](other: RDD[T1],
                          searchQueryGen: T1 => String,
                          topK: Int = DefaultTopK,
-                         linkerMethod: String)
+                         linkerMethod: String = getLinkerMethod)
     : RDD[(T1, Array[SparkScoreDoc])] = {
     logInfo("Linkage requested")
 

From ce4ef953fac2b999e3f4840d6b37da3e088c24ce Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Thu, 14 Sep 2017 08:49:25 +0200
Subject: [PATCH 18/47] code hygiene

---
 .../zouzias/spark/lucenerdd/LuceneRDD.scala   | 26 ++++++++++++++-----
 .../spatial/shape/ShapeLuceneRDD.scala        | 19 ++++++++++----
 2 files changed, 33 insertions(+), 12 deletions(-)

diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/LuceneRDD.scala b/src/main/scala/org/zouzias/spark/lucenerdd/LuceneRDD.scala
index 0e356111..4fa68722 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/LuceneRDD.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/LuceneRDD.scala
@@ -36,10 +36,14 @@ import org.zouzias.spark.lucenerdd.versioning.Versionable
 import scala.reflect.ClassTag
 
 /**
- * Spark RDD with Lucene's query capabilities (term, prefix, fuzzy, phrase query)
- *
- * @tparam T
- */
+  * Spark RDD with Lucene's query capabilities (term, prefix, fuzzy, phrase query)
+  *
+  * @param partitionsRDD Partitions of RDD
+  * @param indexAnalyzer Analyzer during indexing time
+  * @param queryAnalyzer Analyzer during query time
+  * @param similarity Query similarity (TF-IDF / BM25)
+  * @tparam T
+  */
 class LuceneRDD[T: ClassTag](protected val partitionsRDD: RDD[AbstractLuceneRDDPartition[T]],
                              protected val indexAnalyzer: String,
                              protected val queryAnalyzer: String,
@@ -379,7 +383,9 @@ object LuceneRDD extends Versionable
    * @tparam T Generic type
    * @return
    */
-  def apply[T : ClassTag](elems: RDD[T], indexAnalyzer: String, queryAnalyzer: String,
+  def apply[T : ClassTag](elems: RDD[T],
+                          indexAnalyzer: String,
+                          queryAnalyzer: String,
                           similarity: String)
     (implicit conv: T => Document): LuceneRDD[T] = {
     val partitions = elems.mapPartitionsWithIndex[AbstractLuceneRDDPartition[T]](
@@ -407,7 +413,10 @@ object LuceneRDD extends Versionable
    * @return
    */
   def apply[T : ClassTag]
-  (elems: Iterable[T], indexAnalyzer: String, queryAnalyzer: String, similarity: String)
+  (elems: Iterable[T],
+   indexAnalyzer: String,
+   queryAnalyzer: String,
+   similarity: String)
   (implicit sc: SparkContext, conv: T => Document)
   : LuceneRDD[T] = {
     apply[T](sc.parallelize[T](elems.toSeq), indexAnalyzer, queryAnalyzer, similarity)
@@ -432,7 +441,10 @@ object LuceneRDD extends Versionable
    * @param similarity Lucene scoring similarity, i.e., BM25 or TF-IDF
     * @return
    */
-  def apply(dataFrame: DataFrame, indexAnalyzer: String, queryAnalyzer: String, similarity: String)
+  def apply(dataFrame: DataFrame,
+            indexAnalyzer: String,
+            queryAnalyzer: String,
+            similarity: String)
   : LuceneRDD[Row] = {
     apply[Row](dataFrame.rdd, indexAnalyzer, queryAnalyzer, similarity)
   }
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala
index e07b3870..16432e91 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDD.scala
@@ -40,6 +40,9 @@ import scala.reflect.ClassTag
  * ShapeLuceneRDD for geospatial and full-text search queries
  *
  * @param partitionsRDD Partition RDD
+ * @param indexAnalyzerName Analyzer during indexing time
+ * @param queryAnalyzerName Analyzer during query time
+ * @param similarity Query similarity (TF-IDF / BM25)
  * @tparam K Type containing the geospatial information (must be implicitly converted to [[Shape]])
  * @tparam V Type containing remaining information (must be implicitly converted to [[Document]])
  */
@@ -318,7 +321,9 @@ class ShapeLuceneRDD[K: ClassTag, V: ClassTag]
    * @param operationName Intersect, contained, etc.
    * @return
    */
-  def bboxSearch(lowerLeft: PointType, upperRight: PointType, k: Int,
+  def bboxSearch(lowerLeft: PointType,
+                 upperRight: PointType,
+                 k: Int,
                  operationName: String)
   : LuceneRDDResponse = {
     logInfo(s"Bounding box with lower left ${lowerLeft}, upper right ${upperRight} and k = ${k}")
@@ -428,7 +433,8 @@ object ShapeLuceneRDD extends Versionable
    * @param docConverter Implicit conversion for Lucene Document
    * @return
    */
-  def apply(df : DataFrame, shapeField: String)
+  def apply(df : DataFrame,
+            shapeField: String)
            (implicit shapeConv: String => Shape, docConverter: Row => Document)
   : ShapeLuceneRDD[String, Row] = {
     apply(df, shapeField,
@@ -436,14 +442,17 @@ object ShapeLuceneRDD extends Versionable
       getOrElseClassic())
   }
 
-  def apply(df : DataFrame, shapeField: String, indexAnalyzer: String, queryAnalyzer: String,
-            similairty: String)
+  def apply(df : DataFrame,
+            shapeField: String,
+            indexAnalyzer: String,
+            queryAnalyzer: String,
+            similarity: String)
            (implicit shapeConv: String => Shape, docConverter: Row => Document)
   : ShapeLuceneRDD[String, Row] = {
     val partitions = df.rdd.map(row => (row.getString(row.fieldIndex(shapeField)), row))
       .mapPartitions[AbstractShapeLuceneRDDPartition[String, Row]](
       iter => Iterator(ShapeLuceneRDDPartition[String, Row](iter, indexAnalyzer, queryAnalyzer)),
       preservesPartitioning = true)
-    new ShapeLuceneRDD(partitions, indexAnalyzer, queryAnalyzer, similairty)
+    new ShapeLuceneRDD(partitions, indexAnalyzer, queryAnalyzer, similarity)
   }
 }

From 6306ea0bb36c1ef33fedd06cdb2cb8dbf6148f7d Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Thu, 14 Sep 2017 08:54:04 +0200
Subject: [PATCH 19/47] code hygiene: add types to protected members

---
 .../lucenerdd/spatial/shape/context/ContextLoader.scala    | 7 ++++---
 .../lucenerdd/spatial/shape/grids/PrefixTreeLoader.scala   | 4 ++--
 .../spatial/shape/strategies/SpatialStrategy.scala         | 4 ++--
 3 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/context/ContextLoader.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/context/ContextLoader.scala
index 9269768b..c99ed537 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/context/ContextLoader.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/context/ContextLoader.scala
@@ -19,16 +19,17 @@ package org.zouzias.spark.lucenerdd.spatial.shape.context
 import java.io.{StringReader, StringWriter}
 
 import org.locationtech.spatial4j.context.jts.JtsSpatialContext
+import org.locationtech.spatial4j.io.{ShapeReader, ShapeWriter}
 import org.locationtech.spatial4j.shape.Shape
 import org.zouzias.spark.lucenerdd.config.ShapeLuceneRDDConfigurable
 
 trait ContextLoader extends ShapeLuceneRDDConfigurable{
 
-  protected val LocationDefaultField = getLocationFieldName
+  protected val LocationDefaultField: String = getLocationFieldName
 
-  protected lazy val shapeReader = ctx.getFormats.getReader(getShapeFormat)
+  protected lazy val shapeReader: ShapeReader = ctx.getFormats.getReader(getShapeFormat)
 
-  protected lazy val shapeWriter = ctx.getFormats.getWriter(getShapeFormat)
+  protected lazy val shapeWriter: ShapeWriter = ctx.getFormats.getWriter(getShapeFormat)
 
   protected def shapeToString(shape: Shape): String = {
     val writer = new StringWriter()
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/grids/PrefixTreeLoader.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/grids/PrefixTreeLoader.scala
index f1569bf8..19497ac4 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/grids/PrefixTreeLoader.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/grids/PrefixTreeLoader.scala
@@ -16,7 +16,7 @@
  */
 package org.zouzias.spark.lucenerdd.spatial.shape.grids
 
-import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTreeFactory
+import org.apache.lucene.spatial.prefix.tree.{SpatialPrefixTree, SpatialPrefixTreeFactory}
 import org.zouzias.spark.lucenerdd.config.ShapeLuceneRDDConfigurable
 import org.zouzias.spark.lucenerdd.spatial.shape.context.ContextLoader
 
@@ -35,7 +35,7 @@ trait PrefixTreeLoader extends ContextLoader
   protected val prefixTreeMaxDistErr: Double = getPrefixTreeMaxDistErr
 
   // This can also be constructed from SpatialPrefixTreeFactory
-  protected val grid = SpatialPrefixTreeFactory.makeSPT(
+  protected val grid: SpatialPrefixTree = SpatialPrefixTreeFactory.makeSPT(
     Map("prefixTree" -> prefixTreeName,
     "maxLevels" -> maxLevels.toString,
     "maxDistErr" -> prefixTreeMaxDistErr.toString).asJava,
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/strategies/SpatialStrategy.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/strategies/SpatialStrategy.scala
index 1018a570..5e62a870 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/strategies/SpatialStrategy.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/strategies/SpatialStrategy.scala
@@ -16,7 +16,7 @@
  */
 package org.zouzias.spark.lucenerdd.spatial.shape.strategies
 
-import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy
+import org.apache.lucene.spatial.prefix.{PrefixTreeStrategy, RecursivePrefixTreeStrategy}
 import org.zouzias.spark.lucenerdd.spatial.shape.grids.PrefixTreeLoader
 
 trait SpatialStrategy extends PrefixTreeLoader {
@@ -29,6 +29,6 @@ trait SpatialStrategy extends PrefixTreeLoader {
    * <p />
    * Note that these are initialized with a field name.
    */
-  protected val strategy = new RecursivePrefixTreeStrategy(grid,
+  protected val strategy: PrefixTreeStrategy = new RecursivePrefixTreeStrategy(grid,
     LocationDefaultField)
 }

From fc279dfd8ce1dbafe69f25b2a6cd9af7f27b8737 Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Tue, 26 Sep 2017 19:46:28 +0200
Subject: [PATCH 20/47] sbt: algebird update to 0.13.2

---
 build.sbt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.sbt b/build.sbt
index 220b2ab3..573b6e36 100644
--- a/build.sbt
+++ b/build.sbt
@@ -101,7 +101,7 @@ val scalatest                 = "org.scalatest"                  %% "scalatest"
 
 val joda_time                 = "joda-time"                      % "joda-time"                 % "2.9.9"
 val joda_convert              = "org.joda"                       % "joda-convert"              % "1.8.3"
-val algebird                  = "com.twitter"                    %% "algebird-core"            % "0.13.0"
+val algebird                  = "com.twitter"                    %% "algebird-core"            % "0.13.2"
 val spatial4j                 = "org.locationtech.spatial4j"     % "spatial4j"                 % "0.6"
 
 val typesafe_config           = "com.typesafe"                   % "config"                    % "1.3.1"

From 769c1bada3cc7da9cb42fcf28f45ccce19c6d5f4 Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@users.noreply.github.com>
Date: Fri, 29 Sep 2017 11:09:06 +0200
Subject: [PATCH 21/47] Update to Lucene 7 (#125)

* sbt: Lucene version update 7.0.0

* Lucene: query: remove queryParser.setLowercaseExpandedTerms

* sbt: update joda-convert
---
 build.sbt                                                    | 4 ++--
 .../zouzias/spark/lucenerdd/query/LuceneQueryHelpers.scala   | 5 -----
 2 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/build.sbt b/build.sbt
index 573b6e36..2841da6a 100644
--- a/build.sbt
+++ b/build.sbt
@@ -79,7 +79,7 @@ pomExtra := (
   </developers>
 )
 
-val luceneV = "6.6.1"
+val luceneV = "7.0.0"
 
 spName := "zouzias/spark-lucenerdd"
 sparkVersion := "2.2.0"
@@ -100,8 +100,8 @@ val scalactic                 = "org.scalactic"                  %% "scalactic"
 val scalatest                 = "org.scalatest"                  %% "scalatest"                % "3.0.4" % "test"
 
 val joda_time                 = "joda-time"                      % "joda-time"                 % "2.9.9"
-val joda_convert              = "org.joda"                       % "joda-convert"              % "1.8.3"
 val algebird                  = "com.twitter"                    %% "algebird-core"            % "0.13.2"
+val joda_convert              = "org.joda"                       % "joda-convert"              % "1.9.2"
 val spatial4j                 = "org.locationtech.spatial4j"     % "spatial4j"                 % "0.6"
 
 val typesafe_config           = "com.typesafe"                   % "config"                    % "1.3.1"
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/query/LuceneQueryHelpers.scala b/src/main/scala/org/zouzias/spark/lucenerdd/query/LuceneQueryHelpers.scala
index 5f8b928a..64387a34 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/query/LuceneQueryHelpers.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/query/LuceneQueryHelpers.scala
@@ -87,11 +87,6 @@ object LuceneQueryHelpers extends Serializable {
    */
   def parseQueryString(searchString: String, analyzer: Analyzer): Query = {
     val queryParser = new QueryParser(QueryParserDefaultField, analyzer)
-
-    // See http://goo.gl/L8sbrB
-    if (analyzer.getClass.getCanonicalName.toLowerCase().contains("whitespace")) {
-      queryParser.setLowercaseExpandedTerms(false)
-    }
     queryParser.parse(searchString)
   }
 

From f44a85f7495dbb7069a0c0cfc320a4ba85dcc2c0 Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@users.noreply.github.com>
Date: Fri, 29 Sep 2017 19:55:03 +0200
Subject: [PATCH 22/47] ShapeLuceneRDD: fix kryo registration (#124)

* ShapeLuceneRDD: fix kryo registration

* ShapeLuceneRDD: kryo: proper use of twitter chill

* ShapeLuceneRDD: kryo: add all classes required
---
 .../shape/ShapeLuceneRDDKryoRegistrator.scala | 50 +++++++++++++++++--
 1 file changed, 46 insertions(+), 4 deletions(-)

diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDKryoRegistrator.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDKryoRegistrator.scala
index e42366ff..eb2b5269 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDKryoRegistrator.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDKryoRegistrator.scala
@@ -17,11 +17,12 @@
 package org.zouzias.spark.lucenerdd.spatial.shape
 
 import com.twitter.algebird.TopK
-import com.twitter.chill._
+import com.twitter.chill.Kryo
 import org.apache.spark.SparkConf
 import org.apache.spark.serializer.{KryoRegistrator, KryoSerializer}
+import org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema
+import org.apache.spark.sql.types._
 import org.zouzias.spark.lucenerdd.models.{SparkDoc, SparkFacetResult, SparkScoreDoc}
-import org.zouzias.spark.lucenerdd.response.{LuceneRDDResponse, LuceneRDDResponsePartition}
 import org.zouzias.spark.lucenerdd.spatial.shape.partition.ShapeLuceneRDDPartition
 
 
@@ -30,11 +31,52 @@ class ShapeLuceneRDDKryoRegistrator extends KryoRegistrator {
     kryo.register(classOf[ShapeLuceneRDD[_, _]])
     kryo.register(classOf[ShapeLuceneRDDPartition[_, _]])
     kryo.register(classOf[SparkDoc])
+    kryo.register(classOf[Number])
+    kryo.register(classOf[java.lang.Double])
+    kryo.register(classOf[java.lang.Float])
+    kryo.register(classOf[java.lang.Integer])
+    kryo.register(classOf[java.lang.Long])
+    kryo.register(classOf[java.lang.Short])
+    kryo.register(classOf[StructType])
+    kryo.register(classOf[StructField])
+    kryo.register(classOf[IntegerType])
+    kryo.register(classOf[IntegerType$])
+    kryo.register(classOf[DoubleType])
+    kryo.register(classOf[DoubleType$])
+    kryo.register(classOf[FloatType])
+    kryo.register(classOf[StringType])
+    kryo.register(classOf[StringType$])
+    kryo.register(classOf[GenericRowWithSchema])
+    kryo.register(classOf[Metadata])
+    kryo.register(classOf[Object])
+    kryo.register(classOf[Array[Object]])
+    kryo.register(classOf[Array[Array[Byte]]])
+    kryo.register(classOf[scala.collection.mutable.WrappedArray$ofRef])
+    kryo.register(classOf[scala.collection.mutable.WrappedArray$ofFloat])
+    kryo.register(classOf[scala.collection.mutable.WrappedArray$ofDouble])
+    kryo.register(classOf[scala.collection.mutable.WrappedArray$ofInt])
+    kryo.register(classOf[scala.collection.mutable.WrappedArray$ofLong])
+    kryo.register(classOf[Array[String]])
+    kryo.register(classOf[Array[Number]])
+    kryo.register(classOf[Array[Float]])
+    kryo.register(classOf[Array[Int]])
+    kryo.register(classOf[Array[Long]])
+    kryo.register(classOf[Array[Double]])
+    kryo.register(classOf[Array[Boolean]])
+    kryo.register(classOf[Array[SparkScoreDoc]])
+    kryo.register(classOf[Array[StructType]])
+    kryo.register(classOf[Array[StructField]])
+    kryo.register(classOf[Range])
+    kryo.register(classOf[scala.collection.immutable.Map[String, String]])
+    kryo.register(classOf[scala.collection.immutable.Map[String, Number]])
+    kryo.register(classOf[scala.collection.immutable.Map$EmptyMap$])
+    kryo.register(classOf[scala.collection.immutable.Set$EmptySet$])
+    kryo.register(classOf[scala.collection.immutable.Map[_, _]])
+    kryo.register(classOf[Array[scala.collection.immutable.Map[_, _]]])
     kryo.register(classOf[SparkFacetResult])
-    kryo.register(classOf[LuceneRDDResponse])
-    kryo.register(classOf[LuceneRDDResponsePartition])
     kryo.register(classOf[SparkScoreDoc])
     kryo.register(classOf[TopK[_]])
+
     ()
   }
 }

From d12ef965d8ae27a7ed2682670381ab4d30ef6836 Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Sun, 15 Oct 2017 20:57:42 +0200
Subject: [PATCH 23/47] sbt: Lucene update to 7.0.1

---
 build.sbt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.sbt b/build.sbt
index 2841da6a..e6968e78 100644
--- a/build.sbt
+++ b/build.sbt
@@ -79,7 +79,7 @@ pomExtra := (
   </developers>
 )
 
-val luceneV = "7.0.0"
+val luceneV = "7.0.1"
 
 spName := "zouzias/spark-lucenerdd"
 sparkVersion := "2.2.0"

From aa21ab8f97c158b0b10ba8766a5e59fb8bdf084a Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Sun, 15 Oct 2017 20:58:02 +0200
Subject: [PATCH 24/47] sbt: update algebird to 0.13.3

---
 build.sbt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.sbt b/build.sbt
index e6968e78..0c293c4a 100644
--- a/build.sbt
+++ b/build.sbt
@@ -100,7 +100,7 @@ val scalactic                 = "org.scalactic"                  %% "scalactic"
 val scalatest                 = "org.scalatest"                  %% "scalatest"                % "3.0.4" % "test"
 
 val joda_time                 = "joda-time"                      % "joda-time"                 % "2.9.9"
-val algebird                  = "com.twitter"                    %% "algebird-core"            % "0.13.2"
+val algebird                  = "com.twitter"                    %% "algebird-core"            % "0.13.3"
 val joda_convert              = "org.joda"                       % "joda-convert"              % "1.9.2"
 val spatial4j                 = "org.locationtech.spatial4j"     % "spatial4j"                 % "0.6"
 

From 714cb21640386b0486bfc001eb53496d40b864b3 Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Wed, 18 Oct 2017 20:47:56 +0200
Subject: [PATCH 25/47] sbt: update Lucene to 7.1.0

---
 build.sbt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.sbt b/build.sbt
index 0c293c4a..1714865f 100644
--- a/build.sbt
+++ b/build.sbt
@@ -79,7 +79,7 @@ pomExtra := (
   </developers>
 )
 
-val luceneV = "7.0.1"
+val luceneV = "7.1.0"
 
 spName := "zouzias/spark-lucenerdd"
 sparkVersion := "2.2.0"

From 3762fc4e25bd776942dbef343ed9bced996862d4 Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Fri, 20 Oct 2017 19:48:34 +0200
Subject: [PATCH 26/47] sbt: plugin: release version update to 1.0.6

---
 project/plugins.sbt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/project/plugins.sbt b/project/plugins.sbt
index 65bfd12a..c5882af1 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -19,7 +19,7 @@ resolvers += "bintray-spark-packages" at "https://dl.bintray.com/spark-packages/
 
 addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.4")
 
-addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.5")
+addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.6")
 
 addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "0.8.0")
 

From 0b1b68246d7f89ac5f9d8c8d3bdecb82681854c5 Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Wed, 25 Oct 2017 14:45:39 +0100
Subject: [PATCH 27/47] sbt: update typesafe config version

---
 build.sbt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.sbt b/build.sbt
index 1714865f..e75030ed 100644
--- a/build.sbt
+++ b/build.sbt
@@ -104,7 +104,7 @@ val algebird                  = "com.twitter"                    %% "algebird-co
 val joda_convert              = "org.joda"                       % "joda-convert"              % "1.9.2"
 val spatial4j                 = "org.locationtech.spatial4j"     % "spatial4j"                 % "0.6"
 
-val typesafe_config           = "com.typesafe"                   % "config"                    % "1.3.1"
+val typesafe_config           = "com.typesafe"                   % "config"                    % "1.3.2"
 
 val lucene_facet              = "org.apache.lucene"              % "lucene-facet"              % luceneV
 val lucene_analyzers          = "org.apache.lucene"              % "lucene-analyzers-common"   % luceneV

From 08aed0129b0caceac9b4084e80b96cf086befed6 Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Thu, 2 Nov 2017 19:06:23 +0100
Subject: [PATCH 28/47] sbt: plugins: sbt-spark-package update to 0.2.6

---
 project/plugins.sbt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/project/plugins.sbt b/project/plugins.sbt
index c5882af1..ff7bae91 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -31,4 +31,4 @@ addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "1.1")
 
 addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.0.0")
 
-addSbtPlugin("org.spark-packages" % "sbt-spark-package" % "0.2.5")
+addSbtPlugin("org.spark-packages" % "sbt-spark-package" % "0.2.6")

From c17e049da16fd462243b4e293ed57535bda47623 Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@users.noreply.github.com>
Date: Fri, 9 Feb 2018 22:34:39 +0100
Subject: [PATCH 29/47] sbt: version updates (#129)

---
 build.sbt           | 14 +++++++-------
 project/plugins.sbt |  2 +-
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/build.sbt b/build.sbt
index e75030ed..98ab2628 100644
--- a/build.sbt
+++ b/build.sbt
@@ -17,8 +17,8 @@
 
 name := "spark-lucenerdd"
 organization := "org.zouzias"
-scalaVersion := "2.11.11"
-crossScalaVersions := Seq("2.11.11")
+scalaVersion := "2.11.12"
+crossScalaVersions := Seq("2.11.12")
 licenses := Seq("Apache-2.0" -> url("http://www.apache.org/licenses/LICENSE-2.0.html"))
 homepage := Some(url("https://github.com/zouzias/spark-lucenerdd"))
 
@@ -79,10 +79,10 @@ pomExtra := (
   </developers>
 )
 
-val luceneV = "7.1.0"
+val luceneV = "7.2.1"
 
 spName := "zouzias/spark-lucenerdd"
-sparkVersion := "2.2.0"
+sparkVersion := "2.2.1"
 spShortDescription := "Spark RDD with Lucene's query capabilities"
 sparkComponents ++= Seq("core", "sql", "mllib")
 spAppendScalaVersion := true
@@ -96,8 +96,8 @@ testSparkVersion := sys.props.get("spark.testVersion").getOrElse(sparkVersion.va
 
 
 // scalastyle:off
-val scalactic                 = "org.scalactic"                  %% "scalactic" % "3.0.4"
-val scalatest                 = "org.scalatest"                  %% "scalatest"                % "3.0.4" % "test"
+val scalactic                 = "org.scalactic"                  %% "scalactic"                % "3.0.5"
+val scalatest                 = "org.scalatest"                  %% "scalatest"                % "3.0.5" % "test"
 
 val joda_time                 = "joda-time"                      % "joda-time"                 % "2.9.9"
 val algebird                  = "com.twitter"                    %% "algebird-core"            % "0.13.3"
@@ -137,7 +137,7 @@ libraryDependencies ++= Seq(
 libraryDependencies ++= Seq(
   "org.apache.spark" %% "spark-core" % testSparkVersion.value % "test" force(),
   "org.apache.spark" %% "spark-sql" % testSparkVersion.value % "test" force(),
-  "com.holdenkarau"  %% "spark-testing-base" % s"${sparkVersion.value}_0.7.4" % "test" intransitive(),
+  "com.holdenkarau"  %% "spark-testing-base" % s"2.2.0_0.8.0" % "test" intransitive(),
   "org.scala-lang"    % "scala-library" % scalaVersion.value % "compile"
 )
 
diff --git a/project/plugins.sbt b/project/plugins.sbt
index ff7bae91..74f12f97 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -17,7 +17,7 @@
 
 resolvers += "bintray-spark-packages" at "https://dl.bintray.com/spark-packages/maven/"
 
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.4")
+addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.6")
 
 addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.6")
 

From f36847e4e592dd2dcc8637a3285f52885b68ed1f Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Fri, 13 Apr 2018 12:28:54 +0200
Subject: [PATCH 30/47] sbt: patch updates

---
 build.sbt | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/build.sbt b/build.sbt
index 98ab2628..4810bd56 100644
--- a/build.sbt
+++ b/build.sbt
@@ -100,11 +100,11 @@ val scalactic                 = "org.scalactic"                  %% "scalactic"
 val scalatest                 = "org.scalatest"                  %% "scalatest"                % "3.0.5" % "test"
 
 val joda_time                 = "joda-time"                      % "joda-time"                 % "2.9.9"
-val algebird                  = "com.twitter"                    %% "algebird-core"            % "0.13.3"
+val algebird                  = "com.twitter"                    %% "algebird-core"            % "0.13.4"
 val joda_convert              = "org.joda"                       % "joda-convert"              % "1.9.2"
 val spatial4j                 = "org.locationtech.spatial4j"     % "spatial4j"                 % "0.6"
 
-val typesafe_config           = "com.typesafe"                   % "config"                    % "1.3.2"
+val typesafe_config           = "com.typesafe"                   % "config"                    % "1.3.3"
 
 val lucene_facet              = "org.apache.lucene"              % "lucene-facet"              % luceneV
 val lucene_analyzers          = "org.apache.lucene"              % "lucene-analyzers-common"   % luceneV
@@ -137,7 +137,7 @@ libraryDependencies ++= Seq(
 libraryDependencies ++= Seq(
   "org.apache.spark" %% "spark-core" % testSparkVersion.value % "test" force(),
   "org.apache.spark" %% "spark-sql" % testSparkVersion.value % "test" force(),
-  "com.holdenkarau"  %% "spark-testing-base" % s"2.2.0_0.8.0" % "test" intransitive(),
+  "com.holdenkarau"  %% "spark-testing-base" % s"2.2.1_0.9.0" % "test" intransitive(),
   "org.scala-lang"    % "scala-library" % scalaVersion.value % "compile"
 )
 

From 109edaccaae57925768bc5b8748cd0802cfc2956 Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Fri, 13 Apr 2018 12:45:00 +0200
Subject: [PATCH 31/47] sbt: update Lucene version 7.3.0

---
 build.sbt                                                   | 6 +++---
 .../org/zouzias/spark/lucenerdd/spatial/shape/package.scala | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/build.sbt b/build.sbt
index 4810bd56..2edb95ef 100644
--- a/build.sbt
+++ b/build.sbt
@@ -79,7 +79,7 @@ pomExtra := (
   </developers>
 )
 
-val luceneV = "7.2.1"
+val luceneV = "7.3.0"
 
 spName := "zouzias/spark-lucenerdd"
 sparkVersion := "2.2.1"
@@ -102,7 +102,7 @@ val scalatest                 = "org.scalatest"                  %% "scalatest"
 val joda_time                 = "joda-time"                      % "joda-time"                 % "2.9.9"
 val algebird                  = "com.twitter"                    %% "algebird-core"            % "0.13.4"
 val joda_convert              = "org.joda"                       % "joda-convert"              % "1.9.2"
-val spatial4j                 = "org.locationtech.spatial4j"     % "spatial4j"                 % "0.6"
+val spatial4j                 = "org.locationtech.spatial4j"     % "spatial4j"                 % "0.7"
 
 val typesafe_config           = "com.typesafe"                   % "config"                    % "1.3.3"
 
@@ -113,7 +113,7 @@ val lucene_expressions        = "org.apache.lucene"              % "lucene-expre
 val lucene_spatial            = "org.apache.lucene"              % "lucene-spatial"            % luceneV
 val lucene_spatial_extras     = "org.apache.lucene"              % "lucene-spatial-extras"     % luceneV
 
-val jts                       = "com.vividsolutions"             % "jts"                       % "1.13"
+val jts                       = "org.locationtech.jts"           % "jts-core"                  % "1.15.0"
 // scalastyle:on
 
 
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/package.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/package.scala
index 3ca6c12d..b7744729 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/package.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/package.scala
@@ -18,7 +18,7 @@ package org.zouzias.spark.lucenerdd.spatial
 
 import java.io.StringReader
 
-import com.vividsolutions.jts.geom.{Coordinate, GeometryFactory}
+import org.locationtech.jts.geom.{Coordinate, GeometryFactory}
 import org.locationtech.spatial4j.shape.Shape
 import org.zouzias.spark.lucenerdd.spatial.shape.context.ContextLoader
 

From 91555b15a1fa7dea27e1dbcb31bdde572fe69bcd Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Sun, 15 Oct 2017 20:56:47 +0200
Subject: [PATCH 32/47] point: prepare packages

---
 .../context/ContextLoader.scala               |  2 +-
 .../grids/PrefixTreeLoader.scala              |  4 ++--
 .../strategies/SpatialStrategy.scala          |  4 ++--
 .../spatial/point/PointLuceneRDD.scala        | 21 +++++++++++++++++++
 .../lucenerdd/spatial/shape/package.scala     |  2 +-
 .../partition/ShapeLuceneRDDPartition.scala   |  2 +-
 .../shape/ShapeLuceneRDDKnnSearchSpec.scala   |  2 +-
 .../shape/ShapeLuceneRDDLinkageSpec.scala     |  2 +-
 .../ShapeLuceneRDDSpatialSearchSpec.scala     |  2 +-
 .../spatial/shape/ShapeLuceneRDDSpec.scala    |  2 +-
 .../ShapeLuceneRDDImplicitsSpec.scala         |  2 +-
 11 files changed, 33 insertions(+), 12 deletions(-)
 rename src/main/scala/org/zouzias/spark/lucenerdd/spatial/{shape => commons}/context/ContextLoader.scala (97%)
 rename src/main/scala/org/zouzias/spark/lucenerdd/spatial/{shape => commons}/grids/PrefixTreeLoader.scala (92%)
 rename src/main/scala/org/zouzias/spark/lucenerdd/spatial/{shape => commons}/strategies/SpatialStrategy.scala (91%)
 create mode 100644 src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala

diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/context/ContextLoader.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/context/ContextLoader.scala
similarity index 97%
rename from src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/context/ContextLoader.scala
rename to src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/context/ContextLoader.scala
index c99ed537..3e3cbc9d 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/context/ContextLoader.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/context/ContextLoader.scala
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.zouzias.spark.lucenerdd.spatial.shape.context
+package org.zouzias.spark.lucenerdd.spatial.commons.context
 
 import java.io.{StringReader, StringWriter}
 
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/grids/PrefixTreeLoader.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/grids/PrefixTreeLoader.scala
similarity index 92%
rename from src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/grids/PrefixTreeLoader.scala
rename to src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/grids/PrefixTreeLoader.scala
index 19497ac4..5ea16a80 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/grids/PrefixTreeLoader.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/grids/PrefixTreeLoader.scala
@@ -14,11 +14,11 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.zouzias.spark.lucenerdd.spatial.shape.grids
+package org.zouzias.spark.lucenerdd.spatial.commons.grids
 
 import org.apache.lucene.spatial.prefix.tree.{SpatialPrefixTree, SpatialPrefixTreeFactory}
 import org.zouzias.spark.lucenerdd.config.ShapeLuceneRDDConfigurable
-import org.zouzias.spark.lucenerdd.spatial.shape.context.ContextLoader
+import org.zouzias.spark.lucenerdd.spatial.commons.context.ContextLoader
 
 import scala.collection.JavaConverters._
 
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/strategies/SpatialStrategy.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/strategies/SpatialStrategy.scala
similarity index 91%
rename from src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/strategies/SpatialStrategy.scala
rename to src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/strategies/SpatialStrategy.scala
index 5e62a870..c3cf6a31 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/strategies/SpatialStrategy.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/strategies/SpatialStrategy.scala
@@ -14,10 +14,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.zouzias.spark.lucenerdd.spatial.shape.strategies
+package org.zouzias.spark.lucenerdd.spatial.commons.strategies
 
 import org.apache.lucene.spatial.prefix.{PrefixTreeStrategy, RecursivePrefixTreeStrategy}
-import org.zouzias.spark.lucenerdd.spatial.shape.grids.PrefixTreeLoader
+import org.zouzias.spark.lucenerdd.spatial.commons.grids.PrefixTreeLoader
 
 trait SpatialStrategy extends PrefixTreeLoader {
 
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala
new file mode 100644
index 00000000..5d098927
--- /dev/null
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.zouzias.spark.lucenerdd.spatial.point
+
+class PointLuceneRDD {
+
+}
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/package.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/package.scala
index b7744729..80e70218 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/package.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/package.scala
@@ -20,7 +20,7 @@ import java.io.StringReader
 
 import org.locationtech.jts.geom.{Coordinate, GeometryFactory}
 import org.locationtech.spatial4j.shape.Shape
-import org.zouzias.spark.lucenerdd.spatial.shape.context.ContextLoader
+import org.zouzias.spark.lucenerdd.spatial.commons.context.ContextLoader
 
 
 package object shape extends ContextLoader{
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/partition/ShapeLuceneRDDPartition.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/partition/ShapeLuceneRDDPartition.scala
index aaeac337..e472c6c5 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/partition/ShapeLuceneRDDPartition.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/partition/ShapeLuceneRDDPartition.scala
@@ -28,7 +28,7 @@ import org.zouzias.spark.lucenerdd.models.SparkScoreDoc
 import org.zouzias.spark.lucenerdd.query.LuceneQueryHelpers
 import org.zouzias.spark.lucenerdd.response.LuceneRDDResponsePartition
 import org.zouzias.spark.lucenerdd.spatial.shape.ShapeLuceneRDD.PointType
-import org.zouzias.spark.lucenerdd.spatial.shape.strategies.SpatialStrategy
+import org.zouzias.spark.lucenerdd.spatial.commons.strategies.SpatialStrategy
 import org.zouzias.spark.lucenerdd.store.IndexWithTaxonomyWriter
 
 import scala.reflect._
diff --git a/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDKnnSearchSpec.scala b/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDKnnSearchSpec.scala
index 29ab1450..53caf7fb 100644
--- a/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDKnnSearchSpec.scala
+++ b/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDKnnSearchSpec.scala
@@ -21,7 +21,7 @@ import com.holdenkarau.spark.testing.SharedSparkContext
 import org.apache.spark.SparkConf
 import org.scalatest.{BeforeAndAfterEach, FlatSpec, Matchers}
 import org.zouzias.spark.lucenerdd._
-import org.zouzias.spark.lucenerdd.spatial.shape.context.ContextLoader
+import org.zouzias.spark.lucenerdd.spatial.commons.context.ContextLoader
 import org.zouzias.spark.lucenerdd.testing.LuceneRDDTestUtils
 
 class ShapeLuceneRDDKnnSearchSpec extends FlatSpec
diff --git a/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDLinkageSpec.scala b/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDLinkageSpec.scala
index d5347077..b57d333b 100644
--- a/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDLinkageSpec.scala
+++ b/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDLinkageSpec.scala
@@ -20,7 +20,7 @@ import com.holdenkarau.spark.testing.SharedSparkContext
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.{Row, SparkSession}
 import org.scalatest.{BeforeAndAfterEach, FlatSpec, Matchers}
-import org.zouzias.spark.lucenerdd.spatial.shape.context.ContextLoader
+import org.zouzias.spark.lucenerdd.spatial.commons.context.ContextLoader
 import org.zouzias.spark.lucenerdd.testing.LuceneRDDTestUtils
 
 // Required for implicit Document conversion
diff --git a/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDSpatialSearchSpec.scala b/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDSpatialSearchSpec.scala
index 4baea509..3fee0f0c 100644
--- a/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDSpatialSearchSpec.scala
+++ b/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDSpatialSearchSpec.scala
@@ -23,7 +23,7 @@ import org.apache.spark.SparkConf
 import org.locationtech.spatial4j.distance.DistanceUtils
 import org.scalatest.{BeforeAndAfterEach, FlatSpec, Matchers}
 import org.zouzias.spark.lucenerdd._
-import org.zouzias.spark.lucenerdd.spatial.shape.context.ContextLoader
+import org.zouzias.spark.lucenerdd.spatial.commons.context.ContextLoader
 import org.zouzias.spark.lucenerdd.testing.LuceneRDDTestUtils
 
 class ShapeLuceneRDDSpatialSearchSpec extends FlatSpec
diff --git a/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDSpec.scala b/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDSpec.scala
index bbbedfb5..bc616b25 100644
--- a/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDSpec.scala
+++ b/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDSpec.scala
@@ -25,7 +25,7 @@ import org.locationtech.spatial4j.distance.DistanceUtils
 import org.scalatest.{BeforeAndAfterEach, FlatSpec, Matchers}
 import org.zouzias.spark.lucenerdd.testing.LuceneRDDTestUtils
 import org.zouzias.spark.lucenerdd._
-import org.zouzias.spark.lucenerdd.spatial.shape.context.ContextLoader
+import org.zouzias.spark.lucenerdd.spatial.commons.context.ContextLoader
 
 class ShapeLuceneRDDSpec extends FlatSpec
   with Matchers
diff --git a/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/implicits/ShapeLuceneRDDImplicitsSpec.scala b/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/implicits/ShapeLuceneRDDImplicitsSpec.scala
index 223e94b7..4464bc88 100644
--- a/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/implicits/ShapeLuceneRDDImplicitsSpec.scala
+++ b/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/implicits/ShapeLuceneRDDImplicitsSpec.scala
@@ -23,7 +23,7 @@ import org.scalatest.{BeforeAndAfterEach, FlatSpec, Matchers}
 import org.zouzias.spark.lucenerdd.spatial.shape.{ShapeLuceneRDD, _}
 import org.zouzias.spark.lucenerdd.testing.LuceneRDDTestUtils
 import org.zouzias.spark.lucenerdd._
-import org.zouzias.spark.lucenerdd.spatial.shape.context.ContextLoader
+import org.zouzias.spark.lucenerdd.spatial.commons.context.ContextLoader
 
 class ShapeLuceneRDDImplicitsSpec extends FlatSpec
   with Matchers

From a36c1202940188f6cfddadc126b9e1380fdd1bdc Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Sun, 15 Oct 2017 21:22:18 +0200
Subject: [PATCH 33/47] point: WIP

---
 .../spatial/point/PointLuceneRDD.scala        | 430 +++++++++++++++++-
 .../AbstractPointLuceneRDDPartition.scala     | 111 +++++
 .../partition/PointLuceneRDDPartition.scala   | 136 ++++++
 3 files changed, 676 insertions(+), 1 deletion(-)
 create mode 100644 src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/AbstractPointLuceneRDDPartition.scala
 create mode 100644 src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/PointLuceneRDDPartition.scala

diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala
index 5d098927..a7f4fb9a 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala
@@ -16,6 +16,434 @@
  */
 package org.zouzias.spark.lucenerdd.spatial.point
 
-class PointLuceneRDD {
+import com.twitter.algebird.TopKMonoid
+import org.apache.lucene.document.Document
+import org.apache.lucene.spatial.query.SpatialOperation
+import org.apache.spark.{OneToOneDependency, Partition, TaskContext}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, Dataset, Row}
+import org.apache.spark.storage.StorageLevel
+import org.locationtech.spatial4j.shape.Shape
+import org.zouzias.spark.lucenerdd.analyzers.AnalyzerConfigurable
+import org.zouzias.spark.lucenerdd.config.ShapeLuceneRDDConfigurable
+import org.zouzias.spark.lucenerdd.models.SparkScoreDoc
+import org.zouzias.spark.lucenerdd.query.{LuceneQueryHelpers, SimilarityConfigurable}
+import org.zouzias.spark.lucenerdd.response.{LuceneRDDResponse, LuceneRDDResponsePartition}
+import org.zouzias.spark.lucenerdd.spatial.point.partition.{AbstractPointLuceneRDDPartition, PointLuceneRDDPartition}
+import org.zouzias.spark.lucenerdd.spatial.shape.ShapeLuceneRDD
+import org.zouzias.spark.lucenerdd.spatial.shape.ShapeLuceneRDD._
+import org.zouzias.spark.lucenerdd.versioning.Versionable
 
+import scala.reflect.ClassTag
+
+class PointLuceneRDD[V: ClassTag]
+  (private val partitionsRDD: RDD[AbstractPointLuceneRDDPartition[V]],
+   val indexAnalyzerName: String,
+   val queryAnalyzerName: String,
+   val similarity: String)
+  extends RDD[V](partitionsRDD.context, List(new OneToOneDependency(partitionsRDD)))
+    with ShapeLuceneRDDConfigurable {
+
+  logInfo("Instance is created...")
+  logInfo(s"Number of partitions: ${partitionsRDD.count()}")
+  setName("PointLuceneRDD")
+
+  override protected def getPartitions: Array[Partition] = partitionsRDD.partitions
+
+  override protected def getPreferredLocations(s: Partition): Seq[String] =
+    partitionsRDD.preferredLocations(s)
+
+  override def cache(): this.type = {
+    this.persist(StorageLevel.MEMORY_ONLY)
+  }
+
+  override def persist(newLevel: StorageLevel): this.type = {
+    partitionsRDD.persist(newLevel)
+    super.persist(newLevel)
+    this
+  }
+
+  override def unpersist(blocking: Boolean = true): this.type = {
+    partitionsRDD.unpersist(blocking)
+    super.unpersist(blocking)
+    this
+  }
+
+  override def setName(_name: String): this.type = {
+    if (partitionsRDD.name != null) {
+      partitionsRDD.setName(partitionsRDD.name + ", " + _name)
+    } else {
+      partitionsRDD.setName(_name)
+    }
+    this
+  }
+
+  setName("ShapeLuceneRDD")
+
+  private def partitionMapper(f: AbstractPointLuceneRDDPartition[V] =>
+    LuceneRDDResponsePartition): LuceneRDDResponse = {
+    new LuceneRDDResponse(partitionsRDD.map(f), SparkScoreDoc.ascending)
+  }
+
+  private def linker[T: ClassTag](that: RDD[T],
+                                  pointFunctor: T => PointType,
+                                  mapper: ( PointType, AbstractPointLuceneRDDPartition[V]) =>
+                                    Iterable[SparkScoreDoc],
+                                  linkerMethod: String)
+  : RDD[(T, Array[SparkScoreDoc])] = {
+    logInfo("Shape Linkage requested")
+
+    val topKMonoid = new TopKMonoid[SparkScoreDoc](MaxDefaultTopKValue)(SparkScoreDoc.ascending)
+    val queries = that.zipWithIndex().map(_.swap)
+
+    val resultsByPart = linkerMethod match {
+      case "cartesian" =>
+        val concatenated = queries.mapValues(pointFunctor).glom()
+
+        concatenated.cartesian(partitionsRDD)
+          .flatMap { case (qs, lucene) =>
+            qs.map { case (ind, query) =>
+              (ind, topKMonoid.build(mapper((query._1, query._2), lucene)))
+            }
+          }
+      case _ =>
+        logInfo("Collecting query points to driver")
+        val collectedQueries = queries.mapValues(pointFunctor).collect()
+        val queriesB = partitionsRDD.context.broadcast(collectedQueries)
+
+        partitionsRDD.mapPartitions { partitions =>
+            partitions.flatMap { partition =>
+              queriesB.value.map { case (index, (x, y)) =>
+                  (index, topKMonoid.build(mapper((x, y), partition)))
+                }
+            }
+        }
+    }
+
+    logInfo("Computing top-k linkage per partition")
+    val results = resultsByPart.reduceByKey(topKMonoid.plus)
+
+    queries.join(results).values
+      .map(joined => (joined._1, joined._2.items.toArray))
+  }
+
+  /**
+   * Link entities based on k-nearest neighbors (Knn)
+   *
+   * Links this and that based on nearest neighbors, returns Knn
+   *
+   *
+   * @param that An RDD of entities to be linked
+   * @param pointFunctor Function that generates a point from each element of other
+   * @param linkerMethod Method to perform linkage
+   * @tparam T A type
+   * @return an RDD of Tuple2 that contains the linked results
+   *
+   * Note: Currently the query coordinates of the other RDD are collected to the driver and
+   * broadcast to the workers.
+   */
+  def linkByKnn[T: ClassTag](that: RDD[T],
+                             pointFunctor: T => PointType,
+                             topK: Int = DefaultTopK,
+                             linkerMethod: String = getShapeLinkerMethod)
+  : RDD[(T, Array[SparkScoreDoc])] = {
+    logInfo("linkByKnn requested")
+    linker[T](that, pointFunctor, (queryPoint, part) =>
+      part.knnSearch(queryPoint, topK, LuceneQueryHelpers.MatchAllDocsString),
+      linkerMethod)
+  }
+
+  /**
+   * Link entities if their shapes are within a distance in kilometers (km)
+   *
+   * Links this and that based on distance threshold
+   *
+   * @param that An RDD of entities to be linked
+   * @param pointFunctor Function that generates a point from each element of other
+   * @param linkerMethod Method to perform linkage
+   * @tparam T A type
+   * @return an RDD of Tuple2 that contains the linked results
+   *
+   * Note: Currently the query coordinates of the other RDD are collected to the driver and
+   * broadcast to the workers.
+   */
+  def linkByRadius[T: ClassTag](that: RDD[T],
+                                pointFunctor: T => PointType,
+                                radius: Double,
+                                topK: Int = DefaultTopK,
+                                spatialOp: String = SpatialOperation.Intersects.getName,
+                                linkerMethod: String = getShapeLinkerMethod)
+  : RDD[(T, Array[SparkScoreDoc])] = {
+    logInfo("linkByRadius requested")
+    linker[T](that, pointFunctor, (queryPoint, part) =>
+      part.circleSearch(queryPoint, radius, topK, spatialOp),
+      linkerMethod)
+  }
+
+
+  /**
+   * Link with DataFrame based on k-nearest neighbors (Knn)
+   *
+   * Links this and that based on nearest neighbors, returns Knn
+   *
+   *
+   * @param other DataFrame of entities to be linked
+   * @param searchQueryGen Function that generates a point from each element of other
+    * @param topK Number of linked items
+   * @param linkerMethod Method to perform linkage
+   * @return
+   */
+  def linkDataFrameByKnn(other: DataFrame,
+                         searchQueryGen: Row => PointType,
+                         topK: Int = DefaultTopK,
+                         linkerMethod: String = getShapeLinkerMethod)
+  : RDD[(Row, Array[SparkScoreDoc])] = {
+    logInfo("linkDataFrameByKnn requested")
+    linkByKnn[Row](other.rdd, searchQueryGen, topK, linkerMethod)
+  }
+
+  /**
+   * Link entities if their shapes are within a distance in kilometers (km)
+   *
+   * Links this and that based on distance threshold
+   *
+   * @param other DataFrame of entities to be linked
+   * @param pointFunctor Function that generates a point from each element of other
+   * @param linkerMethod Method to perform linkage
+   * @return an RDD of Tuple2 that contains the linked results
+   *
+   * Note: Currently the query coordinates of the other RDD are collected to the driver and
+   * broadcast to the workers.
+   */
+  def linkDataFrameByRadius(other: DataFrame,
+                            pointFunctor: Row => PointType,
+                            radius: Double,
+                            topK: Int = DefaultTopK,
+                            spatialOp: String = SpatialOperation.Intersects.getName,
+                            linkerMethod: String = getShapeLinkerMethod)
+  : RDD[(Row, Array[SparkScoreDoc])] = {
+    logInfo("linkDataFrameByRadius requested")
+    linkByRadius[Row](other.rdd, pointFunctor, radius, topK, spatialOp, linkerMethod)
+  }
+
+  /**
+   * K-nearest neighbors search
+   *
+   * @param queryPoint query point (X, Y)
+   * @param k number of nearest neighbor points to return
+   * @param searchString Lucene query string
+   * @return
+   */
+  def knnSearch(queryPoint: PointType, k: Int,
+                searchString: String = LuceneQueryHelpers.MatchAllDocsString)
+  : LuceneRDDResponse = {
+    logInfo(s"Knn search with query ${queryPoint} and search string ${searchString}")
+    partitionMapper(_.knnSearch(queryPoint, k, searchString))
+  }
+
+  /**
+   * Search for points within a circle
+   *
+   * @param center center of circle
+   * @param radius radius of circle in kilometers (KM)
+   * @param k number of points to return
+   * @return
+   */
+  def circleSearch(center: PointType, radius: Double, k: Int)
+  : LuceneRDDResponse = {
+    logInfo(s"Circle search with center ${center} and radius ${radius}")
+    // Points can only intersect
+    partitionMapper(_.circleSearch(center, radius, k,
+      SpatialOperation.Intersects.getName))
+  }
+
+  /**
+   * Spatial search with arbitrary shape
+   *
+   * @param shapeWKT Shape in WKT format
+   * @param k Number of element to return
+   * @param operationName
+   * @return
+   */
+  def spatialSearch(shapeWKT: String, k: Int,
+                    operationName: String = SpatialOperation.Intersects.getName)
+  : LuceneRDDResponse = {
+    logInfo(s"Spatial search with shape ${shapeWKT} and operation ${operationName}")
+    partitionMapper(_.spatialSearch(shapeWKT, k, operationName))
+  }
+
+  /**
+   * Spatial search with a single Point
+   *
+   * @param point
+   * @param k
+   * @param operationName
+   * @return
+   */
+  def spatialSearch(point: PointType, k: Int,
+                    operationName: String)
+  : LuceneRDDResponse = {
+    logInfo(s"Spatial search with point ${point} and operation ${operationName}")
+    partitionMapper(_.spatialSearch(point, k, operationName))
+  }
+
+  /**
+   * Bounding box search with center and radius
+   *
+   * @param center given as (x, y)
+   * @param radius in kilometers (KM)
+   * @param k
+   * @param operationName
+   * @return
+   */
+  def bboxSearch(center: PointType, radius: Double, k: Int,
+                    operationName: String = SpatialOperation.Intersects.getName)
+  : LuceneRDDResponse = {
+    logInfo(s"Bounding box with center ${center}, radius ${radius}, k = ${k}")
+    partitionMapper(_.bboxSearch(center, radius, k, operationName))
+  }
+
+  /**
+   * Bounding box search with rectangle
+   * @param lowerLeft Lower left corner
+   * @param upperRight Upper right corner
+   * @param k Number of results
+   * @param operationName Intersect, contained, etc.
+   * @return
+   */
+  def bboxSearch(lowerLeft: PointType,
+                 upperRight: PointType,
+                 k: Int,
+                 operationName: String)
+  : LuceneRDDResponse = {
+    logInfo(s"Bounding box with lower left ${lowerLeft}, upper right ${upperRight} and k = ${k}")
+    partitionMapper(_.bboxSearch(lowerLeft, upperRight, k, operationName))
+  }
+
+  override def count(): Long = {
+    logInfo("Count requested")
+    partitionsRDD.map(_.size).reduce(_ + _)
+  }
+
+  /** RDD compute method. */
+  override def compute(part: Partition, context: TaskContext): Iterator[V] = {
+    firstParent[AbstractPointLuceneRDDPartition[V]].iterator(part, context).next.iterator
+  }
+
+  override def filter(pred: V => Boolean): PointLuceneRDD[V] = {
+    val newPartitionRDD = partitionsRDD.mapPartitions(partition =>
+      partition.map(_.filter(pred)), preservesPartitioning = true
+    )
+    new PointLuceneRDD(newPartitionRDD, indexAnalyzerName, queryAnalyzerName, similarity)
+  }
+
+  def exists(point: PointType): Boolean = {
+    partitionsRDD.map(_.isDefined(point)).collect().exists(x => x)
+  }
+
+  def close(): Unit = {
+    logInfo(s"Closing...")
+    partitionsRDD.foreach(_.close())
+  }
+}
+
+
+object PointLuceneRDD extends Versionable
+  with AnalyzerConfigurable
+  with SimilarityConfigurable {
+
+  /** Type for a point */
+  type PointType = (Double, Double)
+
+  /**
+    * Instantiate a ShapeLuceneRDD given an RDD[T]
+    *
+    * @param elems RDD of type T
+    * @param indexAnalyzer Index Analyzer name
+    * @param queryAnalyzer Query Analyzer name
+    * @param similarity Lucene scoring similarity, i.e., BM25 or TF-IDF
+    * @return
+    */
+  def apply[V: ClassTag](elems: RDD[V],
+                                      indexAnalyzer: String,
+                                      queryAnalyzer: String,
+                                      similarity: String)
+                                     (implicit docConverter: V => Document)
+  : PointLuceneRDD[V] = {
+    val partitions = elems.mapPartitions[AbstractPointLuceneRDDPartition[V]](
+      iter => Iterator(PointLuceneRDDPartition[V](iter, indexAnalyzer, queryAnalyzer)),
+      preservesPartitioning = true)
+    new PointLuceneRDD(partitions, indexAnalyzer, queryAnalyzer, similarity)
+  }
+
+  def apply[K: ClassTag, V: ClassTag](elems: RDD[V])
+                                     (implicit docConverter: V => Document)
+  : PointLuceneRDD[V] = {
+    apply[V](elems, getOrElseEn(IndexAnalyzerConfigName), getOrElseEn(QueryAnalyzerConfigName),
+      getOrElseClassic())
+  }
+
+  /**
+    * Constructor for [[Dataset]]
+    */
+  def apply[V: ClassTag](elems: Dataset[V],
+                                      indexAnalyzer: String,
+                                      queryAnalyzer: String,
+                                      similarity: String)
+                                     (implicit docConverter: V => Document)
+  : PointLuceneRDD[V] = {
+    val partitions = elems.rdd.mapPartitions[AbstractPointLuceneRDDPartition[V]](
+      iter => Iterator(PointLuceneRDDPartition[V](iter, indexAnalyzer, queryAnalyzer)),
+      preservesPartitioning = true)
+    new PointLuceneRDD(partitions, indexAnalyzer, queryAnalyzer, similarity)
+  }
+
+  /**
+    * Constructor for [[Dataset]]
+    */
+  def apply[V: ClassTag](elems: Dataset[V])
+                                     (implicit docConverter: V => Document)
+  : PointLuceneRDD[V] = {
+    apply[V](elems, getOrElseEn(IndexAnalyzerConfigName), getOrElseEn(QueryAnalyzerConfigName),
+      getOrElseClassic())
+  }
+
+  /**
+    * Instantiate [[PointLuceneRDD]] from DataFrame with spatial column (shape format)
+    *
+    * Shape format can be one of ShapeIO.GeoJSON, ShapeIO.LEGACY, ShapeIO.POLY, ShapeIO.WKT
+    *
+    * {{
+    *  val countries = spark.read.parquet("data/countries-bbox.parquet")
+    *  val lucene = ShapeLuceneRDD(counties, "shape")
+    *
+    * }}
+    * @param df Input dataframe containing Shape as String field named "shapeField"
+    * @param shapeField Name of DataFrame column that contains Shape as String, i.e., WKT
+    * @param shapeConv Implicit convertion for spatial / shape
+    * @param docConverter Implicit conversion for Lucene Document
+    * @return
+    */
+  def apply(df : DataFrame,
+            shapeField: String)
+           (implicit shapeConv: String => Shape, docConverter: Row => Document)
+  : ShapeLuceneRDD[String, Row] = {
+    apply(df, shapeField,
+      getOrElseEn(IndexAnalyzerConfigName), getOrElseEn(QueryAnalyzerConfigName),
+      getOrElseClassic())
+  }
+
+  def apply(df : DataFrame,
+            shapeField: String,
+            indexAnalyzer: String,
+            queryAnalyzer: String,
+            similarity: String)
+           (implicit docConverter: Row => Document)
+  : PointLuceneRDD[Row] = {
+    val partitions = df.rdd.map(row => (row.getString(row.fieldIndex(shapeField)), row))
+      .mapPartitions[AbstractPointLuceneRDDPartition[Row]](
+      iter => Iterator(PointLuceneRDDPartition[Row](iter, indexAnalyzer, queryAnalyzer)),
+      preservesPartitioning = true)
+    new PointLuceneRDD(partitions, indexAnalyzer, queryAnalyzer, similarity)
+  }
 }
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/AbstractPointLuceneRDDPartition.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/AbstractPointLuceneRDDPartition.scala
new file mode 100644
index 00000000..e7225496
--- /dev/null
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/AbstractPointLuceneRDDPartition.scala
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.zouzias.spark.lucenerdd.spatial.point.partition
+
+import org.zouzias.spark.lucenerdd.response.LuceneRDDResponsePartition
+import org.zouzias.spark.lucenerdd.spatial.shape.ShapeLuceneRDD.PointType
+
+import scala.reflect.ClassTag
+
+
+private[shape] abstract class AbstractPointLuceneRDDPartition[V] extends Serializable {
+
+  protected implicit def vTag: ClassTag[V]
+
+  def size: Long
+
+  def iterator: Iterator[V]
+
+  def isDefined(point: PointType): Boolean
+
+  def close(): Unit
+
+  /**
+   * Nearest neighbour search
+   *
+   * @param point query point
+   * @param k number of neighbors to return
+   * @param searchString Lucene Query string
+   * @return
+   */
+  def knnSearch(point: PointType, k: Int, searchString: String): LuceneRDDResponsePartition
+
+  /**
+   * Search for points within a circle
+   *
+   * @param center center of circle
+   * @param radius radius of circle in kilometers (KM)
+   * @param k number of points to return
+   * @return
+   */
+  def circleSearch(center: PointType, radius: Double, k: Int, operationName: String)
+  : LuceneRDDResponsePartition
+
+  /**
+   * Spatial search with arbitrary shape
+   *
+   * @param shapeAsString Shape object represented as String
+   * @param k Number of results to return
+   * @param operationName Operation name, i.e., intersect, within, etc
+   * @return
+   */
+  def spatialSearch(shapeAsString: String, k: Int, operationName: String)
+  : LuceneRDDResponsePartition
+
+  /**
+   * Spatial search with point
+   *
+   * @param point Query point
+   * @param k Number of result to return
+   * @param operationName Operation name, i.e., intersect, within, etc
+   * @return
+   */
+  def spatialSearch(point: PointType, k: Int, operationName: String)
+  : LuceneRDDResponsePartition
+
+  /**
+   * Bounding box search with point and radius
+   *
+   * @param center given as (x, y)
+   * @param radius distance from center in kilometers (KM)
+   * @param k Number of results to return
+   * @param operationName Operation name, i.e., intersect, within, etc
+   * @return
+   */
+  def bboxSearch(center: PointType, radius: Double, k: Int, operationName: String)
+  : LuceneRDDResponsePartition
+
+  /**
+   * Bounding box search with lower left and upper right corners
+   *
+   * @param lowerLeft Lower left point
+   * @param upperRight Upper left point
+   * @param k Number of results
+   * @param operationName Operation name, i.e., intersect, within, etc
+   * @return
+   */
+  def bboxSearch(lowerLeft: PointType, upperRight: PointType, k: Int, operationName: String)
+  : LuceneRDDResponsePartition
+
+  /**
+   * Restricts the entries to those satisfying a predicate
+   *
+   * @param pred Predicate to filter on
+   * @return
+   */
+  def filter(pred: V => Boolean): AbstractPointLuceneRDDPartition[V]
+}
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/PointLuceneRDDPartition.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/PointLuceneRDDPartition.scala
new file mode 100644
index 00000000..5930dfe7
--- /dev/null
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/PointLuceneRDDPartition.scala
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.zouzias.spark.lucenerdd.spatial.point.partition
+
+import org.apache.lucene.document.Document
+import org.zouzias.spark.lucenerdd.spatial.commons.strategies.SpatialStrategy
+import org.zouzias.spark.lucenerdd.store.IndexWithTaxonomyWriter
+
+import scala.reflect.{ClassTag, classTag}
+
+
+private[shape] class PointLuceneRDDPartition[V]
+(private val iter: Iterator[V],
+ private val indexAnalyzerName: String,
+ private val queryAnalyzerName: String)
+(override implicit val vTag: ClassTag[V])
+(implicit docConversion: V => Document)
+  extends AbstractPointLuceneRDDPartition[V]
+    with IndexWithTaxonomyWriter
+    with SpatialStrategy {
+  override def size = ???
+
+  override def iterator = ???
+
+  override def isDefined(point: (Double, Double)) = ???
+
+  /**
+    * Nearest neighbour search
+    *
+    * @param point        query point
+    * @param k            number of neighbors to return
+    * @param searchString Lucene Query string
+    * @return
+    */
+  override def knnSearch(point: (Double, Double), k: Int, searchString: String) = ???
+
+  /**
+    * Search for points within a circle
+    *
+    * @param center center of circle
+    * @param radius radius of circle in kilometers (KM)
+    * @param k      number of points to return
+    * @return
+    */
+  override def circleSearch(center: (Double, Double), radius: Double, k: Int, operationName: String) = ???
+
+  /**
+    * Spatial search with arbitrary shape
+    *
+    * @param shapeAsString Shape object represented as String
+    * @param k             Number of results to return
+    * @param operationName Operation name, i.e., intersect, within, etc
+    * @return
+    */
+  override def spatialSearch(shapeAsString: String, k: Int, operationName: String) = ???
+
+  /**
+    * Spatial search with point
+    *
+    * @param point         Query point
+    * @param k             Number of result to return
+    * @param operationName Operation name, i.e., intersect, within, etc
+    * @return
+    */
+  override def spatialSearch(point: (Double, Double), k: Int, operationName: String) = ???
+
+  /**
+    * Bounding box search with point and radius
+    *
+    * @param center        given as (x, y)
+    * @param radius        distance from center in kilometers (KM)
+    * @param k             Number of results to return
+    * @param operationName Operation name, i.e., intersect, within, etc
+    * @return
+    */
+  override def bboxSearch(center: (Double, Double), radius: Double, k: Int, operationName: String) = ???
+
+  /**
+    * Bounding box search with lower left and upper right corners
+    *
+    * @param lowerLeft     Lower left point
+    * @param upperRight    Upper left point
+    * @param k             Number of results
+    * @param operationName Operation name, i.e., intersect, within, etc
+    * @return
+    */
+  override def bboxSearch(lowerLeft: (Double, Double),
+                          upperRight: (Double, Double),
+                          k: Int, operationName: String) = ???
+
+  /**
+    * Restricts the entries to those satisfying a predicate
+    *
+    * @param pred Predicate to filter on
+    * @return
+    */
+  override def filter(pred: V => Boolean) = ???
+
+  override protected def indexAnalyzer() = ???
+}
+
+object PointLuceneRDDPartition {
+
+  /**
+    * Constructor for [[PointLuceneRDDPartition]]
+    *
+    * @param iter Iterator over data
+    * @param indexAnalyzer Index analyzer
+    * @param queryAnalyzer Query analyzer
+    * @param docConv Implicit convertion to Lucene document
+    * @tparam V
+    * @return
+    */
+  def apply[V: ClassTag](iter: Iterator[V],
+                                      indexAnalyzer: String,
+                                      queryAnalyzer: String)
+                                     (implicit docConv: V => Document)
+  : PointLuceneRDDPartition[V] = {
+    new PointLuceneRDDPartition[V](iter,
+      indexAnalyzer, queryAnalyzer)(classTag[V]) (docConv)
+  }
+}

From f95eda3e15d55b437cd199e49ca21b06ffd93a42 Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Fri, 20 Oct 2017 20:16:48 +0200
Subject: [PATCH 34/47] point_rdd: fix compilation

---
 .../spatial/point/PointLuceneRDD.scala        |  29 ++-
 .../AbstractPointLuceneRDDPartition.scala     |   6 +-
 .../partition/PointLuceneRDDPartition.scala   | 225 ++++++++++++------
 3 files changed, 174 insertions(+), 86 deletions(-)

diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala
index a7f4fb9a..6929659b 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala
@@ -23,15 +23,13 @@ import org.apache.spark.{OneToOneDependency, Partition, TaskContext}
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.{DataFrame, Dataset, Row}
 import org.apache.spark.storage.StorageLevel
-import org.locationtech.spatial4j.shape.Shape
 import org.zouzias.spark.lucenerdd.analyzers.AnalyzerConfigurable
 import org.zouzias.spark.lucenerdd.config.ShapeLuceneRDDConfigurable
 import org.zouzias.spark.lucenerdd.models.SparkScoreDoc
 import org.zouzias.spark.lucenerdd.query.{LuceneQueryHelpers, SimilarityConfigurable}
 import org.zouzias.spark.lucenerdd.response.{LuceneRDDResponse, LuceneRDDResponsePartition}
+import org.zouzias.spark.lucenerdd.spatial.point.PointLuceneRDD.PointType
 import org.zouzias.spark.lucenerdd.spatial.point.partition.{AbstractPointLuceneRDDPartition, PointLuceneRDDPartition}
-import org.zouzias.spark.lucenerdd.spatial.shape.ShapeLuceneRDD
-import org.zouzias.spark.lucenerdd.spatial.shape.ShapeLuceneRDD._
 import org.zouzias.spark.lucenerdd.versioning.Versionable
 
 import scala.reflect.ClassTag
@@ -41,7 +39,7 @@ class PointLuceneRDD[V: ClassTag]
    val indexAnalyzerName: String,
    val queryAnalyzerName: String,
    val similarity: String)
-  extends RDD[V](partitionsRDD.context, List(new OneToOneDependency(partitionsRDD)))
+  extends RDD[(PointType, V)](partitionsRDD.context, List(new OneToOneDependency(partitionsRDD)))
     with ShapeLuceneRDDConfigurable {
 
   logInfo("Instance is created...")
@@ -326,15 +324,15 @@ class PointLuceneRDD[V: ClassTag]
   }
 
   /** RDD compute method. */
-  override def compute(part: Partition, context: TaskContext): Iterator[V] = {
+  override def compute(part: Partition, context: TaskContext): Iterator[(PointType, V)] = {
     firstParent[AbstractPointLuceneRDDPartition[V]].iterator(part, context).next.iterator
   }
 
-  override def filter(pred: V => Boolean): PointLuceneRDD[V] = {
+  def filter(pred: (PointType, V) => Boolean): PointLuceneRDD[V] = {
     val newPartitionRDD = partitionsRDD.mapPartitions(partition =>
       partition.map(_.filter(pred)), preservesPartitioning = true
     )
-    new PointLuceneRDD(newPartitionRDD, indexAnalyzerName, queryAnalyzerName, similarity)
+    new PointLuceneRDD[V](newPartitionRDD, indexAnalyzerName, queryAnalyzerName, similarity)
   }
 
   def exists(point: PointType): Boolean = {
@@ -364,7 +362,7 @@ object PointLuceneRDD extends Versionable
     * @param similarity Lucene scoring similarity, i.e., BM25 or TF-IDF
     * @return
     */
-  def apply[V: ClassTag](elems: RDD[V],
+  def apply[V: ClassTag](elems: RDD[(PointType, V)],
                                       indexAnalyzer: String,
                                       queryAnalyzer: String,
                                       similarity: String)
@@ -376,7 +374,7 @@ object PointLuceneRDD extends Versionable
     new PointLuceneRDD(partitions, indexAnalyzer, queryAnalyzer, similarity)
   }
 
-  def apply[K: ClassTag, V: ClassTag](elems: RDD[V])
+  def apply[V: ClassTag](elems: RDD[(PointType, V)])
                                      (implicit docConverter: V => Document)
   : PointLuceneRDD[V] = {
     apply[V](elems, getOrElseEn(IndexAnalyzerConfigName), getOrElseEn(QueryAnalyzerConfigName),
@@ -386,7 +384,7 @@ object PointLuceneRDD extends Versionable
   /**
     * Constructor for [[Dataset]]
     */
-  def apply[V: ClassTag](elems: Dataset[V],
+  def apply[V: ClassTag](elems: Dataset[(PointType, V)],
                                       indexAnalyzer: String,
                                       queryAnalyzer: String,
                                       similarity: String)
@@ -401,7 +399,7 @@ object PointLuceneRDD extends Versionable
   /**
     * Constructor for [[Dataset]]
     */
-  def apply[V: ClassTag](elems: Dataset[V])
+  def apply[V: ClassTag](elems: Dataset[(PointType, V)])
                                      (implicit docConverter: V => Document)
   : PointLuceneRDD[V] = {
     apply[V](elems, getOrElseEn(IndexAnalyzerConfigName), getOrElseEn(QueryAnalyzerConfigName),
@@ -426,21 +424,22 @@ object PointLuceneRDD extends Versionable
     */
   def apply(df : DataFrame,
             shapeField: String)
-           (implicit shapeConv: String => Shape, docConverter: Row => Document)
-  : ShapeLuceneRDD[String, Row] = {
+           (implicit shapeConv: String => PointType, docConverter: Row => Document)
+  : PointLuceneRDD[Row] = {
     apply(df, shapeField,
       getOrElseEn(IndexAnalyzerConfigName), getOrElseEn(QueryAnalyzerConfigName),
       getOrElseClassic())
   }
 
+
   def apply(df : DataFrame,
             shapeField: String,
             indexAnalyzer: String,
             queryAnalyzer: String,
             similarity: String)
-           (implicit docConverter: Row => Document)
+           (implicit  shapeConv: String => PointType, docConverter: Row => Document)
   : PointLuceneRDD[Row] = {
-    val partitions = df.rdd.map(row => (row.getString(row.fieldIndex(shapeField)), row))
+    val partitions = df.rdd.map(row => (shapeConv(row.getString(row.fieldIndex(shapeField))), row))
       .mapPartitions[AbstractPointLuceneRDDPartition[Row]](
       iter => Iterator(PointLuceneRDDPartition[Row](iter, indexAnalyzer, queryAnalyzer)),
       preservesPartitioning = true)
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/AbstractPointLuceneRDDPartition.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/AbstractPointLuceneRDDPartition.scala
index e7225496..c1d0b108 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/AbstractPointLuceneRDDPartition.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/AbstractPointLuceneRDDPartition.scala
@@ -22,13 +22,13 @@ import org.zouzias.spark.lucenerdd.spatial.shape.ShapeLuceneRDD.PointType
 import scala.reflect.ClassTag
 
 
-private[shape] abstract class AbstractPointLuceneRDDPartition[V] extends Serializable {
+private[point] abstract class AbstractPointLuceneRDDPartition[V] extends Serializable {
 
   protected implicit def vTag: ClassTag[V]
 
   def size: Long
 
-  def iterator: Iterator[V]
+  def iterator: Iterator[(PointType, V)]
 
   def isDefined(point: PointType): Boolean
 
@@ -107,5 +107,5 @@ private[shape] abstract class AbstractPointLuceneRDDPartition[V] extends Seriali
    * @param pred Predicate to filter on
    * @return
    */
-  def filter(pred: V => Boolean): AbstractPointLuceneRDDPartition[V]
+  def filter(pred: (PointType, V) => Boolean): AbstractPointLuceneRDDPartition[V]
 }
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/PointLuceneRDDPartition.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/PointLuceneRDDPartition.scala
index 5930dfe7..556cc40c 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/PointLuceneRDDPartition.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/PointLuceneRDDPartition.scala
@@ -16,15 +16,26 @@
  */
 package org.zouzias.spark.lucenerdd.spatial.point.partition
 
-import org.apache.lucene.document.Document
+import org.apache.lucene.analysis.Analyzer
+import org.apache.lucene.document.{Document, StoredField}
+import org.apache.lucene.index.DirectoryReader
+import org.apache.lucene.search.{IndexSearcher, ScoreDoc, Sort}
+import org.apache.lucene.spatial.query.{SpatialArgs, SpatialOperation}
+import org.joda.time.DateTime
+import org.locationtech.spatial4j.distance.DistanceUtils
+import org.locationtech.spatial4j.shape.Shape
+import org.zouzias.spark.lucenerdd.models.SparkScoreDoc
+import org.zouzias.spark.lucenerdd.query.LuceneQueryHelpers
+import org.zouzias.spark.lucenerdd.response.LuceneRDDResponsePartition
 import org.zouzias.spark.lucenerdd.spatial.commons.strategies.SpatialStrategy
+import org.zouzias.spark.lucenerdd.spatial.shape.ShapeLuceneRDD.PointType
 import org.zouzias.spark.lucenerdd.store.IndexWithTaxonomyWriter
 
 import scala.reflect.{ClassTag, classTag}
 
 
-private[shape] class PointLuceneRDDPartition[V]
-(private val iter: Iterator[V],
+private[point] class PointLuceneRDDPartition[V]
+(private val iter: Iterator[(PointType, V)],
  private val indexAnalyzerName: String,
  private val queryAnalyzerName: String)
 (override implicit val vTag: ClassTag[V])
@@ -32,85 +43,163 @@ private[shape] class PointLuceneRDDPartition[V]
   extends AbstractPointLuceneRDDPartition[V]
     with IndexWithTaxonomyWriter
     with SpatialStrategy {
-  override def size = ???
 
-  override def iterator = ???
+  override def indexAnalyzer(): Analyzer = getAnalyzer(Some(indexAnalyzerName))
 
-  override def isDefined(point: (Double, Double)) = ???
+  private val QueryAnalyzer: Analyzer = getAnalyzer(Some(queryAnalyzerName))
 
-  /**
-    * Nearest neighbour search
-    *
-    * @param point        query point
-    * @param k            number of neighbors to return
-    * @param searchString Lucene Query string
-    * @return
-    */
-  override def knnSearch(point: (Double, Double), k: Int, searchString: String) = ???
+  private def decorateWithLocation(doc: Document, shapes: Iterable[Shape]): Document = {
 
-  /**
-    * Search for points within a circle
-    *
-    * @param center center of circle
-    * @param radius radius of circle in kilometers (KM)
-    * @param k      number of points to return
-    * @return
-    */
-  override def circleSearch(center: (Double, Double), radius: Double, k: Int, operationName: String) = ???
+    // Potentially more than one shape in this field is supported by some
+    // strategies; see the Javadoc of the SpatialStrategy impl to see.
+    shapes.foreach{ case shape =>
+      strategy.createIndexableFields(shape).foreach{ case field =>
+        doc.add(field)
+      }
 
-  /**
-    * Spatial search with arbitrary shape
-    *
-    * @param shapeAsString Shape object represented as String
-    * @param k             Number of results to return
-    * @param operationName Operation name, i.e., intersect, within, etc
-    * @return
-    */
-  override def spatialSearch(shapeAsString: String, k: Int, operationName: String) = ???
+      doc.add(new StoredField(strategy.getFieldName, shapeToString(shape)))
+    }
 
-  /**
-    * Spatial search with point
-    *
-    * @param point         Query point
-    * @param k             Number of result to return
-    * @param operationName Operation name, i.e., intersect, within, etc
-    * @return
-    */
-  override def spatialSearch(point: (Double, Double), k: Int, operationName: String) = ???
+    doc
+  }
 
-  /**
-    * Bounding box search with point and radius
-    *
-    * @param center        given as (x, y)
-    * @param radius        distance from center in kilometers (KM)
-    * @param k             Number of results to return
-    * @param operationName Operation name, i.e., intersect, within, etc
-    * @return
-    */
-  override def bboxSearch(center: (Double, Double), radius: Double, k: Int, operationName: String) = ???
+  private val (iterOriginal, iterIndex) = iter.duplicate
 
-  /**
-    * Bounding box search with lower left and upper right corners
-    *
-    * @param lowerLeft     Lower left point
-    * @param upperRight    Upper left point
-    * @param k             Number of results
-    * @param operationName Operation name, i.e., intersect, within, etc
-    * @return
-    */
-  override def bboxSearch(lowerLeft: (Double, Double),
-                          upperRight: (Double, Double),
-                          k: Int, operationName: String) = ???
+  private val startTime = new DateTime(System.currentTimeMillis())
+  logInfo(s"Indexing process initiated at ${startTime}...")
+  iterIndex.foreach { case (p, value) =>
+    // (implicitly) convert type K to Shape and V to a Lucene document
+    val doc = docConversion(value)
+    val docWithLocation = decorateWithLocation(doc, Iterable(ctx.makePoint(p._1, p._2)))
+    indexWriter.addDocument(FacetsConfig.build(taxoWriter, docWithLocation))
+  }
+  private val endTime = new DateTime(System.currentTimeMillis())
+  logInfo(s"Indexing process completed at ${endTime}...")
+  logInfo(s"Indexing process took ${(endTime.getMillis - startTime.getMillis) / 1000} seconds...")
+
+  // Close the indexWriter and taxonomyWriter (for faceted search)
+  closeAllWriters()
+
+  private val indexReader = DirectoryReader.open(IndexDir)
+  private val indexSearcher = new IndexSearcher(indexReader)
+
+  override def size: Long = iterOriginal.size.toLong
 
   /**
     * Restricts the entries to those satisfying a predicate
     *
-    * @param pred Predicate to filter on
+    * @param pred
     * @return
     */
-  override def filter(pred: V => Boolean) = ???
+  override def filter(pred: (PointType, V) => Boolean):
+  AbstractPointLuceneRDDPartition[V] = {
+    PointLuceneRDDPartition(iterOriginal.filter(x => pred(x._1, x._2)),
+      indexAnalyzerName, queryAnalyzerName)
+  }
+
+  override def isDefined(key: PointType): Boolean = iterOriginal.exists(_._1 == key)
+
+  override def iterator: Iterator[(PointType, V)] = iterOriginal
+
+  private def docLocation(scoreDoc: ScoreDoc): Option[Shape] = {
+    val shapeString = indexReader.document(scoreDoc.doc)
+      .getField(strategy.getFieldName)
+      .stringValue()
+
+    try{
+      Some(stringToShape(shapeString))
+    }
+    catch {
+      case _: Throwable => None
+    }
+  }
+
+  override def circleSearch(center: PointType, radius: Double, k: Int, operationName: String)
+  : LuceneRDDResponsePartition = {
+    logInfo(s"circleSearch [center:${center}, operation:${operationName}]")
+    val args = new SpatialArgs(SpatialOperation.get(operationName),
+      ctx.makeCircle(center._1, center._2,
+        DistanceUtils.dist2Degrees(radius, DistanceUtils.EARTH_MEAN_RADIUS_KM)))
+
+    val query = strategy.makeQuery(args)
+    val docs = indexSearcher.search(query, k)
+    LuceneRDDResponsePartition(docs.scoreDocs.map(SparkScoreDoc(indexSearcher, _)).toIterator)
+  }
+
+  override def knnSearch(point: PointType, k: Int, searchString: String)
+  : LuceneRDDResponsePartition = {
+    logInfo(s"knnSearch [center:${point}, searchQuery:${searchString}]")
+
+    // Match all, order by distance ascending
+    val pt = ctx.makePoint(point._1, point._2)
+
+    // the distance (in km)
+    val valueSource = strategy.makeDistanceValueSource(pt, DistanceUtils.DEG_TO_KM)
+
+    // false = ascending dist
+    val distSort = new Sort(valueSource.getSortField(false)).rewrite(indexSearcher)
+
+    val query = LuceneQueryHelpers.parseQueryString(searchString, QueryAnalyzer)
+    val docs = indexSearcher.search(query, k, distSort)
+
+    // Here we sorted on it, and the distance will get
+    // computed redundantly.  If the distance is only needed for the top-X
+    // search results then that's not a big deal. Alternatively, try wrapping
+    // the ValueSource with CachingDoubleValueSource then retrieve the value
+    // from the ValueSource now. See LUCENE-4541 for an example.
+    val result = docs.scoreDocs.map { scoreDoc => {
+      val location = docLocation(scoreDoc)
+      location match {
+        case Some(shape) =>
+          SparkScoreDoc(indexSearcher, scoreDoc,
+            ctx.calcDistance(pt, shape.getCenter).toFloat)
+        case None =>
+          SparkScoreDoc(indexSearcher, scoreDoc, -1F)
+      }
+    }
+    }
+
+    LuceneRDDResponsePartition(result.toIterator)
+  }
+
+  override def spatialSearch(shapeAsString: String, k: Int, operationName: String)
+  : LuceneRDDResponsePartition = {
+    logInfo(s"spatialSearch [shape:${shapeAsString} and operation:${operationName}]")
+    val shape = stringToShape(shapeAsString)
+    spatialSearch(shape, k, operationName)
+  }
+
+  private def spatialSearch(shape: Shape, k: Int, operationName: String)
+  : LuceneRDDResponsePartition = {
+    val args = new SpatialArgs(SpatialOperation.get(operationName), shape)
+    val query = strategy.makeQuery(args)
+    val docs = indexSearcher.search(query, k)
+    LuceneRDDResponsePartition(docs.scoreDocs.map(SparkScoreDoc(indexSearcher, _)).toIterator)
+  }
+
+  override def spatialSearch(point: PointType, k: Int, operationName: String)
+  : LuceneRDDResponsePartition = {
+    val shape = ctx.makePoint(point._1, point._2)
+    spatialSearch(shape, k, operationName)
+  }
 
-  override protected def indexAnalyzer() = ???
+  override def bboxSearch(center: PointType, radius: Double, k: Int, operationName: String)
+  : LuceneRDDResponsePartition = {
+    logInfo(s"bboxSearch [center:${center}, radius: ${radius} and operation:${operationName}]")
+    val x = center._1
+    val y = center._2
+    val radiusKM = DistanceUtils.dist2Degrees(radius, DistanceUtils.EARTH_MEAN_RADIUS_KM)
+    val shape = ctx.makeRectangle(x - radiusKM, x + radiusKM, y - radiusKM, y + radiusKM)
+    spatialSearch(shape, k, operationName)
+  }
+
+  override def bboxSearch(lowerLeft: PointType, upperRight: PointType, k: Int, opName: String)
+  : LuceneRDDResponsePartition = {
+    val lowerLeftPt = ctx.makePoint(lowerLeft._1, lowerLeft._2)
+    val upperRightPt = ctx.makePoint(upperRight._1, upperRight._2)
+    val shape = ctx.makeRectangle(lowerLeftPt, upperRightPt)
+    spatialSearch(shape, k, opName)
+  }
 }
 
 object PointLuceneRDDPartition {
@@ -125,7 +214,7 @@ object PointLuceneRDDPartition {
     * @tparam V
     * @return
     */
-  def apply[V: ClassTag](iter: Iterator[V],
+  def apply[V: ClassTag](iter: Iterator[(PointType, V)],
                                       indexAnalyzer: String,
                                       queryAnalyzer: String)
                                      (implicit docConv: V => Document)

From b908213aa3f3f9caec03c3b1c0a0e3b1695634f0 Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Fri, 27 Oct 2017 20:20:40 +0200
Subject: [PATCH 35/47] point_rdd: cleanup and set only point in spatial
 strategy

---
 .../partition/PointLuceneRDDPartition.scala   | 20 +++++++++----------
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/PointLuceneRDDPartition.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/PointLuceneRDDPartition.scala
index 556cc40c..d33779dd 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/PointLuceneRDDPartition.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/PointLuceneRDDPartition.scala
@@ -23,7 +23,7 @@ import org.apache.lucene.search.{IndexSearcher, ScoreDoc, Sort}
 import org.apache.lucene.spatial.query.{SpatialArgs, SpatialOperation}
 import org.joda.time.DateTime
 import org.locationtech.spatial4j.distance.DistanceUtils
-import org.locationtech.spatial4j.shape.Shape
+import org.locationtech.spatial4j.shape.{Point, Shape}
 import org.zouzias.spark.lucenerdd.models.SparkScoreDoc
 import org.zouzias.spark.lucenerdd.query.LuceneQueryHelpers
 import org.zouzias.spark.lucenerdd.response.LuceneRDDResponsePartition
@@ -44,20 +44,18 @@ private[point] class PointLuceneRDDPartition[V]
     with IndexWithTaxonomyWriter
     with SpatialStrategy {
 
+  /* We index only points in [[PointLuceneRDDPartition]] */
+  strategy.setPointsOnly(true)
+
   override def indexAnalyzer(): Analyzer = getAnalyzer(Some(indexAnalyzerName))
 
   private val QueryAnalyzer: Analyzer = getAnalyzer(Some(queryAnalyzerName))
 
-  private def decorateWithLocation(doc: Document, shapes: Iterable[Shape]): Document = {
+  private def decorateWithLocation(doc: Document, point: Point): Document = {
 
-    // Potentially more than one shape in this field is supported by some
-    // strategies; see the Javadoc of the SpatialStrategy impl to see.
-    shapes.foreach{ case shape =>
-      strategy.createIndexableFields(shape).foreach{ case field =>
+    strategy.createIndexableFields(point).foreach{ field =>
         doc.add(field)
-      }
-
-      doc.add(new StoredField(strategy.getFieldName, shapeToString(shape)))
+        doc.add(new StoredField(strategy.getFieldName, shapeToString(point)))
     }
 
     doc
@@ -68,9 +66,9 @@ private[point] class PointLuceneRDDPartition[V]
   private val startTime = new DateTime(System.currentTimeMillis())
   logInfo(s"Indexing process initiated at ${startTime}...")
   iterIndex.foreach { case (p, value) =>
-    // (implicitly) convert type K to Shape and V to a Lucene document
+    // (implicitly) convert type V to a Lucene document
     val doc = docConversion(value)
-    val docWithLocation = decorateWithLocation(doc, Iterable(ctx.makePoint(p._1, p._2)))
+    val docWithLocation = decorateWithLocation(doc, ctx.makePoint(p._1, p._2))
     indexWriter.addDocument(FacetsConfig.build(taxoWriter, docWithLocation))
   }
   private val endTime = new DateTime(System.currentTimeMillis())

From 29e9b09cc820930018cecfa29bded4d34498a6de Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Fri, 27 Oct 2017 20:59:08 +0200
Subject: [PATCH 36/47] point_rdd: introduct bounds() method

---
 .../spark/lucenerdd/spatial/point/PointLuceneRDD.scala   | 9 +++++++++
 .../partition/AbstractPointLuceneRDDPartition.scala      | 6 ++++++
 .../point/partition/PointLuceneRDDPartition.scala        | 5 +++++
 3 files changed, 20 insertions(+)

diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala
index 6929659b..c71f3482 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala
@@ -318,6 +318,15 @@ class PointLuceneRDD[V: ClassTag]
     partitionMapper(_.bboxSearch(lowerLeft, upperRight, k, operationName))
   }
 
+  /**
+    * Returns the smallest enclosing axis aligned bounding box per partition
+    * @return
+    */
+  def boundsPerPartition(): RDD[(PointType, PointType)] = {
+    logInfo("boundsPerPartition requested")
+    partitionsRDD.map(_.bounds())
+  }
+
   override def count(): Long = {
     logInfo("Count requested")
     partitionsRDD.map(_.size).reduce(_ + _)
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/AbstractPointLuceneRDDPartition.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/AbstractPointLuceneRDDPartition.scala
index c1d0b108..5340ece8 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/AbstractPointLuceneRDDPartition.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/AbstractPointLuceneRDDPartition.scala
@@ -34,6 +34,12 @@ private[point] abstract class AbstractPointLuceneRDDPartition[V] extends Seriali
 
   def close(): Unit
 
+  /**
+    * Return smallest enclosing axis aligned bounding box
+    * @return ( (min_x, min_y), (max_x, max_y))
+    */
+  def bounds(): (PointType, PointType)
+
   /**
    * Nearest neighbour search
    *
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/PointLuceneRDDPartition.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/PointLuceneRDDPartition.scala
index d33779dd..3a8d842c 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/PointLuceneRDDPartition.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/PointLuceneRDDPartition.scala
@@ -112,6 +112,11 @@ private[point] class PointLuceneRDDPartition[V]
     }
   }
 
+  override def bounds(): (PointType, PointType) = {
+    val rect = strategy.getSpatialContext.getWorldBounds
+    ((rect.getMinX, rect.getMinY), (rect.getMaxX, rect.getMaxY))
+  }
+
   override def circleSearch(center: PointType, radius: Double, k: Int, operationName: String)
   : LuceneRDDResponsePartition = {
     logInfo(s"circleSearch [center:${center}, operation:${operationName}]")

From 11b2529f05f117f514e416a723dc661d0b19b2e7 Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Sat, 28 Oct 2017 20:52:13 +0200
Subject: [PATCH 37/47] stringToShape returns Option[Shape]

---
 .../response/LuceneRDDResponsePartition.scala      |  6 ++++++
 .../spatial/commons/context/ContextLoader.scala    | 11 +++++++++--
 .../shape/partition/ShapeLuceneRDDPartition.scala  | 14 ++++++--------
 3 files changed, 21 insertions(+), 10 deletions(-)

diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/response/LuceneRDDResponsePartition.scala b/src/main/scala/org/zouzias/spark/lucenerdd/response/LuceneRDDResponsePartition.scala
index 817d9a94..2b705733 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/response/LuceneRDDResponsePartition.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/response/LuceneRDDResponsePartition.scala
@@ -23,3 +23,9 @@ case class LuceneRDDResponsePartition(results: Iterator[SparkScoreDoc])
   extends Iterable[SparkScoreDoc] {
   override def iterator(): Iterator[SparkScoreDoc] = results
 }
+
+object LuceneRDDResponsePartition {
+  def empty(): LuceneRDDResponsePartition = {
+    LuceneRDDResponsePartition(Iterator.empty)
+  }
+}
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/context/ContextLoader.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/context/ContextLoader.scala
index 3e3cbc9d..45abd291 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/context/ContextLoader.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/context/ContextLoader.scala
@@ -18,7 +18,9 @@ package org.zouzias.spark.lucenerdd.spatial.commons.context
 
 import java.io.{StringReader, StringWriter}
 
+import com.vividsolutions.jts.io.ParseException
 import org.locationtech.spatial4j.context.jts.JtsSpatialContext
+import org.locationtech.spatial4j.exception.InvalidShapeException
 import org.locationtech.spatial4j.io.{ShapeReader, ShapeWriter}
 import org.locationtech.spatial4j.shape.Shape
 import org.zouzias.spark.lucenerdd.config.ShapeLuceneRDDConfigurable
@@ -37,8 +39,13 @@ trait ContextLoader extends ShapeLuceneRDDConfigurable{
     writer.toString
   }
 
-  protected def stringToShape(shapeAsString: String): Shape = {
-   shapeReader.read(new StringReader(shapeAsString))
+  protected def stringToShape(shapeAsString: String): Option[Shape] = {
+   try {
+     Some(shapeReader.read(new StringReader(shapeAsString)))
+   } catch {
+     case _: ParseException => None
+     case _ : InvalidShapeException => None
+   }
   }
 
   /**
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/partition/ShapeLuceneRDDPartition.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/partition/ShapeLuceneRDDPartition.scala
index e472c6c5..cb1228ae 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/partition/ShapeLuceneRDDPartition.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/partition/ShapeLuceneRDDPartition.scala
@@ -107,12 +107,7 @@ private[shape] class ShapeLuceneRDDPartition[K, V]
                   .getField(strategy.getFieldName)
                   .stringValue()
 
-   try{
-     Some(stringToShape(shapeString))
-   }
-    catch {
-      case _: Throwable => None
-    }
+    stringToShape(shapeString)
   }
 
   override def circleSearch(center: PointType, radius: Double, k: Int, operationName: String)
@@ -166,8 +161,11 @@ private[shape] class ShapeLuceneRDDPartition[K, V]
   override def spatialSearch(shapeAsString: String, k: Int, operationName: String)
   : LuceneRDDResponsePartition = {
     logInfo(s"spatialSearch [shape:${shapeAsString} and operation:${operationName}]")
-    val shape = stringToShape(shapeAsString)
-    spatialSearch(shape, k, operationName)
+    val shapeOpt = stringToShape(shapeAsString)
+    shapeOpt match {
+      case shape: Shape => spatialSearch(shape, k, operationName)
+      case _ => LuceneRDDResponsePartition.empty()
+    }
   }
 
   private def spatialSearch(shape: Shape, k: Int, operationName: String)

From 1a3544ac24f6a40c32424f4c00447ebf3c38a20f Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Wed, 1 Nov 2017 21:50:29 +0100
Subject: [PATCH 38/47] point_rdd: fix bounds method

---
 .../spatial/point/PointLuceneRDD.scala        | 123 ++++--------------
 .../AbstractPointLuceneRDDPartition.scala     |  11 --
 .../partition/PointLuceneRDDPartition.scala   |  32 +++--
 3 files changed, 37 insertions(+), 129 deletions(-)

diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala
index c71f3482..b637d8be 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala
@@ -76,16 +76,14 @@ class PointLuceneRDD[V: ClassTag]
     this
   }
 
-  setName("ShapeLuceneRDD")
-
   private def partitionMapper(f: AbstractPointLuceneRDDPartition[V] =>
     LuceneRDDResponsePartition): LuceneRDDResponse = {
     new LuceneRDDResponse(partitionsRDD.map(f), SparkScoreDoc.ascending)
   }
 
-  private def linker[T: ClassTag](that: RDD[T],
-                                  pointFunctor: T => PointType,
-                                  mapper: ( PointType, AbstractPointLuceneRDDPartition[V]) =>
+  private def linker[T: ClassTag, S: ClassTag](that: RDD[T],
+                                  pointFunctor: T => S,
+                                  mapper: (S, AbstractPointLuceneRDDPartition[V]) =>
                                     Iterable[SparkScoreDoc],
                                   linkerMethod: String)
   : RDD[(T, Array[SparkScoreDoc])] = {
@@ -101,7 +99,7 @@ class PointLuceneRDD[V: ClassTag]
         concatenated.cartesian(partitionsRDD)
           .flatMap { case (qs, lucene) =>
             qs.map { case (ind, query) =>
-              (ind, topKMonoid.build(mapper((query._1, query._2), lucene)))
+              (ind, topKMonoid.build(mapper(query, lucene)))
             }
           }
       case _ =>
@@ -111,8 +109,8 @@ class PointLuceneRDD[V: ClassTag]
 
         partitionsRDD.mapPartitions { partitions =>
             partitions.flatMap { partition =>
-              queriesB.value.map { case (index, (x, y)) =>
-                  (index, topKMonoid.build(mapper((x, y), partition)))
+              queriesB.value.map { case (index, query) =>
+                  (index, topKMonoid.build(mapper(query, partition)))
                 }
             }
         }
@@ -125,39 +123,13 @@ class PointLuceneRDD[V: ClassTag]
       .map(joined => (joined._1, joined._2.items.toArray))
   }
 
-  /**
-   * Link entities based on k-nearest neighbors (Knn)
-   *
-   * Links this and that based on nearest neighbors, returns Knn
-   *
-   *
-   * @param that An RDD of entities to be linked
-   * @param pointFunctor Function that generates a point from each element of other
-   * @param linkerMethod Method to perform linkage
-   * @tparam T A type
-   * @return an RDD of Tuple2 that contains the linked results
-   *
-   * Note: Currently the query coordinates of the other RDD are collected to the driver and
-   * broadcast to the workers.
-   */
-  def linkByKnn[T: ClassTag](that: RDD[T],
-                             pointFunctor: T => PointType,
-                             topK: Int = DefaultTopK,
-                             linkerMethod: String = getShapeLinkerMethod)
-  : RDD[(T, Array[SparkScoreDoc])] = {
-    logInfo("linkByKnn requested")
-    linker[T](that, pointFunctor, (queryPoint, part) =>
-      part.knnSearch(queryPoint, topK, LuceneQueryHelpers.MatchAllDocsString),
-      linkerMethod)
-  }
-
   /**
    * Link entities if their shapes are within a distance in kilometers (km)
    *
    * Links this and that based on distance threshold
    *
    * @param that An RDD of entities to be linked
-   * @param pointFunctor Function that generates a point from each element of other
+   * @param shapeFunctor Function that generates a point from each element of other
    * @param linkerMethod Method to perform linkage
    * @tparam T A type
    * @return an RDD of Tuple2 that contains the linked results
@@ -165,63 +137,28 @@ class PointLuceneRDD[V: ClassTag]
    * Note: Currently the query coordinates of the other RDD are collected to the driver and
    * broadcast to the workers.
    */
-  def linkByRadius[T: ClassTag](that: RDD[T],
-                                pointFunctor: T => PointType,
-                                radius: Double,
+  def linkByInstersection[T: ClassTag](that: RDD[T],
+                                shapeFunctor: T => String,
                                 topK: Int = DefaultTopK,
-                                spatialOp: String = SpatialOperation.Intersects.getName,
                                 linkerMethod: String = getShapeLinkerMethod)
   : RDD[(T, Array[SparkScoreDoc])] = {
-    logInfo("linkByRadius requested")
-    linker[T](that, pointFunctor, (queryPoint, part) =>
-      part.circleSearch(queryPoint, radius, topK, spatialOp),
+    logInfo("linkByInstersection requested")
+    linker[T, String](that, shapeFunctor, (queryShape: String, part) =>
+      part.spatialSearch(queryShape, topK, SpatialOperation.Intersects.getName),
       linkerMethod)
   }
 
 
-  /**
-   * Link with DataFrame based on k-nearest neighbors (Knn)
-   *
-   * Links this and that based on nearest neighbors, returns Knn
-   *
-   *
-   * @param other DataFrame of entities to be linked
-   * @param searchQueryGen Function that generates a point from each element of other
-    * @param topK Number of linked items
-   * @param linkerMethod Method to perform linkage
-   * @return
-   */
-  def linkDataFrameByKnn(other: DataFrame,
-                         searchQueryGen: Row => PointType,
-                         topK: Int = DefaultTopK,
-                         linkerMethod: String = getShapeLinkerMethod)
-  : RDD[(Row, Array[SparkScoreDoc])] = {
-    logInfo("linkDataFrameByKnn requested")
-    linkByKnn[Row](other.rdd, searchQueryGen, topK, linkerMethod)
-  }
-
-  /**
-   * Link entities if their shapes are within a distance in kilometers (km)
-   *
-   * Links this and that based on distance threshold
-   *
-   * @param other DataFrame of entities to be linked
-   * @param pointFunctor Function that generates a point from each element of other
-   * @param linkerMethod Method to perform linkage
-   * @return an RDD of Tuple2 that contains the linked results
-   *
-   * Note: Currently the query coordinates of the other RDD are collected to the driver and
-   * broadcast to the workers.
-   */
-  def linkDataFrameByRadius(other: DataFrame,
-                            pointFunctor: Row => PointType,
-                            radius: Double,
-                            topK: Int = DefaultTopK,
-                            spatialOp: String = SpatialOperation.Intersects.getName,
-                            linkerMethod: String = getShapeLinkerMethod)
-  : RDD[(Row, Array[SparkScoreDoc])] = {
-    logInfo("linkDataFrameByRadius requested")
-    linkByRadius[Row](other.rdd, pointFunctor, radius, topK, spatialOp, linkerMethod)
+  def linkByRadius[T: ClassTag](that: RDD[T],
+                                       pointFunctor: T => PointType,
+                                       topK: Int = DefaultTopK,
+                                       radius: Double,
+                                       linkerMethod: String = getShapeLinkerMethod)
+  : RDD[(T, Array[SparkScoreDoc])] = {
+    logInfo("linkByRadius requested")
+    linker[T, PointType](that, pointFunctor, (queryPoint, part) =>
+      part.circleSearch(queryPoint, radius, topK, SpatialOperation.Intersects.getName),
+      linkerMethod)
   }
 
   /**
@@ -266,25 +203,9 @@ class PointLuceneRDD[V: ClassTag]
   def spatialSearch(shapeWKT: String, k: Int,
                     operationName: String = SpatialOperation.Intersects.getName)
   : LuceneRDDResponse = {
-    logInfo(s"Spatial search with shape ${shapeWKT} and operation ${operationName}")
     partitionMapper(_.spatialSearch(shapeWKT, k, operationName))
   }
 
-  /**
-   * Spatial search with a single Point
-   *
-   * @param point
-   * @param k
-   * @param operationName
-   * @return
-   */
-  def spatialSearch(point: PointType, k: Int,
-                    operationName: String)
-  : LuceneRDDResponse = {
-    logInfo(s"Spatial search with point ${point} and operation ${operationName}")
-    partitionMapper(_.spatialSearch(point, k, operationName))
-  }
-
   /**
    * Bounding box search with center and radius
    *
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/AbstractPointLuceneRDDPartition.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/AbstractPointLuceneRDDPartition.scala
index 5340ece8..476837fb 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/AbstractPointLuceneRDDPartition.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/AbstractPointLuceneRDDPartition.scala
@@ -72,17 +72,6 @@ private[point] abstract class AbstractPointLuceneRDDPartition[V] extends Seriali
   def spatialSearch(shapeAsString: String, k: Int, operationName: String)
   : LuceneRDDResponsePartition
 
-  /**
-   * Spatial search with point
-   *
-   * @param point Query point
-   * @param k Number of result to return
-   * @param operationName Operation name, i.e., intersect, within, etc
-   * @return
-   */
-  def spatialSearch(point: PointType, k: Int, operationName: String)
-  : LuceneRDDResponsePartition
-
   /**
    * Bounding box search with point and radius
    *
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/PointLuceneRDDPartition.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/PointLuceneRDDPartition.scala
index 3a8d842c..a5937435 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/PointLuceneRDDPartition.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/PointLuceneRDDPartition.scala
@@ -104,17 +104,19 @@ private[point] class PointLuceneRDDPartition[V]
       .getField(strategy.getFieldName)
       .stringValue()
 
-    try{
-      Some(stringToShape(shapeString))
-    }
-    catch {
-      case _: Throwable => None
-    }
+    stringToShape(shapeString)
   }
 
   override def bounds(): (PointType, PointType) = {
-    val rect = strategy.getSpatialContext.getWorldBounds
-    ((rect.getMinX, rect.getMinY), (rect.getMaxX, rect.getMaxY))
+    var (minX, maxX, minY, maxY) = (Double.MaxValue, Double.MinValue,
+      Double.MaxValue, Double.MinValue)
+    iterOriginal.map(_._1).foreach{ case (x, y) =>
+        minX = if (x < minX) x else minX
+        maxX = if (x > maxX) x else maxX
+        minY = if (y < minY) y else minY
+        maxY = if (y > maxY) y else maxY
+    }
+    ((minX, minY), (maxX, maxY))
   }
 
   override def circleSearch(center: PointType, radius: Double, k: Int, operationName: String)
@@ -167,9 +169,11 @@ private[point] class PointLuceneRDDPartition[V]
 
   override def spatialSearch(shapeAsString: String, k: Int, operationName: String)
   : LuceneRDDResponsePartition = {
-    logInfo(s"spatialSearch [shape:${shapeAsString} and operation:${operationName}]")
-    val shape = stringToShape(shapeAsString)
-    spatialSearch(shape, k, operationName)
+    val shapeOpt = stringToShape(shapeAsString)
+    shapeOpt match {
+      case shape: Shape => spatialSearch(shape, k, operationName)
+      case _ => LuceneRDDResponsePartition.empty()
+    }
   }
 
   private def spatialSearch(shape: Shape, k: Int, operationName: String)
@@ -180,12 +184,6 @@ private[point] class PointLuceneRDDPartition[V]
     LuceneRDDResponsePartition(docs.scoreDocs.map(SparkScoreDoc(indexSearcher, _)).toIterator)
   }
 
-  override def spatialSearch(point: PointType, k: Int, operationName: String)
-  : LuceneRDDResponsePartition = {
-    val shape = ctx.makePoint(point._1, point._2)
-    spatialSearch(shape, k, operationName)
-  }
-
   override def bboxSearch(center: PointType, radius: Double, k: Int, operationName: String)
   : LuceneRDDResponsePartition = {
     logInfo(s"bboxSearch [center:${center}, radius: ${radius} and operation:${operationName}]")

From b7422d6e9ee1a2708308ac8d4589231d4e0518d1 Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Thu, 2 Nov 2017 19:19:24 +0100
Subject: [PATCH 39/47] tests: ShapeLuceneRDDSpec: minor fix

---
 .../spark/lucenerdd/spatial/shape/ShapeLuceneRDDSpec.scala      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDSpec.scala b/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDSpec.scala
index bc616b25..01a2740b 100644
--- a/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDSpec.scala
+++ b/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDSpec.scala
@@ -150,7 +150,7 @@ class ShapeLuceneRDDSpec extends FlatSpec
   }
 
   "ShapeLuceneRDD.version" should "return project sbt build information" in {
-    val map = LuceneRDD.version()
+    val map = ShapeLuceneRDD.version()
     map.contains("name") should equal(true)
     map.contains("builtAtMillis") should equal(true)
     map.contains("scalaVersion") should equal(true)

From 8ada5a45480b724a6b1481959223cb32ef5e3b29 Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Fri, 3 Nov 2017 08:34:48 +0100
Subject: [PATCH 40/47] ShapeLuceneRDD: hotfix spatialSearch

---
 .../spatial/shape/partition/ShapeLuceneRDDPartition.scala   | 6 ++++--
 .../spatial/shape/ShapeLuceneRDDSpatialSearchSpec.scala     | 2 +-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/partition/ShapeLuceneRDDPartition.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/partition/ShapeLuceneRDDPartition.scala
index cb1228ae..d441dc57 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/partition/ShapeLuceneRDDPartition.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/partition/ShapeLuceneRDDPartition.scala
@@ -163,8 +163,10 @@ private[shape] class ShapeLuceneRDDPartition[K, V]
     logInfo(s"spatialSearch [shape:${shapeAsString} and operation:${operationName}]")
     val shapeOpt = stringToShape(shapeAsString)
     shapeOpt match {
-      case shape: Shape => spatialSearch(shape, k, operationName)
-      case _ => LuceneRDDResponsePartition.empty()
+      case Some(shape) => spatialSearch(shape, k, operationName)
+      case None =>
+        log.error("Input shape does not have a valid format. Returning empty results")
+        LuceneRDDResponsePartition.empty()
     }
   }
 
diff --git a/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDSpatialSearchSpec.scala b/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDSpatialSearchSpec.scala
index 3fee0f0c..ccea6e1d 100644
--- a/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDSpatialSearchSpec.scala
+++ b/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDSpatialSearchSpec.scala
@@ -135,7 +135,7 @@ class ShapeLuceneRDDSpatialSearchSpec extends FlatSpec
     // Bern, Laussanne and Zurich is within 300km
     val results = pointLuceneRDD.spatialSearch(rectangleWKT, k).collect()
 
-    results.size should equal(3)
+    results.length should equal(3)
 
     results.exists(x => docTextFieldEq(x.doc, "_1", Bern._2)) should equal(true)
     results.exists(x => docTextFieldEq(x.doc, "_1", Zurich._2)) should equal(true)

From e6bae9bd6ee156b8bdf3bf7cbc1b5cbd48fa85d6 Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Fri, 3 Nov 2017 08:55:53 +0100
Subject: [PATCH 41/47] point_rdd: WIP

---
 .../response/LuceneRDDResponse.scala          |  11 +-
 .../lucenerdd/spatial/point/package.scala     |  50 ++++++
 .../AbstractPointLuceneRDDPartition.scala     |   2 +-
 .../partition/PointLuceneRDDPartition.scala   |   7 +-
 .../spatial/point/PointLuceneRDDSpec.scala    | 168 ++++++++++++++++++
 5 files changed, 232 insertions(+), 6 deletions(-)
 create mode 100644 src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/package.scala
 create mode 100644 src/test/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDDSpec.scala

diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/response/LuceneRDDResponse.scala b/src/main/scala/org/zouzias/spark/lucenerdd/response/LuceneRDDResponse.scala
index 31ff662e..4a1195b5 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/response/LuceneRDDResponse.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/response/LuceneRDDResponse.scala
@@ -76,9 +76,14 @@ private[lucenerdd] class LuceneRDDResponse
 
   override def collect(): Array[SparkScoreDoc] = {
     val sz = partitionsRDD.map(_.size).sum().toInt
-    val monoid = new TopKMonoid[SparkScoreDoc](sz)(ordering)
-    partitionsRDD.map(monoid.build(_))
-      .reduce(monoid.plus).items.toArray
+    sz match {
+      case 0 => Array.empty[SparkScoreDoc]
+      case s =>
+        // Total size must be positive for [[TopKMonoid]]
+        val monoid = new TopKMonoid[SparkScoreDoc](s)(ordering)
+        partitionsRDD.map(monoid.build(_))
+          .reduce(monoid.plus).items.toArray
+    }
   }
 
   /**
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/package.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/package.scala
new file mode 100644
index 00000000..04b09a58
--- /dev/null
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/package.scala
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.zouzias.spark.lucenerdd.spatial
+
+import java.io.StringReader
+
+import org.locationtech.spatial4j.shape.{Point, Shape}
+import org.zouzias.spark.lucenerdd.spatial.commons.context.ContextLoader
+import org.zouzias.spark.lucenerdd.spatial.point.PointLuceneRDD.PointType
+
+package object point extends ContextLoader{
+
+
+  implicit def convertToPoint(point: (Double, Double)): Point = {
+    ctx.makePoint(point._1, point._2)
+  }
+
+  /**
+    * ***Experimental***
+    *
+    * Implicitly convert shape from its string representation
+    *
+    * @param shapeAsString
+    * @return
+    */
+  implicit def wktToShape(shapeAsString: String): PointType = {
+    try {
+      val centroid = shapeReader.read(new StringReader(shapeAsString))
+                      .getCenter
+      (centroid.getX, centroid.getY)
+    }
+    catch {
+      case _: Exception => (0.0, 0.0)
+    }
+  }
+}
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/AbstractPointLuceneRDDPartition.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/AbstractPointLuceneRDDPartition.scala
index 476837fb..3744b4dc 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/AbstractPointLuceneRDDPartition.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/AbstractPointLuceneRDDPartition.scala
@@ -36,7 +36,7 @@ private[point] abstract class AbstractPointLuceneRDDPartition[V] extends Seriali
 
   /**
     * Return smallest enclosing axis aligned bounding box
-    * @return ( (min_x, min_y), (max_x, max_y))
+    * @return ( (minX, minY), (maxX, maxY))
     */
   def bounds(): (PointType, PointType)
 
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/PointLuceneRDDPartition.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/PointLuceneRDDPartition.scala
index a5937435..7fafd9fc 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/PointLuceneRDDPartition.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/PointLuceneRDDPartition.scala
@@ -108,6 +108,7 @@ private[point] class PointLuceneRDDPartition[V]
   }
 
   override def bounds(): (PointType, PointType) = {
+    // TODO: Perform the following using monoids
     var (minX, maxX, minY, maxY) = (Double.MaxValue, Double.MinValue,
       Double.MaxValue, Double.MinValue)
     iterOriginal.map(_._1).foreach{ case (x, y) =>
@@ -171,8 +172,10 @@ private[point] class PointLuceneRDDPartition[V]
   : LuceneRDDResponsePartition = {
     val shapeOpt = stringToShape(shapeAsString)
     shapeOpt match {
-      case shape: Shape => spatialSearch(shape, k, operationName)
-      case _ => LuceneRDDResponsePartition.empty()
+      case Some(shape) => spatialSearch(shape, k, operationName)
+      case _ =>
+        log.error("Input shape does not have a valid format. Returning empty results")
+        LuceneRDDResponsePartition.empty()
     }
   }
 
diff --git a/src/test/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDDSpec.scala b/src/test/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDDSpec.scala
new file mode 100644
index 00000000..d3a3ca61
--- /dev/null
+++ b/src/test/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDDSpec.scala
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.zouzias.spark.lucenerdd.spatial.point
+
+import java.io.StringWriter
+
+import com.holdenkarau.spark.testing.SharedSparkContext
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.SparkSession
+import org.locationtech.spatial4j.distance.DistanceUtils
+import org.scalatest.{BeforeAndAfterEach, FlatSpec, Matchers}
+import org.zouzias.spark.lucenerdd._
+import org.zouzias.spark.lucenerdd.spatial.commons.context.ContextLoader
+import org.zouzias.spark.lucenerdd.spatial.shape.ShapeLuceneRDDKryoRegistrator
+import org.zouzias.spark.lucenerdd.testing.LuceneRDDTestUtils
+
+class PointLuceneRDDSpec extends FlatSpec
+  with Matchers
+  with BeforeAndAfterEach
+  with SharedSparkContext
+  with ContextLoader
+  with LuceneRDDTestUtils {
+
+  val k = 6
+
+  val Radius: Double = 5D
+
+  var pointLuceneRDD: PointLuceneRDD[_] = _
+
+  override def afterEach() {
+    pointLuceneRDD.close()
+  }
+
+  override val conf = ShapeLuceneRDDKryoRegistrator.registerKryoClasses(new SparkConf().
+    setMaster("local[*]").
+    setAppName("test").
+    set("spark.ui.enabled", "false").
+    set("spark.app.id", appID))
+
+  "PointLuceneRDD.circleSearch" should "return correct results" in {
+    val rdd = sc.parallelize(cities)
+    pointLuceneRDD = PointLuceneRDD(rdd)
+
+    // Bern, Laussanne and Zurich is within 300km
+    val results = pointLuceneRDD.circleSearch(Bern._1, 300, k).collect()
+
+    results.size should equal(3)
+
+    results.exists(x => docTextFieldEq(x.doc, "_1", Bern._2)) should equal(true)
+    results.exists(x => docTextFieldEq(x.doc, "_1", Zurich._2)) should equal(true)
+    results.exists(x => docTextFieldEq(x.doc, "_1", Laussanne._2)) should equal(true)
+
+    results.exists(x => docTextFieldEq(x.doc, "_1", Milan._2)) should equal(false)
+    results.exists(x => docTextFieldEq(x.doc, "_1", Athens._2)) should equal(false)
+    results.exists(x => docTextFieldEq(x.doc, "_1", Toronto._2)) should equal(false)
+  }
+
+  "PointLuceneRDD.spatialSearch(circle)" should "return correct results" in {
+    val rdd = sc.parallelize(cities)
+    pointLuceneRDD = PointLuceneRDD(rdd)
+
+    val point = ctx.makePoint(Bern._1._1, Bern._1._2)
+    val circle = ctx.makeCircle(point,
+      DistanceUtils.dist2Degrees(300, DistanceUtils.EARTH_MEAN_RADIUS_KM))
+
+    val writer = new StringWriter()
+    shapeWriter.write(writer, circle)
+    val circleWKT = writer.getBuffer.toString
+
+    // Bern, Laussanne and Zurich is within 300km
+    val results = pointLuceneRDD.spatialSearch(circleWKT, k).collect()
+
+    results.size should equal(3)
+
+    results.exists(x => docTextFieldEq(x.doc, "_1", Bern._2)) should equal(true)
+    results.exists(x => docTextFieldEq(x.doc, "_1", Zurich._2)) should equal(true)
+    results.exists(x => docTextFieldEq(x.doc, "_1", Laussanne._2)) should equal(true)
+
+    results.exists(x => docTextFieldEq(x.doc, "_1", Milan._2)) should equal(false)
+    results.exists(x => docTextFieldEq(x.doc, "_1", Athens._2)) should equal(false)
+    results.exists(x => docTextFieldEq(x.doc, "_1", Toronto._2)) should equal(false)
+  }
+
+  "PointLuceneRDD.bboxSearch((Double, Double), Double)" should "return correct results" in {
+    val rdd = sc.parallelize(cities)
+    pointLuceneRDD = PointLuceneRDD(rdd)
+
+    val x = Bern._1._1
+    val y = Bern._1._2
+    val radius = 150.0
+
+    // Bern, Laussanne and Zurich is within 300km
+    val results = pointLuceneRDD.bboxSearch((x, y), radius, k).collect()
+
+    results.size should equal(3)
+
+    results.exists(x => docTextFieldEq(x.doc, "_1", Bern._2)) should equal(true)
+    results.exists(x => docTextFieldEq(x.doc, "_1", Zurich._2)) should equal(true)
+    results.exists(x => docTextFieldEq(x.doc, "_1", Laussanne._2)) should equal(true)
+
+    results.exists(x => docTextFieldEq(x.doc, "_1", Milan._2)) should equal(false)
+    results.exists(x => docTextFieldEq(x.doc, "_1", Athens._2)) should equal(false)
+    results.exists(x => docTextFieldEq(x.doc, "_1", Toronto._2)) should equal(false)
+  }
+
+  "PointLuceneRDD.bboxSearch(lowerLeft, upperRight)" should "return correct results" in {
+    val rdd = sc.parallelize(cities)
+    pointLuceneRDD = PointLuceneRDD(rdd)
+
+    val x = Bern._1._1
+    val y = Bern._1._2
+    val width = DistanceUtils.dist2Degrees(150, DistanceUtils.EARTH_MEAN_RADIUS_KM)
+
+    // Bern, Laussanne and Zurich is within 300km
+    val results = pointLuceneRDD.bboxSearch((x - width, y - width), (x + width, y + width), k, "Intersects")
+      .collect()
+
+    results.length should equal(3)
+
+    results.exists(x => docTextFieldEq(x.doc, "_1", Bern._2)) should equal(true)
+    results.exists(x => docTextFieldEq(x.doc, "_1", Zurich._2)) should equal(true)
+    results.exists(x => docTextFieldEq(x.doc, "_1", Laussanne._2)) should equal(true)
+
+    results.exists(x => docTextFieldEq(x.doc, "_1", Milan._2)) should equal(false)
+    results.exists(x => docTextFieldEq(x.doc, "_1", Athens._2)) should equal(false)
+    results.exists(x => docTextFieldEq(x.doc, "_1", Toronto._2)) should equal(false)
+  }
+
+  "PointLuceneRDD.apply(DF, shapeField)" should
+    "instantiate ShapeLuceneRDD from DataFrame and shape field name" in {
+    val sparkSession = SparkSession.builder.getOrCreate()
+    val capitals = sparkSession.read.parquet("data/capitals.parquet")
+    pointLuceneRDD = PointLuceneRDD(capitals, "shape")
+
+    pointLuceneRDD.count() > 0 should equal(true)
+  }
+
+  "PointLuceneRDD.version" should "return project sbt build information" in {
+    val map = PointLuceneRDD.version()
+    map.contains("name") should equal(true)
+    map.contains("builtAtMillis") should equal(true)
+    map.contains("scalaVersion") should equal(true)
+    map.contains("version") should equal(true)
+    map.contains("sbtVersion") should equal(true)
+    map.contains("builtAtString") should equal(true)
+  }
+
+  "PointLuceneRDD.bounds" should "return correct bounds" in {
+    val rdd = sc.parallelize(cities)
+    pointLuceneRDD = PointLuceneRDD(rdd)
+    pointLuceneRDD.boundsPerPartition()
+    true should equal(true)
+  }
+}

From 8b246f3310bc5908720de5ebd822856f50a9304a Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Wed, 29 Nov 2017 21:12:25 +0100
Subject: [PATCH 42/47] point_rdd: use algebird Aggregators

---
 .../lucenerdd/spatial/point/PointLuceneRDD.scala   | 14 +++++++++++++-
 .../point/partition/PointLuceneRDDPartition.scala  | 13 +++----------
 .../spatial/point/PointLuceneRDDSpec.scala         | 12 ++++++++++--
 3 files changed, 26 insertions(+), 13 deletions(-)

diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala
index b637d8be..ead6a13d 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala
@@ -16,7 +16,7 @@
  */
 package org.zouzias.spark.lucenerdd.spatial.point
 
-import com.twitter.algebird.TopKMonoid
+import com.twitter.algebird._
 import org.apache.lucene.document.Document
 import org.apache.lucene.spatial.query.SpatialOperation
 import org.apache.spark.{OneToOneDependency, Partition, TaskContext}
@@ -248,6 +248,14 @@ class PointLuceneRDD[V: ClassTag]
     partitionsRDD.map(_.bounds())
   }
 
+  def bounds(): (PointType, PointType) = {
+    logInfo("bounds requested")
+    import com.twitter.algebird.GeneratedTupleAggregator._
+    map(_._1)
+      .mapPartitions(iter => Iterator(PointLuceneRDD.boundingBoxAgg(iter)))
+      .reduce(PointLuceneRDD.boundingBoxAgg.reduce)
+  }
+
   override def count(): Long = {
     logInfo("Count requested")
     partitionsRDD.map(_.size).reduce(_ + _)
@@ -375,4 +383,8 @@ object PointLuceneRDD extends Versionable
       preservesPartitioning = true)
     new PointLuceneRDD(partitions, indexAnalyzer, queryAnalyzer, similarity)
   }
+
+  /** Algebird bounding box aggregator */
+  val boundingBoxAgg = Tuple2(Aggregator.min[PointType], Aggregator.max[PointType])
+
 }
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/PointLuceneRDDPartition.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/PointLuceneRDDPartition.scala
index 7fafd9fc..1096301a 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/PointLuceneRDDPartition.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/PointLuceneRDDPartition.scala
@@ -28,6 +28,7 @@ import org.zouzias.spark.lucenerdd.models.SparkScoreDoc
 import org.zouzias.spark.lucenerdd.query.LuceneQueryHelpers
 import org.zouzias.spark.lucenerdd.response.LuceneRDDResponsePartition
 import org.zouzias.spark.lucenerdd.spatial.commons.strategies.SpatialStrategy
+import org.zouzias.spark.lucenerdd.spatial.point.PointLuceneRDD
 import org.zouzias.spark.lucenerdd.spatial.shape.ShapeLuceneRDD.PointType
 import org.zouzias.spark.lucenerdd.store.IndexWithTaxonomyWriter
 
@@ -108,16 +109,8 @@ private[point] class PointLuceneRDDPartition[V]
   }
 
   override def bounds(): (PointType, PointType) = {
-    // TODO: Perform the following using monoids
-    var (minX, maxX, minY, maxY) = (Double.MaxValue, Double.MinValue,
-      Double.MaxValue, Double.MinValue)
-    iterOriginal.map(_._1).foreach{ case (x, y) =>
-        minX = if (x < minX) x else minX
-        maxX = if (x > maxX) x else maxX
-        minY = if (y < minY) y else minY
-        maxY = if (y > maxY) y else maxY
-    }
-    ((minX, minY), (maxX, maxY))
+    import com.twitter.algebird.GeneratedTupleAggregator._
+    PointLuceneRDD.boundingBoxAgg(iterOriginal.map(_._1))
   }
 
   override def circleSearch(center: PointType, radius: Double, k: Int, operationName: String)
diff --git a/src/test/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDDSpec.scala b/src/test/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDDSpec.scala
index d3a3ca61..b22f7f44 100644
--- a/src/test/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDDSpec.scala
+++ b/src/test/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDDSpec.scala
@@ -160,9 +160,17 @@ class PointLuceneRDDSpec extends FlatSpec
   }
 
   "PointLuceneRDD.bounds" should "return correct bounds" in {
+    val Bern = ( (7.45, 46.95), "Bern")
+    val Zurich = ( (8.55, 47.366667), "Zurich")
+    val Laussanne = ( (6.6335, 46.519833), "Laussanne")
+    val Athens = ((23.716667, 37.966667), "Athens")
+    val Toronto = ((-79.4, 43.7), "Toronto")
+    val Milan = ((45.4646, 9.198), "Milan")
+
     val rdd = sc.parallelize(cities)
     pointLuceneRDD = PointLuceneRDD(rdd)
-    pointLuceneRDD.boundsPerPartition()
-    true should equal(true)
+    val (minBounds, maxBounds) = pointLuceneRDD.bounds()
+    minBounds should equal((-79.4, 9.198))
+    maxBounds should equal((45.4646, 47.366667))
   }
 }

From 3ab3a4fe9589c3fca541432de03c9b8251f65356 Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Fri, 1 Dec 2017 11:32:21 +0100
Subject: [PATCH 43/47] point_rdd: use custom monoids

---
 .../spatial/commons/MaxPointMonoid.scala      | 28 +++++++++++++++++++
 .../spatial/commons/MinPointMonoid.scala      | 28 +++++++++++++++++++
 .../spatial/point/PointLuceneRDD.scala        | 14 ++++++----
 .../partition/PointLuceneRDDPartition.scala   |  4 +--
 4 files changed, 66 insertions(+), 8 deletions(-)
 create mode 100644 src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/MaxPointMonoid.scala
 create mode 100644 src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/MinPointMonoid.scala

diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/MaxPointMonoid.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/MaxPointMonoid.scala
new file mode 100644
index 00000000..6277f2a1
--- /dev/null
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/MaxPointMonoid.scala
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.zouzias.spark.lucenerdd.spatial.commons
+
+import com.twitter.algebird.Monoid
+import org.zouzias.spark.lucenerdd.spatial.point.PointLuceneRDD.PointType
+
+object MaxPointMonoid extends Monoid[PointType] {
+  override def zero: PointType = (Double.MaxValue, Double.MaxValue)
+
+  override def plus(x: PointType, y: PointType): PointType = {
+    (Math.max(x._1, y._1), Math.max(x._2, y._2))
+  }
+}
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/MinPointMonoid.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/MinPointMonoid.scala
new file mode 100644
index 00000000..4f074a93
--- /dev/null
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/MinPointMonoid.scala
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.zouzias.spark.lucenerdd.spatial.commons
+
+import com.twitter.algebird.Monoid
+import org.zouzias.spark.lucenerdd.spatial.point.PointLuceneRDD.PointType
+
+object MinPointMonoid extends Monoid[PointType] {
+  override def zero: PointType = (Double.MinValue, Double.MinValue)
+
+  override def plus(x: PointType, y: PointType): PointType = {
+    (Math.min(x._1, y._1), Math.min(x._2, y._2))
+  }
+}
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala
index ead6a13d..41b2a04f 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala
@@ -28,6 +28,7 @@ import org.zouzias.spark.lucenerdd.config.ShapeLuceneRDDConfigurable
 import org.zouzias.spark.lucenerdd.models.SparkScoreDoc
 import org.zouzias.spark.lucenerdd.query.{LuceneQueryHelpers, SimilarityConfigurable}
 import org.zouzias.spark.lucenerdd.response.{LuceneRDDResponse, LuceneRDDResponsePartition}
+import org.zouzias.spark.lucenerdd.spatial.commons.{MaxPointMonoid, MinPointMonoid}
 import org.zouzias.spark.lucenerdd.spatial.point.PointLuceneRDD.PointType
 import org.zouzias.spark.lucenerdd.spatial.point.partition.{AbstractPointLuceneRDDPartition, PointLuceneRDDPartition}
 import org.zouzias.spark.lucenerdd.versioning.Versionable
@@ -87,7 +88,7 @@ class PointLuceneRDD[V: ClassTag]
                                     Iterable[SparkScoreDoc],
                                   linkerMethod: String)
   : RDD[(T, Array[SparkScoreDoc])] = {
-    logInfo("Shape Linkage requested")
+    logInfo("Point Linkage requested")
 
     val topKMonoid = new TopKMonoid[SparkScoreDoc](MaxDefaultTopKValue)(SparkScoreDoc.ascending)
     val queries = that.zipWithIndex().map(_.swap)
@@ -250,10 +251,10 @@ class PointLuceneRDD[V: ClassTag]
 
   def bounds(): (PointType, PointType) = {
     logInfo("bounds requested")
-    import com.twitter.algebird.GeneratedTupleAggregator._
-    map(_._1)
-      .mapPartitions(iter => Iterator(PointLuceneRDD.boundingBoxAgg(iter)))
-      .reduce(PointLuceneRDD.boundingBoxAgg.reduce)
+    boundsPerPartition()
+      .reduce { case (x, y) =>
+        (MinPointMonoid.plus(x._1, y._1), MaxPointMonoid.plus(x._2, y._2))
+      }
   }
 
   override def count(): Long = {
@@ -385,6 +386,7 @@ object PointLuceneRDD extends Versionable
   }
 
   /** Algebird bounding box aggregator */
-  val boundingBoxAgg = Tuple2(Aggregator.min[PointType], Aggregator.max[PointType])
+
+  val boundingBoxMonoid = new Tuple2Monoid()(MinPointMonoid, MaxPointMonoid)
 
 }
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/PointLuceneRDDPartition.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/PointLuceneRDDPartition.scala
index 1096301a..c1495282 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/PointLuceneRDDPartition.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/PointLuceneRDDPartition.scala
@@ -109,8 +109,8 @@ private[point] class PointLuceneRDDPartition[V]
   }
 
   override def bounds(): (PointType, PointType) = {
-    import com.twitter.algebird.GeneratedTupleAggregator._
-    PointLuceneRDD.boundingBoxAgg(iterOriginal.map(_._1))
+    iterOriginal.map(_._1).map(x => (x, x))
+        .reduce(PointLuceneRDD.boundingBoxMonoid.plus)
   }
 
   override def circleSearch(center: PointType, radius: Double, k: Int, operationName: String)

From 46990df239182a1bfed7a015d1eec0c6907c6bcb Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Fri, 1 Dec 2017 14:51:33 +0100
Subject: [PATCH 44/47] point_rdd: sort by x-axis

---
 .../spatial/point/PointLuceneRDD.scala        | 23 +++++++++++--------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala
index 41b2a04f..a0371231 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala
@@ -307,7 +307,9 @@ object PointLuceneRDD extends Versionable
                                       similarity: String)
                                      (implicit docConverter: V => Document)
   : PointLuceneRDD[V] = {
-    val partitions = elems.mapPartitions[AbstractPointLuceneRDDPartition[V]](
+    val partitions = elems
+      .sortBy(item => item._1._1) // Sort by x-axis
+      .mapPartitions[AbstractPointLuceneRDDPartition[V]](
       iter => Iterator(PointLuceneRDDPartition[V](iter, indexAnalyzer, queryAnalyzer)),
       preservesPartitioning = true)
     new PointLuceneRDD(partitions, indexAnalyzer, queryAnalyzer, similarity)
@@ -329,9 +331,11 @@ object PointLuceneRDD extends Versionable
                                       similarity: String)
                                      (implicit docConverter: V => Document)
   : PointLuceneRDD[V] = {
-    val partitions = elems.rdd.mapPartitions[AbstractPointLuceneRDDPartition[V]](
-      iter => Iterator(PointLuceneRDDPartition[V](iter, indexAnalyzer, queryAnalyzer)),
-      preservesPartitioning = true)
+    val partitions = elems.rdd
+        .sortBy(item => item._1._1) // Sort by x-axis
+        .mapPartitions[AbstractPointLuceneRDDPartition[V]](
+          iter => Iterator(PointLuceneRDDPartition[V](iter, indexAnalyzer, queryAnalyzer)),
+          preservesPartitioning = true)
     new PointLuceneRDD(partitions, indexAnalyzer, queryAnalyzer, similarity)
   }
 
@@ -355,13 +359,13 @@ object PointLuceneRDD extends Versionable
     *  val lucene = ShapeLuceneRDD(counties, "shape")
     *
     * }}
-    * @param df Input dataframe containing Shape as String field named "shapeField"
+    * @param df Input DataFrame containing Shape as String field named "shapeField"
     * @param shapeField Name of DataFrame column that contains Shape as String, i.e., WKT
-    * @param shapeConv Implicit convertion for spatial / shape
+    * @param shapeConv Implicit conversion for spatial / shape
     * @param docConverter Implicit conversion for Lucene Document
     * @return
     */
-  def apply(df : DataFrame,
+  def apply(df: DataFrame,
             shapeField: String)
            (implicit shapeConv: String => PointType, docConverter: Row => Document)
   : PointLuceneRDD[Row] = {
@@ -371,14 +375,15 @@ object PointLuceneRDD extends Versionable
   }
 
 
-  def apply(df : DataFrame,
+  def apply(df: DataFrame,
             shapeField: String,
             indexAnalyzer: String,
             queryAnalyzer: String,
             similarity: String)
-           (implicit  shapeConv: String => PointType, docConverter: Row => Document)
+           (implicit shapeConv: String => PointType, docConverter: Row => Document)
   : PointLuceneRDD[Row] = {
     val partitions = df.rdd.map(row => (shapeConv(row.getString(row.fieldIndex(shapeField))), row))
+        .sortBy(item => item._1._1) // Sort by x-axis
       .mapPartitions[AbstractPointLuceneRDDPartition[Row]](
       iter => Iterator(PointLuceneRDDPartition[Row](iter, indexAnalyzer, queryAnalyzer)),
       preservesPartitioning = true)

From ab5d0cdc5acec9209fd0eebb3c73a66c02a5b7e8 Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Sun, 3 Dec 2017 13:24:20 +0100
Subject: [PATCH 45/47] point_rdd: cleanup

---
 .../MaxPointMonoid.scala                      |  8 +++-
 .../MinPointMonoid.scala                      |  8 +++-
 .../commons/SpatialByXPartitioner.scala       | 40 +++++++++++++++++++
 .../spatial/point/PointLuceneRDD.scala        | 20 +++++++---
 4 files changed, 69 insertions(+), 7 deletions(-)
 rename src/main/scala/org/zouzias/spark/lucenerdd/{spatial/commons => aggregate}/MaxPointMonoid.scala (87%)
 rename src/main/scala/org/zouzias/spark/lucenerdd/{spatial/commons => aggregate}/MinPointMonoid.scala (88%)
 create mode 100644 src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/SpatialByXPartitioner.scala

diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/MaxPointMonoid.scala b/src/main/scala/org/zouzias/spark/lucenerdd/aggregate/MaxPointMonoid.scala
similarity index 87%
rename from src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/MaxPointMonoid.scala
rename to src/main/scala/org/zouzias/spark/lucenerdd/aggregate/MaxPointMonoid.scala
index 6277f2a1..712537db 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/MaxPointMonoid.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/aggregate/MaxPointMonoid.scala
@@ -14,12 +14,18 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.zouzias.spark.lucenerdd.spatial.commons
+package org.zouzias.spark.lucenerdd.aggregate
 
 import com.twitter.algebird.Monoid
 import org.zouzias.spark.lucenerdd.spatial.point.PointLuceneRDD.PointType
 
+/**
+  * Maximum point [[Monoid]] used for spatial linkage
+  *
+  * Keeps the maximum value per coordinate
+  */
 object MaxPointMonoid extends Monoid[PointType] {
+
   override def zero: PointType = (Double.MaxValue, Double.MaxValue)
 
   override def plus(x: PointType, y: PointType): PointType = {
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/MinPointMonoid.scala b/src/main/scala/org/zouzias/spark/lucenerdd/aggregate/MinPointMonoid.scala
similarity index 88%
rename from src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/MinPointMonoid.scala
rename to src/main/scala/org/zouzias/spark/lucenerdd/aggregate/MinPointMonoid.scala
index 4f074a93..7b557e42 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/MinPointMonoid.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/aggregate/MinPointMonoid.scala
@@ -14,12 +14,18 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.zouzias.spark.lucenerdd.spatial.commons
+package org.zouzias.spark.lucenerdd.aggregate
 
 import com.twitter.algebird.Monoid
 import org.zouzias.spark.lucenerdd.spatial.point.PointLuceneRDD.PointType
 
+/**
+  * Minimum point Monoid used for spatial linkage
+  *
+  * Keeps the minimum value per coordinate
+  */
 object MinPointMonoid extends Monoid[PointType] {
+
   override def zero: PointType = (Double.MinValue, Double.MinValue)
 
   override def plus(x: PointType, y: PointType): PointType = {
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/SpatialByXPartitioner.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/SpatialByXPartitioner.scala
new file mode 100644
index 00000000..3c58d794
--- /dev/null
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/SpatialByXPartitioner.scala
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.zouzias.spark.lucenerdd.spatial.commons
+
+import org.apache.spark.Partitioner
+import org.zouzias.spark.lucenerdd.spatial.point.PointLuceneRDD.PointType
+
+import scala.util.Random
+
+/**
+  * Spark RDD [[Partitioner]] based on the x-axis and bounds per partition
+  * @param boundsPerPart Bounds of x-axis (minX, maxX) per RDD's partition
+  */
+case class SpatialByXPartitioner(boundsPerPart: Array[(Double, Double)]) extends Partitioner {
+  override def numPartitions: Int = boundsPerPart.length
+
+  override def getPartition(key: Any): Int = {
+    val keyPoint = key.asInstanceOf[PointType]
+    val indexOpt = boundsPerPart.indexWhere{ case (minX, maxX) =>
+      minX <= keyPoint._1 && keyPoint._1 <= maxX
+    }
+
+    // If key is not assigned to a partition, randomly assign the key
+    if (indexOpt == -1) Random.nextInt(numPartitions) else indexOpt
+  }
+}
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala
index a0371231..6c3dfb82 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala
@@ -19,16 +19,17 @@ package org.zouzias.spark.lucenerdd.spatial.point
 import com.twitter.algebird._
 import org.apache.lucene.document.Document
 import org.apache.lucene.spatial.query.SpatialOperation
-import org.apache.spark.{OneToOneDependency, Partition, TaskContext}
+import org.apache.spark.{OneToOneDependency, Partition, Partitioner, TaskContext}
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.{DataFrame, Dataset, Row}
 import org.apache.spark.storage.StorageLevel
+import org.zouzias.spark.lucenerdd.aggregate.{MaxPointMonoid, MinPointMonoid}
 import org.zouzias.spark.lucenerdd.analyzers.AnalyzerConfigurable
 import org.zouzias.spark.lucenerdd.config.ShapeLuceneRDDConfigurable
 import org.zouzias.spark.lucenerdd.models.SparkScoreDoc
 import org.zouzias.spark.lucenerdd.query.{LuceneQueryHelpers, SimilarityConfigurable}
 import org.zouzias.spark.lucenerdd.response.{LuceneRDDResponse, LuceneRDDResponsePartition}
-import org.zouzias.spark.lucenerdd.spatial.commons.{MaxPointMonoid, MinPointMonoid}
+import org.zouzias.spark.lucenerdd.spatial.commons.SpatialByXPartitioner
 import org.zouzias.spark.lucenerdd.spatial.point.PointLuceneRDD.PointType
 import org.zouzias.spark.lucenerdd.spatial.point.partition.{AbstractPointLuceneRDDPartition, PointLuceneRDDPartition}
 import org.zouzias.spark.lucenerdd.versioning.Versionable
@@ -198,7 +199,7 @@ class PointLuceneRDD[V: ClassTag]
    *
    * @param shapeWKT Shape in WKT format
    * @param k Number of element to return
-   * @param operationName
+   * @param operationName Spatial operation name, i.e., intersection, contained, etc.
    * @return
    */
   def spatialSearch(shapeWKT: String, k: Int,
@@ -213,7 +214,7 @@ class PointLuceneRDD[V: ClassTag]
    * @param center given as (x, y)
    * @param radius in kilometers (KM)
    * @param k
-   * @param operationName
+   * @param operationName Spatial operation name, i.e., intersection, contained, etc.
    * @return
    */
   def bboxSearch(center: PointType, radius: Double, k: Int,
@@ -228,7 +229,7 @@ class PointLuceneRDD[V: ClassTag]
    * @param lowerLeft Lower left corner
    * @param upperRight Upper right corner
    * @param k Number of results
-   * @param operationName Intersect, contained, etc.
+   * @param operationName Spatial operation name, i.e., intersection, contained, etc.
    * @return
    */
   def bboxSearch(lowerLeft: PointType,
@@ -257,6 +258,15 @@ class PointLuceneRDD[V: ClassTag]
       }
   }
 
+  /**
+    * Returns an x-axis spatial [[Partitioner]] that is used in link related methods
+    * @return Spark [[Partitioner]]
+    */
+  def spatialPartitioner(): Partitioner = {
+    val bpp = boundsPerPartition().map(x => (x._1._1, x._2._1)).collect()
+    SpatialByXPartitioner(bpp)
+  }
+
   override def count(): Long = {
     logInfo("Count requested")
     partitionsRDD.map(_.size).reduce(_ + _)

From 9f68e93f144afa73c5d7204842a92aaaf3c9f9a6 Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Tue, 9 Jan 2018 21:57:07 +0100
Subject: [PATCH 46/47] point_rdd: WIP

---
 .../spatial/point/PointLuceneRDD.scala        | 91 ++++---------------
 1 file changed, 20 insertions(+), 71 deletions(-)

diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala
index 6c3dfb82..b796e413 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala
@@ -83,84 +83,34 @@ class PointLuceneRDD[V: ClassTag]
     new LuceneRDDResponse(partitionsRDD.map(f), SparkScoreDoc.ascending)
   }
 
-  private def linker[T: ClassTag, S: ClassTag](that: RDD[T],
-                                  pointFunctor: T => S,
-                                  mapper: (S, AbstractPointLuceneRDDPartition[V]) =>
-                                    Iterable[SparkScoreDoc],
-                                  linkerMethod: String)
+  def linkByRadius[T: ClassTag](that: RDD[T],
+                                       pointFunctor: T => PointType,
+                                       topK: Int = DefaultTopK,
+                                       radius: Double,
+                                       linkerMethod: String = getShapeLinkerMethod)
   : RDD[(T, Array[SparkScoreDoc])] = {
-    logInfo("Point Linkage requested")
+    logInfo("linkByRadius requested")
+
+    val partitioner = SpatialByXPartitioner(boundsPerPartition()
+      .map(x => (x._1._1, x._2._1)).collect()
+    )
 
-    val topKMonoid = new TopKMonoid[SparkScoreDoc](MaxDefaultTopKValue)(SparkScoreDoc.ascending)
     val queries = that.zipWithIndex().map(_.swap)
+    val queriesPart = queries.mapValues(pointFunctor).partitionBy(partitioner)
 
-    val resultsByPart = linkerMethod match {
-      case "cartesian" =>
-        val concatenated = queries.mapValues(pointFunctor).glom()
+    val coGrouped = partitionsRDD.zipPartitions(queriesPart, preservesPartitioning = true)
+      {case tp =>
+        val queries = tp._2.toArray
 
-        concatenated.cartesian(partitionsRDD)
-          .flatMap { case (qs, lucene) =>
-            qs.map { case (ind, query) =>
-              (ind, topKMonoid.build(mapper(query, lucene)))
-            }
+        tp._1.flatMap{lucene =>
+          queries.map{q =>
+            (q._1,
+              lucene.circleSearch(q._2, radius, topK, linkerMethod).toArray)
           }
-      case _ =>
-        logInfo("Collecting query points to driver")
-        val collectedQueries = queries.mapValues(pointFunctor).collect()
-        val queriesB = partitionsRDD.context.broadcast(collectedQueries)
-
-        partitionsRDD.mapPartitions { partitions =>
-            partitions.flatMap { partition =>
-              queriesB.value.map { case (index, query) =>
-                  (index, topKMonoid.build(mapper(query, partition)))
-                }
-            }
         }
-    }
-
-    logInfo("Computing top-k linkage per partition")
-    val results = resultsByPart.reduceByKey(topKMonoid.plus)
-
-    queries.join(results).values
-      .map(joined => (joined._1, joined._2.items.toArray))
-  }
-
-  /**
-   * Link entities if their shapes are within a distance in kilometers (km)
-   *
-   * Links this and that based on distance threshold
-   *
-   * @param that An RDD of entities to be linked
-   * @param shapeFunctor Function that generates a point from each element of other
-   * @param linkerMethod Method to perform linkage
-   * @tparam T A type
-   * @return an RDD of Tuple2 that contains the linked results
-   *
-   * Note: Currently the query coordinates of the other RDD are collected to the driver and
-   * broadcast to the workers.
-   */
-  def linkByInstersection[T: ClassTag](that: RDD[T],
-                                shapeFunctor: T => String,
-                                topK: Int = DefaultTopK,
-                                linkerMethod: String = getShapeLinkerMethod)
-  : RDD[(T, Array[SparkScoreDoc])] = {
-    logInfo("linkByInstersection requested")
-    linker[T, String](that, shapeFunctor, (queryShape: String, part) =>
-      part.spatialSearch(queryShape, topK, SpatialOperation.Intersects.getName),
-      linkerMethod)
-  }
-
+      }
 
-  def linkByRadius[T: ClassTag](that: RDD[T],
-                                       pointFunctor: T => PointType,
-                                       topK: Int = DefaultTopK,
-                                       radius: Double,
-                                       linkerMethod: String = getShapeLinkerMethod)
-  : RDD[(T, Array[SparkScoreDoc])] = {
-    logInfo("linkByRadius requested")
-    linker[T, PointType](that, pointFunctor, (queryPoint, part) =>
-      part.circleSearch(queryPoint, radius, topK, SpatialOperation.Intersects.getName),
-      linkerMethod)
+    queries.join(coGrouped).values
   }
 
   /**
@@ -403,5 +353,4 @@ object PointLuceneRDD extends Versionable
   /** Algebird bounding box aggregator */
 
   val boundingBoxMonoid = new Tuple2Monoid()(MinPointMonoid, MaxPointMonoid)
-
 }

From bfc7fd454f3f265f3fd971ef1e005f043eeefb15 Mon Sep 17 00:00:00 2001
From: Anastasios Zouzias <zouzias@gmail.com>
Date: Fri, 2 Mar 2018 16:41:21 +0100
Subject: [PATCH 47/47] point_rdd: introduce BoundingBox

---
 .../aggregate/BoundingBoxMonoid.scala         | 38 +++++++++++++++++++
 .../lucenerdd/aggregate/MaxPointMonoid.scala  |  6 +--
 .../lucenerdd/aggregate/MinPointMonoid.scala  |  2 +-
 .../spark/lucenerdd/models/BoundingBox.scala  | 27 +++++++++++++
 .../commons/SpatialByXPartitioner.scala       |  2 +-
 .../spatial/point/PointLuceneRDD.scala        | 34 +++++++++--------
 .../AbstractPointLuceneRDDPartition.scala     |  5 ++-
 .../partition/PointLuceneRDDPartition.scala   | 17 +++++----
 .../spatial/point/PointLuceneRDDSpec.scala    | 19 +++-------
 9 files changed, 106 insertions(+), 44 deletions(-)
 create mode 100644 src/main/scala/org/zouzias/spark/lucenerdd/aggregate/BoundingBoxMonoid.scala
 create mode 100644 src/main/scala/org/zouzias/spark/lucenerdd/models/BoundingBox.scala

diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/aggregate/BoundingBoxMonoid.scala b/src/main/scala/org/zouzias/spark/lucenerdd/aggregate/BoundingBoxMonoid.scala
new file mode 100644
index 00000000..14886ad4
--- /dev/null
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/aggregate/BoundingBoxMonoid.scala
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.zouzias.spark.lucenerdd.aggregate
+
+import com.twitter.algebird.Monoid
+import org.zouzias.spark.lucenerdd.models.BoundingBox
+import org.zouzias.spark.lucenerdd.spatial.point.PointLuceneRDD.PointType
+
+
+/**
+  * Bounding box monoid based on [[BoundingBox]]
+  */
+object BoundingBoxMonoid extends Monoid[BoundingBox] {
+
+  override def zero: BoundingBox = {
+    BoundingBox((Double.MaxValue, Double.MaxValue), (Double.MinValue, Double.MinValue))
+  }
+
+  override def plus(x: BoundingBox,
+                    y: BoundingBox): BoundingBox = {
+    BoundingBox(MinPointMonoid.plus(x.lowerLeft, y.lowerLeft),
+      MaxPointMonoid.plus(x.upperRight, y.upperRight))
+  }
+}
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/aggregate/MaxPointMonoid.scala b/src/main/scala/org/zouzias/spark/lucenerdd/aggregate/MaxPointMonoid.scala
index 712537db..836eab5f 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/aggregate/MaxPointMonoid.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/aggregate/MaxPointMonoid.scala
@@ -26,9 +26,9 @@ import org.zouzias.spark.lucenerdd.spatial.point.PointLuceneRDD.PointType
   */
 object MaxPointMonoid extends Monoid[PointType] {
 
-  override def zero: PointType = (Double.MaxValue, Double.MaxValue)
+  override def zero: PointType = (Double.MinValue, Double.MinValue)
 
-  override def plus(x: PointType, y: PointType): PointType = {
-    (Math.max(x._1, y._1), Math.max(x._2, y._2))
+  override def plus(a: PointType, b: PointType): PointType = {
+    (Math.max(a._1, b._1), Math.max(a._2, b._2))
   }
 }
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/aggregate/MinPointMonoid.scala b/src/main/scala/org/zouzias/spark/lucenerdd/aggregate/MinPointMonoid.scala
index 7b557e42..ae539121 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/aggregate/MinPointMonoid.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/aggregate/MinPointMonoid.scala
@@ -26,7 +26,7 @@ import org.zouzias.spark.lucenerdd.spatial.point.PointLuceneRDD.PointType
   */
 object MinPointMonoid extends Monoid[PointType] {
 
-  override def zero: PointType = (Double.MinValue, Double.MinValue)
+  override def zero: PointType = (Double.MaxValue, Double.MaxValue)
 
   override def plus(x: PointType, y: PointType): PointType = {
     (Math.min(x._1, y._1), Math.min(x._2, y._2))
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/models/BoundingBox.scala b/src/main/scala/org/zouzias/spark/lucenerdd/models/BoundingBox.scala
new file mode 100644
index 00000000..7c3be78d
--- /dev/null
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/models/BoundingBox.scala
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.zouzias.spark.lucenerdd.models
+
+import org.zouzias.spark.lucenerdd.spatial.point.PointLuceneRDD.PointType
+
+/**
+  * Bounding box described as lower-left and upper right point
+  *
+  * @param lowerLeft The lowest-left point, i.e., (min_X, min_Y)
+  * @param upperRight The upper-right point, i.e., (max_X, max_Y)
+  */
+case class BoundingBox(lowerLeft: PointType, upperRight: PointType)
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/SpatialByXPartitioner.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/SpatialByXPartitioner.scala
index 3c58d794..1813bbb9 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/SpatialByXPartitioner.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/SpatialByXPartitioner.scala
@@ -25,7 +25,7 @@ import scala.util.Random
   * Spark RDD [[Partitioner]] based on the x-axis and bounds per partition
   * @param boundsPerPart Bounds of x-axis (minX, maxX) per RDD's partition
   */
-case class SpatialByXPartitioner(boundsPerPart: Array[(Double, Double)]) extends Partitioner {
+case class SpatialByXPartitioner(boundsPerPart: Array[PointType]) extends Partitioner {
   override def numPartitions: Int = boundsPerPart.length
 
   override def getPartition(key: Any): Int = {
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala
index b796e413..11497793 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala
@@ -23,10 +23,10 @@ import org.apache.spark.{OneToOneDependency, Partition, Partitioner, TaskContext
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.{DataFrame, Dataset, Row}
 import org.apache.spark.storage.StorageLevel
-import org.zouzias.spark.lucenerdd.aggregate.{MaxPointMonoid, MinPointMonoid}
+import org.zouzias.spark.lucenerdd.aggregate.{BoundingBoxMonoid, MaxPointMonoid, MinPointMonoid}
 import org.zouzias.spark.lucenerdd.analyzers.AnalyzerConfigurable
 import org.zouzias.spark.lucenerdd.config.ShapeLuceneRDDConfigurable
-import org.zouzias.spark.lucenerdd.models.SparkScoreDoc
+import org.zouzias.spark.lucenerdd.models.{BoundingBox, SparkScoreDoc}
 import org.zouzias.spark.lucenerdd.query.{LuceneQueryHelpers, SimilarityConfigurable}
 import org.zouzias.spark.lucenerdd.response.{LuceneRDDResponse, LuceneRDDResponsePartition}
 import org.zouzias.spark.lucenerdd.spatial.commons.SpatialByXPartitioner
@@ -87,25 +87,29 @@ class PointLuceneRDD[V: ClassTag]
                                        pointFunctor: T => PointType,
                                        topK: Int = DefaultTopK,
                                        radius: Double,
-                                       linkerMethod: String = getShapeLinkerMethod)
+                                       linkerMethod: String)
   : RDD[(T, Array[SparkScoreDoc])] = {
     logInfo("linkByRadius requested")
 
-    val partitioner = SpatialByXPartitioner(boundsPerPartition()
-      .map(x => (x._1._1, x._2._1)).collect()
-    )
+    // Get bounding box per LuceneRDD's partitions points
+    val bounds: Array[PointType] = boundsPerPartition()
+      .map(x => (x.lowerLeft._1, x.upperRight._1)).collect()
+    val partitioner = SpatialByXPartitioner(bounds)
 
+    // Index queries and repartition them by above bounding boxes
     val queries = that.zipWithIndex().map(_.swap)
-    val queriesPart = queries.mapValues(pointFunctor).partitionBy(partitioner)
+    val queriesPart = queries.mapValues(pointFunctor)
+        .map(_.swap)
+        .partitionBy(partitioner)
 
     val coGrouped = partitionsRDD.zipPartitions(queriesPart, preservesPartitioning = true)
       {case tp =>
-        val queries = tp._2.toArray
+        val queries = tp._2
 
         tp._1.flatMap{lucene =>
           queries.map{q =>
-            (q._1,
-              lucene.circleSearch(q._2, radius, topK, linkerMethod).toArray)
+            (q._2,
+              lucene.circleSearch(q._1, radius, topK, linkerMethod).toArray)
           }
         }
       }
@@ -195,17 +199,15 @@ class PointLuceneRDD[V: ClassTag]
     * Returns the smallest enclosing axis aligned bounding box per partition
     * @return
     */
-  def boundsPerPartition(): RDD[(PointType, PointType)] = {
+  def boundsPerPartition(): RDD[BoundingBox] = {
     logInfo("boundsPerPartition requested")
     partitionsRDD.map(_.bounds())
   }
 
-  def bounds(): (PointType, PointType) = {
+  def bounds(): BoundingBox = {
     logInfo("bounds requested")
     boundsPerPartition()
-      .reduce { case (x, y) =>
-        (MinPointMonoid.plus(x._1, y._1), MaxPointMonoid.plus(x._2, y._2))
-      }
+      .reduce(BoundingBoxMonoid.plus)
   }
 
   /**
@@ -213,7 +215,7 @@ class PointLuceneRDD[V: ClassTag]
     * @return Spark [[Partitioner]]
     */
   def spatialPartitioner(): Partitioner = {
-    val bpp = boundsPerPartition().map(x => (x._1._1, x._2._1)).collect()
+    val bpp = boundsPerPartition().map(x => (x.lowerLeft._1, x.upperRight._1)).collect()
     SpatialByXPartitioner(bpp)
   }
 
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/AbstractPointLuceneRDDPartition.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/AbstractPointLuceneRDDPartition.scala
index 3744b4dc..565080b5 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/AbstractPointLuceneRDDPartition.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/AbstractPointLuceneRDDPartition.scala
@@ -16,6 +16,7 @@
  */
 package org.zouzias.spark.lucenerdd.spatial.point.partition
 
+import org.zouzias.spark.lucenerdd.models.BoundingBox
 import org.zouzias.spark.lucenerdd.response.LuceneRDDResponsePartition
 import org.zouzias.spark.lucenerdd.spatial.shape.ShapeLuceneRDD.PointType
 
@@ -36,9 +37,9 @@ private[point] abstract class AbstractPointLuceneRDDPartition[V] extends Seriali
 
   /**
     * Return smallest enclosing axis aligned bounding box
-    * @return ( (minX, minY), (maxX, maxY))
+    * @return Smallest enclosing bounding box
     */
-  def bounds(): (PointType, PointType)
+  def bounds(): BoundingBox
 
   /**
    * Nearest neighbour search
diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/PointLuceneRDDPartition.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/PointLuceneRDDPartition.scala
index c1495282..b119096a 100644
--- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/PointLuceneRDDPartition.scala
+++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/PointLuceneRDDPartition.scala
@@ -24,7 +24,8 @@ import org.apache.lucene.spatial.query.{SpatialArgs, SpatialOperation}
 import org.joda.time.DateTime
 import org.locationtech.spatial4j.distance.DistanceUtils
 import org.locationtech.spatial4j.shape.{Point, Shape}
-import org.zouzias.spark.lucenerdd.models.SparkScoreDoc
+import org.zouzias.spark.lucenerdd.aggregate.BoundingBoxMonoid
+import org.zouzias.spark.lucenerdd.models.{BoundingBox, SparkScoreDoc}
 import org.zouzias.spark.lucenerdd.query.LuceneQueryHelpers
 import org.zouzias.spark.lucenerdd.response.LuceneRDDResponsePartition
 import org.zouzias.spark.lucenerdd.spatial.commons.strategies.SpatialStrategy
@@ -66,16 +67,21 @@ private[point] class PointLuceneRDDPartition[V]
 
   private val startTime = new DateTime(System.currentTimeMillis())
   logInfo(s"Indexing process initiated at ${startTime}...")
-  iterIndex.foreach { case (p, value) =>
+  private val boundingBox_ = iterIndex.map { case (p, value) =>
     // (implicitly) convert type V to a Lucene document
     val doc = docConversion(value)
     val docWithLocation = decorateWithLocation(doc, ctx.makePoint(p._1, p._2))
     indexWriter.addDocument(FacetsConfig.build(taxoWriter, docWithLocation))
-  }
+
+    BoundingBox(p, p)
+  }.reduce(BoundingBoxMonoid.plus)
   private val endTime = new DateTime(System.currentTimeMillis())
+
   logInfo(s"Indexing process completed at ${endTime}...")
   logInfo(s"Indexing process took ${(endTime.getMillis - startTime.getMillis) / 1000} seconds...")
 
+  override def bounds(): BoundingBox = boundingBox_
+
   // Close the indexWriter and taxonomyWriter (for faceted search)
   closeAllWriters()
 
@@ -108,11 +114,6 @@ private[point] class PointLuceneRDDPartition[V]
     stringToShape(shapeString)
   }
 
-  override def bounds(): (PointType, PointType) = {
-    iterOriginal.map(_._1).map(x => (x, x))
-        .reduce(PointLuceneRDD.boundingBoxMonoid.plus)
-  }
-
   override def circleSearch(center: PointType, radius: Double, k: Int, operationName: String)
   : LuceneRDDResponsePartition = {
     logInfo(s"circleSearch [center:${center}, operation:${operationName}]")
diff --git a/src/test/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDDSpec.scala b/src/test/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDDSpec.scala
index b22f7f44..89c7ce68 100644
--- a/src/test/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDDSpec.scala
+++ b/src/test/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDDSpec.scala
@@ -45,7 +45,7 @@ class PointLuceneRDDSpec extends FlatSpec
     pointLuceneRDD.close()
   }
 
-  override val conf = ShapeLuceneRDDKryoRegistrator.registerKryoClasses(new SparkConf().
+  override val conf: SparkConf = ShapeLuceneRDDKryoRegistrator.registerKryoClasses(new SparkConf().
     setMaster("local[*]").
     setAppName("test").
     set("spark.ui.enabled", "false").
@@ -58,7 +58,7 @@ class PointLuceneRDDSpec extends FlatSpec
     // Bern, Laussanne and Zurich is within 300km
     val results = pointLuceneRDD.circleSearch(Bern._1, 300, k).collect()
 
-    results.size should equal(3)
+    results.length should equal(3)
 
     results.exists(x => docTextFieldEq(x.doc, "_1", Bern._2)) should equal(true)
     results.exists(x => docTextFieldEq(x.doc, "_1", Zurich._2)) should equal(true)
@@ -84,7 +84,7 @@ class PointLuceneRDDSpec extends FlatSpec
     // Bern, Laussanne and Zurich is within 300km
     val results = pointLuceneRDD.spatialSearch(circleWKT, k).collect()
 
-    results.size should equal(3)
+    results.length should equal(3)
 
     results.exists(x => docTextFieldEq(x.doc, "_1", Bern._2)) should equal(true)
     results.exists(x => docTextFieldEq(x.doc, "_1", Zurich._2)) should equal(true)
@@ -160,17 +160,10 @@ class PointLuceneRDDSpec extends FlatSpec
   }
 
   "PointLuceneRDD.bounds" should "return correct bounds" in {
-    val Bern = ( (7.45, 46.95), "Bern")
-    val Zurich = ( (8.55, 47.366667), "Zurich")
-    val Laussanne = ( (6.6335, 46.519833), "Laussanne")
-    val Athens = ((23.716667, 37.966667), "Athens")
-    val Toronto = ((-79.4, 43.7), "Toronto")
-    val Milan = ((45.4646, 9.198), "Milan")
-
     val rdd = sc.parallelize(cities)
     pointLuceneRDD = PointLuceneRDD(rdd)
-    val (minBounds, maxBounds) = pointLuceneRDD.bounds()
-    minBounds should equal((-79.4, 9.198))
-    maxBounds should equal((45.4646, 47.366667))
+    val boundingBox = pointLuceneRDD.bounds()
+    boundingBox.lowerLeft should equal((-79.4, 9.198))
+    boundingBox.upperRight should equal((45.4646, 47.366667))
   }
 }