diff --git a/build.sbt b/build.sbt index 1714865f..2edb95ef 100644 --- a/build.sbt +++ b/build.sbt @@ -17,8 +17,8 @@ name := "spark-lucenerdd" organization := "org.zouzias" -scalaVersion := "2.11.11" -crossScalaVersions := Seq("2.11.11") +scalaVersion := "2.11.12" +crossScalaVersions := Seq("2.11.12") licenses := Seq("Apache-2.0" -> url("http://www.apache.org/licenses/LICENSE-2.0.html")) homepage := Some(url("https://github.com/zouzias/spark-lucenerdd")) @@ -79,10 +79,10 @@ pomExtra := ( ) -val luceneV = "7.1.0" +val luceneV = "7.3.0" spName := "zouzias/spark-lucenerdd" -sparkVersion := "2.2.0" +sparkVersion := "2.2.1" spShortDescription := "Spark RDD with Lucene's query capabilities" sparkComponents ++= Seq("core", "sql", "mllib") spAppendScalaVersion := true @@ -96,15 +96,15 @@ testSparkVersion := sys.props.get("spark.testVersion").getOrElse(sparkVersion.va // scalastyle:off -val scalactic = "org.scalactic" %% "scalactic" % "3.0.4" -val scalatest = "org.scalatest" %% "scalatest" % "3.0.4" % "test" +val scalactic = "org.scalactic" %% "scalactic" % "3.0.5" +val scalatest = "org.scalatest" %% "scalatest" % "3.0.5" % "test" val joda_time = "joda-time" % "joda-time" % "2.9.9" -val algebird = "com.twitter" %% "algebird-core" % "0.13.3" +val algebird = "com.twitter" %% "algebird-core" % "0.13.4" val joda_convert = "org.joda" % "joda-convert" % "1.9.2" -val spatial4j = "org.locationtech.spatial4j" % "spatial4j" % "0.6" +val spatial4j = "org.locationtech.spatial4j" % "spatial4j" % "0.7" -val typesafe_config = "com.typesafe" % "config" % "1.3.1" +val typesafe_config = "com.typesafe" % "config" % "1.3.3" val lucene_facet = "org.apache.lucene" % "lucene-facet" % luceneV val lucene_analyzers = "org.apache.lucene" % "lucene-analyzers-common" % luceneV @@ -113,7 +113,7 @@ val lucene_expressions = "org.apache.lucene" % "lucene-expre val lucene_spatial = "org.apache.lucene" % "lucene-spatial" % luceneV val lucene_spatial_extras = "org.apache.lucene" % "lucene-spatial-extras" % luceneV -val jts = "com.vividsolutions" % "jts" % "1.13" +val jts = "org.locationtech.jts" % "jts-core" % "1.15.0" // scalastyle:on @@ -137,7 +137,7 @@ libraryDependencies ++= Seq( libraryDependencies ++= Seq( "org.apache.spark" %% "spark-core" % testSparkVersion.value % "test" force(), "org.apache.spark" %% "spark-sql" % testSparkVersion.value % "test" force(), - "com.holdenkarau" %% "spark-testing-base" % s"${sparkVersion.value}_0.7.4" % "test" intransitive(), + "com.holdenkarau" %% "spark-testing-base" % s"2.2.1_0.9.0" % "test" intransitive(), "org.scala-lang" % "scala-library" % scalaVersion.value % "compile" ) diff --git a/project/plugins.sbt b/project/plugins.sbt index 65bfd12a..74f12f97 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -17,9 +17,9 @@ resolvers += "bintray-spark-packages" at "https://dl.bintray.com/spark-packages/maven/" -addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.4") +addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.6") -addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.5") +addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.6") addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "0.8.0") @@ -31,4 +31,4 @@ addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "1.1") addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.0.0") -addSbtPlugin("org.spark-packages" % "sbt-spark-package" % "0.2.5") +addSbtPlugin("org.spark-packages" % "sbt-spark-package" % "0.2.6") diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/aggregate/BoundingBoxMonoid.scala b/src/main/scala/org/zouzias/spark/lucenerdd/aggregate/BoundingBoxMonoid.scala new file mode 100644 index 00000000..14886ad4 --- /dev/null +++ b/src/main/scala/org/zouzias/spark/lucenerdd/aggregate/BoundingBoxMonoid.scala @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.zouzias.spark.lucenerdd.aggregate + +import com.twitter.algebird.Monoid +import org.zouzias.spark.lucenerdd.models.BoundingBox +import org.zouzias.spark.lucenerdd.spatial.point.PointLuceneRDD.PointType + + +/** + * Bounding box monoid based on [[BoundingBox]] + */ +object BoundingBoxMonoid extends Monoid[BoundingBox] { + + override def zero: BoundingBox = { + BoundingBox((Double.MaxValue, Double.MaxValue), (Double.MinValue, Double.MinValue)) + } + + override def plus(x: BoundingBox, + y: BoundingBox): BoundingBox = { + BoundingBox(MinPointMonoid.plus(x.lowerLeft, y.lowerLeft), + MaxPointMonoid.plus(x.upperRight, y.upperRight)) + } +} diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/aggregate/MaxPointMonoid.scala b/src/main/scala/org/zouzias/spark/lucenerdd/aggregate/MaxPointMonoid.scala new file mode 100644 index 00000000..836eab5f --- /dev/null +++ b/src/main/scala/org/zouzias/spark/lucenerdd/aggregate/MaxPointMonoid.scala @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.zouzias.spark.lucenerdd.aggregate + +import com.twitter.algebird.Monoid +import org.zouzias.spark.lucenerdd.spatial.point.PointLuceneRDD.PointType + +/** + * Maximum point [[Monoid]] used for spatial linkage + * + * Keeps the maximum value per coordinate + */ +object MaxPointMonoid extends Monoid[PointType] { + + override def zero: PointType = (Double.MinValue, Double.MinValue) + + override def plus(a: PointType, b: PointType): PointType = { + (Math.max(a._1, b._1), Math.max(a._2, b._2)) + } +} diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/aggregate/MinPointMonoid.scala b/src/main/scala/org/zouzias/spark/lucenerdd/aggregate/MinPointMonoid.scala new file mode 100644 index 00000000..ae539121 --- /dev/null +++ b/src/main/scala/org/zouzias/spark/lucenerdd/aggregate/MinPointMonoid.scala @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.zouzias.spark.lucenerdd.aggregate + +import com.twitter.algebird.Monoid +import org.zouzias.spark.lucenerdd.spatial.point.PointLuceneRDD.PointType + +/** + * Minimum point Monoid used for spatial linkage + * + * Keeps the minimum value per coordinate + */ +object MinPointMonoid extends Monoid[PointType] { + + override def zero: PointType = (Double.MaxValue, Double.MaxValue) + + override def plus(x: PointType, y: PointType): PointType = { + (Math.min(x._1, y._1), Math.min(x._2, y._2)) + } +} diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/models/BoundingBox.scala b/src/main/scala/org/zouzias/spark/lucenerdd/models/BoundingBox.scala new file mode 100644 index 00000000..7c3be78d --- /dev/null +++ b/src/main/scala/org/zouzias/spark/lucenerdd/models/BoundingBox.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.zouzias.spark.lucenerdd.models + +import org.zouzias.spark.lucenerdd.spatial.point.PointLuceneRDD.PointType + +/** + * Bounding box described as lower-left and upper right point + * + * @param lowerLeft The lowest-left point, i.e., (min_X, min_Y) + * @param upperRight The upper-right point, i.e., (max_X, max_Y) + */ +case class BoundingBox(lowerLeft: PointType, upperRight: PointType) diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/response/LuceneRDDResponse.scala b/src/main/scala/org/zouzias/spark/lucenerdd/response/LuceneRDDResponse.scala index 31ff662e..4a1195b5 100644 --- a/src/main/scala/org/zouzias/spark/lucenerdd/response/LuceneRDDResponse.scala +++ b/src/main/scala/org/zouzias/spark/lucenerdd/response/LuceneRDDResponse.scala @@ -76,9 +76,14 @@ private[lucenerdd] class LuceneRDDResponse override def collect(): Array[SparkScoreDoc] = { val sz = partitionsRDD.map(_.size).sum().toInt - val monoid = new TopKMonoid[SparkScoreDoc](sz)(ordering) - partitionsRDD.map(monoid.build(_)) - .reduce(monoid.plus).items.toArray + sz match { + case 0 => Array.empty[SparkScoreDoc] + case s => + // Total size must be positive for [[TopKMonoid]] + val monoid = new TopKMonoid[SparkScoreDoc](s)(ordering) + partitionsRDD.map(monoid.build(_)) + .reduce(monoid.plus).items.toArray + } } /** diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/response/LuceneRDDResponsePartition.scala b/src/main/scala/org/zouzias/spark/lucenerdd/response/LuceneRDDResponsePartition.scala index 817d9a94..2b705733 100644 --- a/src/main/scala/org/zouzias/spark/lucenerdd/response/LuceneRDDResponsePartition.scala +++ b/src/main/scala/org/zouzias/spark/lucenerdd/response/LuceneRDDResponsePartition.scala @@ -23,3 +23,9 @@ case class LuceneRDDResponsePartition(results: Iterator[SparkScoreDoc]) extends Iterable[SparkScoreDoc] { override def iterator(): Iterator[SparkScoreDoc] = results } + +object LuceneRDDResponsePartition { + def empty(): LuceneRDDResponsePartition = { + LuceneRDDResponsePartition(Iterator.empty) + } +} diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/SpatialByXPartitioner.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/SpatialByXPartitioner.scala new file mode 100644 index 00000000..1813bbb9 --- /dev/null +++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/SpatialByXPartitioner.scala @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.zouzias.spark.lucenerdd.spatial.commons + +import org.apache.spark.Partitioner +import org.zouzias.spark.lucenerdd.spatial.point.PointLuceneRDD.PointType + +import scala.util.Random + +/** + * Spark RDD [[Partitioner]] based on the x-axis and bounds per partition + * @param boundsPerPart Bounds of x-axis (minX, maxX) per RDD's partition + */ +case class SpatialByXPartitioner(boundsPerPart: Array[PointType]) extends Partitioner { + override def numPartitions: Int = boundsPerPart.length + + override def getPartition(key: Any): Int = { + val keyPoint = key.asInstanceOf[PointType] + val indexOpt = boundsPerPart.indexWhere{ case (minX, maxX) => + minX <= keyPoint._1 && keyPoint._1 <= maxX + } + + // If key is not assigned to a partition, randomly assign the key + if (indexOpt == -1) Random.nextInt(numPartitions) else indexOpt + } +} diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/context/ContextLoader.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/context/ContextLoader.scala similarity index 84% rename from src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/context/ContextLoader.scala rename to src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/context/ContextLoader.scala index c99ed537..45abd291 100644 --- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/context/ContextLoader.scala +++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/context/ContextLoader.scala @@ -14,11 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.zouzias.spark.lucenerdd.spatial.shape.context +package org.zouzias.spark.lucenerdd.spatial.commons.context import java.io.{StringReader, StringWriter} +import com.vividsolutions.jts.io.ParseException import org.locationtech.spatial4j.context.jts.JtsSpatialContext +import org.locationtech.spatial4j.exception.InvalidShapeException import org.locationtech.spatial4j.io.{ShapeReader, ShapeWriter} import org.locationtech.spatial4j.shape.Shape import org.zouzias.spark.lucenerdd.config.ShapeLuceneRDDConfigurable @@ -37,8 +39,13 @@ trait ContextLoader extends ShapeLuceneRDDConfigurable{ writer.toString } - protected def stringToShape(shapeAsString: String): Shape = { - shapeReader.read(new StringReader(shapeAsString)) + protected def stringToShape(shapeAsString: String): Option[Shape] = { + try { + Some(shapeReader.read(new StringReader(shapeAsString))) + } catch { + case _: ParseException => None + case _ : InvalidShapeException => None + } } /** diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/grids/PrefixTreeLoader.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/grids/PrefixTreeLoader.scala similarity index 92% rename from src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/grids/PrefixTreeLoader.scala rename to src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/grids/PrefixTreeLoader.scala index 19497ac4..5ea16a80 100644 --- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/grids/PrefixTreeLoader.scala +++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/grids/PrefixTreeLoader.scala @@ -14,11 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.zouzias.spark.lucenerdd.spatial.shape.grids +package org.zouzias.spark.lucenerdd.spatial.commons.grids import org.apache.lucene.spatial.prefix.tree.{SpatialPrefixTree, SpatialPrefixTreeFactory} import org.zouzias.spark.lucenerdd.config.ShapeLuceneRDDConfigurable -import org.zouzias.spark.lucenerdd.spatial.shape.context.ContextLoader +import org.zouzias.spark.lucenerdd.spatial.commons.context.ContextLoader import scala.collection.JavaConverters._ diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/strategies/SpatialStrategy.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/strategies/SpatialStrategy.scala similarity index 91% rename from src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/strategies/SpatialStrategy.scala rename to src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/strategies/SpatialStrategy.scala index 5e62a870..c3cf6a31 100644 --- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/strategies/SpatialStrategy.scala +++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/commons/strategies/SpatialStrategy.scala @@ -14,10 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.zouzias.spark.lucenerdd.spatial.shape.strategies +package org.zouzias.spark.lucenerdd.spatial.commons.strategies import org.apache.lucene.spatial.prefix.{PrefixTreeStrategy, RecursivePrefixTreeStrategy} -import org.zouzias.spark.lucenerdd.spatial.shape.grids.PrefixTreeLoader +import org.zouzias.spark.lucenerdd.spatial.commons.grids.PrefixTreeLoader trait SpatialStrategy extends PrefixTreeLoader { diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala new file mode 100644 index 00000000..11497793 --- /dev/null +++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDD.scala @@ -0,0 +1,358 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.zouzias.spark.lucenerdd.spatial.point + +import com.twitter.algebird._ +import org.apache.lucene.document.Document +import org.apache.lucene.spatial.query.SpatialOperation +import org.apache.spark.{OneToOneDependency, Partition, Partitioner, TaskContext} +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.{DataFrame, Dataset, Row} +import org.apache.spark.storage.StorageLevel +import org.zouzias.spark.lucenerdd.aggregate.{BoundingBoxMonoid, MaxPointMonoid, MinPointMonoid} +import org.zouzias.spark.lucenerdd.analyzers.AnalyzerConfigurable +import org.zouzias.spark.lucenerdd.config.ShapeLuceneRDDConfigurable +import org.zouzias.spark.lucenerdd.models.{BoundingBox, SparkScoreDoc} +import org.zouzias.spark.lucenerdd.query.{LuceneQueryHelpers, SimilarityConfigurable} +import org.zouzias.spark.lucenerdd.response.{LuceneRDDResponse, LuceneRDDResponsePartition} +import org.zouzias.spark.lucenerdd.spatial.commons.SpatialByXPartitioner +import org.zouzias.spark.lucenerdd.spatial.point.PointLuceneRDD.PointType +import org.zouzias.spark.lucenerdd.spatial.point.partition.{AbstractPointLuceneRDDPartition, PointLuceneRDDPartition} +import org.zouzias.spark.lucenerdd.versioning.Versionable + +import scala.reflect.ClassTag + +class PointLuceneRDD[V: ClassTag] + (private val partitionsRDD: RDD[AbstractPointLuceneRDDPartition[V]], + val indexAnalyzerName: String, + val queryAnalyzerName: String, + val similarity: String) + extends RDD[(PointType, V)](partitionsRDD.context, List(new OneToOneDependency(partitionsRDD))) + with ShapeLuceneRDDConfigurable { + + logInfo("Instance is created...") + logInfo(s"Number of partitions: ${partitionsRDD.count()}") + setName("PointLuceneRDD") + + override protected def getPartitions: Array[Partition] = partitionsRDD.partitions + + override protected def getPreferredLocations(s: Partition): Seq[String] = + partitionsRDD.preferredLocations(s) + + override def cache(): this.type = { + this.persist(StorageLevel.MEMORY_ONLY) + } + + override def persist(newLevel: StorageLevel): this.type = { + partitionsRDD.persist(newLevel) + super.persist(newLevel) + this + } + + override def unpersist(blocking: Boolean = true): this.type = { + partitionsRDD.unpersist(blocking) + super.unpersist(blocking) + this + } + + override def setName(_name: String): this.type = { + if (partitionsRDD.name != null) { + partitionsRDD.setName(partitionsRDD.name + ", " + _name) + } else { + partitionsRDD.setName(_name) + } + this + } + + private def partitionMapper(f: AbstractPointLuceneRDDPartition[V] => + LuceneRDDResponsePartition): LuceneRDDResponse = { + new LuceneRDDResponse(partitionsRDD.map(f), SparkScoreDoc.ascending) + } + + def linkByRadius[T: ClassTag](that: RDD[T], + pointFunctor: T => PointType, + topK: Int = DefaultTopK, + radius: Double, + linkerMethod: String) + : RDD[(T, Array[SparkScoreDoc])] = { + logInfo("linkByRadius requested") + + // Get bounding box per LuceneRDD's partitions points + val bounds: Array[PointType] = boundsPerPartition() + .map(x => (x.lowerLeft._1, x.upperRight._1)).collect() + val partitioner = SpatialByXPartitioner(bounds) + + // Index queries and repartition them by above bounding boxes + val queries = that.zipWithIndex().map(_.swap) + val queriesPart = queries.mapValues(pointFunctor) + .map(_.swap) + .partitionBy(partitioner) + + val coGrouped = partitionsRDD.zipPartitions(queriesPart, preservesPartitioning = true) + {case tp => + val queries = tp._2 + + tp._1.flatMap{lucene => + queries.map{q => + (q._2, + lucene.circleSearch(q._1, radius, topK, linkerMethod).toArray) + } + } + } + + queries.join(coGrouped).values + } + + /** + * K-nearest neighbors search + * + * @param queryPoint query point (X, Y) + * @param k number of nearest neighbor points to return + * @param searchString Lucene query string + * @return + */ + def knnSearch(queryPoint: PointType, k: Int, + searchString: String = LuceneQueryHelpers.MatchAllDocsString) + : LuceneRDDResponse = { + logInfo(s"Knn search with query ${queryPoint} and search string ${searchString}") + partitionMapper(_.knnSearch(queryPoint, k, searchString)) + } + + /** + * Search for points within a circle + * + * @param center center of circle + * @param radius radius of circle in kilometers (KM) + * @param k number of points to return + * @return + */ + def circleSearch(center: PointType, radius: Double, k: Int) + : LuceneRDDResponse = { + logInfo(s"Circle search with center ${center} and radius ${radius}") + // Points can only intersect + partitionMapper(_.circleSearch(center, radius, k, + SpatialOperation.Intersects.getName)) + } + + /** + * Spatial search with arbitrary shape + * + * @param shapeWKT Shape in WKT format + * @param k Number of element to return + * @param operationName Spatial operation name, i.e., intersection, contained, etc. + * @return + */ + def spatialSearch(shapeWKT: String, k: Int, + operationName: String = SpatialOperation.Intersects.getName) + : LuceneRDDResponse = { + partitionMapper(_.spatialSearch(shapeWKT, k, operationName)) + } + + /** + * Bounding box search with center and radius + * + * @param center given as (x, y) + * @param radius in kilometers (KM) + * @param k + * @param operationName Spatial operation name, i.e., intersection, contained, etc. + * @return + */ + def bboxSearch(center: PointType, radius: Double, k: Int, + operationName: String = SpatialOperation.Intersects.getName) + : LuceneRDDResponse = { + logInfo(s"Bounding box with center ${center}, radius ${radius}, k = ${k}") + partitionMapper(_.bboxSearch(center, radius, k, operationName)) + } + + /** + * Bounding box search with rectangle + * @param lowerLeft Lower left corner + * @param upperRight Upper right corner + * @param k Number of results + * @param operationName Spatial operation name, i.e., intersection, contained, etc. + * @return + */ + def bboxSearch(lowerLeft: PointType, + upperRight: PointType, + k: Int, + operationName: String) + : LuceneRDDResponse = { + logInfo(s"Bounding box with lower left ${lowerLeft}, upper right ${upperRight} and k = ${k}") + partitionMapper(_.bboxSearch(lowerLeft, upperRight, k, operationName)) + } + + /** + * Returns the smallest enclosing axis aligned bounding box per partition + * @return + */ + def boundsPerPartition(): RDD[BoundingBox] = { + logInfo("boundsPerPartition requested") + partitionsRDD.map(_.bounds()) + } + + def bounds(): BoundingBox = { + logInfo("bounds requested") + boundsPerPartition() + .reduce(BoundingBoxMonoid.plus) + } + + /** + * Returns an x-axis spatial [[Partitioner]] that is used in link related methods + * @return Spark [[Partitioner]] + */ + def spatialPartitioner(): Partitioner = { + val bpp = boundsPerPartition().map(x => (x.lowerLeft._1, x.upperRight._1)).collect() + SpatialByXPartitioner(bpp) + } + + override def count(): Long = { + logInfo("Count requested") + partitionsRDD.map(_.size).reduce(_ + _) + } + + /** RDD compute method. */ + override def compute(part: Partition, context: TaskContext): Iterator[(PointType, V)] = { + firstParent[AbstractPointLuceneRDDPartition[V]].iterator(part, context).next.iterator + } + + def filter(pred: (PointType, V) => Boolean): PointLuceneRDD[V] = { + val newPartitionRDD = partitionsRDD.mapPartitions(partition => + partition.map(_.filter(pred)), preservesPartitioning = true + ) + new PointLuceneRDD[V](newPartitionRDD, indexAnalyzerName, queryAnalyzerName, similarity) + } + + def exists(point: PointType): Boolean = { + partitionsRDD.map(_.isDefined(point)).collect().exists(x => x) + } + + def close(): Unit = { + logInfo(s"Closing...") + partitionsRDD.foreach(_.close()) + } +} + + +object PointLuceneRDD extends Versionable + with AnalyzerConfigurable + with SimilarityConfigurable { + + /** Type for a point */ + type PointType = (Double, Double) + + /** + * Instantiate a ShapeLuceneRDD given an RDD[T] + * + * @param elems RDD of type T + * @param indexAnalyzer Index Analyzer name + * @param queryAnalyzer Query Analyzer name + * @param similarity Lucene scoring similarity, i.e., BM25 or TF-IDF + * @return + */ + def apply[V: ClassTag](elems: RDD[(PointType, V)], + indexAnalyzer: String, + queryAnalyzer: String, + similarity: String) + (implicit docConverter: V => Document) + : PointLuceneRDD[V] = { + val partitions = elems + .sortBy(item => item._1._1) // Sort by x-axis + .mapPartitions[AbstractPointLuceneRDDPartition[V]]( + iter => Iterator(PointLuceneRDDPartition[V](iter, indexAnalyzer, queryAnalyzer)), + preservesPartitioning = true) + new PointLuceneRDD(partitions, indexAnalyzer, queryAnalyzer, similarity) + } + + def apply[V: ClassTag](elems: RDD[(PointType, V)]) + (implicit docConverter: V => Document) + : PointLuceneRDD[V] = { + apply[V](elems, getOrElseEn(IndexAnalyzerConfigName), getOrElseEn(QueryAnalyzerConfigName), + getOrElseClassic()) + } + + /** + * Constructor for [[Dataset]] + */ + def apply[V: ClassTag](elems: Dataset[(PointType, V)], + indexAnalyzer: String, + queryAnalyzer: String, + similarity: String) + (implicit docConverter: V => Document) + : PointLuceneRDD[V] = { + val partitions = elems.rdd + .sortBy(item => item._1._1) // Sort by x-axis + .mapPartitions[AbstractPointLuceneRDDPartition[V]]( + iter => Iterator(PointLuceneRDDPartition[V](iter, indexAnalyzer, queryAnalyzer)), + preservesPartitioning = true) + new PointLuceneRDD(partitions, indexAnalyzer, queryAnalyzer, similarity) + } + + /** + * Constructor for [[Dataset]] + */ + def apply[V: ClassTag](elems: Dataset[(PointType, V)]) + (implicit docConverter: V => Document) + : PointLuceneRDD[V] = { + apply[V](elems, getOrElseEn(IndexAnalyzerConfigName), getOrElseEn(QueryAnalyzerConfigName), + getOrElseClassic()) + } + + /** + * Instantiate [[PointLuceneRDD]] from DataFrame with spatial column (shape format) + * + * Shape format can be one of ShapeIO.GeoJSON, ShapeIO.LEGACY, ShapeIO.POLY, ShapeIO.WKT + * + * {{ + * val countries = spark.read.parquet("data/countries-bbox.parquet") + * val lucene = ShapeLuceneRDD(counties, "shape") + * + * }} + * @param df Input DataFrame containing Shape as String field named "shapeField" + * @param shapeField Name of DataFrame column that contains Shape as String, i.e., WKT + * @param shapeConv Implicit conversion for spatial / shape + * @param docConverter Implicit conversion for Lucene Document + * @return + */ + def apply(df: DataFrame, + shapeField: String) + (implicit shapeConv: String => PointType, docConverter: Row => Document) + : PointLuceneRDD[Row] = { + apply(df, shapeField, + getOrElseEn(IndexAnalyzerConfigName), getOrElseEn(QueryAnalyzerConfigName), + getOrElseClassic()) + } + + + def apply(df: DataFrame, + shapeField: String, + indexAnalyzer: String, + queryAnalyzer: String, + similarity: String) + (implicit shapeConv: String => PointType, docConverter: Row => Document) + : PointLuceneRDD[Row] = { + val partitions = df.rdd.map(row => (shapeConv(row.getString(row.fieldIndex(shapeField))), row)) + .sortBy(item => item._1._1) // Sort by x-axis + .mapPartitions[AbstractPointLuceneRDDPartition[Row]]( + iter => Iterator(PointLuceneRDDPartition[Row](iter, indexAnalyzer, queryAnalyzer)), + preservesPartitioning = true) + new PointLuceneRDD(partitions, indexAnalyzer, queryAnalyzer, similarity) + } + + /** Algebird bounding box aggregator */ + + val boundingBoxMonoid = new Tuple2Monoid()(MinPointMonoid, MaxPointMonoid) +} diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/package.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/package.scala new file mode 100644 index 00000000..04b09a58 --- /dev/null +++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/package.scala @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.zouzias.spark.lucenerdd.spatial + +import java.io.StringReader + +import org.locationtech.spatial4j.shape.{Point, Shape} +import org.zouzias.spark.lucenerdd.spatial.commons.context.ContextLoader +import org.zouzias.spark.lucenerdd.spatial.point.PointLuceneRDD.PointType + +package object point extends ContextLoader{ + + + implicit def convertToPoint(point: (Double, Double)): Point = { + ctx.makePoint(point._1, point._2) + } + + /** + * ***Experimental*** + * + * Implicitly convert shape from its string representation + * + * @param shapeAsString + * @return + */ + implicit def wktToShape(shapeAsString: String): PointType = { + try { + val centroid = shapeReader.read(new StringReader(shapeAsString)) + .getCenter + (centroid.getX, centroid.getY) + } + catch { + case _: Exception => (0.0, 0.0) + } + } +} diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/AbstractPointLuceneRDDPartition.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/AbstractPointLuceneRDDPartition.scala new file mode 100644 index 00000000..565080b5 --- /dev/null +++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/AbstractPointLuceneRDDPartition.scala @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.zouzias.spark.lucenerdd.spatial.point.partition + +import org.zouzias.spark.lucenerdd.models.BoundingBox +import org.zouzias.spark.lucenerdd.response.LuceneRDDResponsePartition +import org.zouzias.spark.lucenerdd.spatial.shape.ShapeLuceneRDD.PointType + +import scala.reflect.ClassTag + + +private[point] abstract class AbstractPointLuceneRDDPartition[V] extends Serializable { + + protected implicit def vTag: ClassTag[V] + + def size: Long + + def iterator: Iterator[(PointType, V)] + + def isDefined(point: PointType): Boolean + + def close(): Unit + + /** + * Return smallest enclosing axis aligned bounding box + * @return Smallest enclosing bounding box + */ + def bounds(): BoundingBox + + /** + * Nearest neighbour search + * + * @param point query point + * @param k number of neighbors to return + * @param searchString Lucene Query string + * @return + */ + def knnSearch(point: PointType, k: Int, searchString: String): LuceneRDDResponsePartition + + /** + * Search for points within a circle + * + * @param center center of circle + * @param radius radius of circle in kilometers (KM) + * @param k number of points to return + * @return + */ + def circleSearch(center: PointType, radius: Double, k: Int, operationName: String) + : LuceneRDDResponsePartition + + /** + * Spatial search with arbitrary shape + * + * @param shapeAsString Shape object represented as String + * @param k Number of results to return + * @param operationName Operation name, i.e., intersect, within, etc + * @return + */ + def spatialSearch(shapeAsString: String, k: Int, operationName: String) + : LuceneRDDResponsePartition + + /** + * Bounding box search with point and radius + * + * @param center given as (x, y) + * @param radius distance from center in kilometers (KM) + * @param k Number of results to return + * @param operationName Operation name, i.e., intersect, within, etc + * @return + */ + def bboxSearch(center: PointType, radius: Double, k: Int, operationName: String) + : LuceneRDDResponsePartition + + /** + * Bounding box search with lower left and upper right corners + * + * @param lowerLeft Lower left point + * @param upperRight Upper left point + * @param k Number of results + * @param operationName Operation name, i.e., intersect, within, etc + * @return + */ + def bboxSearch(lowerLeft: PointType, upperRight: PointType, k: Int, operationName: String) + : LuceneRDDResponsePartition + + /** + * Restricts the entries to those satisfying a predicate + * + * @param pred Predicate to filter on + * @return + */ + def filter(pred: (PointType, V) => Boolean): AbstractPointLuceneRDDPartition[V] +} diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/PointLuceneRDDPartition.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/PointLuceneRDDPartition.scala new file mode 100644 index 00000000..b119096a --- /dev/null +++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/point/partition/PointLuceneRDDPartition.scala @@ -0,0 +1,223 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.zouzias.spark.lucenerdd.spatial.point.partition + +import org.apache.lucene.analysis.Analyzer +import org.apache.lucene.document.{Document, StoredField} +import org.apache.lucene.index.DirectoryReader +import org.apache.lucene.search.{IndexSearcher, ScoreDoc, Sort} +import org.apache.lucene.spatial.query.{SpatialArgs, SpatialOperation} +import org.joda.time.DateTime +import org.locationtech.spatial4j.distance.DistanceUtils +import org.locationtech.spatial4j.shape.{Point, Shape} +import org.zouzias.spark.lucenerdd.aggregate.BoundingBoxMonoid +import org.zouzias.spark.lucenerdd.models.{BoundingBox, SparkScoreDoc} +import org.zouzias.spark.lucenerdd.query.LuceneQueryHelpers +import org.zouzias.spark.lucenerdd.response.LuceneRDDResponsePartition +import org.zouzias.spark.lucenerdd.spatial.commons.strategies.SpatialStrategy +import org.zouzias.spark.lucenerdd.spatial.point.PointLuceneRDD +import org.zouzias.spark.lucenerdd.spatial.shape.ShapeLuceneRDD.PointType +import org.zouzias.spark.lucenerdd.store.IndexWithTaxonomyWriter + +import scala.reflect.{ClassTag, classTag} + + +private[point] class PointLuceneRDDPartition[V] +(private val iter: Iterator[(PointType, V)], + private val indexAnalyzerName: String, + private val queryAnalyzerName: String) +(override implicit val vTag: ClassTag[V]) +(implicit docConversion: V => Document) + extends AbstractPointLuceneRDDPartition[V] + with IndexWithTaxonomyWriter + with SpatialStrategy { + + /* We index only points in [[PointLuceneRDDPartition]] */ + strategy.setPointsOnly(true) + + override def indexAnalyzer(): Analyzer = getAnalyzer(Some(indexAnalyzerName)) + + private val QueryAnalyzer: Analyzer = getAnalyzer(Some(queryAnalyzerName)) + + private def decorateWithLocation(doc: Document, point: Point): Document = { + + strategy.createIndexableFields(point).foreach{ field => + doc.add(field) + doc.add(new StoredField(strategy.getFieldName, shapeToString(point))) + } + + doc + } + + private val (iterOriginal, iterIndex) = iter.duplicate + + private val startTime = new DateTime(System.currentTimeMillis()) + logInfo(s"Indexing process initiated at ${startTime}...") + private val boundingBox_ = iterIndex.map { case (p, value) => + // (implicitly) convert type V to a Lucene document + val doc = docConversion(value) + val docWithLocation = decorateWithLocation(doc, ctx.makePoint(p._1, p._2)) + indexWriter.addDocument(FacetsConfig.build(taxoWriter, docWithLocation)) + + BoundingBox(p, p) + }.reduce(BoundingBoxMonoid.plus) + private val endTime = new DateTime(System.currentTimeMillis()) + + logInfo(s"Indexing process completed at ${endTime}...") + logInfo(s"Indexing process took ${(endTime.getMillis - startTime.getMillis) / 1000} seconds...") + + override def bounds(): BoundingBox = boundingBox_ + + // Close the indexWriter and taxonomyWriter (for faceted search) + closeAllWriters() + + private val indexReader = DirectoryReader.open(IndexDir) + private val indexSearcher = new IndexSearcher(indexReader) + + override def size: Long = iterOriginal.size.toLong + + /** + * Restricts the entries to those satisfying a predicate + * + * @param pred + * @return + */ + override def filter(pred: (PointType, V) => Boolean): + AbstractPointLuceneRDDPartition[V] = { + PointLuceneRDDPartition(iterOriginal.filter(x => pred(x._1, x._2)), + indexAnalyzerName, queryAnalyzerName) + } + + override def isDefined(key: PointType): Boolean = iterOriginal.exists(_._1 == key) + + override def iterator: Iterator[(PointType, V)] = iterOriginal + + private def docLocation(scoreDoc: ScoreDoc): Option[Shape] = { + val shapeString = indexReader.document(scoreDoc.doc) + .getField(strategy.getFieldName) + .stringValue() + + stringToShape(shapeString) + } + + override def circleSearch(center: PointType, radius: Double, k: Int, operationName: String) + : LuceneRDDResponsePartition = { + logInfo(s"circleSearch [center:${center}, operation:${operationName}]") + val args = new SpatialArgs(SpatialOperation.get(operationName), + ctx.makeCircle(center._1, center._2, + DistanceUtils.dist2Degrees(radius, DistanceUtils.EARTH_MEAN_RADIUS_KM))) + + val query = strategy.makeQuery(args) + val docs = indexSearcher.search(query, k) + LuceneRDDResponsePartition(docs.scoreDocs.map(SparkScoreDoc(indexSearcher, _)).toIterator) + } + + override def knnSearch(point: PointType, k: Int, searchString: String) + : LuceneRDDResponsePartition = { + logInfo(s"knnSearch [center:${point}, searchQuery:${searchString}]") + + // Match all, order by distance ascending + val pt = ctx.makePoint(point._1, point._2) + + // the distance (in km) + val valueSource = strategy.makeDistanceValueSource(pt, DistanceUtils.DEG_TO_KM) + + // false = ascending dist + val distSort = new Sort(valueSource.getSortField(false)).rewrite(indexSearcher) + + val query = LuceneQueryHelpers.parseQueryString(searchString, QueryAnalyzer) + val docs = indexSearcher.search(query, k, distSort) + + // Here we sorted on it, and the distance will get + // computed redundantly. If the distance is only needed for the top-X + // search results then that's not a big deal. Alternatively, try wrapping + // the ValueSource with CachingDoubleValueSource then retrieve the value + // from the ValueSource now. See LUCENE-4541 for an example. + val result = docs.scoreDocs.map { scoreDoc => { + val location = docLocation(scoreDoc) + location match { + case Some(shape) => + SparkScoreDoc(indexSearcher, scoreDoc, + ctx.calcDistance(pt, shape.getCenter).toFloat) + case None => + SparkScoreDoc(indexSearcher, scoreDoc, -1F) + } + } + } + + LuceneRDDResponsePartition(result.toIterator) + } + + override def spatialSearch(shapeAsString: String, k: Int, operationName: String) + : LuceneRDDResponsePartition = { + val shapeOpt = stringToShape(shapeAsString) + shapeOpt match { + case Some(shape) => spatialSearch(shape, k, operationName) + case _ => + log.error("Input shape does not have a valid format. Returning empty results") + LuceneRDDResponsePartition.empty() + } + } + + private def spatialSearch(shape: Shape, k: Int, operationName: String) + : LuceneRDDResponsePartition = { + val args = new SpatialArgs(SpatialOperation.get(operationName), shape) + val query = strategy.makeQuery(args) + val docs = indexSearcher.search(query, k) + LuceneRDDResponsePartition(docs.scoreDocs.map(SparkScoreDoc(indexSearcher, _)).toIterator) + } + + override def bboxSearch(center: PointType, radius: Double, k: Int, operationName: String) + : LuceneRDDResponsePartition = { + logInfo(s"bboxSearch [center:${center}, radius: ${radius} and operation:${operationName}]") + val x = center._1 + val y = center._2 + val radiusKM = DistanceUtils.dist2Degrees(radius, DistanceUtils.EARTH_MEAN_RADIUS_KM) + val shape = ctx.makeRectangle(x - radiusKM, x + radiusKM, y - radiusKM, y + radiusKM) + spatialSearch(shape, k, operationName) + } + + override def bboxSearch(lowerLeft: PointType, upperRight: PointType, k: Int, opName: String) + : LuceneRDDResponsePartition = { + val lowerLeftPt = ctx.makePoint(lowerLeft._1, lowerLeft._2) + val upperRightPt = ctx.makePoint(upperRight._1, upperRight._2) + val shape = ctx.makeRectangle(lowerLeftPt, upperRightPt) + spatialSearch(shape, k, opName) + } +} + +object PointLuceneRDDPartition { + + /** + * Constructor for [[PointLuceneRDDPartition]] + * + * @param iter Iterator over data + * @param indexAnalyzer Index analyzer + * @param queryAnalyzer Query analyzer + * @param docConv Implicit convertion to Lucene document + * @tparam V + * @return + */ + def apply[V: ClassTag](iter: Iterator[(PointType, V)], + indexAnalyzer: String, + queryAnalyzer: String) + (implicit docConv: V => Document) + : PointLuceneRDDPartition[V] = { + new PointLuceneRDDPartition[V](iter, + indexAnalyzer, queryAnalyzer)(classTag[V]) (docConv) + } +} diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/package.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/package.scala index 3ca6c12d..80e70218 100644 --- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/package.scala +++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/package.scala @@ -18,9 +18,9 @@ package org.zouzias.spark.lucenerdd.spatial import java.io.StringReader -import com.vividsolutions.jts.geom.{Coordinate, GeometryFactory} +import org.locationtech.jts.geom.{Coordinate, GeometryFactory} import org.locationtech.spatial4j.shape.Shape -import org.zouzias.spark.lucenerdd.spatial.shape.context.ContextLoader +import org.zouzias.spark.lucenerdd.spatial.commons.context.ContextLoader package object shape extends ContextLoader{ diff --git a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/partition/ShapeLuceneRDDPartition.scala b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/partition/ShapeLuceneRDDPartition.scala index aaeac337..d441dc57 100644 --- a/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/partition/ShapeLuceneRDDPartition.scala +++ b/src/main/scala/org/zouzias/spark/lucenerdd/spatial/shape/partition/ShapeLuceneRDDPartition.scala @@ -28,7 +28,7 @@ import org.zouzias.spark.lucenerdd.models.SparkScoreDoc import org.zouzias.spark.lucenerdd.query.LuceneQueryHelpers import org.zouzias.spark.lucenerdd.response.LuceneRDDResponsePartition import org.zouzias.spark.lucenerdd.spatial.shape.ShapeLuceneRDD.PointType -import org.zouzias.spark.lucenerdd.spatial.shape.strategies.SpatialStrategy +import org.zouzias.spark.lucenerdd.spatial.commons.strategies.SpatialStrategy import org.zouzias.spark.lucenerdd.store.IndexWithTaxonomyWriter import scala.reflect._ @@ -107,12 +107,7 @@ private[shape] class ShapeLuceneRDDPartition[K, V] .getField(strategy.getFieldName) .stringValue() - try{ - Some(stringToShape(shapeString)) - } - catch { - case _: Throwable => None - } + stringToShape(shapeString) } override def circleSearch(center: PointType, radius: Double, k: Int, operationName: String) @@ -166,8 +161,13 @@ private[shape] class ShapeLuceneRDDPartition[K, V] override def spatialSearch(shapeAsString: String, k: Int, operationName: String) : LuceneRDDResponsePartition = { logInfo(s"spatialSearch [shape:${shapeAsString} and operation:${operationName}]") - val shape = stringToShape(shapeAsString) - spatialSearch(shape, k, operationName) + val shapeOpt = stringToShape(shapeAsString) + shapeOpt match { + case Some(shape) => spatialSearch(shape, k, operationName) + case None => + log.error("Input shape does not have a valid format. Returning empty results") + LuceneRDDResponsePartition.empty() + } } private def spatialSearch(shape: Shape, k: Int, operationName: String) diff --git a/src/test/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDDSpec.scala b/src/test/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDDSpec.scala new file mode 100644 index 00000000..89c7ce68 --- /dev/null +++ b/src/test/scala/org/zouzias/spark/lucenerdd/spatial/point/PointLuceneRDDSpec.scala @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.zouzias.spark.lucenerdd.spatial.point + +import java.io.StringWriter + +import com.holdenkarau.spark.testing.SharedSparkContext +import org.apache.spark.SparkConf +import org.apache.spark.sql.SparkSession +import org.locationtech.spatial4j.distance.DistanceUtils +import org.scalatest.{BeforeAndAfterEach, FlatSpec, Matchers} +import org.zouzias.spark.lucenerdd._ +import org.zouzias.spark.lucenerdd.spatial.commons.context.ContextLoader +import org.zouzias.spark.lucenerdd.spatial.shape.ShapeLuceneRDDKryoRegistrator +import org.zouzias.spark.lucenerdd.testing.LuceneRDDTestUtils + +class PointLuceneRDDSpec extends FlatSpec + with Matchers + with BeforeAndAfterEach + with SharedSparkContext + with ContextLoader + with LuceneRDDTestUtils { + + val k = 6 + + val Radius: Double = 5D + + var pointLuceneRDD: PointLuceneRDD[_] = _ + + override def afterEach() { + pointLuceneRDD.close() + } + + override val conf: SparkConf = ShapeLuceneRDDKryoRegistrator.registerKryoClasses(new SparkConf(). + setMaster("local[*]"). + setAppName("test"). + set("spark.ui.enabled", "false"). + set("spark.app.id", appID)) + + "PointLuceneRDD.circleSearch" should "return correct results" in { + val rdd = sc.parallelize(cities) + pointLuceneRDD = PointLuceneRDD(rdd) + + // Bern, Laussanne and Zurich is within 300km + val results = pointLuceneRDD.circleSearch(Bern._1, 300, k).collect() + + results.length should equal(3) + + results.exists(x => docTextFieldEq(x.doc, "_1", Bern._2)) should equal(true) + results.exists(x => docTextFieldEq(x.doc, "_1", Zurich._2)) should equal(true) + results.exists(x => docTextFieldEq(x.doc, "_1", Laussanne._2)) should equal(true) + + results.exists(x => docTextFieldEq(x.doc, "_1", Milan._2)) should equal(false) + results.exists(x => docTextFieldEq(x.doc, "_1", Athens._2)) should equal(false) + results.exists(x => docTextFieldEq(x.doc, "_1", Toronto._2)) should equal(false) + } + + "PointLuceneRDD.spatialSearch(circle)" should "return correct results" in { + val rdd = sc.parallelize(cities) + pointLuceneRDD = PointLuceneRDD(rdd) + + val point = ctx.makePoint(Bern._1._1, Bern._1._2) + val circle = ctx.makeCircle(point, + DistanceUtils.dist2Degrees(300, DistanceUtils.EARTH_MEAN_RADIUS_KM)) + + val writer = new StringWriter() + shapeWriter.write(writer, circle) + val circleWKT = writer.getBuffer.toString + + // Bern, Laussanne and Zurich is within 300km + val results = pointLuceneRDD.spatialSearch(circleWKT, k).collect() + + results.length should equal(3) + + results.exists(x => docTextFieldEq(x.doc, "_1", Bern._2)) should equal(true) + results.exists(x => docTextFieldEq(x.doc, "_1", Zurich._2)) should equal(true) + results.exists(x => docTextFieldEq(x.doc, "_1", Laussanne._2)) should equal(true) + + results.exists(x => docTextFieldEq(x.doc, "_1", Milan._2)) should equal(false) + results.exists(x => docTextFieldEq(x.doc, "_1", Athens._2)) should equal(false) + results.exists(x => docTextFieldEq(x.doc, "_1", Toronto._2)) should equal(false) + } + + "PointLuceneRDD.bboxSearch((Double, Double), Double)" should "return correct results" in { + val rdd = sc.parallelize(cities) + pointLuceneRDD = PointLuceneRDD(rdd) + + val x = Bern._1._1 + val y = Bern._1._2 + val radius = 150.0 + + // Bern, Laussanne and Zurich is within 300km + val results = pointLuceneRDD.bboxSearch((x, y), radius, k).collect() + + results.size should equal(3) + + results.exists(x => docTextFieldEq(x.doc, "_1", Bern._2)) should equal(true) + results.exists(x => docTextFieldEq(x.doc, "_1", Zurich._2)) should equal(true) + results.exists(x => docTextFieldEq(x.doc, "_1", Laussanne._2)) should equal(true) + + results.exists(x => docTextFieldEq(x.doc, "_1", Milan._2)) should equal(false) + results.exists(x => docTextFieldEq(x.doc, "_1", Athens._2)) should equal(false) + results.exists(x => docTextFieldEq(x.doc, "_1", Toronto._2)) should equal(false) + } + + "PointLuceneRDD.bboxSearch(lowerLeft, upperRight)" should "return correct results" in { + val rdd = sc.parallelize(cities) + pointLuceneRDD = PointLuceneRDD(rdd) + + val x = Bern._1._1 + val y = Bern._1._2 + val width = DistanceUtils.dist2Degrees(150, DistanceUtils.EARTH_MEAN_RADIUS_KM) + + // Bern, Laussanne and Zurich is within 300km + val results = pointLuceneRDD.bboxSearch((x - width, y - width), (x + width, y + width), k, "Intersects") + .collect() + + results.length should equal(3) + + results.exists(x => docTextFieldEq(x.doc, "_1", Bern._2)) should equal(true) + results.exists(x => docTextFieldEq(x.doc, "_1", Zurich._2)) should equal(true) + results.exists(x => docTextFieldEq(x.doc, "_1", Laussanne._2)) should equal(true) + + results.exists(x => docTextFieldEq(x.doc, "_1", Milan._2)) should equal(false) + results.exists(x => docTextFieldEq(x.doc, "_1", Athens._2)) should equal(false) + results.exists(x => docTextFieldEq(x.doc, "_1", Toronto._2)) should equal(false) + } + + "PointLuceneRDD.apply(DF, shapeField)" should + "instantiate ShapeLuceneRDD from DataFrame and shape field name" in { + val sparkSession = SparkSession.builder.getOrCreate() + val capitals = sparkSession.read.parquet("data/capitals.parquet") + pointLuceneRDD = PointLuceneRDD(capitals, "shape") + + pointLuceneRDD.count() > 0 should equal(true) + } + + "PointLuceneRDD.version" should "return project sbt build information" in { + val map = PointLuceneRDD.version() + map.contains("name") should equal(true) + map.contains("builtAtMillis") should equal(true) + map.contains("scalaVersion") should equal(true) + map.contains("version") should equal(true) + map.contains("sbtVersion") should equal(true) + map.contains("builtAtString") should equal(true) + } + + "PointLuceneRDD.bounds" should "return correct bounds" in { + val rdd = sc.parallelize(cities) + pointLuceneRDD = PointLuceneRDD(rdd) + val boundingBox = pointLuceneRDD.bounds() + boundingBox.lowerLeft should equal((-79.4, 9.198)) + boundingBox.upperRight should equal((45.4646, 47.366667)) + } +} diff --git a/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDKnnSearchSpec.scala b/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDKnnSearchSpec.scala index 29ab1450..53caf7fb 100644 --- a/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDKnnSearchSpec.scala +++ b/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDKnnSearchSpec.scala @@ -21,7 +21,7 @@ import com.holdenkarau.spark.testing.SharedSparkContext import org.apache.spark.SparkConf import org.scalatest.{BeforeAndAfterEach, FlatSpec, Matchers} import org.zouzias.spark.lucenerdd._ -import org.zouzias.spark.lucenerdd.spatial.shape.context.ContextLoader +import org.zouzias.spark.lucenerdd.spatial.commons.context.ContextLoader import org.zouzias.spark.lucenerdd.testing.LuceneRDDTestUtils class ShapeLuceneRDDKnnSearchSpec extends FlatSpec diff --git a/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDLinkageSpec.scala b/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDLinkageSpec.scala index d5347077..b57d333b 100644 --- a/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDLinkageSpec.scala +++ b/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDLinkageSpec.scala @@ -20,7 +20,7 @@ import com.holdenkarau.spark.testing.SharedSparkContext import org.apache.spark.SparkConf import org.apache.spark.sql.{Row, SparkSession} import org.scalatest.{BeforeAndAfterEach, FlatSpec, Matchers} -import org.zouzias.spark.lucenerdd.spatial.shape.context.ContextLoader +import org.zouzias.spark.lucenerdd.spatial.commons.context.ContextLoader import org.zouzias.spark.lucenerdd.testing.LuceneRDDTestUtils // Required for implicit Document conversion diff --git a/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDSpatialSearchSpec.scala b/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDSpatialSearchSpec.scala index 4baea509..ccea6e1d 100644 --- a/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDSpatialSearchSpec.scala +++ b/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDSpatialSearchSpec.scala @@ -23,7 +23,7 @@ import org.apache.spark.SparkConf import org.locationtech.spatial4j.distance.DistanceUtils import org.scalatest.{BeforeAndAfterEach, FlatSpec, Matchers} import org.zouzias.spark.lucenerdd._ -import org.zouzias.spark.lucenerdd.spatial.shape.context.ContextLoader +import org.zouzias.spark.lucenerdd.spatial.commons.context.ContextLoader import org.zouzias.spark.lucenerdd.testing.LuceneRDDTestUtils class ShapeLuceneRDDSpatialSearchSpec extends FlatSpec @@ -135,7 +135,7 @@ class ShapeLuceneRDDSpatialSearchSpec extends FlatSpec // Bern, Laussanne and Zurich is within 300km val results = pointLuceneRDD.spatialSearch(rectangleWKT, k).collect() - results.size should equal(3) + results.length should equal(3) results.exists(x => docTextFieldEq(x.doc, "_1", Bern._2)) should equal(true) results.exists(x => docTextFieldEq(x.doc, "_1", Zurich._2)) should equal(true) diff --git a/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDSpec.scala b/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDSpec.scala index bbbedfb5..01a2740b 100644 --- a/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDSpec.scala +++ b/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/ShapeLuceneRDDSpec.scala @@ -25,7 +25,7 @@ import org.locationtech.spatial4j.distance.DistanceUtils import org.scalatest.{BeforeAndAfterEach, FlatSpec, Matchers} import org.zouzias.spark.lucenerdd.testing.LuceneRDDTestUtils import org.zouzias.spark.lucenerdd._ -import org.zouzias.spark.lucenerdd.spatial.shape.context.ContextLoader +import org.zouzias.spark.lucenerdd.spatial.commons.context.ContextLoader class ShapeLuceneRDDSpec extends FlatSpec with Matchers @@ -150,7 +150,7 @@ class ShapeLuceneRDDSpec extends FlatSpec } "ShapeLuceneRDD.version" should "return project sbt build information" in { - val map = LuceneRDD.version() + val map = ShapeLuceneRDD.version() map.contains("name") should equal(true) map.contains("builtAtMillis") should equal(true) map.contains("scalaVersion") should equal(true) diff --git a/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/implicits/ShapeLuceneRDDImplicitsSpec.scala b/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/implicits/ShapeLuceneRDDImplicitsSpec.scala index 223e94b7..4464bc88 100644 --- a/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/implicits/ShapeLuceneRDDImplicitsSpec.scala +++ b/src/test/scala/org/zouzias/spark/lucenerdd/spatial/shape/implicits/ShapeLuceneRDDImplicitsSpec.scala @@ -23,7 +23,7 @@ import org.scalatest.{BeforeAndAfterEach, FlatSpec, Matchers} import org.zouzias.spark.lucenerdd.spatial.shape.{ShapeLuceneRDD, _} import org.zouzias.spark.lucenerdd.testing.LuceneRDDTestUtils import org.zouzias.spark.lucenerdd._ -import org.zouzias.spark.lucenerdd.spatial.shape.context.ContextLoader +import org.zouzias.spark.lucenerdd.spatial.commons.context.ContextLoader class ShapeLuceneRDDImplicitsSpec extends FlatSpec with Matchers