Merge branch 'main' into jdbc-join-operator

Author: mohit-devlogs

.github/workflows/documentation.yml

@@ -55,4 +55,5 @@ jobs:
           key: ${{ runner.os }}-maven
       - name: Generate the documentation
         run: |
-          ./mvnw clean gem:exec@bundle-install gem:exec@generate-documentation gem:exec@generate-documentation-as-latest site -pl wayang-docs -Pweb-documentation          
+          ./mvnw clean install -DskipTests
+          ./mvnw gem:exec@bundle-install gem:exec@generate-documentation gem:exec@generate-documentation-as-latest site -pl wayang-docs -Pweb-documentation          

.gitignore

@@ -34,3 +34,4 @@ pom.xml.*
 
 # Scala Plugin for VSCode
 .metals
+.bloop/

README.md

@@ -55,7 +55,6 @@ Apache Wayang aims at freeing data engineers and software developers from the bu
 - [Apache Spark](https://spark.apache.org/)
 - [Apache Flink](https://flink.apache.org/)
 - [Apache Giraph](https://giraph.apache.org/)
-- [GraphChi](https://github.com/GraphChi/graphchi-java)
 - [Postgres](http://www.postgresql.org)
 - [SQLite](https://www.sqlite.org/)
 - [Apache Kafka](https://kafka.apache.org)
@@ -101,13 +100,13 @@ tar -xvf wayang-1.0.1-SNAPSHOT.tar.gz
 cd wayang-1.0.1-SNAPSHOT
 ```
 
-In linux
+In Linux
 ```shell
 echo "export WAYANG_HOME=$(pwd)" >> ~/.bashrc
 echo "export PATH=${PATH}:${WAYANG_HOME}/bin" >> ~/.bashrc
 source ~/.bashrc
 ```
-In MacOS
+In macOS
 ```shell
 echo "export WAYANG_HOME=$(pwd)" >> ~/.zshrc
 echo "export PATH=${PATH}:${WAYANG_HOME}/bin" >> ~/.zshrc
@@ -193,7 +192,7 @@ Spark 3.4.4, Compatible with Scala 2.12.
 Maven
 ```
 
-> **NOTE:** In windows, you need to define the variable `HADOOP_HOME` with the winutils.exe, an not official option to obtain [this repository](https://github.com/steveloughran/winutils), or you can generate your winutils.exe following the instructions in the repository. Also, you may need to install [msvcr100.dll](https://www.microsoft.com/en-us/download/details.aspx?id=26999)
+>**NOTE:** In Windows, you need to define the variable `HADOOP_HOME` with the winutils.exe. An unofficial option to obtain it is from [this repository](https://github.com/steveloughran/winutils), or you can generate your winutils.exe following the instructions in the repository. Also, you may need to install [msvcr100.dll](https://www.microsoft.com/en-us/download/details.aspx?id=26999)
 
 > **NOTE:** Make sure that the JAVA_HOME environment variable is set correctly to Java 17 as the prerequisite checker script currently supports up to Java 17 and checks the latest version of Java if you have higher version installed. In Linux, it is preferably to use the export JAVA_HOME method inside the project folder. It is also recommended running './mvnw clean install' before opening the project using IntelliJ.
 

guides/windows-setup.md

@@ -0,0 +1,134 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+# Running Apache Wayang on Windows
+
+This guide helps Windows users successfully build and run Apache Wayang.
+
+Many users encounter issues related to Hadoop, winutils, and environment variables.  
+Follow these steps carefully.
+
+---
+
+## 1. Install Java 17
+
+Download and install:
+
+https://adoptium.net/
+
+After installation verify:
+java -version
+
+---
+
+## 2. Install Maven
+
+Download:
+
+https://maven.apache.org/download.cgi
+
+Extract and add Maven `/bin` to your **System PATH**.
+
+Verify:
+mvn -version
+
+---
+
+## 3. Install Hadoop winutils
+
+Wayang requires Hadoop utilities on Windows.
+
+### Download winutils
+
+Download from:
+
+https://github.com/steveloughran/winutils
+
+Choose a version matching Hadoop 3.x.
+
+### Setup
+
+Create directory: C:\hadoop\bin
+
+Place: winutils.exe
+
+inside `bin`.
+
+---
+
+## 4. Set Environment Variables
+
+Open:
+
+**System Properties → Environment Variables**
+
+### Add:
+
+#### HADOOP_HOME
+C:\hadoop
+
+#### Add to PATH
+C:\hadoop\bin
+
+Restart terminal after saving.
+
+---
+
+## 5. Verify Hadoop setup
+
+Run:
+winutils.exe ls
+
+
+If no error appears → setup is correct.
+
+---
+
+## 6. Build Wayang
+
+From project root:
+./mvnw clean install -DskipTests
+
+---
+
+## 7. Common Issues
+
+### ❌ winutils.exe not found
+Ensure:
+
+• file exists in `C:\hadoop\bin`  
+• PATH includes the bin folder  
+
+### ❌ HADOOP_HOME not set
+Verify environment variable.
+
+### ❌ Access denied errors
+Run terminal as Administrator.
+
+---
+
+## 8. Notes
+
+• Windows support requires winutils.  
+• WSL (Windows Subsystem for Linux) can be used as an alternative.
+
+---
+
+You are now ready to run Apache Wayang on Windows.
+

pom.xml

@@ -96,6 +96,7 @@
         <scala.version>2.12.17</scala.version>
         <scala.mayor.version>2.12</scala.mayor.version>
         <spark.version>3.4.4</spark.version>
+        <sedona.version>1.6.1</sedona.version>
         <flink.version>1.20.0</flink.version>
         <calcite.version>1.39.0</calcite.version>
 

python/src/pywy/requirements.txt

@@ -18,5 +18,5 @@
 # we support python 3.11
 numpy==1.24.0
 cloudpickle==3.1.2
-pytest==8.3.5
-requests==2.32.4
+pytest==9.0.3
+requests==2.33.0

python/src/pywy/tests/resources/sample_data.md

@@ -246,7 +246,7 @@ The list of [contributors](https://github.com/apache/incubator-wayang/graphs/con
 ## License
 All files in this repository are licensed under the Apache Software License 2.0
 
-Copyright 2020 - 2025 The Apache Software Foundation.
+Copyright 2020 - 2026 The Apache Software Foundation.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

python/src/pywy/tests/test_word_count.py

@@ -55,7 +55,7 @@ def config(pytestconfig):
 
 
 # TODO: implement tuple types support
-@pytest.mark.skip(reason="missing implementation for tuple types in operators")
+# @pytest.mark.skip(reason="missing implementation for tuple types in operators")
 def test_wordcount(config):
     with resources.path(resources_folder, "sample_data.md") as resource_path, \
          resources.path(resources_folder, "wordcount_out_python.txt") as output_path, \

python/src/pywy/types.py

@@ -15,6 +15,7 @@
 #  limitations under the License.
 #
 
+import collections
 import re
 
 from inspect import signature
@@ -175,7 +176,7 @@ def get_type_flatmap_function(call: FlatmapFunction) -> (type, type):
             )
         )
 
-    if type(sig.return_annotation) != type(Iterable):
+    if get_origin(sig.return_annotation) is not collections.abc.Iterable:
         raise PywyException(
             "the return for the FlatmapFunction is not Iterable, {}".format(
                 str(sig.return_annotation)
@@ -196,11 +197,15 @@ def typecheck(input_type: Type[ConstrainedOperatorType]):
 
     if isinstance(input_type, List) and args:
         typecheck(args[0])
-    elif isinstance(input_type, Tuple):
+    elif get_origin(input_type) is tuple:
         if all(arg in allowed_types for arg in args):
             return
         else:
             raise TypeError(f"Unsupported Operator type: {input_type}")
+    elif isinstance(input_type,tuple):
+        for arg in input_type:
+            typecheck(arg)
+        return
     else:
         raise TypeError(f"Unsupported Operator type: {input_type}, {origin}, {args}")
 

wayang-api/wayang-api-json/REAME.md

@@ -0,0 +1,37 @@
+<!--
+  - Licensed to the Apache Software Foundation (ASF) under one
+  - or more contributor license agreements.  See the NOTICE file
+  - distributed with this work for additional information
+  - regarding copyright ownership.  The ASF licenses this file
+  - to you under the Apache License, Version 2.0 (the
+  - "License"); you may not use this file except in compliance
+  - with the License.  You may obtain a copy of the License at
+  -
+  -   http://www.apache.org/licenses/LICENSE-2.0
+  -
+  - Unless required by applicable law or agreed to in writing,
+  - software distributed under the License is distributed on an
+  - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  - KIND, either express or implied.  See the License for the
+  - specific language governing permissions and limitations
+  - under the License.
+  -->
+  
+# Wayang JSON REST API
+
+## Getting Started
+
+### 1. Package the Project
+
+```bash
+./mvnw clean package -pl :wayang-assembly -Pdistribution
+```
+
+### 2. Starting the REST API
+
+```bash
+cd wayang-assembly/target/
+tar -xvf apache-wayang-assembly-1.1.1-dist.tar.gz
+cd wayang-1.1.1
+./bin/wayang-submit org.apache.wayang.api.json.Main
+```