Skip to content

Commit 2c03a20

Browse files
msprucmspruc
committed
merge
2 parents 7cc0449 + 990fe21 commit 2c03a20

11 files changed

Lines changed: 385 additions & 32 deletions

File tree

‎conf/wayang-defaults.properties‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
18+
# Configure statistics collection.
19+
wayang.core.log.enabled = true
20+
21+
wayang.core.explain.enabled = false
22+
wayang.core.explain.directrory = ~/.wayang/

‎pom.xml‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@
106106
<scala.mayor.version>2.12</scala.mayor.version>
107107
<spark.version>3.4.4</spark.version>
108108
<flink.version>1.20.0</flink.version>
109-
<calcite.version>1.39.0</calcite.version>
109+
<calcite.version>1.39.0</calcite.version>
110110

111111
<java.version>17</java.version>
112112
<source.level>17</source.level>
@@ -785,6 +785,12 @@
785785
<artifactId>guava</artifactId>
786786
<version>${guava.version}</version>
787787
</dependency>
788+
<!-- https://mvnrepository.com/artifact/commons-cli/commons-cli -->
789+
<dependency>
790+
<groupId>commons-cli</groupId>
791+
<artifactId>commons-cli</artifactId>
792+
<version>1.3.1</version>
793+
</dependency>
788794
</dependencies>
789795
</dependencyManagement>
790796

‎wayang-api/pom.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
<module>wayang-api-python</module>
4141
<module>wayang-api-sql</module>
4242
<module>wayang-api-json</module>
43+
<module>wayang-api-utils</module>
4344
</modules>
4445

4546
</project>

‎wayang-api/wayang-api-scala-java/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,11 @@
105105
<artifactId>wayang-spark</artifactId>
106106
<version>1.0.1-SNAPSHOT</version>
107107
</dependency>
108+
<dependency>
109+
<groupId>org.apache.wayang</groupId>
110+
<artifactId>wayang-tensorflow</artifactId>
111+
<version>1.0.1-SNAPSHOT</version>
112+
</dependency>
108113
<dependency>
109114
<groupId>org.apache.spark</groupId>
110115
<artifactId>spark-core_2.12</artifactId>

‎wayang-api/wayang-api-sql/pom.xml‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
<version>1.0.1-SNAPSHOT</version>
2626
</parent>
2727
<modelVersion>4.0.0</modelVersion>
28-
28+
2929
<artifactId>wayang-api-sql</artifactId>
3030
<dependencies>
3131
<dependency>
@@ -49,6 +49,12 @@
4949
<version>1.0.1-SNAPSHOT</version>
5050
<scope>compile</scope>
5151
</dependency>
52+
<dependency>
53+
<groupId>org.apache.wayang</groupId>
54+
<artifactId>wayang-api-utils</artifactId>
55+
<version>1.0.1-SNAPSHOT</version>
56+
<scope>compile</scope>
57+
</dependency>
5258
<dependency>
5359
<groupId>org.apache.wayang</groupId>
5460
<artifactId>wayang-postgres</artifactId>

‎wayang-api/wayang-api-sql/src/main/java/org/apache/wayang/api/sql/context/SqlContext.java‎

Lines changed: 54 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -36,16 +36,27 @@
3636
import org.apache.wayang.basic.data.Record;
3737
import org.apache.wayang.core.api.Configuration;
3838
import org.apache.wayang.core.plugin.Plugin;
39+
import org.apache.wayang.api.utils.Parameters;
3940
import org.apache.wayang.core.api.WayangContext;
41+
import org.apache.wayang.core.util.ReflectionUtils;
4042
import org.apache.wayang.core.plan.wayangplan.WayangPlan;
4143
import org.apache.wayang.java.Java;
4244
import org.apache.wayang.postgres.Postgres;
4345
import org.apache.wayang.spark.Spark;
46+
import org.apache.commons.cli.*;
4447

48+
import com.google.common.io.Resources;
49+
50+
import org.json.simple.JSONObject;
51+
import org.json.simple.parser.JSONParser;
52+
53+
import scala.collection.JavaConversions;
4554
import java.io.BufferedWriter;
4655
import java.io.IOException;
4756
import java.nio.file.Files;
57+
import java.nio.charset.Charset;
4858
import java.nio.file.Paths;
59+
import java.net.URL;
4960
import java.sql.SQLException;
5061
import java.util.ArrayList;
5162
import java.util.Arrays;
@@ -87,43 +98,60 @@ public SqlContext(final Configuration configuration, final List<Plugin> plugins)
8798
/**
8899
* Entry point for executing SQL statements while providing arguments.
89100
* You need to provide at least a JDBC source.
90-
* Sample:
91-
* @param args args[0] = config file path, args[1] = query file path, args[2] = output path
101+
*
102+
* @param args args[0] = SQL statement path, args[1] = JDBC driver, args[2] =
103+
* JDBC URL, args[3] = JDBC user,
104+
*              args[4] = JDBC password, args[5] = outputPath,
105+
* args[6...] = platforms
92106
*/
93107
public static void main(final String[] args) throws Exception {
94108
if (args.length < 3)
95109
throw new IllegalArgumentException(
96110
"Usage: ./bin/wayang-submit org.apache.wayang.api.sql.SqlContext <configuration path> <SQL statement path> <output path> [platforms...]");
97111

98-
final String configurationPath = args[0];
99-
final String queryPath = args[1];
100-
final String outputPath = args[2];
112+
//Specify the named arguments
113+
Options options = new Options();
114+
options.addOption("p", "platforms", true, "[platforms...]");
115+
options.addOption("s", "schema", true, "Schema path");
116+
options.addOption("q", "query", true, "SQL statement path");
117+
options.addOption("o", "outputPath", true, "Output path");
118+
options.addOption("d", "data", true, "Data path for file-based schema");
119+
options.addOption("c", "config", true, "File path for config file");
120+
options.addOption("jdbcDriver", true, "JDBC driver");
121+
options.addOption("jdbcUrl", true, "JDBC URL");
122+
options.addOption("jdbcPassword", true, "JDBC URL");
123+
124+
CommandLineParser parser = new DefaultParser();
125+
CommandLine cmd = parser.parse(options, args);
126+
127+
final String queryPath = cmd.getOptionValue("q");
128+
final String jdbcDriver = cmd.getOptionValue("jdbcDriver");
129+
final String jdbcUrl = cmd.getOptionValue("jdbcUrl");
130+
final String jdbcUser = cmd.getOptionValue("jdbcUser");
131+
final String jdbcPassword = cmd.getOptionValue("jdbcPassword");
132+
final String outputPath = cmd.getOptionValue("o");
133+
final String dataPath = cmd.getOptionValue("d");
134+
final String schemaPath = cmd.getOptionValue("s");
101135

102136
final String query = StringUtils.chop(Files.readString(Paths.get(queryPath)).stripTrailing());
137+
final Configuration configuration = new Configuration();
103138

104-
final Configuration configuration = new Configuration(configurationPath);
105-
106-
final SqlContext context = new SqlContext(configuration,
107-
List.of(Java.channelConversionPlugin(),
108-
Postgres.conversionPlugin()));
109-
110-
for (int i = 3; i < args.length; i++) {
111-
final String platform = args[i];
112-
switch (platform.toLowerCase()) {
113-
case "spark":
114-
context.withPlugin(Spark.basicPlugin());
115-
break;
116-
case "java":
117-
context.withPlugin(Java.basicPlugin());
118-
break;
119-
case "postgres":
120-
context.withPlugin(Postgres.plugin());
121-
break;
122-
default:
123-
throw new IllegalArgumentException("platform not supported " + platform);
124-
}
139+
if (cmd.hasOption("c")) {
140+
configuration.load(cmd.getOptionValue("c"));
125141
}
126142

143+
final String calciteModel = Resources.toString(
144+
new URL(schemaPath),
145+
Charset.defaultCharset()
146+
);
147+
148+
final JSONObject calciteModelJSON = (JSONObject) new JSONParser().parse(calciteModel);
149+
150+
final SqlContext context = new SqlContext(configuration, List.of(Java.channelConversionPlugin(), Postgres.conversionPlugin()));
151+
152+
final List<Plugin> plugins = JavaConversions.seqAsJavaList(Parameters.loadPlugins(cmd.getOptionValue("p")));
153+
plugins.stream().forEach(context::register);
154+
127155
final Properties configProperties = Optimizer.ConfigProperties.getDefaults();
128156
final RelDataTypeFactory relDataTypeFactory = new JavaTypeFactoryImpl();
129157

‎wayang-api/wayang-api-utils/pom.xml‎

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Licensed to the Apache Software Foundation (ASF) under one
4+
or more contributor license agreements. See the NOTICE file
5+
distributed with this work for additional information
6+
regarding copyright ownership. The ASF licenses this file
7+
to you under the Apache License, Version 2.0 (the
8+
"License"); you may not use this file except in compliance
9+
with the License. You may obtain a copy of the License at
10+
11+
http://www.apache.org/licenses/LICENSE-2.0
12+
13+
Unless required by applicable law or agreed to in writing,
14+
software distributed under the License is distributed on an
15+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
KIND, either express or implied. See the License for the
17+
specific language governing permissions and limitations
18+
under the License.
19+
-->
20+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
21+
<modelVersion>4.0.0</modelVersion>
22+
23+
<parent>
24+
<artifactId>wayang-api</artifactId>
25+
<groupId>org.apache.wayang</groupId>
26+
<version>1.0.1-SNAPSHOT</version>
27+
</parent>
28+
29+
<artifactId>wayang-api-utils</artifactId>
30+
<version>1.0.1-SNAPSHOT</version>
31+
32+
<name>Wayang API Utils</name>
33+
<description>Wayang implementation of an API of Scala-Java to be enable to work in functional style</description>
34+
35+
<properties>
36+
<java-module-name>org.apache.wayang.api</java-module-name>
37+
<tensorflow.version>1.0.0</tensorflow.version>
38+
</properties>
39+
40+
<dependencyManagement>
41+
<dependencies>
42+
<dependency>
43+
<groupId>org.apache.wayang</groupId>
44+
<artifactId>wayang-commons</artifactId>
45+
<version>1.0.1-SNAPSHOT</version>
46+
<type>pom</type>
47+
<scope>import</scope>
48+
</dependency>
49+
</dependencies>
50+
</dependencyManagement>
51+
52+
<dependencies>
53+
<dependency>
54+
<groupId>org.apache.wayang</groupId>
55+
<artifactId>wayang-core</artifactId>
56+
<version>1.0.1-SNAPSHOT</version>
57+
</dependency>
58+
<dependency>
59+
<groupId>org.apache.wayang</groupId>
60+
<artifactId>wayang-basic</artifactId>
61+
<version>1.0.1-SNAPSHOT</version>
62+
</dependency>
63+
<dependency>
64+
<groupId>org.apache.wayang</groupId>
65+
<artifactId>wayang-utils-profile-db</artifactId>
66+
<version>1.0.1-SNAPSHOT</version>
67+
</dependency>
68+
<dependency>
69+
<groupId>org.apache.wayang</groupId>
70+
<artifactId>wayang-api-python</artifactId>
71+
<version>1.0.1-SNAPSHOT</version>
72+
</dependency>
73+
<dependency>
74+
<groupId>org.apache.commons</groupId>
75+
<artifactId>commons-lang3</artifactId>
76+
<version>3.12.0</version>
77+
</dependency>
78+
<dependency>
79+
<groupId>org.apache.wayang</groupId>
80+
<artifactId>wayang-java</artifactId>
81+
<version>1.0.1-SNAPSHOT</version>
82+
</dependency>
83+
<dependency>
84+
<groupId>org.apache.wayang</groupId>
85+
<artifactId>wayang-sqlite3</artifactId>
86+
<version>1.0.1-SNAPSHOT</version>
87+
</dependency>
88+
<dependency>
89+
<groupId>org.apache.wayang</groupId>
90+
<artifactId>wayang-postgres</artifactId>
91+
<version>1.0.1-SNAPSHOT</version>
92+
</dependency>
93+
<dependency>
94+
<groupId>org.apache.hadoop</groupId>
95+
<artifactId>hadoop-common</artifactId>
96+
<scope>test</scope>
97+
</dependency>
98+
<dependency>
99+
<groupId>org.apache.hadoop</groupId>
100+
<artifactId>hadoop-hdfs</artifactId>
101+
<scope>test</scope>
102+
</dependency>
103+
<dependency>
104+
<groupId>org.apache.wayang</groupId>
105+
<artifactId>wayang-spark</artifactId>
106+
<version>1.0.1-SNAPSHOT</version>
107+
</dependency>
108+
<dependency>
109+
<groupId>org.apache.wayang</groupId>
110+
<artifactId>wayang-tensorflow</artifactId>
111+
<version>1.0.1-SNAPSHOT</version>
112+
</dependency>
113+
<dependency>
114+
<groupId>org.apache.spark</groupId>
115+
<artifactId>spark-core_2.12</artifactId>
116+
<version>${spark.version}</version>
117+
<scope>test</scope>
118+
</dependency>
119+
<dependency>
120+
<groupId>org.apache.spark</groupId>
121+
<artifactId>spark-graphx_2.12</artifactId>
122+
<version>${spark.version}</version>
123+
<scope>test</scope>
124+
</dependency>
125+
<dependency>
126+
<groupId>org.apache.spark</groupId>
127+
<artifactId>spark-mllib_2.12</artifactId>
128+
<version>${spark.version}</version>
129+
</dependency>
130+
<dependency>
131+
<groupId>org.scala-lang</groupId>
132+
<artifactId>scala-library</artifactId>
133+
<version>${scala.version}</version>
134+
<scope>provided</scope>
135+
</dependency>
136+
<dependency>
137+
<groupId>io.netty</groupId>
138+
<artifactId>netty-all</artifactId>
139+
<version>4.1.89.Final</version>
140+
</dependency>
141+
<dependency>
142+
<groupId>com.google.protobuf</groupId>
143+
<artifactId>protobuf-java</artifactId>
144+
<version>3.25.5</version>
145+
</dependency>
146+
<dependency>
147+
<groupId>com.fasterxml.jackson.core</groupId>
148+
<artifactId>jackson-databind</artifactId>
149+
<version>2.15.4</version>
150+
</dependency>
151+
<dependency>
152+
<groupId>com.fasterxml.jackson.module</groupId>
153+
<artifactId>jackson-module-scala_2.12</artifactId>
154+
</dependency>
155+
<dependency>
156+
<groupId>org.tensorflow</groupId>
157+
<artifactId>tensorflow-core-platform</artifactId>
158+
<version>${tensorflow.version}</version>
159+
</dependency>
160+
<dependency>
161+
<groupId>org.tensorflow</groupId>
162+
<artifactId>tensorflow-framework</artifactId>
163+
<version>${tensorflow.version}</version>
164+
</dependency>
165+
</dependencies>
166+
</project>

0 commit comments

Comments
 (0)