Skip to content

Commit 93d5c84

Browse files
novatechflownovatechflow
committed
Implement Docker support with image creation and configuration updates
1 parent fd208c7 commit 93d5c84

8 files changed

Lines changed: 269 additions & 94 deletions

File tree

‎.dockerignore‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
*
17+
!Dockerfile
18+
!wayang-assembly/
19+
wayang-assembly/**
20+
!wayang-assembly/target/
21+
wayang-assembly/target/**
22+
!wayang-assembly/target/*-dist.tar.gz

‎.github/workflows/docker.yml‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
18+
name: Docker
19+
20+
on:
21+
push:
22+
branches: [ main ]
23+
pull_request:
24+
branches: [ main ]
25+
26+
concurrency:
27+
group: docker-${{ github.event.pull_request.number || github.ref }}
28+
cancel-in-progress: true
29+
30+
jobs:
31+
build:
32+
name: Build Docker image
33+
runs-on: ubuntu-latest
34+
steps:
35+
- uses: actions/checkout@v5
36+
37+
- name: Set up JDK 17
38+
uses: actions/setup-java@v5
39+
with:
40+
java-version: 17
41+
distribution: temurin
42+
cache: maven
43+
44+
- name: Build Wayang distribution
45+
run: |
46+
./mvnw clean install -B -Dmaven.test.skip=true
47+
./mvnw package -B -pl :wayang-assembly -Pdistribution -Dmaven.test.skip=true
48+
49+
- name: Build Docker image
50+
run: docker build -t apache-wayang:ci .
51+
52+
- name: Smoke test Java platform
53+
run: docker run --rm apache-wayang:ci

‎Dockerfile‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
FROM eclipse-temurin:17-jre
17+
18+
ARG WAYANG_DIST=wayang-assembly/target/*-dist.tar.gz
19+
20+
ENV WAYANG_HOME=/opt/wayang
21+
ENV PATH="${WAYANG_HOME}/bin:${PATH}"
22+
23+
COPY ${WAYANG_DIST} /tmp/wayang-dist.tar.gz
24+
25+
RUN mkdir -p /opt /root/.wayang \
26+
&& tar -xzf /tmp/wayang-dist.tar.gz -C /opt \
27+
&& extracted_dir="$(find /opt -mindepth 1 -maxdepth 1 -type d -name 'wayang-*' | head -n 1)" \
28+
&& test -n "${extracted_dir}" \
29+
&& ln -s "${extracted_dir}" "${WAYANG_HOME}" \
30+
&& rm /tmp/wayang-dist.tar.gz
31+
32+
WORKDIR /opt/wayang
33+
34+
ENTRYPOINT ["/opt/wayang/bin/wayang-submit"]
35+
CMD ["org.apache.wayang.apps.pi.PiEstimation", "java", "1"]

‎bin/wayang-submit‎

Lines changed: 55 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -41,21 +41,6 @@ if [ -z "${WAYANG_HOME}" ]; then
4141
export WAYANG_HOME=$( get_realpath ${BASH_SOURCE[0]} )
4242
fi
4343

44-
if [ -z "${SPARK_HOME}" ]; then
45-
echo "The variable SPARK_HOME it needs to be setup" >&2
46-
exit 1
47-
fi
48-
49-
#if [ -z "${FLINK_HOME}" ]; then
50-
# echo "The variable FLINK_HOME it needs to be setup" >&2
51-
# exit 1
52-
#fi
53-
54-
if [ -z "${HADOOP_HOME}" ]; then
55-
echo "The variable HADOOP_HOME it needs to be setup" >&2
56-
exit 1
57-
fi
58-
5944
# Find the java binary
6045
if [ -n "${JAVA_HOME}" ]; then
6146
RUNNER="${JAVA_HOME}/bin/java"
@@ -68,57 +53,77 @@ else
6853
fi
6954
fi
7055

71-
# Find Spark jars.
72-
if [ -d "${SPARK_HOME}" ]; then
73-
SPARK_JARS_DIR="${SPARK_HOME}/jars"
74-
fi
56+
WAYANG_CODE="${WAYANG_HOME}/jars"
7557

76-
# Find Hadoop jars.
77-
if [ -d "${HADOOP_HOME}" ]; then
78-
HADOOP_JARS_DIR="${HADOOP_HOME}/share/hadoop/common/*:${HADOOP_HOME}/share/hadoop/common/lib/*"
79-
fi
58+
WAYANG_LIBS="${WAYANG_HOME}/libs"
8059

81-
if [ "$(ls ${SPARK_JARS_DIR} | grep ^hadoop | wc -l)" == "0" ]; then
60+
WAYANG_CONF="${WAYANG_HOME}/conf"
8261

83-
HADOOP_JARS_DIR="${HADOOP_JARS_DIR}:${HADOOP_HOME}/share/hadoop/mapreduce/$(ls ${HADOOP_HOME}/share/hadoop/mapreduce | grep ^hadoop-mapreduce-client-common | grep -v tests | head -n 1)"
84-
HADOOP_JARS_DIR="${HADOOP_JARS_DIR}:${HADOOP_HOME}/share/hadoop/mapreduce/$(ls ${HADOOP_HOME}/share/hadoop/mapreduce | grep ^hadoop-mapreduce-client-core | grep -v tests | head -n 1)"
85-
HADOOP_JARS_DIR="${HADOOP_JARS_DIR}:${HADOOP_HOME}/share/hadoop/mapreduce/$(ls ${HADOOP_HOME}/share/hadoop/mapreduce | grep ^hadoop-mapreduce-client-jobclient | grep -v tests | head -n 1)"
86-
HADOOP_JARS_DIR="${HADOOP_JARS_DIR}:${HADOOP_HOME}/share/hadoop/hdfs/$(ls ${HADOOP_HOME}/share/hadoop/hdfs | grep ^hadoop-hdfs-client | grep -v tests | head -n 1)"
87-
HADOOP_JARS_DIR="${HADOOP_JARS_DIR}:${HADOOP_HOME}/share/hadoop/hdfs/lib/$(ls ${HADOOP_HOME}/share/hadoop/hdfs/lib | grep ^hadoop-annotations | grep -v tests | head -n 1)"
88-
HADOOP_JARS_DIR="${HADOOP_JARS_DIR}:${HADOOP_HOME}/share/hadoop/hdfs/lib/$(ls ${HADOOP_HOME}/share/hadoop/hdfs/lib | grep ^hadoop-auth | grep -v tests | head -n 1)"
62+
# Bootstrap the classpath.
63+
WAYANG_CLASSPATH="${WAYANG_CONF}/*:${WAYANG_CODE}/*:${WAYANG_LIBS}/*"
8964

90-
fi
65+
append_classpath() {
66+
if [ -n "$1" ]; then
67+
WAYANG_CLASSPATH="${WAYANG_CLASSPATH}:$1"
68+
fi
69+
}
9170

71+
append_first_matching_jar() {
72+
local dir="$1"
73+
local pattern="$2"
74+
local jar
9275

93-
WAYANG_CODE="${WAYANG_HOME}/jars"
76+
if [ -d "${dir}" ]; then
77+
jar=$(find "${dir}" -maxdepth 1 -name "${pattern}" ! -name "*tests*" | head -n 1)
78+
append_classpath "${jar}"
79+
fi
80+
}
9481

95-
WAYANG_LIBS="${WAYANG_HOME}/libs"
82+
if [ -n "${SPARK_HOME}" ]; then
83+
if [ ! -d "${SPARK_HOME}/jars" ]; then
84+
echo "SPARK_HOME is set but ${SPARK_HOME}/jars does not exist" >&2
85+
exit 1
86+
fi
87+
append_classpath "${SPARK_HOME}/jars/*"
88+
fi
9689

97-
WAYANG_CONF="${WAYANG_HOME}/conf"
90+
if [ -n "${HADOOP_HOME}" ]; then
91+
if [ ! -d "${HADOOP_HOME}/share/hadoop/common" ]; then
92+
echo "HADOOP_HOME is set but ${HADOOP_HOME}/share/hadoop/common does not exist" >&2
93+
exit 1
94+
fi
95+
append_classpath "${HADOOP_HOME}/share/hadoop/common/*"
96+
append_classpath "${HADOOP_HOME}/share/hadoop/common/lib/*"
97+
append_first_matching_jar "${HADOOP_HOME}/share/hadoop/mapreduce" "hadoop-mapreduce-client-common*.jar"
98+
append_first_matching_jar "${HADOOP_HOME}/share/hadoop/mapreduce" "hadoop-mapreduce-client-core*.jar"
99+
append_first_matching_jar "${HADOOP_HOME}/share/hadoop/mapreduce" "hadoop-mapreduce-client-jobclient*.jar"
100+
append_first_matching_jar "${HADOOP_HOME}/share/hadoop/hdfs" "hadoop-hdfs-client*.jar"
101+
append_first_matching_jar "${HADOOP_HOME}/share/hadoop/hdfs/lib" "hadoop-annotations*.jar"
102+
append_first_matching_jar "${HADOOP_HOME}/share/hadoop/hdfs/lib" "hadoop-auth*.jar"
103+
fi
98104

99-
# Bootstrap the classpath.
100-
WAYANG_CLASSPATH="${WAYANG_CONF}/*:${WAYANG_CODE}/*:${WAYANG_LIBS}/*"
101-
WAYANG_CLASSPATH="${WAYANG_CLASSPATH}:${SPARK_JARS_DIR}/*:${HADOOP_JARS_DIR}"
105+
if [ -n "${FLINK_HOME}" ]; then
106+
if [ ! -d "${FLINK_HOME}/lib" ]; then
107+
echo "FLINK_HOME is set but ${FLINK_HOME}/lib does not exist" >&2
108+
exit 1
109+
fi
110+
append_classpath "${FLINK_HOME}/lib/*"
111+
fi
112+
113+
append_classpath "${WAYANG_EXTRA_CLASSPATH}"
102114

103-
FLAGS=""
115+
FLAGS=()
104116
if [ "${FLAG_LOG}" = "true" ]; then
105-
FLAGS="${FLAGS} -Dlog4j.configuration=file://${WAYANG_CONF}/log4j.properties"
117+
FLAGS+=("-Dlog4j.configuration=file://${WAYANG_CONF}/log4j.properties")
106118
fi
107119

108120
if [ "${FLAG_WAYANG}" = "true" ]; then
109-
FLAGS="${FLAGS} -Dwayang.configuration=file://${WAYANG_CONF}/wayang.properties"
121+
FLAGS+=("-Dwayang.configuration=file://${WAYANG_CONF}/wayang.properties")
110122
fi
111123

112124
if [ -n "${OTHER_FLAGS}" ]; then
113-
FLAGS="${FLAGS} ${OTHER_FLAGS}"
125+
read -r -a OTHER_FLAGS_ARRAY <<< "${OTHER_FLAGS}"
126+
FLAGS+=("${OTHER_FLAGS_ARRAY[@]}")
114127
fi
115128

116-
# Wrap args in quotes to be able to execute args with parenthesis, spaces, etc
117-
ARGS=""
118-
for arg in $(echo ${@:2})
119-
do
120-
ARGS="$ARGS \"${arg}\""
121-
done
122-
123-
eval "$RUNNER $FLAGS -cp "${WAYANG_CLASSPATH}" $CLASS ${ARGS}"
124-
129+
exec "$RUNNER" "${FLAGS[@]}" -cp "${WAYANG_CLASSPATH}" "$CLASS" "${@:2}"

‎wayang-assembly/README.md‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,3 +41,52 @@ To execute the Wayang Assembly you need to execute the following command in the
4141
./mvnw clean install -DskipTests
4242
./mvnw clean package -pl :wayang-assembly -Pdistribution
4343
```
44+
45+
# Docker Image
46+
47+
The project Docker image packages the Wayang assembly with a Java 17 runtime. It
48+
does not distribute external execution platforms such as Apache Spark, Apache
49+
Hadoop, Apache Flink, or database drivers. Those runtimes stay outside the image
50+
and can be connected at container runtime.
51+
52+
Build the assembly first, then build the image from the project root. Do not use
53+
the Maven `standalone` profile for this image, because that profile changes
54+
external platform dependencies from `provided` to `compile`.
55+
56+
```shell
57+
./mvnw clean install -Dmaven.test.skip=true
58+
./mvnw package -pl :wayang-assembly -Pdistribution -Dmaven.test.skip=true
59+
docker build -t apache-wayang:local .
60+
```
61+
62+
Run the default Java platform smoke test:
63+
64+
```shell
65+
docker run --rm apache-wayang:local
66+
```
67+
68+
Run a specific Wayang application by passing the main class and its arguments to
69+
`wayang-submit`:
70+
71+
```shell
72+
docker run --rm apache-wayang:local \
73+
org.apache.wayang.apps.pi.PiEstimation java 1
74+
```
75+
76+
To connect external platforms, mount or install their runtimes separately and
77+
set the corresponding environment variables:
78+
79+
```shell
80+
docker run --rm \
81+
-v /path/to/spark:/opt/spark:ro \
82+
-v /path/to/hadoop:/opt/hadoop:ro \
83+
-e SPARK_HOME=/opt/spark \
84+
-e HADOOP_HOME=/opt/hadoop \
85+
apache-wayang:local \
86+
org.apache.wayang.apps.pi.PiEstimation spark 1
87+
```
88+
89+
Additional platform libraries, such as JDBC drivers, can be supplied with
90+
`WAYANG_EXTRA_CLASSPATH`. A custom Wayang configuration can be mounted at
91+
`${WAYANG_HOME}/conf/wayang.properties` and activated with `FLAG_WAYANG=true`.
92+
Applications still need to register the platform plugins they use.

0 commit comments

Comments
 (0)