Skip to content

Commit 4f052d6

Browse files
novatechflownovatechflow
committed
Add Docker support for Python API and enhance Java integration
- Updated Dockerfile for Python API with necessary configurations. - Created entrypoint script for Python container to ensure API availability. - Added example word count script for demonstration. - Enhanced Java integration with dynamic API URL configuration. - Updated dependencies in setup.cfg for compatibility. - Modified GitHub Actions workflow to include Python image build and tests.
1 parent 93d5c84 commit 4f052d6

12 files changed

Lines changed: 330 additions & 30 deletions

File tree

‎.dockerignore‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,20 @@
1515

1616
*
1717
!Dockerfile
18+
!python/
19+
python/**
20+
!python/Dockerfile
21+
!python/docker/
22+
!python/docker/**
23+
!python/examples/
24+
!python/examples/**
25+
!python/pyproject.toml
26+
!python/setup.cfg
27+
!python/README.md
28+
!python/src/
29+
python/src/**
30+
!python/src/pywy/
31+
!python/src/pywy/**
1832
!wayang-assembly/
1933
wayang-assembly/**
2034
!wayang-assembly/target/

‎.github/workflows/docker.yml‎

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,30 @@ jobs:
4747
./mvnw package -B -pl :wayang-assembly -Pdistribution -Dmaven.test.skip=true
4848
4949
- name: Build Docker image
50-
run: docker build -t apache-wayang:ci .
50+
run: docker buildx build --load -t apache-wayang:ci .
5151

52-
- name: Smoke test Java platform
53-
run: docker run --rm apache-wayang:ci
52+
- name: Smoke test WordCount on Java platform
53+
run: |
54+
docker run --rm apache-wayang:ci \
55+
org.apache.wayang.apps.wordcount.Main \
56+
java \
57+
file:///opt/wayang/smoke/wordcount.txt
58+
59+
- name: Build Python API Docker image
60+
run: docker buildx build --load -f python/Dockerfile -t apache-wayang-python:ci .
61+
62+
- name: Smoke test Python API against REST host
63+
run: |
64+
set -e
65+
docker network create wayang-python-smoke
66+
cleanup() {
67+
docker rm -f wayang-rest >/dev/null 2>&1 || true
68+
docker network rm wayang-python-smoke >/dev/null 2>&1 || true
69+
}
70+
trap cleanup EXIT
71+
docker run -d --name wayang-rest --network wayang-python-smoke apache-wayang-python:ci
72+
docker run --rm --network wayang-python-smoke \
73+
--entrypoint /usr/local/bin/wayang-python-entrypoint \
74+
-e WAYANG_API_HOST=wayang-rest \
75+
apache-wayang-python:ci \
76+
python python/examples/wordcount.py

‎Dockerfile‎

Lines changed: 18 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,22 +13,32 @@
1313
# See the License for the specific language governing permissions and
1414
# limitations under the License.
1515

16-
FROM eclipse-temurin:17-jre
16+
FROM eclipse-temurin:17-jre AS wayang-assembler
1717

1818
ARG WAYANG_DIST=wayang-assembly/target/*-dist.tar.gz
1919

20-
ENV WAYANG_HOME=/opt/wayang
21-
ENV PATH="${WAYANG_HOME}/bin:${PATH}"
22-
2320
COPY ${WAYANG_DIST} /tmp/wayang-dist.tar.gz
2421

25-
RUN mkdir -p /opt /root/.wayang \
26-
&& tar -xzf /tmp/wayang-dist.tar.gz -C /opt \
27-
&& extracted_dir="$(find /opt -mindepth 1 -maxdepth 1 -type d -name 'wayang-*' | head -n 1)" \
22+
RUN mkdir -p /opt/wayang-root /opt/wayang /opt/wayang/smoke \
23+
&& tar -xzf /tmp/wayang-dist.tar.gz -C /opt/wayang-root \
24+
&& extracted_dir="$(find /opt/wayang-root -mindepth 1 -maxdepth 1 -type d -name 'wayang-*' | head -n 1)" \
2825
&& test -n "${extracted_dir}" \
29-
&& ln -s "${extracted_dir}" "${WAYANG_HOME}" \
26+
&& cp -a "${extracted_dir}/." /opt/wayang/ \
27+
&& find /opt/wayang -type f \( -name "*-sources.jar" -o -name "*-javadoc.jar" -o -name "README.md" \) -delete \
28+
&& printf "apache wayang docker smoke test\n" > /opt/wayang/smoke/wordcount.txt \
3029
&& rm /tmp/wayang-dist.tar.gz
3130

31+
FROM eclipse-temurin:17-jre
32+
33+
ENV WAYANG_HOME=/opt/wayang
34+
ENV FLAG_WAYANG=true
35+
ENV PATH="${WAYANG_HOME}/bin:${PATH}"
36+
37+
COPY --from=wayang-assembler /opt/wayang /opt/wayang
38+
39+
RUN mkdir -p /root/.wayang /opt/wayang/conf \
40+
&& touch /opt/wayang/conf/wayang.properties
41+
3242
WORKDIR /opt/wayang
3343

3444
ENTRYPOINT ["/opt/wayang/bin/wayang-submit"]

‎bin/wayang-submit‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,21 @@ fi
113113
append_classpath "${WAYANG_EXTRA_CLASSPATH}"
114114

115115
FLAGS=()
116+
FLAGS+=(
117+
"--add-exports=java.base/sun.nio.ch=ALL-UNNAMED"
118+
"--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED"
119+
"--add-opens=java.base/java.nio=ALL-UNNAMED"
120+
"--add-opens=java.base/sun.nio.ch=ALL-UNNAMED"
121+
"--add-opens=java.base/java.lang=ALL-UNNAMED"
122+
"--add-opens=java.base/java.util=ALL-UNNAMED"
123+
"--add-opens=java.base/java.io=ALL-UNNAMED"
124+
"--add-opens=java.base/java.lang.reflect=ALL-UNNAMED"
125+
"--add-opens=java.base/sun.reflect.annotation=ALL-UNNAMED"
126+
"--add-opens=java.base/java.util.concurrent=ALL-UNNAMED"
127+
"--add-opens=java.base/java.net=ALL-UNNAMED"
128+
"--add-opens=java.base/java.lang.invoke=ALL-UNNAMED"
129+
)
130+
116131
if [ "${FLAG_LOG}" = "true" ]; then
117132
FLAGS+=("-Dlog4j.configuration=file://${WAYANG_CONF}/log4j.properties")
118133
fi

‎python/Dockerfile‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
ARG WAYANG_IMAGE=apache-wayang:ci
17+
FROM ${WAYANG_IMAGE} AS wayang-runtime
18+
19+
FROM python:3.11-slim
20+
21+
ENV PYTHONUNBUFFERED=1
22+
ENV WAYANG_API_HOST=wayang-rest
23+
ENV WAYANG_API_PORT=8080
24+
ENV FLAG_WAYANG=true
25+
ENV JAVA_HOME=/opt/java/openjdk
26+
ENV WAYANG_HOME=/opt/wayang
27+
ENV PATH=/opt/wayang/bin:/opt/java/openjdk/bin:/opt/wayang/python/.venv/bin:${PATH}
28+
29+
USER root
30+
31+
COPY --from=wayang-runtime /opt/java/openjdk /opt/java/openjdk
32+
COPY --from=wayang-runtime /opt/wayang /opt/wayang
33+
34+
WORKDIR /opt/wayang/python
35+
36+
COPY python/ /opt/wayang/python/
37+
38+
RUN mkdir -p /root/.wayang \
39+
&& python3 -m venv /opt/wayang/python/.venv \
40+
&& python -m pip install --no-cache-dir -r src/pywy/requirements.txt \
41+
&& python -m pip install --no-cache-dir . \
42+
&& printf '%s\n' \
43+
'wayang.api.python.worker = /opt/wayang/python/src/pywy/execution/worker.py' \
44+
'wayang.api.python.path = /opt/wayang/python/.venv/bin/python' \
45+
'wayang.api.python.env.path = /opt/wayang/python/src' \
46+
>> /opt/wayang/conf/wayang.properties
47+
48+
COPY python/docker/entrypoint.sh /usr/local/bin/wayang-python-entrypoint
49+
50+
RUN chmod +x /usr/local/bin/wayang-python-entrypoint
51+
52+
WORKDIR /opt/wayang
53+
54+
ENTRYPOINT ["bin/wayang-submit"]
55+
CMD ["org.apache.wayang.api.json.Main", "8080"]

‎python/README.md‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,5 +96,55 @@ if __name__ == "__main__":
9696
word_count
9797
```
9898

99+
By default, `pywy` submits plans to
100+
`http://localhost:8080/wayang-api-json/submit-plan/json`. For a remote REST API
101+
or another container, configure either the full URL:
102+
103+
```python
104+
from pywy.configuration import Configuration
105+
from pywy.dataquanta import WayangContext
106+
107+
configuration = Configuration()
108+
configuration.set_property("wayang.api.python.url", "http://wayang-rest:8080/wayang-api-json/submit-plan/json")
109+
ctx = WayangContext(configuration)
110+
```
111+
112+
or configure host and port separately:
113+
114+
```python
115+
configuration.set_property("wayang.api.python.host", "wayang-rest")
116+
configuration.set_property("wayang.api.python.port", "8080")
117+
```
118+
119+
The same values can be supplied through environment variables:
120+
`WAYANG_API_URL`, or `WAYANG_API_HOST` and `WAYANG_API_PORT`.
121+
122+
## Docker
123+
124+
The Python API can be run with the Wayang REST API as a separate container.
125+
Build the assembly first, then build the base Wayang image and the optional
126+
Python-enabled image from the repository root:
127+
128+
```shell
129+
./mvnw clean install -Dmaven.test.skip=true
130+
./mvnw package -pl :wayang-assembly -Pdistribution -Dmaven.test.skip=true
131+
docker buildx build --load -t apache-wayang:ci .
132+
docker buildx build --load -f python/Dockerfile -t apache-wayang-python:ci .
133+
```
134+
135+
Start the REST API and run the Python example on the same Docker network:
136+
137+
```shell
138+
docker network create wayang-python
139+
docker run -d --name wayang-rest --network wayang-python apache-wayang-python:ci
140+
docker run --rm --network wayang-python \
141+
--entrypoint /usr/local/bin/wayang-python-entrypoint \
142+
-e WAYANG_API_HOST=wayang-rest \
143+
apache-wayang-python:ci \
144+
python python/examples/wordcount.py
145+
docker rm -f wayang-rest
146+
docker network rm wayang-python
147+
```
148+
99149
### Testing python code
100150
You can run the python tests by using pytest, the requirements for the tests are listed in `python/src/pywy/requirements.txt`. To run the tests navigate to the base wayang folder, e.g. `/var/www/html` and run `pytest -s python/src/pywy` if you need to pass a specific configuration for your use case you can also add a config flag `pytest -s --config=pathToYourConfig python/src/pywy/`

‎python/docker/entrypoint.sh‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
#!/usr/bin/env sh
2+
#
3+
# Licensed to the Apache Software Foundation (ASF) under one or more
4+
# contributor license agreements. See the NOTICE file distributed with
5+
# this work for additional information regarding copyright ownership.
6+
# The ASF licenses this file to You under the Apache License, Version 2.0
7+
# (the "License"); you may not use this file except in compliance with
8+
# the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
18+
set -eu
19+
20+
python - <<'PY'
21+
import os
22+
import socket
23+
import sys
24+
import time
25+
import urllib.parse
26+
27+
base_url = os.environ.get("WAYANG_API_URL")
28+
if base_url is not None:
29+
parsed_url = urllib.parse.urlparse(base_url)
30+
host = parsed_url.hostname
31+
port = parsed_url.port or 80
32+
else:
33+
host = os.environ.get("WAYANG_API_HOST", "localhost")
34+
port = int(os.environ.get("WAYANG_API_PORT", "8080"))
35+
36+
deadline = time.time() + int(os.environ.get("WAYANG_API_WAIT_TIMEOUT", "60"))
37+
while time.time() < deadline:
38+
try:
39+
with socket.create_connection((host, port), timeout=2):
40+
pass
41+
break
42+
except Exception:
43+
time.sleep(1)
44+
else:
45+
print(f"Timed out waiting for Wayang REST API at {host}:{port}", file=sys.stderr)
46+
sys.exit(1)
47+
PY
48+
49+
exec "$@"

‎python/examples/wordcount.py‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
18+
from pywy.dataquanta import WayangContext
19+
from pywy.platforms.java import JavaPlugin
20+
21+
22+
def word_count():
23+
WayangContext() \
24+
.register({JavaPlugin}) \
25+
.textfile("file:///opt/wayang/smoke/wordcount.txt") \
26+
.flatmap(lambda line: line.split(), str, str) \
27+
.filter(lambda word: word.strip() != "", str) \
28+
.map(lambda word: (word.lower(), 1), str, (str, int)) \
29+
.reduce_by_key(lambda item: item[0], lambda left, right: (left[0], int(left[1]) + int(right[1])), (str, int)) \
30+
.store_textfile("file:///tmp/wayang-python-wordcount.txt", (str, int))
31+
32+
33+
if __name__ == "__main__":
34+
word_count()

‎python/setup.cfg‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,9 @@ package_dir =
3737
packages = find:
3838
python_requires = >=3.6
3939
install_requires =
40-
cloudpickle ==3.0.0
41-
requests ==2.31.0
42-
numpy ==1.19.5
40+
cloudpickle ==3.1.2
41+
requests ==2.33.0
42+
numpy ==1.24.0
4343

4444
tests_require =
4545
unitest ==1.3.5

‎python/src/pywy/core/core.py‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
from typing import Set, Iterable, Dict
1818
import json
19+
import os
1920
import requests
2021

2122
from pywy.configuration import Configuration
@@ -126,10 +127,21 @@ def execute(self):
126127
json_data["operators"].append(serializer.serialize(operator))
127128
pipeline = []
128129

129-
port = self.configuration.get_property("wayang.api.python.port") or 8080
130+
api_url = self.configuration.get_property("wayang.api.python.url") or os.environ.get("WAYANG_API_URL")
131+
if api_url is None:
132+
host = (
133+
self.configuration.get_property("wayang.api.python.host")
134+
or os.environ.get("WAYANG_API_HOST")
135+
or "localhost"
136+
)
137+
port = (
138+
self.configuration.get_property("wayang.api.python.port")
139+
or os.environ.get("WAYANG_API_PORT")
140+
or 8080
141+
)
142+
api_url = f"http://{host}:{port}/wayang-api-json/submit-plan/json"
130143

131-
url = f'http://localhost:{port}/wayang-api-json/submit-plan/json'
132144
headers = {'Content-type': 'application/json'}
133145
json_body = json.dumps(json_data)
134146
print(json_body)
135-
response = requests.post(url, headers=headers, json=json_data)
147+
response = requests.post(api_url, headers=headers, json=json_data)

0 commit comments

Comments
 (0)