Merge pull request #602 from mspruc/main

fix wrongful configuration getter, core.py should now fetch set port …

Author: juripetersen

python/src/pywy/configuration.py

@@ -0,0 +1,31 @@
+
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+from typing import Dict
+
+class Configuration:
+    entries: Dict[str, str]
+
+    def __init__(self):
+        self.entries = {}
+
+    def set_property(self, key: str, value: str):
+        self.entries[key] = value
+
+    def get_property(self, key: str):
+        return self.entries.get(key)

python/src/pywy/core/core.py

@@ -18,6 +18,7 @@
 import json
 import requests
 
+from pywy.configuration import Configuration
 from pywy.core.platform import Platform
 from pywy.core.serializer import JSONSerializer
 from pywy.graph.graph import WayangGraph
@@ -68,7 +69,7 @@ class PywyPlan:
     """
     graph: WayangGraph
 
-    def __init__(self, plugins: Set[Plugin], configuration: Dict[str, str], sinks: Iterable[SinkOperator]):
+    def __init__(self, plugins: Set[Plugin], configuration: Configuration, sinks: Iterable[SinkOperator]):
         """basic Constructor of PywyPlan
 
         this constructor set the plugins and sinks element, and it prepares
@@ -100,7 +101,7 @@ def execute(self):
         context = {}
         context["origin"] = "python"
         context["platforms"] = {}
-        context["configuration"] = self.configuration
+        context["configuration"] = self.configuration.entries
 
         if len(self.plugins) > 0:
             context["platforms"] = list(map(lambda pl: next(iter(pl.platforms)).name, self.plugins))
@@ -125,10 +126,7 @@ def execute(self):
                 json_data["operators"].append(serializer.serialize(operator))
                 pipeline = []
 
-        port = self.configuration.get_property("wayang.api.python.port")
-
-        if port is None: 
-            port = 8080
+        port = self.configuration.get_property("wayang.api.python.port") or 8080
 
         url = f'http://localhost:{port}/wayang-api-json/submit-plan/json'
         headers = {'Content-type': 'application/json'}

python/src/pywy/dataquanta.py

@@ -15,7 +15,7 @@
 #  limitations under the License.
 #
 
-from typing import Dict, Set, List, Optional, cast
+from typing import Set, List, Optional, cast
 
 from pywy.core.core import Plugin, PywyPlan
 from pywy.operators.base import PO_T
@@ -24,18 +24,7 @@
 from pywy.basic.data.record import Record
 from pywy.basic.model.option import Option
 from pywy.basic.model.models import (Model, LogisticRegression, DecisionTreeRegression, LinearSVC)
-
-
-
-class Configuration:
-    entries: Dict[str, str]
-
-    def __init__(self):
-        self.entries = {}
-
-    def set_property(self, key: str, value: str):
-        self.entries[key] = value
-
+from pywy.configuration import Configuration
 
 class WayangContext:
     """
@@ -242,7 +231,7 @@ def store_textfile(self: "DataQuanta[In]", path: str, input_type: GenericTco = N
                 )
             )
         ]
-        PywyPlan(self.context.plugins, self.context.configuration.entries, last).execute()
+        PywyPlan(self.context.plugins, self.context.configuration, last).execute()
 
     def _connect(self, op: PO_T, port_op: int = 0) -> PywyOperator:
         self.operator.connect(0, op, port_op)

python/src/pywy/tests/word_count.py

@@ -16,9 +16,8 @@
 #
 
 import unittest
-from typing import Tuple, Callable, Iterable
+from typing import Iterable
 from pywy.dataquanta import WayangContext, Configuration
-from unittest.mock import Mock
 from pywy.platforms.java import JavaPlugin
 from pywy.platforms.spark import SparkPlugin
 

python/src/pywy/types.py

@@ -16,7 +16,7 @@
 #
 
 import re
-from ast import literal_eval
+
 from inspect import signature
 from typing import (
     Generic, TypeVar, Callable, Hashable, Iterable, Type, Union, Tuple, get_args, get_origin, List, Dict, Any