Merge pull request #16 from arenadata/ADCM-1133

Task ADCM-1133

Author: acmnu

.gitignore

@@ -110,3 +110,6 @@ dmypy.json
 
 # vscode
 .vscode
+
+# venv
+/env/

adcm_client/base.py

@@ -37,6 +37,10 @@ def strip_none_keys(args):
     return {k: v for k, v in args.items() if v is not None}
 
 
+class TooManyArguments(Exception):
+    pass
+
+
 class NoSuchEndpoint(Exception):
     pass
 

adcm_client/objects.py

@@ -18,7 +18,7 @@
 from version_utils import rpm
 
 from .base import (ADCMApiError, BaseAPIListObject, BaseAPIObject,
-                   ObjectNotFound, strip_none_keys)
+                   ObjectNotFound, TooManyArguments, strip_none_keys)
 
 # If we are running the client from tests with Allure we expected that code
 # to trace steps in Allure UI.
@@ -937,8 +937,22 @@ def _upload(self, bundle_stream) -> Bundle:
         return self.bundle(bundle_id=data['id'])
 
     @allure_step('Upload bundle from "{1}"')
-    def upload_from_fs(self, dirname) -> Bundle:
-        return self._upload(stream.file(dirname))
+    def upload_from_fs(self, dirname, **args) -> Bundle:
+        streams = stream.file(dirname, **args)
+        if len(streams) > 1:
+            raise TooManyArguments('upload_bundle_from_fs is not capable with building multiple \
+                bundle editions from one dir. Use upload_from_fs_all instead.')
+        return self._upload(streams[0])
+
+    @allure_step('Upload bundles from "{1}"')
+    def upload_from_fs_all(self, dirname, **args) -> BundleList:
+        streams = stream.file(dirname, **args)
+        # Create empty bundle list by filtering on wittingly nonexisting field
+        # and value.
+        result = BundleList(self._api, empty_bundlelist='not_existing_bundle')
+        for st in streams:
+            result.append(self._upload(st))
+        return result
 
     @allure_step('Upload bundle from "{1}"')
     def upload_from_url(self, url) -> Bundle:

adcm_client/packer/add_to_tar.py

@@ -26,7 +26,7 @@ def compare_helper(version, except_list):
     if version is None:
         except_list.extend([
             'spec.yaml', 'pylintrc', '.[0-9a-zA-Z]*', '*pycache*',
-            'README.md', '*test*', '*requirements*', '*.gz', '*.md'])
+            'README.md', '*requirements*', '*.gz', '*.md'])
 
         def v_None(sub: DirEntry):
             for n in except_list:

adcm_client/packer/bundle_build.py

@@ -40,7 +40,6 @@ def _prepare_result_dir(workspace, tarball_path):
 
 
 def _pack(reponame, repopaths, tarpaths, spec: SpecFile, **kwargs):
-    tarballs = {}
     for edition in spec.data['editions']:
         name = edition.get('name')
         tarpath = tarpaths[name] if isinstance(tarpaths, dict) else tarpaths
@@ -56,12 +55,9 @@ def _pack(reponame, repopaths, tarpaths, spec: SpecFile, **kwargs):
         logging.info("#######\n Edition %s \n#######", name)
         logging.info("#######\n Packed files list:\n%s\n#######", "\n".join(tar.getnames()))
         tar.close()
+        stream.seek(0)
         # saving tarball
-        tarballs.update({name: os.path.join(tarpath, tarname)})
-        with open(tarballs[name], 'wb') as file:
-            stream.seek(0)
-            file.write(stream.read())
-    return {'tarballs': tarballs}
+        yield os.path.join(tarpath, tarname), stream
 
 
 def _clean_ws(path):
@@ -72,44 +68,59 @@ def _clean_ws(path):
         remove_tree(path)
 
 
-def build(reponame, repopath,  # pylint: disable=W0102, R0913
-          workspace='/tmp', tarball_path=None, loglevel='ERROR',
-          clean_ws=True, master_branches=['master'], autotest=False):
+def build(reponame=None, repopath=None, workspace='/tmp',  # pylint: disable=R0913
+          tarball_path=None, loglevel='ERROR',
+          clean_ws=True, master_branches=None, **args):
     """Moves sources to workspace inside of temporary directory. \
     Some operations over sources cant be proceed concurent(for exemple in pytest with xdist \
     plugin) that why each thread need is own tmp dir with sources. \
     Also when there is complex docker containers launching to process some information there is \
     necessery to share same workspace with every used container.
     Proceed spec file.
     Writes build number to bundle config file.
-    Recursively add files to tarball.
+    Recursively add files to bytes stream.
+    Cleanup tmp dirs.
 
-    :param reponame: arenadata repository name. Used for naming aftifact and tmp dir.
-    :type reponame: str
     :param repopath: Where bundle sources are
     :type repopath: str
+    :param reponame: arenadata repository name. Used for naming aftifact and tmp dir.
+    :type reponame: str
     :param workspace: where build operations will be performed, defaults to /tmp.
     :type workspace: str, optional
     :param tarball_path: where to copy builded bundle, defaults to None.
     None means that tarball will be left in temporary directory inside of workspace.
     :type tarball_path: str, optional
     :param loglevel: lower or equal to INFO will be stdout
     :type loglevel: str, optional
-    :return: return a dict. Keys:
-        tarball - path to aftifact
+    :return: return a dict.
+    Keys - path and name of tarball to save.
+    Value - stream of bytes.
     :rtype: dict
     """
+    if not master_branches:
+        master_branches = ['master']
+    if not repopath:
+        raise ValueError('path to source should be defined')
+    if not reponame:
+        reponame = os.path.basename(os.path.realpath(repopath))
+
     logging.basicConfig(stream=sys.stdout, level=getattr(logging, loglevel))
     spec = SpecFile(os.path.join(repopath, 'spec.yaml'))
     spec.normalize_spec()
+
     ws_tepm_dir, work_dir_paths = _prepare_ws(reponame, workspace, repopath, spec)
-    if autotest:
-        tarpath = _prepare_result_dir(work_dir_paths, tarball_path)
-    else:
-        tarpath = _prepare_result_dir(workspace, tarball_path)
+
+    tarpath = _prepare_result_dir(workspace, tarball_path)
     spec_processing(spec, work_dir_paths, workspace)
 
-    out = _pack(reponame, work_dir_paths, tarpath, spec, master_branches=master_branches)
+    out = dict(
+        _pack(
+            reponame,
+            work_dir_paths,
+            tarpath,
+            spec,
+            master_branches=master_branches))
+
     if clean_ws:
         _clean_ws(ws_tepm_dir)
 

adcm_client/packer/naming_rules.py

@@ -39,6 +39,13 @@ def write_version(file, old_version, new_version):
             config.truncate()
             config.write(data)
 
+    edition = "community" if edition is None or edition == "None" else edition
+    bundle = ConfigData(catalog=path)
+    version = bundle.get_data('version', 'catalog', explict_raw=True)
+
+    if version is None:
+        raise NoVersionFound('No version detected').with_traceback(sys.exc_info()[2])
+
     try:
         git = Repo(path).git
         if git.describe('--all').split('/')[0] == 'tags':
@@ -58,18 +65,12 @@ def write_version(file, old_version, new_version):
             branch = '-' + branch.replace("-", "_")
     except InvalidGitRepositoryError:
         branch = ''
-
-    bundle = ConfigData(catalog=path, branch='origin/master')
-    version = bundle.get_data('version', 'catalog', explict_raw=True)
-
-    if version is None:
-        raise NoVersionFound('No version detected').with_traceback(sys.exc_info()[2])
-    if '-' in version:
-        raise RestrictedSymbol('Version contains restricted symbol \
-            "-" in position %s' % version.index('-')).with_traceback(sys.exc_info()[2])
-
-    if edition is None or edition == "None":
-        edition = "ce"
-    if branch:
+    else:
+        if not isinstance(version, str):
+            raise TypeError('Bundle version must be string').with_traceback(sys.exc_info()[2])
+        if '-' in version:
+            raise RestrictedSymbol('Version contains restricted symbol \
+                "-" in position %s' % version.index('-')).with_traceback(sys.exc_info()[2])
         write_version(bundle.file, version, version + branch)
+
     return str(reponame) + '_v' + str(version) + branch + '_' + edition + '.tgz'

adcm_client/util/stream.py

@@ -12,27 +12,21 @@
 import gzip
 import io
 import os
-import tarfile
 
 import requests
+from adcm_client.packer.bundle_build import build
 
 
-def file(path):
-    stream = None
+def file(path, **args):
     if os.path.isdir(path):
-        stream = io.BytesIO()
-        tar = tarfile.TarFile(fileobj=stream, mode="w")
-        for sub in os.listdir(path):
-            tar.add(os.path.join(path, sub), arcname=sub)
-        tar.close()
+        return list(build(repopath=path, **args).values())
     else:
         stream = io.BytesIO()
         try:
             stream.write(gzip.open(path, 'rb').read())
         except OSError:
             stream.write(io.open(path, 'rb').read())
-    stream.seek(0)
-    return stream
+        return [stream.seek(0)]
 
 
 def web(url):

bin/adcm_sdk_pack

@@ -17,7 +17,7 @@ from adcm_client.packer.bundle_build import build
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description='Builds bundles from sources.')
-    parser.add_argument('-n', '--name', type=str, help='bundle name', required=True)
+    parser.add_argument('-n', '--name', type=str, help='bundle name', required=False)
     parser.add_argument('-p', '--path', type=str, help='path where to seek configs', required=True)
     parser.add_argument('-t', '--tarpath', type=str, help='path where to save', default=None)
     parser.add_argument('-w', '--workspace', type=str, help='workspace where to build',
@@ -26,8 +26,16 @@ if __name__ == "__main__":
                         action="store_false")
     parser.add_argument('-m', '--master-branches', nargs='+',
                         help='non default master branches', default=['master'])
+    parser.add_argument('-v', '--verbose', action="store_true", help='show info', required=False)
     args = parser.parse_args()
-    build(args.name, args.path,
-          workspace=args.workspace, tarball_path=args.tarpath,
-          loglevel='INFO', clean_ws=args.dont_clean_ws,
-          master_branches=args.master_branches)
+
+    loglevel = 'INFO' if args.verbose else 'ERROR'
+
+    tarballs = build(args.name, args.path,
+                     workspace=args.workspace, tarball_path=args.tarpath,
+                     loglevel=loglevel, clean_ws=args.dont_clean_ws,
+                     master_branches=args.master_branches)
+
+    for edition in tarballs:
+        with open(edition, 'wb') as file:
+            file.write(tarballs[edition].read())

setup.py

@@ -20,7 +20,7 @@
 
 setuptools.setup(
     name="adcm_client",
-    version="2020.02.07.13",
+    version="2020.02.11.12",
     author="Anton Chevychalov",
     author_email="cab@arenadata.io",
     description="ArenaData Cluster Manager Client",