cloud-init-dev team mailing list archive

[Chad Smith <chad.smith@xxxxxxxxxxxxx>]

Addressed review comments plus:
  - dropped argparse filtering from read-dependencies and brpm now that master has baked argparse direclty into the spec for py26
  - added dry-run param to read-dependencies to allow for viewing intended install actions
  - made sure Requires in redhat and suse specs no longer contained any dependencies from test-requirements.txt

Diff comments:

> diff --git a/Makefile b/Makefile
> index a3bfaf7..faa391c 100644
> --- a/Makefile
> +++ b/Makefile
> @@ -78,6 +86,7 @@ clean_pyc:
>  clean: clean_pyc
>  	rm -rf /var/log/cloud-init.log /var/lib/cloud/
>  
> +

accidental, dropped.

>  yaml:
>  	@$(PYVER) $(CWD)/tools/validate-yaml.py $(YAML_FILES)
>  
> diff --git a/tools/read-dependencies b/tools/read-dependencies
> index f434905..149c2b0 100755
> --- a/tools/read-dependencies
> +++ b/tools/read-dependencies
> @@ -1,43 +1,194 @@
>  #!/usr/bin/env python
> +"""List pip dependencies or system package dependencies for cloud-init."""
>  
>  # You might be tempted to rewrite this as a shell script, but you
>  # would be surprised to discover that things like 'egrep' or 'sed' may
>  # differ between Linux and *BSD.
>  
> +try:
> +    from argparse import ArgumentParser
> +except ImportError:
> +    raise RuntimeError(
> +        'Could not import python-argparse. Please install python-argparse '
> +        'package to continue')
> +
> +import json
>  import os
>  import re
> -import sys
>  import subprocess
> +import sys
>  
> -if 'CLOUD_INIT_TOP_D' in os.environ:
> -    topd = os.path.realpath(os.environ.get('CLOUD_INIT_TOP_D'))
> -else:
> -    topd = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
>  
> -for fname in ("setup.py", "requirements.txt"):
> -    if not os.path.isfile(os.path.join(topd, fname)):
> -        sys.stderr.write("Unable to locate '%s' file that should "
> -                         "exist in cloud-init root directory." % fname)
> -        sys.exit(1)
> -
> -if len(sys.argv) > 1:
> -    reqfile = sys.argv[1]
> -else:
> -    reqfile = "requirements.txt"
> -
> -with open(os.path.join(topd, reqfile), "r") as fp:
> -    for line in fp:
> -        line = line.strip()
> -        if not line or line.startswith("#"):
> +# Map the appropriate package dir needed for each distro choice
> +DISTRO_PKG_TYPE_MAP = {
> +    'centos': 'redhat',
> +    'redhat': 'redhat',
> +    'debian': 'debian',
> +    'ubuntu': 'debian',
> +    'opensuse': 'suse',
> +    'suse': 'suse'
> +}
> +
> +DISTRO_INSTALL_PKG_CMD = {
> +    'centos': ['yum', 'install', '--assumeyes'],
> +    'redhat': ['yum', 'install', '--assumeyes'],
> +    'debian': ['apt', 'install', '-y'],
> +    'ubuntu': ['apt', 'install', '-y'],
> +    'opensuse': ['zypper', 'install'],
> +    'suse': ['zypper', 'install']
> +}
> +
> +PY_26_ONLY = ['argparse']
> +
> +# List of base system packages required to start using make
> +EXTRA_SYSTEM_BASE_PKGS = ['make', 'sudo', 'tar']
> +
> +
> +# JSON definition of distro-specific package dependencies
> +DISTRO_PKG_DEPS_PATH = "packages/pkg-deps.json"
> +
> +
> +def get_parser():
> +    """Return an argument parser for this command."""
> +    parser = ArgumentParser(description=__doc__)
> +    parser.add_argument(
> +        '-r', '--requirements-file', type=str, dest='req_file',
> +        default='requirements.txt', help='The pip-style requirements file')
> +    parser.add_argument(
> +        '-d', '--distro', type=str, choices=DISTRO_PKG_TYPE_MAP.keys(),
> +        help='The name of the distro to generate package deps for.')
> +    parser.add_argument(
> +        '-s', '--system-pkg-names', action='store_true', default=False,
> +        dest='system_pkg_names',
> +        help='The name of the distro to generate package deps for.')
> +    parser.add_argument(
> +        '-i', '--install', action='store_true', default=False,
> +        dest='install',
> +        help='When specified, install the required system packages.')
> +    parser.add_argument(
> +        '-v', '--python-version', type=str, dest='python_version', default="2",
> +        choices=["2", "3"],
> +        help='The version of python we want to generate system package '
> +             'dependencies for.')
> +    return parser
> +
> +
> +def get_package_deps_from_json(topdir, distro):
> +    """Get a dict of build and runtime package requirements for a distro.
> +
> +    @param topdir: The root directory in which to search for the
> +        DISTRO_PKG_DEPS_PATH json blob of package requirements information.
> +    @param distro: The specific distribution shortname to pull dependencies
> +        for.
> +    @return: Dict containing "requires", "build-requires" and "rename" lists
> +        for a given distribution.
> +    """
> +    with open(os.path.join(topdir, DISTRO_PKG_DEPS_PATH), 'r') as stream:
> +        deps = json.loads(stream.read())
> +    if distro is None:
> +        return {}
> +    return deps[DISTRO_PKG_TYPE_MAP[distro]]
> +
> +
> +def parse_pip_requirements(requirements_path):
> +    """Return the pip requirement names from pip-style requirements_path."""
> +    dep_names = []
> +    with open(requirements_path, "r") as fp:
> +        for line in fp:
> +            line = line.strip()
> +            if not line or line.startswith("#"):
> +                continue
> +
> +            # remove pip-style markers
> +            dep = line.split(';')[0]
> +
> +            # remove version requirements
> +            if re.search('[>=.<]+', dep):
> +                dep_names.append(re.split("[>=.<]*", dep)[0].strip())
> +            else:
> +                dep_names.append(dep)
> +    return dep_names
> +
> +
> +def translate_pip_to_system_pkg(pip_requires, renames, python_ver="2"):
> +    """Translate pip package names to distro-specific package names.
> +
> +    @param pip_requires: List of versionless pip package names to translate.
> +    @param renames: Dict containg special case renames from pip name to system
> +        package name for the distro.
> +    """
> +    if python_ver == "2":
> +        prefix = "python-"
> +    else:
> +        prefix = "python3-"
> +    standard_pkg_name = "{0}{1}"
> +    translated_names = []
> +    for pip_name in pip_requires:
> +        pip_name = pip_name.lower()
> +        if pip_name in PY_26_ONLY and sys.version_info[0:2] > (2, 6):
> +            # Skip PY26 pkg deps unless we are actually on python 2.6
>              continue
> +        # Find a rename if present for the distro package and python version
> +        rename = renames.get(pip_name, {}).get(python_ver, None)
> +        if rename:
> +            translated_names.append(rename)
> +        else:
> +            translated_names.append(
> +                standard_pkg_name.format(prefix, pip_name))
> +    return translated_names
> +
> +
> +def main(distro):
> +    parser = get_parser()
> +    args = parser.parse_args()
> +    if 'CLOUD_INIT_TOP_D' in os.environ:
> +        topd = os.path.realpath(os.environ.get('CLOUD_INIT_TOP_D'))
> +    else:
> +        topd = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
> +    req_path = os.path.join(topd, args.req_file)
> +    if not os.path.isfile(req_path):
> +        sys.stderr.write("Unable to locate '%s' file that should "
> +                         "exist in cloud-init root directory." % req_path)
> +        return 1
> +    pip_pkg_names = parse_pip_requirements(req_path)
> +    deps_from_json = get_package_deps_from_json(topd, args.distro)
> +    renames = deps_from_json.get('renames', {})
> +    translated_pip_names = translate_pip_to_system_pkg(
> +        pip_pkg_names, renames, args.python_version)
> +    all_deps = []
> +    if args.distro:
> +        all_deps.extend(
> +            translated_pip_names + deps_from_json['requires'] +
> +            deps_from_json['build-requires'])
> +    else:
> +        if args.system_pkg_names:
> +            all_deps = translated_pip_names
> +        else:
> +            all_deps = pip_pkg_names
> +    if args.install:
> +        pkg_install(all_deps, args.distro)
> +    else:
> +        print('\n'.join(all_deps))
> +
>  
> -        # remove pip-style markers
> -        dep = line.split(';')[0]
> +def pkg_install(pkg_list, distro):
> +    """Install a list of packages using the DISTRO_INSTALL_PKG_CMD"""
> +    print("Installing deps:", ' '.join(pkg_list))
> +    pkg_list.extend(EXTRA_SYSTEM_BASE_PKGS)
> +    if distro == 'centos':

fixed.

> +        # CentOs needs epel-release to access oauthlib and jsonschema

fixed.

> +        subprocess.check_call(DISTRO_INSTALL_PKG_CMD[distro] + ['epel-release'])
> +    if distro in ['suse', 'opensuse', 'redhat', 'centos']:
> +        pkg_list.append('rpm-build')
> +    cmd = DISTRO_INSTALL_PKG_CMD[distro] + pkg_list
> +    if os.getuid() != 0:
> +        cmd.insert(0, 'sudo')  # We aren't root so let's try sudo
> +    subprocess.check_call(cmd)
>  
> -        # remove version requirements
> -        dep = re.split("[>=.<]*", dep)[0].strip()
> -        print(dep)
>  
> -sys.exit(0)
> +if __name__ == "__main__":
> +    parser = get_parser()
> +    args = parser.parse_args()
> +    sys.exit(main(args.distro))
>  
>  # vi: ts=4 expandtab


-- 
https://code.launchpad.net/~chad.smith/cloud-init/+git/cloud-init/+merge/325342
Your team cloud-init commiters is requested to review the proposed merge of ~chad.smith/cloud-init:ci-deps into cloud-init:master.