diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index bf018ccdfedb48b6bfe927483242fa44cb888251..29c4be0267154630dd6f106d4cab2dc6b22aee9f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,8 +1,83 @@
stages:
- test
+ - build
+ - check
+ - deploy
mdcheck:
stage: test
image: it4innovations/docker-mdcheck:latest
script:
- mdl *.md
+
+pycodestyle:
+ stage: test
+ image: it4innovations/docker-pycheck:latest
+ script:
+ - pycodestyle --max-line-length=100 *.py
+
+pylint:
+ stage: test
+ image: it4innovations/docker-pycheck:latest
+ before_script:
+ - export PYTHONIOENCODING=UTF-8
+ - export LC_CTYPE=en_US.UTF-8
+ - python setup.py egg_info
+ - pip install $(paste -d " " -s pipdeps.egg-info/requires.txt)
+ script:
+ - pylint $(find . -type f -not -path "*/.eggs/*" -name "*.py")
+
+build:
+ stage: build
+ image: it4innovations/docker-pypi:latest
+ artifacts:
+ expire_in: 1 day
+ paths:
+ - dist/pipdeps*tar.gz
+ before_script:
+ - export PYTHONIOENCODING=UTF-8
+ - export LC_CTYPE=en_US.UTF-8
+ - pip install mustache pystache
+ script:
+ - echo "output_engine = mustache(\"restructuredtext\")" >> .gitchangelog.rc
+ - gitchangelog >> HISTORY.txt
+ - cat HISTORY.txt
+ - echo "output_engine = mustache(\"markdown\")" >> .gitchangelog.rc
+ - echo >> README.md
+ - gitchangelog | sed -r '1,/^# Changelog$/s/^(# Changelog)$/#\1/' >> README.md
+ - python setup.py --version
+ - echo "version = '$(python setup.py --version)'" >> version.py
+ - python setup.py sdist
+
+Install test:
+ stage: check
+ image: it4innovations/docker-pypi:latest
+ script:
+ - pip install dist/pipdeps*tar.gz
+ - pipdeps -l || true
+ - pipdeps -u
+
+versioncheck:
+ stage: check
+ image: it4innovations/docker-pypi:latest
+ before_script:
+ - pip install packaging
+ script:
+ - python setup.py --version
+ - export BUILD_VERSION="$(python setup.py --version)"
+ - >-
+ export PUBLISHED_VERSION="$(pip install pipdeps== 2>&1 | grep -oE "\(from versions: .*)" | sed "s/(from versions: //" | sed "s/)//" | tr ", " "\n" | tail -n1)"
+ #- CMP_VERSION="$(cmp-version $BUILD_VERSION $PUBLISHED_VERSION)"
+ #- if [ $CMP_VERSION -eq 1 ]; then true; else echo 'Git tag is older/same version as module already available from public pypi repository. Please run git tag -a <version> -m "<version>"'; false; fi
+ - >-
+ echo -e "import sys\nfrom packaging.version import Version, LegacyVersion\nver1=Version('$BUILD_VERSION')\nver2=Version('$PUBLISHED_VERSION')\nif ver1>ver2: sys.exit(0)\nif ver1<ver2: sys.exit(1)\nif ver1==ver2: sys.exit(2)" | python
+ - if [ $? -eq 0 ]; then true; else echo 'Git tag is older/same version as module already available from public pypi repository. Please run git tag -a <version> -m "<version>"'; false; fi
+
+upload:
+ stage: deploy
+ image: it4innovations/docker-pypi:latest
+ script:
+ - twine upload -u "$PYPI_USERNAME" -p "$PYPI_PASSWORD" dist/pipdeps*tar.gz
+ only:
+ - master
+ when: manual
diff --git a/LICENSE b/LICENSE
index e76b29762f914dea666f9a46fece03dc993d630c..4b457196f03b30eb739a4d6afa6a41bad35f4074 100644
--- a/LICENSE
+++ b/LICENSE
@@ -631,7 +631,7 @@ to attach them to the start of each source file to most effectively
state the exclusion of warranty; and each file should have at least
the "copyright" line and a pointer to where the full notice is found.
- pip-upgradedependencies
+ pipdeps
Copyright (C) 2019 IT4Innovations National Supercomputing Center,
VSB - Technical University of Ostrava, Czech Republic
@@ -653,7 +653,7 @@ Also add information on how to contact you by electronic and paper mail.
If the program does terminal interaction, make it output a short
notice like this when it starts in an interactive mode:
- pip-upgradedependencies Copyright (C) 2019
+ pipdeps Copyright (C) 2019
IT4Innovations National Supercomputing Center,
VSB - Technical University of Ostrava, Czech Republic
This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
diff --git a/README.md b/README.md
index a20bc7b24abf2ed06c644564b4337926e42dca32..c3914badc5e24445ee8c2edec41850a9d18d6560 100644
--- a/README.md
+++ b/README.md
@@ -1,11 +1,22 @@
-# pip-upgradedependencies
+# pipdeps
-Pip-upgradedependencies upgrades all outdated packages with respect to existing dependencies.
+Pipdeps shows/upgrades outdated packages with respect to existing dependencies.
Python 2.7 is required.
## Usage
```console
-pip-upgradedependencies
+$ pipdeps.py --help
+usage: pipdeps.py [-h] (-l | -u | -s SHOW [SHOW ...])
+
+Pipdeps shows/upgrades outdated packages with respect to existing
+dependencies.
+
+optional arguments:
+ -h, --help show this help message and exit
+ -l, --list show upgradeable packages and versions
+ -u, --upgrade upgrade upgradeable packages
+ -s SHOW [SHOW ...], --show SHOW [SHOW ...]
+ show detailed info about upgradeable packages
```
diff --git a/pipdeps/__init__.py b/pipdeps/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..c2da04136d4d43e6cab79b88e7d2e8c882b7972e
--- /dev/null
+++ b/pipdeps/__init__.py
@@ -0,0 +1,4 @@
+"""
+pipdeps init
+"""
+__import__('pkg_resources').declare_namespace(__name__)
diff --git a/pipdeps/pipdeps.py b/pipdeps/pipdeps.py
new file mode 100644
index 0000000000000000000000000000000000000000..073a2168e8dd82b30c0cf9aa023e038eb7e0ea6e
--- /dev/null
+++ b/pipdeps/pipdeps.py
@@ -0,0 +1,374 @@
+"""
+pipdeps
+"""
+import argparse
+import json
+import distutils.version
+import os
+import pprint
+import re
+import subprocess
+import sys
+import urllib2
+import tarfile
+import tempfile
+import zipfile
+
+import tabulate
+import packaging.specifiers
+import packaging.version
+
+def arg_parse():
+ """
+ argument parser
+ """
+ parser = argparse.ArgumentParser(
+ description="Pipdeps shows/upgrades outdated packages with respect to existing \
+ dependencies."
+ )
+ group = parser.add_mutually_exclusive_group(required=True)
+ group.add_argument('-l', '--list',
+ action='store_true',
+ help="show upgradeable packages and versions")
+ group.add_argument('-u', '--upgrade',
+ action='store_true',
+ help="upgrade upgradeable packages")
+ group.add_argument('-s', '--show',
+ nargs='+',
+ help="show detailed info about upgradeable packages")
+ return parser.parse_args()
+
+def get_pyver():
+ """
+ return running python version
+ """
+ return ".".join(map(str, sys.version_info[:3]))
+
+def is_strict_version(version):
+ """
+ Return true if version is strict, otherwise return false
+ """
+ try:
+ distutils.version.StrictVersion(version)
+ except ValueError:
+ return False
+ return True
+
+def version_conform_specifiers(version, specifiers):
+ """
+ check if version conforms specifiers
+ """
+ if not specifiers:
+ return True
+ elif version is None:
+ return True
+ else:
+ ver = packaging.version.Version(version)
+ spec = packaging.specifiers.SpecifierSet(",".join(specifiers))
+ if spec.contains(ver):
+ return True
+ return False
+
+def upgrade_package(package, versions):
+ """
+ pip install --upgrade "<package><versions>"
+ """
+ subprocess.check_call(
+ ["pip", "install", "--upgrade", "%s==%s" % (package, "".join(versions))],
+ stderr=subprocess.STDOUT
+ )
+
+def get_pip_list():
+ """
+ pip list
+ """
+ outdated_packages = subprocess.check_output(["pip", "list"])
+ return [line.split()[0] for line in outdated_packages.strip().split("\n")[2:]]
+
+def file_download(url):
+ """
+ Download file from url as temporary file
+ It returns file object
+ """
+ tmp_file = tempfile.NamedTemporaryFile(delete=False)
+ rfile = urllib2.urlopen(url)
+ with tmp_file as output:
+ output.write(rfile.read())
+ return tmp_file
+
+def get_jsonpipdeptree():
+ """
+ pipdeptree --json-tree
+ """
+ pipdeptree = subprocess.check_output(
+ ["pipdeptree", "--json-tree"],
+ stderr=subprocess.STDOUT
+ )
+ return json.loads(pipdeptree.strip())
+
+def get_json(url):
+ """
+ Return url json
+ """
+ return json.load(urllib2.urlopen(urllib2.Request(url)))
+
+def json_search(jsonpipdeptree, package, key):
+ """
+ find package dependencies in json tree
+ """
+ if isinstance(jsonpipdeptree, dict):
+ keys = jsonpipdeptree.keys()
+ if 'package_name' in keys and key in keys:
+ if re.search(r'^%s$' % package, jsonpipdeptree['package_name'], re.IGNORECASE):
+ yield jsonpipdeptree[key]
+ for child_val in json_search(jsonpipdeptree['dependencies'], package, key):
+ yield child_val
+ elif isinstance(jsonpipdeptree, list):
+ for item in jsonpipdeptree:
+ for item_val in json_search(item, package, key):
+ yield item_val
+
+def get_highest_version(package, data):
+ """
+ Return upgradeable version if possible, otherwise return installed version
+ """
+ try:
+ version = data[package]['upgradeable_version']
+ except KeyError:
+ version = data[package]['installed_version']
+ return version
+
+def find_available_vers(package_name, pyver):
+ """
+ Return descending list of available strict version
+ """
+ versions = []
+ data = get_json("https://pypi.python.org/pypi/%s/json" % (package_name,))
+ releases = data["releases"].keys()
+ for release in releases:
+ requires_python = []
+ for item in data["releases"][release]:
+ if item['requires_python'] is not None:
+ requires_python.append(item['requires_python'])
+ if is_strict_version(release) and version_conform_specifiers(pyver, requires_python):
+ versions.append(release)
+ return sorted(versions, key=distutils.version.StrictVersion, reverse=True)
+
+def get_newer_vers(available_version, required_version, installed_version=None):
+ """
+ Return list of newer versions which conforms pipdeptree dependencies, otherwise return none.
+ """
+ if [rver for rver in required_version if re.search(r'(^==.*|^\d.*)', rver) is not None]:
+ return None
+ result = []
+ av_version = list(available_version)
+ while True:
+ try:
+ version = av_version.pop(0)
+ except IndexError:
+ break
+ aver = packaging.version.Version(version)
+ rver = packaging.specifiers.SpecifierSet(",".join(required_version))
+ if rver.contains(aver):
+ if installed_version is not None:
+ iver = packaging.version.Version(installed_version)
+ if aver == iver:
+ break
+ elif aver > iver:
+ result.append(version)
+ else:
+ result.append(version)
+ if result:
+ return sorted(result, key=distutils.version.StrictVersion, reverse=True)
+ return None
+
+def parse_requires_txt(package, version):
+ """
+ Return content of requires.txt until first [ appears
+ """
+ content = None
+ release_data = get_json("https://pypi.python.org/pypi/%s/%s/json" % (package, version,))
+ for item in release_data['releases'][version]:
+ if item['packagetype'] == 'sdist':
+ tmp_file = file_download(item['url'])
+ try:
+ tar_file = tarfile.open(tmp_file.name, 'r')
+ for member in tar_file.getmembers():
+ if 'requires.txt' in member.name:
+ content = tar_file.extractfile(member)
+ except tarfile.ReadError:
+ zip_file = zipfile.ZipFile(tmp_file.name, 'r')
+ for member in zip_file.namelist():
+ if 'requires.txt' in member:
+ content = zip_file.read(member)
+ if content is not None:
+ par = []
+ for line in content:
+ if '[' in line:
+ break
+ else:
+ par.append(line.strip())
+ content = "\n".join(par)
+ os.unlink(tmp_file.name)
+ return content
+
+def find_new_dependencies(package, version, package_list, pyver):
+ """
+ Return package dependencies parsed from pypi json
+ """
+ content = parse_requires_txt(package, version)
+ if content is not None:
+ for line in content.split("\n"):
+ try:
+ pkg = re.search(r'^([a-zA-Z0-9_.-]+)', line).group(0)
+ dep = line.replace(pkg, '').strip()
+ if not dep:
+ dep = None
+ if pkg in package_list:
+ yield (pkg, dep)
+ else:
+ for child in find_new_dependencies(
+ pkg,
+ get_newer_vers(find_available_vers(pkg, pyver), dep, None)[0],
+ package_list,
+ pyver
+ ):
+ yield child
+ except AttributeError:
+ pass
+
+def depless_vers(res):
+ """
+ If there is no dependencies or versionless dependencies, return the upgradeable version,
+ otherwise return None
+ """
+ depless = []
+ for ver, deps in res.iteritems():
+ if not deps:
+ depless.append(ver)
+ else:
+ if not [dep for dep in deps if dep[1] is not None]:
+ depless.append(ver)
+ if depless:
+ depless = sorted(depless, key=distutils.version.StrictVersion, reverse=True)[0]
+ else:
+ depless = None
+ return depless
+
+def collect_packages(package_list, jsonpipdeptree, pyver=None):
+ """
+ Collect data about packages as dictionary
+ """
+ result = {}
+ for package in package_list:
+ installed_version = "".join(list(set(
+ [_ for _ in json_search(jsonpipdeptree, package, 'installed_version')])))
+ required_version = []
+ for dep in list(set(
+ [_ for _ in json_search(jsonpipdeptree, package, 'required_version')]
+ )):
+ if 'Any' not in dep:
+ required_version.append(dep)
+ available_version = find_available_vers(package, pyver)
+ newer_version = get_newer_vers(available_version, required_version, installed_version)
+ rev = {'installed_version': installed_version,
+ 'required_version': required_version,
+ 'available_version': available_version}
+ if newer_version is not None:
+ res = {}
+ for version in newer_version:
+ res[version] = [
+ _ for _ in find_new_dependencies(package, version, package_list, pyver)]
+ rev['newer_version'] = res
+
+ depless = depless_vers(res)
+ if depless:
+ rev['upgradeable_version'] = depless
+
+ result[package] = rev
+ return result
+
+def check_deps(deps, packages_data):
+ """
+ Return true, if all package dependencies conforms
+ """
+ ndeps = []
+ for item in deps:
+ if item[1] is not None:
+ ndeps.append(
+ version_conform_specifiers(
+ get_highest_version(item[0], packages_data),
+ packages_data[item[0]]['required_version']+[item[1]]
+ )
+ )
+ return all(ndeps)
+
+def select_pkgs(packages_data, rkey):
+ """
+ Return data packages having requested key
+ """
+ result = {}
+ for pkg, pkg_data in packages_data.iteritems():
+ if rkey in pkg_data.keys():
+ result[pkg] = pkg_data
+ return result
+
+def print_list(upgradeable_pkgs):
+ """
+ Provides list option
+ """
+ if upgradeable_pkgs:
+ data = []
+ for pkg, pkg_data in sorted(upgradeable_pkgs.iteritems(), key=lambda x: x[0].lower()):
+ data.append([pkg, pkg_data['installed_version'], pkg_data['upgradeable_version']])
+ print tabulate.tabulate(
+ data,
+ ['package', 'installed_version', 'upgradeable_version']
+ )
+ sys.exit(1)
+ else:
+ print "There is nothing to upgrade."
+ sys.exit(0)
+
+def main():
+ """
+ main function
+ """
+ os.environ["PYTHONWARNINGS"] = "ignore:DEPRECATION"
+ arguments = arg_parse()
+ pyver = get_pyver()
+ pkglist = get_pip_list()
+ jsonpipdeptree = get_jsonpipdeptree()
+ packages_data = collect_packages(pkglist, jsonpipdeptree, pyver=pyver)
+ for pkg, pkg_data in sorted(
+ select_pkgs(packages_data, 'newer_version').iteritems(), key=lambda x: x[0].lower()
+ ):
+ pkg_keys = pkg_data.keys()
+ if 'newer_version' in pkg_keys and 'upgradeable_version' not in pkg_keys:
+ for ver, deps in sorted(
+ pkg_data['newer_version'].iteritems(),
+ key=lambda x: distutils.version.StrictVersion(x[0]),
+ reverse=True
+ ):
+ ndeps = check_deps(deps, packages_data)
+ if ndeps:
+ packages_data[pkg]['upgradeable_version'] = ver
+ break
+ upgradeable_pkgs = select_pkgs(packages_data, 'upgradeable_version')
+
+ if arguments.list:
+ print_list(upgradeable_pkgs)
+ if arguments.show:
+ for pkg in arguments.show:
+ pprint.pprint({pkg: packages_data[pkg]})
+ sys.exit(0)
+ if arguments.upgrade:
+ if 'pip' in upgradeable_pkgs.keys():
+ upgrade_package('pip', upgradeable_pkgs['pip']['upgradeable_version'])
+ del upgradeable_pkgs['pip']
+ for pkg, pkg_data in sorted(upgradeable_pkgs.iteritems(), key=lambda x: x[0].lower()):
+ upgrade_package(pkg, pkg_data['upgradeable_version'])
+ print "Done."
+
+if __name__ == "__main__":
+ main()
diff --git a/setup.py b/setup.py
new file mode 100644
index 0000000000000000000000000000000000000000..10457bd205c4ae45701f746c1cb4486a018d732a
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,38 @@
+"""
+pipdeps setup
+"""
+from setuptools import setup, find_packages
+
+setup(
+ name='pipdeps',
+
+ description='Pipdeps shows/upgrades outdated packages with respect to existing \
+ dependencies.',
+ classifiers=[
+ 'Operating System :: POSIX :: Linux',
+ 'Programming Language :: Python',
+ 'Programming Language :: Python :: 2.7',
+ 'Topic :: Software Development :: Libraries :: Python Modules',
+ ],
+
+ author='IT4Innovations',
+ author_email='support@it4i.cz',
+ url='https://code.it4i.cz/sccs/pip-deps',
+ license='GPLv3+',
+ packages=find_packages(),
+ namespace_packages=['pipdeps'],
+ zip_safe=False,
+ version_format='{tag}',
+ long_description_markdown_filename='README.md',
+ setup_requires=['mustache', 'pystache', 'setuptools-git-version', 'setuptools-markdown'],
+ install_requires=[
+ 'packaging',
+ 'pipdeptree',
+ 'tabulate',
+ ],
+ entry_points={
+ 'console_scripts': [
+ 'pipdeps = pipdeps.pipdeps:main',
+ ]
+ }
+)