From 4516501d3d761b1206290a02945e6458254a7f6e Mon Sep 17 00:00:00 2001
From: Marek Chrastina <marek.chrastina@vsb.cz>
Date: Thu, 29 Aug 2019 10:47:46 +0200
Subject: [PATCH] Case vanished module; all deps are taken from metadata;
hotfix extras
---
pipdeps/pipdeps.py | 157 ++++++++++++++++++++++++++++++++++-----------
1 file changed, 118 insertions(+), 39 deletions(-)
diff --git a/pipdeps/pipdeps.py b/pipdeps/pipdeps.py
index 073a216..d5571de 100644
--- a/pipdeps/pipdeps.py
+++ b/pipdeps/pipdeps.py
@@ -13,6 +13,7 @@ import urllib2
import tarfile
import tempfile
import zipfile
+import wheel.metadata
import tabulate
import packaging.specifiers
@@ -143,7 +144,11 @@ def find_available_vers(package_name, pyver):
Return descending list of available strict version
"""
versions = []
- data = get_json("https://pypi.python.org/pypi/%s/json" % (package_name,))
+ try:
+ data = get_json("https://pypi.python.org/pypi/%s/json" % (package_name,))
+ except urllib2.HTTPError, err:
+ print "%s %s" % (err, err.url)
+ raise urllib2.HTTPError(err.url, err.code, None, err.hdrs, err.fp)
releases = data["releases"].keys()
for release in releases:
requires_python = []
@@ -158,6 +163,9 @@ def get_newer_vers(available_version, required_version, installed_version=None):
"""
Return list of newer versions which conforms pipdeptree dependencies, otherwise return none.
"""
+ if required_version is None:
+ result = [aver for aver in list(available_version)]
+ return sorted(result, key=distutils.version.StrictVersion, reverse=True)
if [rver for rver in required_version if re.search(r'(^==.*|^\d.*)', rver) is not None]:
return None
result = []
@@ -182,60 +190,128 @@ def get_newer_vers(available_version, required_version, installed_version=None):
return sorted(result, key=distutils.version.StrictVersion, reverse=True)
return None
-def parse_requires_txt(package, version):
+def write_metadata(tmp_file):
+ """
+ Write package metadata
+ """
+ try:
+ tar_file = tarfile.open(tmp_file.name, 'r')
+ for member in tar_file.getmembers():
+ if 'requires.txt' in member.name:
+ with open('/tmp/requires.txt', 'w') as tmpf:
+ tmpf.write(tar_file.extractfile(member).read())
+ if 'PKG-INFO' in member.name:
+ with open('/tmp/PKG-INFO', 'w') as tmpf:
+ tmpf.write(tar_file.extractfile(member).read())
+ except tarfile.ReadError:
+ zip_file = zipfile.ZipFile(tmp_file.name, 'r')
+ for member in zip_file.namelist():
+ if 'requires.txt' in member:
+ with open('/tmp/requires.txt', 'w') as tmpf:
+ tmpf.write(zip_file.read(member))
+ if 'PKG-INFO' in member:
+ with open('/tmp/PKG-INFO', 'w') as tmpf:
+ tmpf.write(zip_file.read(member))
+
+def get_metadata(package, version):
"""
- Return content of requires.txt until first [ appears
+ Return package metadata
"""
- content = None
- release_data = get_json("https://pypi.python.org/pypi/%s/%s/json" % (package, version,))
- for item in release_data['releases'][version]:
+ metadata = None
+ for item in get_json("https://pypi.python.org/pypi/%s/%s/json" % (package, version,)) \
+ ['releases'][version]:
if item['packagetype'] == 'sdist':
tmp_file = file_download(item['url'])
+ write_metadata(tmp_file)
+ if os.path.isfile('/tmp/requires.txt') and os.path.isfile('/tmp/PKG-INFO'):
+ metadata = [
+ line.decode('utf-8') \
+ for line in wheel.metadata.pkginfo_to_metadata('/tmp', '/tmp/PKG-INFO') \
+ .as_string().splitlines()]
+ try:
+ os.unlink('/tmp/requires.txt')
+ except OSError:
+ pass
try:
- tar_file = tarfile.open(tmp_file.name, 'r')
- for member in tar_file.getmembers():
- if 'requires.txt' in member.name:
- content = tar_file.extractfile(member)
- except tarfile.ReadError:
- zip_file = zipfile.ZipFile(tmp_file.name, 'r')
- for member in zip_file.namelist():
- if 'requires.txt' in member:
- content = zip_file.read(member)
- if content is not None:
- par = []
- for line in content:
- if '[' in line:
+ os.unlink('/tmp/PKG-INFO')
+ except OSError:
+ pass
+ os.unlink(tmp_file.name)
+ if metadata:
break
- else:
- par.append(line.strip())
- content = "\n".join(par)
- os.unlink(tmp_file.name)
- return content
+ elif item['packagetype'] == 'bdist_wheel':
+ tmp_file = file_download(item['url'])
+ zip_file = zipfile.ZipFile(tmp_file.name, 'r')
+ for member in zip_file.namelist():
+ if 'METADATA' in member:
+ metadata = [line.decode('utf-8') for line in zip_file.read(member).splitlines()]
+ os.unlink(tmp_file.name)
+ break
+ return metadata
+
+def parse_metadata(metadata, pyver):
+ """
+ Return dependencies parsed from metadata
+ """
+ for line in metadata:
+ if 'Metadata-Version' in line.decode('utf-8'):
+ metadata_version = line.replace('Metadata-Version:', '').strip()
+ break
+ if packaging.version.Version(metadata_version) >= packaging.version.Version('2.0'):
+ out = []
+ for dep in [
+ line.replace('Requires-Dist:', '').strip() \
+ for line in metadata if re.search(r'^Requires-Dist:', line)]:
+ if ';' in dep:
+ dep = dep.split(';')
+ if 'python_version' in dep[1]:
+ if packaging.specifiers.SpecifierSet(
+ dep[1].replace('python_version', '').replace('"', '').strip()) \
+ .contains(packaging.version.Version(pyver)):
+ dep = dep[0]
+ else:
+ continue
+ else:
+ continue
+ dep = dep.split()
+ try:
+ pkg = re.search(r'(.*)(\[.*\])', dep[0]).group(1)
+ except AttributeError:
+ pkg = dep[0]
+ try:
+ pkg = re.search(r'(^[\w\.\-]*)(.*)', dep[0]).group(1)
+ dep.append(re.search(r'(^[\w\.\-]*)(.*)', dep[0]).group(2))
+ except AttributeError:
+ pkg = dep[0]
+ try:
+ ver = dep[1].replace('(', '').replace(')', '').replace(';', '')
+ except IndexError:
+ ver = None
+ out.append((pkg, ver))
+ return out
def find_new_dependencies(package, version, package_list, pyver):
"""
Return package dependencies parsed from pypi json
"""
- content = parse_requires_txt(package, version)
- if content is not None:
- for line in content.split("\n"):
- try:
- pkg = re.search(r'^([a-zA-Z0-9_.-]+)', line).group(0)
- dep = line.replace(pkg, '').strip()
- if not dep:
- dep = None
- if pkg in package_list:
- yield (pkg, dep)
- else:
+ content = parse_metadata(get_metadata(package, version), pyver)
+ for pkg, ver in content:
+ try:
+ if pkg in package_list:
+ yield (pkg, ver)
+ else:
+ try:
for child in find_new_dependencies(
pkg,
- get_newer_vers(find_available_vers(pkg, pyver), dep, None)[0],
+ get_newer_vers(find_available_vers(pkg, pyver), ver, None)[0],
package_list,
pyver
):
yield child
- except AttributeError:
- pass
+ except TypeError:
+ pass
+ except AttributeError:
+ pass
def depless_vers(res):
"""
@@ -269,7 +345,10 @@ def collect_packages(package_list, jsonpipdeptree, pyver=None):
)):
if 'Any' not in dep:
required_version.append(dep)
- available_version = find_available_vers(package, pyver)
+ try:
+ available_version = find_available_vers(package, pyver)
+ except urllib2.HTTPError:
+ available_version = [installed_version]
newer_version = get_newer_vers(available_version, required_version, installed_version)
rev = {'installed_version': installed_version,
'required_version': required_version,
--
GitLab