Case vanished module; all deps are taken from metadata; hotfix extras

pipdeps/pipdeps.py

@@ -13,6 +13,7 @@ import urllib2
 import tarfile
 import tempfile
 import zipfile
+import wheel.metadata
 
 import tabulate
 import packaging.specifiers
@@ -143,7 +144,11 @@ def find_available_vers(package_name, pyver):
     Return descending list of available strict version
     """
     versions = []
-    data = get_json("https://pypi.python.org/pypi/%s/json" % (package_name,))
+    try:
+        data = get_json("https://pypi.python.org/pypi/%s/json" % (package_name,))
+    except urllib2.HTTPError, err:
+        print "%s %s" % (err, err.url)
+        raise urllib2.HTTPError(err.url, err.code, None, err.hdrs, err.fp)
     releases = data["releases"].keys()
     for release in releases:
         requires_python = []
@@ -158,6 +163,9 @@ def get_newer_vers(available_version, required_version, installed_version=None):
     """
     Return list of newer versions which conforms pipdeptree dependencies, otherwise return none.
     """
+    if required_version is None:
+        result = [aver for aver in list(available_version)]
+        return sorted(result, key=distutils.version.StrictVersion, reverse=True)
     if [rver for rver in required_version if re.search(r'(^==.*|^\d.*)', rver) is not None]:
         return None
     result = []
@@ -182,11 +190,11 @@ def get_newer_vers(available_version, required_version, installed_version=None):
         return sorted(result, key=distutils.version.StrictVersion, reverse=True)
     return None
 
-def parse_requires_txt(package, version):
+def parse_requires_txt(package, version, pyver):
     """
     Return content of requires.txt until first [ appears
     """
-    content = None
+    metadata = None
     release_data = get_json("https://pypi.python.org/pypi/%s/%s/json" % (package, version,))
     for item in release_data['releases'][version]:
         if item['packagetype'] == 'sdist':
@@ -195,47 +203,84 @@ def parse_requires_txt(package, version):
                 tar_file = tarfile.open(tmp_file.name, 'r')
                 for member in tar_file.getmembers():
                     if 'requires.txt' in member.name:
-                        content = tar_file.extractfile(member)
+                        with open('/tmp/requires.txt', 'w') as tmpf:
+                            tmpf.write(tar_file.extractfile(member).read())
+                    if 'PKG-INFO' in member.name:
+                        with open('/tmp/PKG-INFO', 'w') as tmpf:
+                            tmpf.write(tar_file.extractfile(member).read())
             except tarfile.ReadError:
                 zip_file = zipfile.ZipFile(tmp_file.name, 'r')
                 for member in zip_file.namelist():
                     if 'requires.txt' in member:
-                        content = zip_file.read(member)
-    if content is not None:
-        par = []
-        for line in content:
-            if '[' in line:
+                        with open('/tmp/requires.txt', 'w') as tmpf:
+                            tmpf.write(zip_file.read(member))
+                    if 'PKG-INFO' in member:
+                        with open('/tmp/PKG-INFO', 'w') as tmpf:
+                            tmpf.write(zip_file.read(member))
+            if os.path.isfile('/tmp/requires.txt') and os.path.isfile('/tmp/PKG-INFO'):
+                metadata = [line.decode('utf-8') for line in wheel.metadata.pkginfo_to_metadata('/tmp', '/tmp/PKG-INFO').as_string().splitlines()]
+                os.unlink('/tmp/requires.txt')
+                os.unlink('/tmp/PKG-INFO')
+                os.unlink(tmp_file.name)
+        elif item['packagetype'] == 'bdist_wheel':
+            tmp_file = file_download(item['url'])
+            zip_file = zipfile.ZipFile(tmp_file.name, 'r')
+            for member in zip_file.namelist():
+                if 'METADATA' in member:
+                    metadata = [line.decode('utf-8') for line in zip_file.read(member).splitlines()]
+            os.unlink(tmp_file.name)
+    if metadata is not None:
+        for line in metadata:
+            if 'Metadata-Version' in line.decode('utf-8'):
+                metadata_version = line.replace('Metadata-Version:', '').strip()
                 break
-            else:
-                par.append(line.strip())
-        content = "\n".join(par)
-    os.unlink(tmp_file.name)
-    return content
+        if packaging.version.Version(metadata_version) >= packaging.version.Version('2.0'):
+            out = []
+            deps = [line.replace('Requires-Dist:', '').strip() for line in metadata if re.search(r'^Requires-Dist:', line)]
+            for dep in deps:
+                if ';' in dep:
+                    dep = dep.split(';')
+                    if 'python_version' in dep[1]:
+                        if packaging.specifiers.SpecifierSet(dep[1].replace('python_version', '').replace('"', '').strip()).contains(packaging.version.Version(pyver)):
+                            dep = dep[0]
+                        else:
+                            continue
+                    else:
+                        continue
+                dep = dep.split()
+                try:
+                    pkg = re.search(r'(.*)(\[.*\])', dep[0]).group(1)
+                except AttributeError:
+                    pkg = dep[0]
+                try:
+                    ver = dep[1].replace('(', '').replace(')', '').replace(';', '')
+                except IndexError:
+                    ver = None
+                out.append((pkg, ver))
+    return out
 
 def find_new_dependencies(package, version, package_list, pyver):
     """
     Return package dependencies parsed from pypi json
     """
-    content = parse_requires_txt(package, version)
-    if content is not None:
-        for line in content.split("\n"):
-            try:
-                pkg = re.search(r'^([a-zA-Z0-9_.-]+)', line).group(0)
-                dep = line.replace(pkg, '').strip()
-                if not dep:
-                    dep = None
-                if pkg in package_list:
-                    yield (pkg, dep)
-                else:
+    content = parse_requires_txt(package, version, pyver)
+    for pkg, ver in content:
+        try:
+            if pkg in package_list:
+                yield (pkg, ver)
+            else:
+                try:
                     for child in find_new_dependencies(
                             pkg,
-                            get_newer_vers(find_available_vers(pkg, pyver), dep, None)[0],
+                            get_newer_vers(find_available_vers(pkg, pyver), ver, None)[0],
                             package_list,
                             pyver
                         ):
                         yield child
-            except AttributeError:
-                pass
+                except TypeError:
+                    pass
+        except AttributeError:
+            pass
 
 def depless_vers(res):
     """
@@ -269,7 +314,10 @@ def collect_packages(package_list, jsonpipdeptree, pyver=None):
             )):
             if 'Any' not in dep:
                 required_version.append(dep)
-        available_version = find_available_vers(package, pyver)
+        try:
+            available_version = find_available_vers(package, pyver)
+        except urllib2.HTTPError:
+            available_version = [installed_version]
         newer_version = get_newer_vers(available_version, required_version, installed_version)
         rev = {'installed_version': installed_version,
                'required_version': required_version,