Case vanished module; all deps are taken from metadata; hotfix extras

pipdeps/pipdeps.py

@@ -13,6 +13,7 @@ import urllib2
 import tarfile
 import tempfile
 import zipfile
+import wheel.metadata
 
 import tabulate
 import packaging.specifiers
@@ -143,7 +144,11 @@ def find_available_vers(package_name, pyver):
     Return descending list of available strict version
     """
     versions = []
-    data = get_json("https://pypi.python.org/pypi/%s/json" % (package_name,))
+    try:
+        data = get_json("https://pypi.python.org/pypi/%s/json" % (package_name,))
+    except urllib2.HTTPError, err:
+        print "%s %s" % (err, err.url)
+        raise urllib2.HTTPError(err.url, err.code, None, err.hdrs, err.fp)
     releases = data["releases"].keys()
     for release in releases:
         requires_python = []
@@ -158,6 +163,9 @@ def get_newer_vers(available_version, required_version, installed_version=None):
     """
     Return list of newer versions which conforms pipdeptree dependencies, otherwise return none.
     """
+    if required_version is None:
+        result = [aver for aver in list(available_version)]
+        return sorted(result, key=distutils.version.StrictVersion, reverse=True)
     if [rver for rver in required_version if re.search(r'(^==.*|^\d.*)', rver) is not None]:
         return None
     result = []
@@ -182,60 +190,128 @@ def get_newer_vers(available_version, required_version, installed_version=None):
         return sorted(result, key=distutils.version.StrictVersion, reverse=True)
     return None
 
-def parse_requires_txt(package, version):
+def write_metadata(tmp_file):
+    """
+    Write package metadata
+    """
+    try:
+        tar_file = tarfile.open(tmp_file.name, 'r')
+        for member in tar_file.getmembers():
+            if 'requires.txt' in member.name:
+                with open('/tmp/requires.txt', 'w') as tmpf:
+                    tmpf.write(tar_file.extractfile(member).read())
+            if 'PKG-INFO' in member.name:
+                with open('/tmp/PKG-INFO', 'w') as tmpf:
+                    tmpf.write(tar_file.extractfile(member).read())
+    except tarfile.ReadError:
+        zip_file = zipfile.ZipFile(tmp_file.name, 'r')
+        for member in zip_file.namelist():
+            if 'requires.txt' in member:
+                with open('/tmp/requires.txt', 'w') as tmpf:
+                    tmpf.write(zip_file.read(member))
+            if 'PKG-INFO' in member:
+                with open('/tmp/PKG-INFO', 'w') as tmpf:
+                    tmpf.write(zip_file.read(member))
+
+def get_metadata(package, version):
     """
-    Return content of requires.txt until first [ appears
+    Return package metadata
     """
-    content = None
-    release_data = get_json("https://pypi.python.org/pypi/%s/%s/json" % (package, version,))
-    for item in release_data['releases'][version]:
+    metadata = None
+    for item in get_json("https://pypi.python.org/pypi/%s/%s/json" % (package, version,)) \
+        ['releases'][version]:
         if item['packagetype'] == 'sdist':
             tmp_file = file_download(item['url'])
+            write_metadata(tmp_file)
+            if os.path.isfile('/tmp/requires.txt') and os.path.isfile('/tmp/PKG-INFO'):
+                metadata = [
+                    line.decode('utf-8') \
+                    for line in wheel.metadata.pkginfo_to_metadata('/tmp', '/tmp/PKG-INFO') \
+                    .as_string().splitlines()]
+            try:
+                os.unlink('/tmp/requires.txt')
+            except OSError:
+                pass
             try:
-                tar_file = tarfile.open(tmp_file.name, 'r')
-                for member in tar_file.getmembers():
-                    if 'requires.txt' in member.name:
-                        content = tar_file.extractfile(member)
-            except tarfile.ReadError:
-                zip_file = zipfile.ZipFile(tmp_file.name, 'r')
-                for member in zip_file.namelist():
-                    if 'requires.txt' in member:
-                        content = zip_file.read(member)
-    if content is not None:
-        par = []
-        for line in content:
-            if '[' in line:
+                os.unlink('/tmp/PKG-INFO')
+            except OSError:
+                pass
+            os.unlink(tmp_file.name)
+            if metadata:
                 break
-            else:
-                par.append(line.strip())
-        content = "\n".join(par)
-    os.unlink(tmp_file.name)
-    return content
+        elif item['packagetype'] == 'bdist_wheel':
+            tmp_file = file_download(item['url'])
+            zip_file = zipfile.ZipFile(tmp_file.name, 'r')
+            for member in zip_file.namelist():
+                if 'METADATA' in member:
+                    metadata = [line.decode('utf-8') for line in zip_file.read(member).splitlines()]
+            os.unlink(tmp_file.name)
+            break
+    return metadata
+
+def parse_metadata(metadata, pyver):
+    """
+    Return dependencies parsed from metadata
+    """
+    for line in metadata:
+        if 'Metadata-Version' in line.decode('utf-8'):
+            metadata_version = line.replace('Metadata-Version:', '').strip()
+            break
+    if packaging.version.Version(metadata_version) >= packaging.version.Version('2.0'):
+        out = []
+        for dep in [
+                line.replace('Requires-Dist:', '').strip() \
+                for line in metadata if re.search(r'^Requires-Dist:', line)]:
+            if ';' in dep:
+                dep = dep.split(';')
+                if 'python_version' in dep[1]:
+                    if packaging.specifiers.SpecifierSet(
+                            dep[1].replace('python_version', '').replace('"', '').strip()) \
+                        .contains(packaging.version.Version(pyver)):
+                        dep = dep[0]
+                    else:
+                        continue
+                else:
+                    continue
+            dep = dep.split()
+            try:
+                pkg = re.search(r'(.*)(\[.*\])', dep[0]).group(1)
+            except AttributeError:
+                pkg = dep[0]
+            try:
+                pkg = re.search(r'(^[\w\.\-]*)(.*)', dep[0]).group(1)
+                dep.append(re.search(r'(^[\w\.\-]*)(.*)', dep[0]).group(2))
+            except AttributeError:
+                pkg = dep[0]
+            try:
+                ver = dep[1].replace('(', '').replace(')', '').replace(';', '')
+            except IndexError:
+                ver = None
+            out.append((pkg, ver))
+    return out
 
 def find_new_dependencies(package, version, package_list, pyver):
     """
     Return package dependencies parsed from pypi json
     """
-    content = parse_requires_txt(package, version)
-    if content is not None:
-        for line in content.split("\n"):
-            try:
-                pkg = re.search(r'^([a-zA-Z0-9_.-]+)', line).group(0)
-                dep = line.replace(pkg, '').strip()
-                if not dep:
-                    dep = None
-                if pkg in package_list:
-                    yield (pkg, dep)
-                else:
+    content = parse_metadata(get_metadata(package, version), pyver)
+    for pkg, ver in content:
+        try:
+            if pkg in package_list:
+                yield (pkg, ver)
+            else:
+                try:
                     for child in find_new_dependencies(
                             pkg,
-                            get_newer_vers(find_available_vers(pkg, pyver), dep, None)[0],
+                            get_newer_vers(find_available_vers(pkg, pyver), ver, None)[0],
                             package_list,
                             pyver
                         ):
                         yield child
-            except AttributeError:
-                pass
+                except TypeError:
+                    pass
+        except AttributeError:
+            pass
 
 def depless_vers(res):
     """
@@ -269,7 +345,10 @@ def collect_packages(package_list, jsonpipdeptree, pyver=None):
             )):
             if 'Any' not in dep:
                 required_version.append(dep)
-        available_version = find_available_vers(package, pyver)
+        try:
+            available_version = find_available_vers(package, pyver)
+        except urllib2.HTTPError:
+            available_version = [installed_version]
         newer_version = get_newer_vers(available_version, required_version, installed_version)
         rev = {'installed_version': installed_version,
                'required_version': required_version,