Add sdist support

src/_packagedcode/pypi.py

@@ -19,6 +19,7 @@
 from pathlib import Path
 
 import dparse2
+import packaging
 import pip_requirements_parser
 import pkginfo2
 from commoncode import fileutils
@@ -452,8 +453,9 @@ def parse_metadata(location, datasource_id, package_type):
         type=package_type,
         primary_language='Python',
         name=name,
-        version=version,
-        description=get_description(meta, location),
+        version=version, #TODO: https://github.com/nexB/scancode-toolkit/issues/3014
+        # description=get_description(meta, location),
+        description = "",
         declared_license=get_declared_license(meta),
         keywords=get_keywords(meta),
         parties=get_parties(meta),
@@ -684,10 +686,17 @@ def parse(cls, location):
 
         metadata = {}
         parser = ConfigParser()
+        dependent_packages = []
         with open(location) as f:
             parser.read_file(f)
-
         for section in parser.values():
+            if section.name == 'options':
+                reqs = list(get_requirement_from_section(section=section, sub_section="install_requires"))
+                dependent_packages.extend(cls.parse_reqs(reqs, "install"))
+            if section.name == "options.extras_require":
+                for sub_section in section:
+                    reqs = list(get_requirement_from_section(section=section, sub_section=sub_section))
+                    dependent_packages.extend(cls.parse_reqs(reqs, sub_section))
             if section.name == 'metadata':
                 options = (
                     'name',
@@ -719,10 +728,6 @@ def parse(cls, location):
         if not dependency_type:
             return
 
-        dependencies = parse_with_dparse2(
-            location=location,
-            file_name=dependency_type,
-        )
         yield models.PackageData(
             datasource_id=cls.datasource_id,
             type=cls.default_package_type,
@@ -731,9 +736,37 @@ def parse(cls, location):
             parties=parties,
             homepage_url=metadata.get('url'),
             primary_language=cls.default_primary_language,
-            dependencies=dependencies,
+            dependencies=dependent_packages,
         )
 
+    @classmethod
+    def parse_reqs(cls, reqs, scope):
+        """
+        Parse a list of requirements and return a list of dependencies
+        """ 
+        dependent_packages = []
+        for req in reqs:
+            is_resolved = False
+            req_parsed = packaging.requirements.Requirement(str(req))
+            name = canonicalize_name(req_parsed.name)
+            purl = PackageURL(type="pypi", name=name)
+            specifiers = req_parsed.specifier._specs
+            if len(specifiers) == 1:
+                specifier = list(specifiers)[0]
+                if specifier.operator in ('==', '==='):
+                    is_resolved = True
+                    purl = purl._replace(version=specifier.version)
+            dependent_packages.append(
+                        models.DependentPackage(
+                        purl=str(purl),
+                        scope=scope,
+                        is_runtime=True,
+                        is_optional=False,
+                        is_resolved=is_resolved,
+                        extracted_requirement=req
+                    )
+                )
+        return dependent_packages
 
 class PipfileHandler(BaseDependencyFileHandler):
     datasource_id = 'pipfile'
@@ -1017,10 +1050,11 @@ def get_classifiers(metainfo):
     license_classifiers = []
     other_classifiers = []
     for classifier in classifiers:
-        if classifier.startswith('License'):
-            license_classifiers.append(classifier)
-        else:
-            other_classifiers.append(classifier)
+        if classifier:
+            if classifier.startswith('License'):
+                license_classifiers.append(classifier)
+            else:
+                other_classifiers.append(classifier)
     return license_classifiers, other_classifiers
 
 
@@ -1897,3 +1931,28 @@ def compute_normalized_license(declared_license):
 
     if detected_licenses:
         return combine_expressions(detected_licenses)
+
+
+def get_requirement_from_section(section, sub_section):
+    """
+    Generate requirements from the `sub_section`
+    """
+    content = section.get(sub_section) or ""
+    for req in content.splitlines():
+        if req:
+            #pytest-mypy >= 0.9.1; \
+            req = req.replace("; \\", "")
+            # pip>=19.1 # For proper file:// URLs support.
+            if "#" in req:
+                req , _ = req.rsplit("#")
+            #pure-eval; black; tox
+            req_split_by_semi_colon = req.split(";")
+            req_split_by_semi_colon = [req.strip() for req in req_split_by_semi_colon if req]
+            if len(req_split_by_semi_colon) >= 2 and not(req_split_by_semi_colon[1].startswith("python_version") # pip>=19.1 ;python_version > 3.7 
+            or req_split_by_semi_colon[1].startswith("sys_platform") # pip>=19.1 ;sys_platform = "Windows"
+            or req_split_by_semi_colon[1].startswith("platform_system") # pip>=19.1 ;platform_system = "Windows"
+            or req_split_by_semi_colon[1].startswith("platform_python_implementation")):#pytest-black>=0.3.7; platform_python_implementation != "PyPy"
+                for temp_req in req_split_by_semi_colon:
+                    yield temp_req
+            else:
+                yield req

src/python_inspector/resolution.py

@@ -10,20 +10,27 @@
 import collections
 import operator
 import os
+import tarfile
 from typing import List
 from typing import Sequence
+from zipfile import ZipFile
 
 import packaging.requirements
 import packaging.utils
 import packaging.version
 import requests
 from packageurl import PackageURL
 from packaging.requirements import Requirement
+from packaging.specifiers import SpecifierSet
 from resolvelib import AbstractProvider
 from resolvelib import Resolver
 from resolvelib.reporters import BaseReporter
 
+from _packagedcode.pypi import PipRequirementsFileHandler
 from _packagedcode.pypi import PypiWheelHandler
+from _packagedcode.pypi import PythonSdistPkgInfoFile
+from _packagedcode.pypi import PythonSetupPyHandler
+from _packagedcode.pypi import SetupCfgHandler
 from python_inspector import utils_pypi
 
 Candidate = collections.namedtuple("Candidate", "name version extras")
@@ -62,12 +69,43 @@ def get_python_version_from_env_tag(python_version: str):
     return python_version
 
 
+def get_sdist_file(repos, candidate):
+    """
+    Return the sdist file for a candidate.
+    """
+    sdist = utils_pypi.download_sdist(
+        name=candidate.name,
+        version=str(candidate.version),
+        repos=repos,
+    )
+    sdist_file = None
+
+    if sdist.endswith(".tar.gz"):
+        sdist_file = sdist.rstrip(".tar.gz")
+        with tarfile.open(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, sdist)) as file:
+            file.extractall(
+                os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file)
+            )
+    if sdist.endswith(".zip"):
+        sdist_file = sdist.rstrip(".zip")
+        with ZipFile(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, sdist)) as zip:
+            zip.extractall(
+                os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file)
+            )
+
+    if not sdist_file:
+        raise Exception(f"Unable to extract sdist {sdist}")
+    return sdist_file
+
+
 class PythonInputProvider(AbstractProvider):
-    def __init__(self, environment=None, repos=tuple()):
+    def __init__(self, environment=None, repos=tuple(), resolved_requirements=[]):
         self.environment = environment
         self.repos = repos or []
         self.versions_by_package = {}
         self.dependencies_by_purl = {}
+        self.wheel_or_sdist_by_package = {}
+        self.resolved_requirements = resolved_requirements
 
     def identify(self, requirement_or_candidate):
         """Given a requirement, return an identifier for it. Overridden."""
@@ -104,10 +142,37 @@ def get_versions_for_package_from_repo(self, name, repo):
         Return a list of versions for a package name from a repo
         """
         versions = []
-        for version, package in repo._get_package_versions_map(name).items():
-            wheels = package.get_supported_wheels(environment=self.environment)
-            if list(wheels):
-                versions.append(version)
+        for version, package in repo.get_package_versions(name).items():
+            name = packaging.utils.canonicalize_name(package.name)
+            purl = PackageURL(type="pypi", name=name, version=version)
+            python_version = packaging.version.parse(
+                get_python_version_from_env_tag(self.environment.python_version)
+            )
+            formats = []
+            wheels = list(package.get_supported_wheels(environment=self.environment))
+            if wheels:
+                valid_wheel_present = False
+                for wheel in wheels:
+                    if (
+                        wheel.requires_python
+                        and python_version in SpecifierSet(wheel.requires_python)
+                    ) or not wheel.requires_python:
+                        valid_wheel_present = True
+                if valid_wheel_present:
+                    versions.append(version)
+                    formats.append("Wheel")
+            if package.sdist:
+                valid_sdist_present = False
+                if (
+                    package.sdist.requires_python
+                    and python_version in SpecifierSet(package.sdist.requires_python)
+                    or not package.sdist.requires_python
+                ):
+                    valid_sdist_present = True
+                if valid_sdist_present:
+                    versions.append(version)
+                    formats.append("Sdist")
+            self.wheel_or_sdist_by_package[str(purl)] = formats
         return versions
 
     def get_versions_for_package_from_pypi_json_api(self, name):
@@ -134,24 +199,99 @@ def get_requirements_for_package(self, purl, candidate):
             return self.get_requirements_for_package_from_pypi_json_api(purl)
 
     def get_requirements_for_package_from_pypi_simple(self, candidate):
-        wheels = utils_pypi.download_wheel(
-            name=candidate.name,
-            version=str(candidate.version),
-            environment=self.environment,
-            repos=self.repos,
-        )
-        for wheel in wheels:
-            deps = list(
-                PypiWheelHandler.parse(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, wheel))
-            )
-            assert len(deps) == 1
-            deps = deps[0].dependencies
-            for dep in deps:
-                if dep.scope == "install":
-                    yield packaging.requirements.Requirement(str(dep.extracted_requirement))
+        """
+        Return requirements for a package from the simple repositories.
+        """
+        purl = PackageURL(type="pypi", name=candidate.name, version=str(candidate.version))
 
-    def get_requirements_for_package_from_pypi_json_api(self, purl):
+        formats = self.wheel_or_sdist_by_package[str(purl)]
 
+        for format in formats:
+            if format == "Wheel":
+                wheels = utils_pypi.download_wheel(
+                    name=candidate.name,
+                    version=str(candidate.version),
+                    environment=self.environment,
+                    repos=self.repos,
+                )
+                for wheel in wheels:
+                    deps = list(
+                        PypiWheelHandler.parse(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, wheel))
+                    )
+                    assert len(deps) == 1
+                    deps = deps[0].dependencies
+                    for dep in deps:
+                        if dep.scope == "install":
+                            yield packaging.requirements.Requirement(str(dep.extracted_requirement))
+
+            if format == "Sdist":
+                sdist_file = get_sdist_file(repos=self.repos, candidate=candidate)
+                setup_py_path = os.path.join(
+                    utils_pypi.CACHE_THIRDPARTY_DIR,
+                    "extracted_sdists",
+                    sdist_file,
+                    sdist_file,
+                    "setup.py",
+                )
+                setup_cfg_path = os.path.join(
+                    utils_pypi.CACHE_THIRDPARTY_DIR,
+                    "extracted_sdists",
+                    sdist_file,
+                    sdist_file,
+                    "setup.cfg",
+                )
+                pkg_info_path = os.path.join(
+                    utils_pypi.CACHE_THIRDPARTY_DIR, "extracted_sdists", sdist_file
+                )
+                requirement_path = os.path.join(
+                    utils_pypi.CACHE_THIRDPARTY_DIR,
+                    "extracted_sdists",
+                    sdist_file,
+                    sdist_file,
+                    "requirements.txt",
+                )
+
+                path_by_sdist_parser = {
+                    PythonSdistPkgInfoFile: pkg_info_path,
+                    PythonSetupPyHandler: setup_py_path,
+                    SetupCfgHandler: setup_cfg_path,
+                    PipRequirementsFileHandler: requirement_path,
+                }
+
+                for handler, path in path_by_sdist_parser.items():
+                    if not os.path.exists(path):
+                        continue
+
+                    deps = list(handler.parse(path))
+                    assert len(deps) == 1
+                    dependencies = deps[0].dependencies
+                    for dep in dependencies:
+                        if not dep.purl:
+                            continue
+
+                        dep_purl = PackageURL.from_string(dep.purl)
+                        if not (
+                            dep.scope == "install"
+                            and (
+                                not (dep.is_resolved)
+                                or (
+                                    dep.is_resolved
+                                    and dep_purl.name not in self.resolved_requirements
+                                )
+                            )
+                        ):
+                            continue
+                        if dep.is_resolved:
+                            self.resolved_requirements.append(dep_purl)
+                        # skip the requirement starting with -- like
+                        # --editable, --requirement
+                        if not dep.extracted_requirement.startswith("--"):
+                            yield packaging.requirements.Requirement(str(dep.extracted_requirement))
+
+    def get_requirements_for_package_from_pypi_json_api(self, purl):
+        """
+        Return requirements for a package from the PyPI.org JSON API
+        """
         # if no repos are provided use the incorrect but fast JSON API
         if str(purl) not in self.dependencies_by_purl:
             api_url = f"https://pypi.org/pypi/{purl.name}/{purl.version}/json"
@@ -178,7 +318,7 @@ def _iter_matches(self, identifier, requirements, incompatibilities):
         """
         Yield candidates for the given identifier, requirements and incompatibilities
         """
-        name, _, _extras = identifier.partition("[")
+        name, _, _ = identifier.partition("[")
         bad_versions = {c.version for c in incompatibilities[identifier]}
         extras = {e for r in requirements[identifier] for e in r.extras}
         if not self.repos:
@@ -323,8 +463,15 @@ def get_resolved_dependencies(
     Used the provided ``repos`` list of PypiSimpleRepository.
     If empty, use instead the PyPI.org JSON API exclusively instead
     """
+    resolved_requirements = [
+        packaging.utils.canonicalize_name(r.name)
+        for r in requirements
+        if getattr(r, "is_requirement_resolved", False)
+    ]
     resolver = Resolver(
-        provider=PythonInputProvider(environment=environment, repos=repos),
+        provider=PythonInputProvider(
+            environment=environment, repos=repos, resolved_requirements=resolved_requirements
+        ),
         reporter=BaseReporter(),
     )
     results = resolver.resolve(requirements=requirements, max_rounds=max_rounds)