Add sdist support

src/_packagedcode/pypi.py

@@ -14,14 +14,17 @@
 import os
 import re
 import sys
+from typing import NamedTuple
 import zipfile
 from configparser import ConfigParser
 from pathlib import Path
 
 import dparse2
+import packaging
 import pip_requirements_parser
 import pkginfo2
 from commoncode import fileutils
+from packaging.specifiers import SpecifierSet
 from packageurl import PackageURL
 from packaging import markers
 from packaging.requirements import Requirement
@@ -453,7 +456,8 @@ def parse_metadata(location, datasource_id, package_type):
         primary_language='Python',
         name=name,
         version=version,
-        description=get_description(meta, location),
+        description=get_description( metainfo= meta, location= str(location)),
+        #TODO: https://github.com/nexB/scancode-toolkit/issues/3014
         declared_license=get_declared_license(meta),
         keywords=get_keywords(meta),
         parties=get_parties(meta),
@@ -645,6 +649,14 @@ def parse(cls, location):
         )
 
 
+class ResolvedPurl(NamedTuple):
+    """
+    A resolved PURL
+    """
+    purl: PackageURL
+    is_resolved: bool
+
+
 class BaseDependencyFileHandler(BasePypiHandler):
     """
     Base class for a dependency files parsed with the same library
@@ -684,10 +696,44 @@ def parse(cls, location):
 
         metadata = {}
         parser = ConfigParser()
+        dependent_packages = []
         with open(location) as f:
             parser.read_file(f)
-
         for section in parser.values():
+            if section.name == 'options':
+                scope_by_sub_section = {
+                    "install_requires": "install",
+                    "tests_require": "test",
+                    "setup_requires": "setup",
+                    "python_requires": "python",
+                }
+                for sub_section in scope_by_sub_section:
+                    if sub_section not in section:
+                        continue
+                    scope = scope_by_sub_section[sub_section]
+                    if scope != "python":
+                        reqs = list(get_requirement_from_section(section=section, sub_section=sub_section))
+                        dependent_packages.extend(cls.parse_reqs(reqs, scope))
+                        continue
+                    python_requires = section[sub_section]
+                    purl = PackageURL(
+                        name="python",
+                        type="generic"
+                    )
+                    resolved_purl = is_purl_resolved(purl = purl, specifiers= SpecifierSet(python_requires))
+                    dependent_packages.append(models.DependentPackage(
+                    purl=str(resolved_purl.purl),
+                    scope=scope,
+                    is_runtime=True,
+                    is_optional=False,
+                    is_resolved=resolved_purl.is_resolved,
+                    extracted_requirement=f"python_requires{python_requires}",
+                    ))
+
+            if section.name == "options.extras_require":
+                for sub_section in section:
+                    reqs = list(get_requirement_from_section(section=section, sub_section=sub_section))
+                    dependent_packages.extend(cls.parse_reqs(reqs, sub_section))
             if section.name == 'metadata':
                 options = (
                     'name',
@@ -715,14 +761,7 @@ def parse(cls, location):
                 )
             ]
 
-        dependency_type = get_dparse2_supported_file_name(file_name)
-        if not dependency_type:
-            return
 
-        dependencies = parse_with_dparse2(
-            location=location,
-            file_name=dependency_type,
-        )
         yield models.PackageData(
             datasource_id=cls.datasource_id,
             type=cls.default_package_type,
@@ -731,9 +770,48 @@ def parse(cls, location):
             parties=parties,
             homepage_url=metadata.get('url'),
             primary_language=cls.default_primary_language,
-            dependencies=dependencies,
+            dependencies=dependent_packages,
         )
 
+    @classmethod
+    def parse_reqs(cls, reqs, scope):
+        """
+        Parse a list of requirements and return a list of dependencies
+        """ 
+        dependent_packages = []
+        for req in reqs:
+            req_parsed = packaging.requirements.Requirement(str(req))
+            name = canonicalize_name(req_parsed.name)
+            purl = PackageURL(type="pypi", name=name)
+            specifiers = req_parsed.specifier._specs
+            resolved_purl = is_purl_resolved(purl = purl, specifiers= specifiers)
+            dependent_packages.append(
+                        models.DependentPackage(
+                        purl=str(resolved_purl.purl),
+                        scope=scope,
+                        is_runtime=True,
+                        is_optional=False,
+                        is_resolved=resolved_purl.is_resolved,
+                        extracted_requirement=req
+                    )
+                )
+        return dependent_packages
+
+
+def is_purl_resolved(purl: PackageURL, specifiers: SpecifierSet):
+    """
+    Check if the purl is resolved
+    """
+    is_resolved = False
+    if len(specifiers) == 1:
+        specifier = list(specifiers)[0]
+        if specifier.operator in ('==', '==='):
+            is_resolved = True
+            purl = purl._replace(version=specifier.version)
+    return ResolvedPurl(
+        purl=purl,
+        is_resolved=is_resolved,
+    )    
 
 class PipfileHandler(BaseDependencyFileHandler):
     datasource_id = 'pipfile'
@@ -1017,10 +1095,11 @@ def get_classifiers(metainfo):
     license_classifiers = []
     other_classifiers = []
     for classifier in classifiers:
-        if classifier.startswith('License'):
-            license_classifiers.append(classifier)
-        else:
-            other_classifiers.append(classifier)
+        if classifier:
+            if classifier.startswith('License'):
+                license_classifiers.append(classifier)
+            else:
+                other_classifiers.append(classifier)
     return license_classifiers, other_classifiers
 
 
@@ -1248,7 +1327,6 @@ def get_dparse2_supported_file_name(file_name):
         'Pipfile.lock',
         'Pipfile',
         'conda.yml',
-        'setup.cfg',
     )
 
     for dfile_name in dfile_names:
@@ -1897,3 +1975,27 @@ def compute_normalized_license(declared_license):
 
     if detected_licenses:
         return combine_expressions(detected_licenses)
+
+
+def get_requirement_from_section(section, sub_section):
+    """
+    Generate requirements from the `sub_section`
+    """
+    content = section.get(sub_section) or ""
+    for req in content.splitlines():
+        if not req:
+            continue
+        #pytest-mypy >= 0.9.1; \
+        req = req.replace("; \\", "")
+        # pip>=19.1 # For proper file:// URLs support.
+        if "#" in req:
+            req , _ = req.rsplit("#")
+        try:
+            Requirement(req)
+            yield req
+        except:
+            #pure-eval; black; tox
+            req_split_by_semi_colon = req.split(";")
+            req_split_by_semi_colon = [req.strip() for req in req_split_by_semi_colon if req]
+            for req in req_split_by_semi_colon:
+                yield req

src/python_inspector/__init__.py

@@ -6,3 +6,5 @@
 # See https://github.com/nexB/scancode-toolkit for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
+
+DEFAULT_PYTHON_VERSION = "3.8"