Packagedcode to handle Python(requirements.txt)

src/packagedcode/pypi.py

@@ -83,7 +83,7 @@ def logger_debug(*args):
 
 @attr.s()
 class PythonPackage(models.Package):
-    metafiles = ('metadata.json', '*setup.py', 'PKG-INFO', '*.whl', '*.egg')
+    metafiles = ('metadata.json', '*setup.py', 'PKG-INFO', '*.whl', '*.egg', '*requirements*.txt', '*requirements*.in')
     extensions = ('.egg', '.whl', '.pyz', '.pex',)
     default_type = 'pypi'
     default_primary_language = 'Python'
@@ -112,6 +112,8 @@ def parse(location):
         file_name = fileutils.file_name(location)
         parsers = {
             'setup.py': parse_setup_py,
+            'requirements.txt': parse_requirements_txt,
+            'requirements.in': parse_requirements_txt,
             'metadata.json': parse_metadata,
             'PKG-INFO': parse_pkg_info,
             '.whl': parse_wheel,
@@ -187,12 +189,29 @@ def parse_dependencies(location, package):
             package.dependencies = dependencies
 
 
+dependency_type_by_extensions = {
+    ('.txt', '.in'): 'requirements.txt',
+}
+
+
+def get_dependency_type(file_name, dependency_type_by_extensions=dependency_type_by_extensions):
+    """
+    Return the type of a dependency as a string or None given a `file_name` string.
+    """
+    for extensions, dependency_type in dependency_type_by_extensions.items():
+        if file_name.endswith(extensions):
+            return dependency_type
+
+
 def parse_with_dparse(location):
     is_dir = filetype.is_dir(location)
     if is_dir:
         return
     file_name = fileutils.file_name(location)
-    if file_name not in (filetypes.requirements_txt,
+
+    dependency_type = get_dependency_type(file_name)
+
+    if dependency_type not in (filetypes.requirements_txt,
                          filetypes.conda_yml,
                          filetypes.tox_ini,
                          filetypes.pipfile,
@@ -204,27 +223,54 @@ def parse_with_dparse(location):
         mode = 'r'
     with open(location, mode) as f:
         content = f.read()
-        df = dparse.parse(content, file_type=file_name)
-        df_dependencies = df.dependencies
-        if not df_dependencies:
-            return
-        package_dependencies = []
-        for df_dependency in df_dependencies:
-            specs = df_dependency.specs
-            requirement = None
-            if specs:
-                requirement = str(specs)
-            package_dependencies.append(
-                models.DependentPackage(
-                    purl=PackageURL(
-                        type='pypi', name=df_dependency.name).to_string(),
-                    scope='dependencies',
-                    is_runtime=True,
-                    is_optional=False,
-                    requirement=requirement,
-                )
+
+    df = dparse.parse(content, file_type=dependency_type)
+    df_dependencies = df.dependencies
+
+    if not df_dependencies:
+        return
+
+    package_dependencies = []
+    for df_dependency in df_dependencies:
+        specs = list(df_dependency.specs._specs)
+        is_resolved = False
+        requirement = None
+        purl = PackageURL(
+            type='pypi',
+            name=df_dependency.name
+        ).to_string()
+        if specs:
+            requirement = str(df_dependency.specs)
+            for spec in specs:
+                operator = spec.operator
+                version = spec.version
+                if any(operator == element for element in ('==', '===')):
+                    is_resolved = True
+                    purl = PackageURL(
+                        type='pypi',
+                        name=df_dependency.name,
+                        version=version
+                    ).to_string()
+        package_dependencies.append(
+            models.DependentPackage(
+                purl=purl,
+                scope='dependencies',
+                is_runtime=True,
+                is_optional=False,
+                is_resolved=is_resolved,
+                requirement=requirement
             )
-        return package_dependencies
+        )
+
+    return package_dependencies
+
+
+def parse_requirements_txt(location):
+    """
+    Return a package built from requirements.txt.
+    """
+    package_dependencies = parse_with_dparse(location)
+    return PythonPackage(dependencies=package_dependencies)
 
 
 def parse_setup_py(location):
@@ -260,7 +306,7 @@ def parse_setup_py(location):
                 if isinstance(kw.value, ast.Str):
                     setup_args[arg_name] = kw.value.s
                 if isinstance(kw.value, ast.List):
-                     # We collect the elements of a list if the element is not a function call
+                    # We collect the elements of a list if the element is not a function call
                     setup_args[arg_name] = [elt.s for elt in kw.value.elts if not isinstance(elt, ast.Call)]
 
     description = build_description(
@@ -342,7 +388,7 @@ def parse_metadata(location):
         parties.append(models.Party(type=models.party_person, name=name, role='contact'))
 
     description = build_description(
-        infos.get('summary') ,
+        infos.get('summary'),
         infos.get('description')
     )
 

tests/packagedcode/data/plugin/help.txt

@@ -203,7 +203,7 @@ Package: nuget
 --------------------------------------------
 Package: pypi
   class: packagedcode.pypi:PythonPackage
-  metafiles: metadata.json, *setup.py, PKG-INFO, *.whl, *.egg
+  metafiles: metadata.json, *setup.py, PKG-INFO, *.whl, *.egg, *requirements*.txt, *requirements*.in
   extensions: .egg, .whl, .pyz, .pex
 
 --------------------------------------------

tests/packagedcode/data/pypi/requirements_txt/sample1/output.expected.json

@@ -0,0 +1,60 @@
+{
+  "type": "pypi",
+  "namespace": null,
+  "name": null,
+  "version": null,
+  "qualifiers": {},
+  "subpath": null,
+  "primary_language": "Python",
+  "description": null,
+  "release_date": null,
+  "parties": [],
+  "keywords": [],
+  "homepage_url": null,
+  "download_url": null,
+  "size": null,
+  "sha1": null,
+  "md5": null,
+  "sha256": null,
+  "sha512": null,
+  "bug_tracking_url": null,
+  "code_view_url": null,
+  "vcs_url": null,
+  "copyright": null,
+  "license_expression": null,
+  "declared_license": null,
+  "notice_text": null,
+  "root_path": null,
+  "dependencies": [
+    {
+      "purl": "pkg:pypi/setuptools",
+      "requirement": ">=32.0.0",
+      "scope": "dependencies",
+      "is_runtime": true,
+      "is_optional": false,
+      "is_resolved": false
+    },
+    {
+      "purl": "pkg:pypi/nose",
+      "requirement": ">=1.3.7",
+      "scope": "dependencies",
+      "is_runtime": true,
+      "is_optional": false,
+      "is_resolved": false
+    },
+    {
+      "purl": "pkg:pypi/chardet",
+      "requirement": ">=3.0.4",
+      "scope": "dependencies",
+      "is_runtime": true,
+      "is_optional": false,
+      "is_resolved": false
+    }
+  ],
+  "contains_source_code": null,
+  "source_packages": [],
+  "purl": null,
+  "repository_homepage_url": null,
+  "repository_download_url": null,
+  "api_data_url": null
+}

tests/packagedcode/data/pypi/requirements_txt/sample1/requirements.txt

@@ -0,0 +1,3 @@
+setuptools>=32.0.0
+nose>=1.3.7
+chardet>=3.0.4

tests/packagedcode/data/pypi/requirements_txt/sample2/output.expected.json

@@ -0,0 +1,60 @@
+{
+  "type": "pypi",
+  "namespace": null,
+  "name": null,
+  "version": null,
+  "qualifiers": {},
+  "subpath": null,
+  "primary_language": "Python",
+  "description": null,
+  "release_date": null,
+  "parties": [],
+  "keywords": [],
+  "homepage_url": null,
+  "download_url": null,
+  "size": null,
+  "sha1": null,
+  "md5": null,
+  "sha256": null,
+  "sha512": null,
+  "bug_tracking_url": null,
+  "code_view_url": null,
+  "vcs_url": null,
+  "copyright": null,
+  "license_expression": null,
+  "declared_license": null,
+  "notice_text": null,
+  "root_path": null,
+  "dependencies": [
+    {
+      "purl": "pkg:pypi/aiohttp@3.6.2",
+      "requirement": "==3.6.2",
+      "scope": "dependencies",
+      "is_runtime": true,
+      "is_optional": false,
+      "is_resolved": true
+    },
+    {
+      "purl": "pkg:pypi/async-timeout@3.0.1",
+      "requirement": "==3.0.1",
+      "scope": "dependencies",
+      "is_runtime": true,
+      "is_optional": false,
+      "is_resolved": true
+    },
+    {
+      "purl": "pkg:pypi/attrs@19.3.0",
+      "requirement": "==19.3.0",
+      "scope": "dependencies",
+      "is_runtime": true,
+      "is_optional": false,
+      "is_resolved": true
+    }
+  ],
+  "contains_source_code": null,
+  "source_packages": [],
+  "purl": null,
+  "repository_homepage_url": null,
+  "repository_download_url": null,
+  "api_data_url": null
+}

tests/packagedcode/data/pypi/requirements_txt/sample2/sample-requirements.txt

@@ -0,0 +1,3 @@
+aiohttp==3.6.2
+async-timeout==3.0.1
+attrs==19.3.0

tests/packagedcode/data/pypi/requirements_txt/sample3/output.expected.json

@@ -0,0 +1,60 @@
+{
+  "type": "pypi",
+  "namespace": null,
+  "name": null,
+  "version": null,
+  "qualifiers": {},
+  "subpath": null,
+  "primary_language": "Python",
+  "description": null,
+  "release_date": null,
+  "parties": [],
+  "keywords": [],
+  "homepage_url": null,
+  "download_url": null,
+  "size": null,
+  "sha1": null,
+  "md5": null,
+  "sha256": null,
+  "sha512": null,
+  "bug_tracking_url": null,
+  "code_view_url": null,
+  "vcs_url": null,
+  "copyright": null,
+  "license_expression": null,
+  "declared_license": null,
+  "notice_text": null,
+  "root_path": null,
+  "dependencies": [
+    {
+      "purl": "pkg:pypi/prompt-toolkit",
+      "requirement": ">=1.0.14",
+      "scope": "dependencies",
+      "is_runtime": true,
+      "is_optional": false,
+      "is_resolved": false
+    },
+    {
+      "purl": "pkg:pypi/pygments@2.2.0",
+      "requirement": "==2.2.0",
+      "scope": "dependencies",
+      "is_runtime": true,
+      "is_optional": false,
+      "is_resolved": true
+    },
+    {
+      "purl": "pkg:pypi/regex",
+      "requirement": null,
+      "scope": "dependencies",
+      "is_runtime": true,
+      "is_optional": false,
+      "is_resolved": false
+    }
+  ],
+  "contains_source_code": null,
+  "source_packages": [],
+  "purl": null,
+  "repository_homepage_url": null,
+  "repository_download_url": null,
+  "api_data_url": null
+}

tests/packagedcode/data/pypi/requirements_txt/sample3/requirements-dev.txt

@@ -0,0 +1,3 @@
+prompt_toolkit>=1.0.14
+Pygments==2.2.0
+regex