Add is_direct attribute to dependency model #3780

Adds is_direct attribute to differentiate between direct
dependecy relationships and dependencies listed in lockfiles
which have both direct and transitive dependencies together,
which will have is_direct as False.

Reference: #3780
Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

Author: AyanSinhaMahapatra

src/packagedcode/models.py

@@ -371,6 +371,14 @@ class DependentPackage(ModelMixin):
              'been resolved and this dependency url points to an '
              'exact version.')
 
+    is_direct = Boolean(
+        default=True,
+        label='is direct flag',
+        help='True if this dependency version requirement is '
+             'a direct dependency relation between two packages '
+             'as opposed to a transitive dependency relation, '
+             'which are present in lockfiles/dependency list.')
+
     resolved_package = Mapping(
         label='resolved package data',
         help='A mapping of resolved package data for this dependent package, '
@@ -1072,6 +1080,17 @@ def is_datafile(cls, location, filetypes=tuple(), _bare_filename=False):
                     actual_type = T.filetype_file.lower()
                     return any(ft in actual_type for ft in filetypes)
 
+    @classmethod
+    def is_lockfile(cls):
+        """
+        Return True if this is a lockfile, False otherwise.
+
+        This has to be implemented by datafile handlers classes
+        of lockfiles, to return True, in contrast to the default
+        value False.
+        """
+        return False
+
     @classmethod
     def parse(cls, location, package_only=False):
         """

src/packagedcode/npm.py

@@ -107,6 +107,10 @@ def assemble(cls, package_data, resource, codebase, package_adder):
             yield from yield_dependencies_from_package_resource(resource)
             return
 
+        if codebase.has_single_resource:
+            yield from models.DatafileHandler.assemble(package_data, resource, codebase, package_adder)
+            return
+
         assert len(package_resource.package_data) == 1, f'Invalid package.json for {package_resource.path}'
         pkg_data = package_resource.package_data[0]
         pkg_data = models.PackageData.from_dict(pkg_data)
@@ -204,6 +208,7 @@ def update_dependencies_by_purl(
         is_runtime=False,
         is_optional=False,
         is_resolved=False,
+        is_direct=True,
     ):
 
         metadata_deps = ['peerDependenciesMeta', 'dependenciesMeta']
@@ -221,6 +226,7 @@ def update_dependencies_by_purl(
                     is_runtime=is_runtime,
                     is_optional=is_optional,
                     is_resolved=is_resolved,
+                    is_direct=is_direct,
                 )
                 dependecies_by_purl[dep_purl] = dep_package
 
@@ -244,6 +250,7 @@ def update_dependencies_by_purl(
                             is_runtime=is_runtime,
                             is_optional=metadata.get("optional"),
                             is_resolved=is_resolved,
+                            is_direct=is_direct,
                         )
                         dependecies_by_purl[dep_purl] = dep_package
                     continue
@@ -264,6 +271,7 @@ def update_dependencies_by_purl(
                     is_runtime=is_runtime,
                     is_optional=is_optional,
                     is_resolved=is_resolved,
+                    is_direct=is_direct,
                 )
                 dependecies_by_purl[dep_purl] = dep_package
 
@@ -476,6 +484,10 @@ def parse(cls, location, package_only=False):
 
 class BaseNpmLockHandler(BaseNpmHandler):
 
+    @classmethod
+    def is_lockfile(cls):
+        return True
+
     @classmethod
     def parse(cls, location, package_only=False):
 
@@ -590,6 +602,7 @@ def parse(cls, location, package_only=False):
                 is_runtime=is_runtime,
                 is_optional=is_optional,
                 is_resolved=True,
+                is_direct=False,
             )
 
             # URLs and checksums
@@ -638,6 +651,7 @@ def parse(cls, location, package_only=False):
                 is_runtime=is_runtime,
                 is_optional=is_optional,
                 is_resolved=False,
+                is_direct=True,
             )
 
             resolved_package.dependencies = [
@@ -723,6 +737,10 @@ class YarnLockV2Handler(BaseNpmHandler):
     def is_datafile(cls, location, filetypes=tuple()):
         return super().is_datafile(location, filetypes=filetypes) and is_yarn_v2(location)
 
+    @classmethod
+    def is_lockfile(cls):
+        return True
+
     @classmethod
     def parse(cls, location, package_only=False):
         """
@@ -833,6 +851,10 @@ class YarnLockV1Handler(BaseNpmHandler):
     description = 'yarn.lock lockfile v1 format'
     documentation_url = 'https://classic.yarnpkg.com/lang/en/docs/yarn-lock/'
 
+    @classmethod
+    def is_lockfile(cls):
+        return True
+
     @classmethod
     def is_datafile(cls, location, filetypes=tuple()):
         return super().is_datafile(location, filetypes=filetypes) and not is_yarn_v2(location)
@@ -953,6 +975,7 @@ def parse(cls, location, package_only=False):
                     scope='dependencies',
                     is_optional=False,
                     is_runtime=True,
+                    is_direct=True,
                 )
                 resolved_package_data.dependencies.append(subdep)
 
@@ -972,6 +995,7 @@ def parse(cls, location, package_only=False):
                 scope='dependencies',
                 is_optional=False,
                 is_runtime=True,
+                is_direct=False,
                 resolved_package=resolved_package_data.to_dict(),
             )
             dependencies.append(dep.to_dict())
@@ -988,6 +1012,10 @@ def parse(cls, location, package_only=False):
 
 class BasePnpmLockHandler(BaseNpmHandler):
 
+    @classmethod
+    def is_lockfile(cls):
+        return True
+
     @classmethod
     def parse(cls, location, package_only=False):
         """
@@ -1063,19 +1091,22 @@ def parse(cls, location, package_only=False):
                 scope='dependencies',
                 dependecies_by_purl=deps_for_resolved_by_purl,
                 is_resolved=True,
+                is_direct=False,
             )
             cls.update_dependencies_by_purl(
                 dependencies=peer_dependencies,
                 scope='peerDependencies',
                 dependecies_by_purl=deps_for_resolved_by_purl,
                 is_optional=True,
+                is_direct=False,
             )
             cls.update_dependencies_by_purl(
                 dependencies=optional_dependencies,
                 scope='optionalDependencies',
                 dependecies_by_purl=deps_for_resolved_by_purl,
                 is_resolved=True,
                 is_optional=True,
+                is_direct=False,
             )
             cls.update_dependencies_by_purl(
                 dependencies=peer_dependencies_meta,
@@ -1122,6 +1153,7 @@ def parse(cls, location, package_only=False):
                 is_optional=is_optional,
                 is_runtime=is_runtime,
                 is_resolved=True,
+                is_direct=True,
                 resolved_package=resolved_package.to_dict(),
                 extra_data=extra_data_deps,
             )
@@ -1577,7 +1609,7 @@ def bundle_deps_mapper(bundle_deps, package):
     return package
 
 
-def deps_mapper(deps, package, field_name):
+def deps_mapper(deps, package, field_name, is_direct=True):
     """
     Handle deps such as dependencies, devDependencies, peerDependencies, optionalDependencies
     return a tuple of (dep type, list of deps)
@@ -1630,6 +1662,7 @@ def deps_mapper(deps, package, field_name):
                 purl=purl,
                 scope=field_name,
                 extracted_requirement=requirement,
+                is_direct=is_direct,
                 **dependency_attributes
             )
             dependencies.append(dep)

tests/licensedcode/data/plugin_license/package/package.expected.json

@@ -70,7 +70,11 @@
       "notice_text": null,
       "source_packages": [],
       "is_private": false,
-      "extra_data": {},
+      "extra_data": {
+        "engines": {
+          "node": ">=0.8.0"
+        }
+      },
       "repository_homepage_url": "https://www.npmjs.com/package/busboy",
       "repository_download_url": "https://registry.npmjs.org/busboy/-/busboy-0.2.14.tgz",
       "api_data_url": "https://registry.npmjs.org/busboy/0.2.14",
@@ -92,6 +96,7 @@
       "is_runtime": true,
       "is_optional": false,
       "is_resolved": false,
+      "is_direct": true,
       "resolved_package": {},
       "extra_data": {},
       "dependency_uid": "pkg:npm/dicer?uuid=fixed-uid-done-for-testing-5642512d1758",
@@ -106,6 +111,7 @@
       "is_runtime": true,
       "is_optional": false,
       "is_resolved": false,
+      "is_direct": true,
       "resolved_package": {},
       "extra_data": {},
       "dependency_uid": "pkg:npm/readable-stream?uuid=fixed-uid-done-for-testing-5642512d1758",
@@ -236,7 +242,11 @@
           "source_packages": [],
           "file_references": [],
           "is_private": false,
-          "extra_data": {},
+          "extra_data": {
+            "engines": {
+              "node": ">=0.8.0"
+            }
+          },
           "dependencies": [
             {
               "purl": "pkg:npm/dicer",
@@ -245,6 +255,7 @@
               "is_runtime": true,
               "is_optional": false,
               "is_resolved": false,
+              "is_direct": true,
               "resolved_package": {},
               "extra_data": {}
             },
@@ -255,6 +266,7 @@
               "is_runtime": true,
               "is_optional": false,
               "is_resolved": false,
+              "is_direct": true,
               "resolved_package": {},
               "extra_data": {}
             }