Fix test failures from merging develop

Add node_modules as ignored paths for NPM package instance creation.
Regenerate test expectations after merging from develop.

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

Author: AyanSinhaMahapatra

src/packagedcode/models.py

@@ -802,6 +802,15 @@ class PackageInstance:
         help='List of files provided by this package.'
     )
 
+    @property
+    def ignore_paths(self):
+        """
+        Paths to ignore when looking for other package_data files.
+
+        Override the default empty list by defining for each package ecosystems specifically.
+        """
+        return []
+
     def to_dict(self, **kwargs):
         """
         Return an dict of primitive Python types.
@@ -889,10 +898,19 @@ def get_other_package_data_for_instance(self, resource, codebase):
 
         parent = resource.parent(codebase)
 
+        paths_to_ignore = self.ignore_paths
+
         for resource in parent.walk(codebase):
             if resource.is_dir:
                 continue
 
+            if paths_to_ignore:
+                if any(
+                    path in resource.path
+                    for path in paths_to_ignore
+                ):
+                    continue
+
             filename = file_name(resource.location)
             file_patterns = self.get_file_patterns(manifests=self.manifests)
             if any(fnmatch.fnmatchcase(filename, pattern) for pattern in file_patterns):

src/packagedcode/npm.py

@@ -467,6 +467,12 @@ class NpmPackageInstance(NpmPackage, models.PackageInstance):
     manifests, lockfiles, build scripts and package-like data, with it's files.
     """
 
+    @property
+    def ignore_paths(self):
+        return [
+            "node_modules"
+        ]
+
     @property
     def manifests(self):
         return [

tests/formattedcode/data/csv/packages/expected.csv

@@ -1,4 +1,4 @@
-Resource,type,scan_errors,package__type,package__namespace,package__name,package__version,package__qualifiers,package__subpath,package__primary_language,package__description,package__release_date,package__homepage_url,package__download_url,package__size,package__sha1,package__md5,package__sha256,package__sha512,package__bug_tracking_url,package__code_view_url,package__vcs_url,package__copyright,package__license_expression,package__declared_license,package__notice_text,package__root_path,package__contains_source_code,package__extra_data,package__purl
+path,type,scan_errors,package__type,package__namespace,package__name,package__version,package__qualifiers,package__subpath,package__primary_language,package__description,package__release_date,package__homepage_url,package__download_url,package__size,package__sha1,package__md5,package__sha256,package__sha512,package__bug_tracking_url,package__code_view_url,package__vcs_url,package__copyright,package__license_expression,package__declared_license,package__notice_text,package__root_path,package__contains_source_code,package__extra_data,package__purl
 scan/,directory,,,,,,,,,,,,,,,,,,,,,,,,,,,,
 scan/package.json,file,,,,,,,,,,,,,,,,,,,,,,,,,,,,
 scan/package.json,,,npm,,npm,v 2.13.5,,,JavaScript,a package manager for JavaScript,,https://docs.npmjs.com/,https://registry.npmjs.org/npm/-/npm-2.13.5.tgz,,a124386bce4a90506f28ad4b1d1a804a17baaf32,,,,http://github.com/npm/npm/issues,,git+https://github.com/npm/npm.git@fc7bbf03e39cc48a8924b90696d28345a6a90f3c,,artistic-2.0,Artistic-2.0,,,,,pkg:npm/npm@2.13.5

tests/formattedcode/data/cyclonedx/expected.json

@@ -15,12 +15,7 @@
       "author": "Isaac Z. Schlueter",
       "description": "a package manager for JavaScript",
       "purl": "pkg:npm/npm@2.13.5",
-      "hashes": [
-        {
-          "alg": "SHA-1",
-          "content": "a124386bce4a90506f28ad4b1d1a804a17baaf32"
-        }
-      ],
+      "hashes": [],
       "licenses": [
         {
           "expression": "Artistic-2.0"

tests/formattedcode/data/cyclonedx/expected.xml

@@ -17,9 +17,6 @@
       <author>Isaac Z. Schlueter</author>
       <scope>required</scope>
       <purl>pkg:npm/npm@2.13.5</purl>
-      <hashes>
-        <hash alg="SHA-1">a124386bce4a90506f28ad4b1d1a804a17baaf32</hash>
-      </hashes>
       <licenses>
         <expression>Artistic-2.0</expression>
       </licenses>

…ata/cyclonedx/simple/abbrev/package.json …/simple/node_modules/abbrev/package.jsontests/formattedcode/data/cyclonedx/simple/abbrev/package.json renamed to tests/formattedcode/data/cyclonedx/simple/node_modules/abbrev/package.json tests/formattedcode/data/cyclonedx/simple/abbrev/package.json renamed to tests/formattedcode/data/cyclonedx/simple/node_modules/abbrev/package.json

@@ -15,11 +15,11 @@
       "errors": [],
       "extra_data": {
         "spdx_license_list_version": "3.16",
-        "OUTDATED": "WARNING: Outdated ScanCode Toolkit version! You are using an outdated version of ScanCode Toolkit: 31.0.0 released on: 2021-09-24. A new version is available with important improvements including bug and security fixes, updated license, copyright and package detection, and improved scanning accuracy. Please download and install the latest version of ScanCode. Visit https://github.com/nexB/scancode-toolkit/releases for details.",
         "files_count": 2
       }
     }
   ],
+  "dependencies": [],
   "packages": [
     {
       "type": "npm",
@@ -49,7 +49,7 @@
       "homepage_url": "https://docs.npmjs.com/",
       "download_url": "https://registry.npmjs.org/npm/-/npm-2.13.5.tgz",
       "size": null,
-      "sha1": "a124386bce4a90506f28ad4b1d1a804a17baaf32",
+      "sha1": null,
       "md5": null,
       "sha256": null,
       "sha512": null,
@@ -62,11 +62,19 @@
         "Artistic-2.0 OR MIT"
       ],
       "notice_text": null,
-      "root_path": "scan",
+      "root_path": null,
       "dependencies": [],
       "contains_source_code": null,
       "source_packages": [],
       "extra_data": {},
+      "package_uuid": "pkg:npm/npm@2.13.5",
+      "package_data_paths": [
+        "scan/package.json"
+      ],
+      "files": [
+        "scan/copyr.java",
+        "scan/package.json"
+      ],
       "purl": "pkg:npm/npm@2.13.5",
       "repository_homepage_url": "https://www.npmjs.com/package/npm",
       "repository_download_url": "https://registry.npmjs.org/npm/-/npm-2.13.5.tgz",
@@ -177,7 +185,8 @@
       "licenses": [],
       "license_expressions": [],
       "percentage_of_license_text": 0,
-      "package_manifests": [],
+      "package_data": [],
+      "for_packages": [],
       "scan_errors": []
     },
     {
@@ -308,7 +317,10 @@
         "mit OR bsd-simplified"
       ],
       "percentage_of_license_text": 100.0,
-      "package_manifests": [],
+      "package_data": [],
+      "for_packages": [
+        "pkg:npm/npm@2.13.5"
+      ],
       "scan_errors": []
     },
     {
@@ -358,7 +370,7 @@
         "artistic-2.0"
       ],
       "percentage_of_license_text": 5.0,
-      "package_manifests": [
+      "package_data": [
         {
           "type": "npm",
           "namespace": null,
@@ -400,7 +412,7 @@
             "Artistic-2.0 OR MIT"
           ],
           "notice_text": null,
-          "root_path": "scan",
+          "root_path": null,
           "dependencies": [],
           "contains_source_code": null,
           "source_packages": [],
@@ -411,6 +423,9 @@
           "api_data_url": "https://registry.npmjs.org/npm/2.13.5"
         }
       ],
+      "for_packages": [
+        "pkg:npm/npm@2.13.5"
+      ],
       "scan_errors": []
     }
   ]

tests/licensedcode/data/plugin_licenses_reference/scan.expected.json

@@ -27,5 +27,5 @@ def test_licenses_reference_works():
     run_scan_click(args)
     check_json_scan(
         test_env.get_test_loc('plugin_licenses_reference/scan.expected.json'),
-        result_file, remove_file_date=True, regen=False,
+        result_file, remove_file_date=True, remove_instance_uuid=True, regen=False,
     )