Map unmatched node_modules files

scanpipe/pipelines/deploy_to_develop.py

@@ -54,6 +54,7 @@ def steps(cls):
             cls.map_javascript,
             cls.match_purldb,
             cls.map_javascript_post_purldb_match,
+            cls.map_javascript_npm_lookup,
             cls.map_javascript_path,
             cls.map_javascript_colocation,
             cls.map_path,
@@ -161,6 +162,14 @@ def map_javascript_post_purldb_match(self):
         """Map minified javascript file based on existing PurlDB match."""
         d2d.map_javascript_post_purldb_match(project=self.project, logger=self.log)
 
+    def map_javascript_npm_lookup(self):
+        """Map unmatched ``node_modules`` files."""
+        if not purldb.is_available():
+            self.log("PurlDB is not available. Skipping.")
+            return
+
+        d2d.map_javascript_npm_lookup(project=self.project, logger=self.log)
+
     def map_javascript_path(self):
         """Map javascript file based on path."""
         d2d.map_javascript_path(project=self.project, logger=self.log)

scanpipe/pipes/d2d.py

@@ -989,3 +989,102 @@ def flag_processed_archives(project):
             continue
 
         to_archive.update(status=flag.ARCHIVE_PROCESSED)
+
+
+def map_javascript_npm_lookup(project, logger=None):
+    """Map unmatched ``node_modules`` files."""
+    project_directories = project.codebaseresources.directories()
+    project_files = project.codebaseresources.files()
+
+    to_directories_key = (
+        project_directories.to_codebase()
+        .no_status()
+        .filter(path__regex=r"^.*\/node_modules\/(?!.*\/)")
+        .distinct()
+    )
+
+    to_resources = (
+        project_files.to_codebase()
+        .no_status()
+        .filter(path__regex=r"^.*\/node_modules\/.*$")
+    )
+
+    if not to_directories_key:
+        logger("No unmatched ``node_modules`` file is available. Skipping.")
+        return
+
+    resource_count = to_resources.count()
+
+    if logger:
+        logger(
+            f"Mapping {resource_count:,d} to/ resources using javascript map "
+            f"against from/ codebase"
+        )
+
+    to_resources_index = pathmap.build_index(
+        to_resources.values_list("id", "path"), with_subpaths=True
+    )
+
+    resource_iterator = to_directories_key.iterator(chunk_size=2000)
+    last_percent = 0
+    map_count = 0
+    start_time = timer()
+
+    for resource_index, to_directory in enumerate(resource_iterator):
+        last_percent = pipes.log_progress(
+            logger,
+            resource_index,
+            resource_count,
+            last_percent,
+            increment_percent=10,
+            start_time=start_time,
+        )
+        map_count += _map_javascript_npm_lookup_resource(
+            to_directory,
+            to_resources,
+            to_resources_index,
+            project,
+        )
+
+    logger(f"{map_count:,d} resource(s) mapped")
+
+
+def _map_javascript_npm_lookup_resource(
+    to_directory,
+    to_resources,
+    to_resources_index,
+    project,
+):
+    """Map unmatched ``node_modules`` files."""
+    purl = js.get_purl_from_node_module(to_directory.path)
+    matched = to_resources.filter(path__startswith=to_directory.path)
+    matched_count = matched.count()
+
+    if not matched:
+        return 0
+
+    package = project.discoveredpackages.filter(
+        type=purl.type,
+        namespace="" if not purl.namespace else purl.namespace,
+        name=purl.name,
+        version=purl.version,
+    ).first()
+
+    if package:
+        package.add_resources(matched)
+    else:
+        if results := purldb.fetch_package(purl=str(purl)):
+            package_data = results[0]
+            package_data.pop("uuid", None)
+            package_data.pop("dependencies", None)
+
+            package = pipes.update_or_create_package(
+                project=project,
+                package_data=package_data,
+                codebase_resources=matched,
+            )
+        else:
+            return 0
+
+    matched.update(status=flag.NPM_LOOKUP)
+    return matched_count

scanpipe/pipes/flag.py

@@ -51,6 +51,7 @@
 MATCHED_TO_PURLDB = "matched-to-purldb"
 TOO_MANY_MAPS = "too-many-maps"
 NO_JAVA_SOURCE = "no-java-source"
+NPM_LOOKUP = "npm-lookup"
 
 
 def flag_empty_files(project):

scanpipe/pipes/js.py

@@ -28,6 +28,8 @@
 from django.core.exceptions import MultipleObjectsReturned
 from django.core.exceptions import ObjectDoesNotExist
 
+from packageurl import PackageURL
+
 from scanpipe import pipes
 from scanpipe.models import CodebaseResource
 from scanpipe.pipes import flag
@@ -261,3 +263,21 @@ def map_related_files(to_resources, to_resource, from_resource, map_type, extra_
         match.update(status=flag.MAPPED)
 
     return len(transpiled)
+
+
+def get_purl_from_node_module(node_module_directory):
+    """Return PURL for given a `node_modules` package directory."""
+    path = Path(node_module_directory)
+    path_parts = path.parts
+
+    npm_package = path_parts[-1]
+
+    if "$" in npm_package:
+        _, npm_package = npm_package.split("$")
+
+    # Handle the scoped pacakage.
+    if "%2F" in npm_package:
+        npm_package = npm_package.replace("%2F", "/")
+        npm_package = f"%40{npm_package}"
+
+    return PackageURL.from_string(f"pkg:npm/{npm_package}")

scanpipe/pipes/purldb.py

@@ -127,3 +127,13 @@ def submit_purls(purls, timeout=None, api_url=PURLDB_API_URL):
     )
 
     return response
+
+
+def fetch_package(purl, timeout=None, api_url=PURLDB_API_URL):
+    """Fetch package data for the PURL."""
+    payload = {"purl": purl}
+    response = request_get(url=f"{api_url}packages/", payload=payload, timeout=timeout)
+
+    if response and response.get("count"):
+        results = response["results"]
+        return results

scanpipe/tests/__init__.py

@@ -145,6 +145,55 @@ def make_resource_file(project, path, **extra):
     "version": "3.119",
 }
 
+package_data3 = {
+    "type": "npm",
+    "namespace": "",
+    "name": "luxon",
+    "version": "1.27.0",
+    "qualifiers": "",
+    "subpath": "",
+    "filename": "luxon-1.27.0.tgz",
+    "package_content": "source_archive",
+    "purl": "pkg:npm/luxon@1.27.0",
+    "primary_language": "JavaScript",
+    "description": "Immutable date wrapper",
+    "release_date": None,
+    "parties": [
+        {
+            "type": "person",
+            "role": "author",
+            "name": "Isaac Cambron",
+            "email": None,
+            "url": None,
+        },
+        {
+            "type": "person",
+            "role": "maintainer",
+            "name": "icambron",
+            "email": "icambron@gmail.com",
+            "url": None,
+        },
+    ],
+    "keywords": ["date", "immutable"],
+    "homepage_url": "https://github.com/moment/luxon#readme",
+    "download_url": "https://registry.npmjs.org/luxon/-/luxon-1.27.0.tgz",
+    "bug_tracking_url": "https://github.com/moment/luxon/issues",
+    "code_view_url": None,
+    "vcs_url": "https://packages.vcs.url",
+    "repository_homepage_url": None,
+    "repository_download_url": None,
+    "api_data_url": None,
+    "size": None,
+    "md5": None,
+    "sha1": "ae10c69113d85dab8f15f5e8390d0cbeddf4f00f",
+    "sha256": None,
+    "sha512": None,
+    "copyright": "Copyright (c) JS Foundation and other contributors",
+    "holder": None,
+    "declared_license_expression": "mit",
+    "declared_license_expression_spdx": "MIT",
+}
+
 for_package_uid = "pkg:deb/debian/adduser@3.118?uuid=610bed29-ce39-40e7-92d6-fd8b"
 
 dependency_data1 = {

scanpipe/tests/pipes/test_d2d.py

@@ -37,6 +37,7 @@
 from scanpipe.pipes.input import copy_inputs
 from scanpipe.tests import make_resource_file
 from scanpipe.tests import package_data1
+from scanpipe.tests import package_data3
 
 
 class ScanPipeD2DPipesTest(TestCase):
@@ -817,3 +818,46 @@ def test_scanpipe_pipes_d2d_map_javascript_colocation(self):
 
         self.assertIn(expected, buffer.getvalue())
         self.assertEqual(from_expected, relation[0].from_resource)
+
+    def test_scanpipe_pipes_d2d_map_javascript_npm_lookup(self):
+        to_map = self.data_location / "d2d-javascript" / "to" / "main.js.map"
+        to_mini = self.data_location / "d2d-javascript" / "to" / "main.js"
+        to_dir = (
+            self.project1.codebase_path
+            / "to/project.tar.zst/modules/apps/adaptive-media/"
+            "adaptive-media-web/src/main/resources/META-INF/resources/"
+            "node_modules/@adaptive-media-web$luxon@1.27.0"
+        )
+
+        to_dir.mkdir(parents=True)
+        copy_input(to_mini, to_dir)
+        copy_input(to_map, to_dir)
+
+        d2d.collect_and_create_codebase_resources(self.project1)
+
+        to_map_resource = self.project1.codebaseresources.get(
+            path=(
+                "to/project.tar.zst/modules/apps/adaptive-media/"
+                "adaptive-media-web/src/main/resources/META-INF/resources/"
+                "node_modules/@adaptive-media-web$luxon@1.27.0/main.js.map"
+            )
+        )
+
+        package_data = package_data3.copy()
+        package_data["uuid"] = uuid.uuid4()
+
+        d2d.create_package_from_purldb_data(
+            self.project1, to_map_resource, package_data
+        )
+
+        buffer = io.StringIO()
+        d2d.map_javascript_npm_lookup(
+            self.project1,
+            logger=buffer.write,
+        )
+        expected = "Mapping 1 to/ resources using javascript map against from/ codebase"
+        self.assertIn(expected, buffer.getvalue())
+
+        result = self.project1.codebaseresources.filter(status=flag.NPM_LOOKUP).count()
+
+        self.assertEqual(1, result)

scanpipe/tests/pipes/test_js.py

@@ -319,3 +319,20 @@ def test_scanpipe_pipes_js_get_basename_and_extension(self):
         for ext in js._js_extensions:
             basename, extension = js.get_js_map_basename_and_extension(f"file{ext}")
             self.assertEqual(("file", ext), (basename, extension))
+
+    def test_scanpipe_pipes_js_get_purl_from_node_module(self):
+        node_module_directory1 = "atls!template-util$codemirror@5.65.2/"
+        expected1 = "pkg:npm/codemirror@5.65.2"
+        result1 = str(js.get_purl_from_node_module(node_module_directory1))
+
+        node_module_directory2 = "@atls!sig-local$babel%2Fruntime@7.17.9/"
+        expected2 = "pkg:npm/%40babel/runtime@7.17.9"
+        result2 = str(js.get_purl_from_node_module(node_module_directory2))
+
+        node_module_directory3 = "codemirror@5.65.2/"
+        expected3 = "pkg:npm/codemirror@5.65.2"
+        result3 = str(js.get_purl_from_node_module(node_module_directory3))
+
+        self.assertEqual(expected1, result1)
+        self.assertEqual(expected2, result2)
+        self.assertEqual(expected3, result3)