Map unmatched node_modules files

Signed-off-by: Keshav Priyadarshi <git@keshav.space>

Author: keshav-space

scanpipe/pipelines/deploy_to_develop.py

@@ -54,6 +54,7 @@ def steps(cls):
             cls.map_javascript,
             cls.match_purldb,
             cls.map_javascript_post_purldb_match,
+            cls.map_javascript_npm_lookup,
             cls.map_javascript_path,
             cls.map_path,
             cls.flag_mapped_resources_and_ignored_directories,
@@ -160,6 +161,14 @@ def map_javascript_post_purldb_match(self):
         """Map minified javascript file based on existing PurlDB match."""
         d2d.map_javascript_post_purldb_match(project=self.project, logger=self.log)
 
+    def map_javascript_npm_lookup(self):
+        """Map unmatched ``node_modules`` files."""
+        if not purldb.is_available():
+            self.log("PurlDB is not available. Skipping.")
+            return
+
+        d2d.map_javascript_npm_lookup(project=self.project, logger=self.log)
+
     def map_javascript_path(self):
         """Map javascript file based on path."""
         d2d.map_javascript_path(project=self.project, logger=self.log)

scanpipe/pipes/d2d.py

@@ -29,6 +29,7 @@
 from scanpipe import pipes
 from scanpipe.models import CodebaseRelation
 from scanpipe.models import CodebaseResource
+from scanpipe.models import DiscoveredPackage
 from scanpipe.pipes import flag
 from scanpipe.pipes import get_resource_diff_ratio
 from scanpipe.pipes import js
@@ -840,3 +841,100 @@ def _map_javascript_path_resource(
             extra_data=extra_data,
         )
     return len(transpiled)
+
+
+def map_javascript_npm_lookup(project, logger=None):
+    """Map unmatched ``node_modules`` files."""
+    project_directories = project.codebaseresources.directories().only("path")
+    project_files = project.codebaseresources.files().only("path")
+
+    to_directories_key = (
+        project_directories.to_codebase()
+        .no_status()
+        .filter(path__regex=r"^.*\/node_modules\/(?!.*\/)")
+        .distinct()
+    )
+
+    to_resources = (
+        project_files.to_codebase()
+        .no_status()
+        .filter(path__regex=r"^.*\/node_modules\/.*$")
+    )
+
+    if not to_directories_key:
+        logger("No unmatched ``node_modules`` file is available. Skipping.")
+        return
+
+    resource_count = to_resources.count()
+
+    if logger:
+        logger(
+            f"Mapping {resource_count:,d} to/ resources using javascript map "
+            f"against from/ codebase"
+        )
+
+    to_resources_index = pathmap.build_index(
+        to_resources.values_list("id", "path"), with_subpaths=True
+    )
+
+    resource_iterator = to_directories_key.iterator(chunk_size=2000)
+    last_percent = 0
+    map_count = 0
+    start_time = timer()
+
+    for resource_index, to_directory in enumerate(resource_iterator):
+        last_percent = pipes.log_progress(
+            logger,
+            resource_index,
+            resource_count,
+            last_percent,
+            increment_percent=10,
+            start_time=start_time,
+        )
+        map_count += _map_javascript_npm_lookup_resource(
+            to_directory,
+            to_resources,
+            to_resources_index,
+            project,
+        )
+
+    logger(f"{map_count:,d} resource(s) mapped")
+
+
+def _map_javascript_npm_lookup_resource(
+    to_directory,
+    to_resources,
+    to_resources_index,
+    project,
+):
+    """Map unmatched ``node_modules`` files."""
+    purl = js.get_purl_from_node_module(to_directory)
+    matched = to_resources.filter(path__startswith=to_directory.path)
+    if not matched:
+        return 0
+
+    try:
+        package = project.discoveredpackages.get(
+            type=purl.type,
+            namespace="" if not purl.namespace else purl.namespace,
+            name=purl.name,
+            version=purl.version,
+        )
+        package.add_resources(matched)
+
+    except DiscoveredPackage.DoesNotExist:
+        if results := purldb.fetch_package(purl=str(purl)):
+            package_data = results[0]
+            package_data.pop("uuid", None)
+            package_data.pop("dependencies", None)
+
+            package = pipes.update_or_create_package(
+                project=project,
+                package_data=package_data,
+                codebase_resources=matched,
+            )
+        else:
+            return 0
+
+    matched.update(status=flag.NPM_LOOKUP)
+    return len(matched)

scanpipe/pipes/flag.py

@@ -49,6 +49,7 @@
 MATCHED_TO_PURLDB = "matched-to-purldb"
 TOO_MANY_MAPS = "too-many-maps"
 NO_JAVA_SOURCE = "no-java-source"
+NPM_LOOKUP = "npm-lookup"
 
 
 def flag_empty_files(project):

scanpipe/pipes/js.py

@@ -24,6 +24,8 @@
 import json
 from pathlib import Path
 
+from packageurl import PackageURL
+
 from scanpipe.pipes import get_text_str_diff_ratio
 from scanpipe.pipes import pathmap
 
@@ -204,3 +206,21 @@ def get_js_map_basename_and_extension(filename):
         if filename.endswith(ext):
             basename = filename[: -len(ext)]
             return basename, ext
+
+
+def get_purl_from_node_module(node_module_directory):
+    """Return PURL for given a `node_modules` package directory."""
+    path = Path(node_module_directory.path)
+    path_parts = path.parts
+
+    npm_package = path_parts[-1]
+
+    if "$" in npm_package:
+        _, npm_package = npm_package.split("$")
+
+    # Handle the scoped pacakage.
+    if "%2F" in npm_package:
+        npm_package = npm_package.replace("%2F", "/")
+        npm_package = f"%40{npm_package}"
+
+    return PackageURL.from_string(f"pkg:npm/{npm_package}")

scanpipe/pipes/purldb.py

@@ -127,3 +127,13 @@ def submit_purls(purls, timeout=None, api_url=PURLDB_API_URL):
     )
 
     return response
+
+
+def fetch_package(purl, timeout=None, api_url=PURLDB_API_URL):
+    """Fetch package data for the PURL."""
+    payload = {"purl": purl}
+    response = request_get(url=f"{api_url}packages/", payload=payload, timeout=timeout)
+
+    if response and response.get("count"):
+        results = response["results"]
+        return results