diff --git a/scanpipe/pipelines/deploy_to_develop.py b/scanpipe/pipelines/deploy_to_develop.py index d075f74325..bac13ad44d 100644 --- a/scanpipe/pipelines/deploy_to_develop.py +++ b/scanpipe/pipelines/deploy_to_develop.py @@ -54,6 +54,7 @@ def steps(cls): cls.map_javascript, cls.match_purldb, cls.map_javascript_post_purldb_match, + cls.map_javascript_npm_lookup, cls.map_javascript_path, cls.map_javascript_colocation, cls.map_path, @@ -161,6 +162,14 @@ def map_javascript_post_purldb_match(self): """Map minified javascript file based on existing PurlDB match.""" d2d.map_javascript_post_purldb_match(project=self.project, logger=self.log) + def map_javascript_npm_lookup(self): + """Map unmatched ``node_modules`` files.""" + if not purldb.is_available(): + self.log("PurlDB is not available. Skipping.") + return + + d2d.map_javascript_npm_lookup(project=self.project, logger=self.log) + def map_javascript_path(self): """Map javascript file based on path.""" d2d.map_javascript_path(project=self.project, logger=self.log) diff --git a/scanpipe/pipes/d2d.py b/scanpipe/pipes/d2d.py index 886a476040..d2e00b2c38 100644 --- a/scanpipe/pipes/d2d.py +++ b/scanpipe/pipes/d2d.py @@ -989,3 +989,102 @@ def flag_processed_archives(project): continue to_archive.update(status=flag.ARCHIVE_PROCESSED) + + +def map_javascript_npm_lookup(project, logger=None): + """Map unmatched ``node_modules`` files.""" + project_directories = project.codebaseresources.directories() + project_files = project.codebaseresources.files() + + to_directories_key = ( + project_directories.to_codebase() + .no_status() + .filter(path__regex=r"^.*\/node_modules\/(?!.*\/)") + .distinct() + ) + + to_resources = ( + project_files.to_codebase() + .no_status() + .filter(path__regex=r"^.*\/node_modules\/.*$") + ) + + if not to_directories_key: + logger("No unmatched ``node_modules`` file is available. Skipping.") + return + + resource_count = to_resources.count() + + if logger: + logger( + f"Mapping {resource_count:,d} to/ resources using javascript map " + f"against from/ codebase" + ) + + to_resources_index = pathmap.build_index( + to_resources.values_list("id", "path"), with_subpaths=True + ) + + resource_iterator = to_directories_key.iterator(chunk_size=2000) + last_percent = 0 + map_count = 0 + start_time = timer() + + for resource_index, to_directory in enumerate(resource_iterator): + last_percent = pipes.log_progress( + logger, + resource_index, + resource_count, + last_percent, + increment_percent=10, + start_time=start_time, + ) + map_count += _map_javascript_npm_lookup_resource( + to_directory, + to_resources, + to_resources_index, + project, + ) + + logger(f"{map_count:,d} resource(s) mapped") + + +def _map_javascript_npm_lookup_resource( + to_directory, + to_resources, + to_resources_index, + project, +): + """Map unmatched ``node_modules`` files.""" + purl = js.get_purl_from_node_module(to_directory.path) + matched = to_resources.filter(path__startswith=to_directory.path) + matched_count = matched.count() + + if not matched: + return 0 + + package = project.discoveredpackages.filter( + type=purl.type, + namespace="" if not purl.namespace else purl.namespace, + name=purl.name, + version=purl.version, + ).first() + + if package: + package.add_resources(matched) + else: + if results := purldb.fetch_package(purl=str(purl)): + package_data = results[0] + package_data.pop("uuid", None) + package_data.pop("dependencies", None) + + package = pipes.update_or_create_package( + project=project, + package_data=package_data, + codebase_resources=matched, + ) + else: + return 0 + + matched.update(status=flag.NPM_LOOKUP) + return matched_count diff --git a/scanpipe/pipes/flag.py b/scanpipe/pipes/flag.py index 3fd741b9ac..2c2ceee6b2 100644 --- a/scanpipe/pipes/flag.py +++ b/scanpipe/pipes/flag.py @@ -51,6 +51,7 @@ MATCHED_TO_PURLDB = "matched-to-purldb" TOO_MANY_MAPS = "too-many-maps" NO_JAVA_SOURCE = "no-java-source" +NPM_LOOKUP = "npm-lookup" def flag_empty_files(project): diff --git a/scanpipe/pipes/js.py b/scanpipe/pipes/js.py index 49b6d71333..c5d7926f59 100644 --- a/scanpipe/pipes/js.py +++ b/scanpipe/pipes/js.py @@ -28,6 +28,8 @@ from django.core.exceptions import MultipleObjectsReturned from django.core.exceptions import ObjectDoesNotExist +from packageurl import PackageURL + from scanpipe import pipes from scanpipe.models import CodebaseResource from scanpipe.pipes import flag @@ -261,3 +263,21 @@ def map_related_files(to_resources, to_resource, from_resource, map_type, extra_ match.update(status=flag.MAPPED) return len(transpiled) + + +def get_purl_from_node_module(node_module_directory): + """Return PURL for given a `node_modules` package directory.""" + path = Path(node_module_directory) + path_parts = path.parts + + npm_package = path_parts[-1] + + if "$" in npm_package: + _, npm_package = npm_package.split("$") + + # Handle the scoped pacakage. + if "%2F" in npm_package: + npm_package = npm_package.replace("%2F", "/") + npm_package = f"%40{npm_package}" + + return PackageURL.from_string(f"pkg:npm/{npm_package}") diff --git a/scanpipe/pipes/purldb.py b/scanpipe/pipes/purldb.py index 31dc1ec241..e78ef88203 100644 --- a/scanpipe/pipes/purldb.py +++ b/scanpipe/pipes/purldb.py @@ -127,3 +127,13 @@ def submit_purls(purls, timeout=None, api_url=PURLDB_API_URL): ) return response + + +def fetch_package(purl, timeout=None, api_url=PURLDB_API_URL): + """Fetch package data for the PURL.""" + payload = {"purl": purl} + response = request_get(url=f"{api_url}packages/", payload=payload, timeout=timeout) + + if response and response.get("count"): + results = response["results"] + return results diff --git a/scanpipe/tests/__init__.py b/scanpipe/tests/__init__.py index 644d0cc37a..68b3ef2eaf 100644 --- a/scanpipe/tests/__init__.py +++ b/scanpipe/tests/__init__.py @@ -145,6 +145,55 @@ def make_resource_file(project, path, **extra): "version": "3.119", } +package_data3 = { + "type": "npm", + "namespace": "", + "name": "luxon", + "version": "1.27.0", + "qualifiers": "", + "subpath": "", + "filename": "luxon-1.27.0.tgz", + "package_content": "source_archive", + "purl": "pkg:npm/luxon@1.27.0", + "primary_language": "JavaScript", + "description": "Immutable date wrapper", + "release_date": None, + "parties": [ + { + "type": "person", + "role": "author", + "name": "Isaac Cambron", + "email": None, + "url": None, + }, + { + "type": "person", + "role": "maintainer", + "name": "icambron", + "email": "icambron@gmail.com", + "url": None, + }, + ], + "keywords": ["date", "immutable"], + "homepage_url": "https://github.com/moment/luxon#readme", + "download_url": "https://registry.npmjs.org/luxon/-/luxon-1.27.0.tgz", + "bug_tracking_url": "https://github.com/moment/luxon/issues", + "code_view_url": None, + "vcs_url": "https://packages.vcs.url", + "repository_homepage_url": None, + "repository_download_url": None, + "api_data_url": None, + "size": None, + "md5": None, + "sha1": "ae10c69113d85dab8f15f5e8390d0cbeddf4f00f", + "sha256": None, + "sha512": None, + "copyright": "Copyright (c) JS Foundation and other contributors", + "holder": None, + "declared_license_expression": "mit", + "declared_license_expression_spdx": "MIT", +} + for_package_uid = "pkg:deb/debian/adduser@3.118?uuid=610bed29-ce39-40e7-92d6-fd8b" dependency_data1 = { diff --git a/scanpipe/tests/pipes/test_d2d.py b/scanpipe/tests/pipes/test_d2d.py index 338c0b36dd..fcc7dd031e 100644 --- a/scanpipe/tests/pipes/test_d2d.py +++ b/scanpipe/tests/pipes/test_d2d.py @@ -37,6 +37,7 @@ from scanpipe.pipes.input import copy_inputs from scanpipe.tests import make_resource_file from scanpipe.tests import package_data1 +from scanpipe.tests import package_data3 class ScanPipeD2DPipesTest(TestCase): @@ -817,3 +818,46 @@ def test_scanpipe_pipes_d2d_map_javascript_colocation(self): self.assertIn(expected, buffer.getvalue()) self.assertEqual(from_expected, relation[0].from_resource) + + def test_scanpipe_pipes_d2d_map_javascript_npm_lookup(self): + to_map = self.data_location / "d2d-javascript" / "to" / "main.js.map" + to_mini = self.data_location / "d2d-javascript" / "to" / "main.js" + to_dir = ( + self.project1.codebase_path + / "to/project.tar.zst/modules/apps/adaptive-media/" + "adaptive-media-web/src/main/resources/META-INF/resources/" + "node_modules/@adaptive-media-web$luxon@1.27.0" + ) + + to_dir.mkdir(parents=True) + copy_input(to_mini, to_dir) + copy_input(to_map, to_dir) + + d2d.collect_and_create_codebase_resources(self.project1) + + to_map_resource = self.project1.codebaseresources.get( + path=( + "to/project.tar.zst/modules/apps/adaptive-media/" + "adaptive-media-web/src/main/resources/META-INF/resources/" + "node_modules/@adaptive-media-web$luxon@1.27.0/main.js.map" + ) + ) + + package_data = package_data3.copy() + package_data["uuid"] = uuid.uuid4() + + d2d.create_package_from_purldb_data( + self.project1, to_map_resource, package_data + ) + + buffer = io.StringIO() + d2d.map_javascript_npm_lookup( + self.project1, + logger=buffer.write, + ) + expected = "Mapping 1 to/ resources using javascript map against from/ codebase" + self.assertIn(expected, buffer.getvalue()) + + result = self.project1.codebaseresources.filter(status=flag.NPM_LOOKUP).count() + + self.assertEqual(1, result) diff --git a/scanpipe/tests/pipes/test_js.py b/scanpipe/tests/pipes/test_js.py index 4367f45dd9..7f06c6579c 100644 --- a/scanpipe/tests/pipes/test_js.py +++ b/scanpipe/tests/pipes/test_js.py @@ -319,3 +319,20 @@ def test_scanpipe_pipes_js_get_basename_and_extension(self): for ext in js._js_extensions: basename, extension = js.get_js_map_basename_and_extension(f"file{ext}") self.assertEqual(("file", ext), (basename, extension)) + + def test_scanpipe_pipes_js_get_purl_from_node_module(self): + node_module_directory1 = "atls!template-util$codemirror@5.65.2/" + expected1 = "pkg:npm/codemirror@5.65.2" + result1 = str(js.get_purl_from_node_module(node_module_directory1)) + + node_module_directory2 = "@atls!sig-local$babel%2Fruntime@7.17.9/" + expected2 = "pkg:npm/%40babel/runtime@7.17.9" + result2 = str(js.get_purl_from_node_module(node_module_directory2)) + + node_module_directory3 = "codemirror@5.65.2/" + expected3 = "pkg:npm/codemirror@5.65.2" + result3 = str(js.get_purl_from_node_module(node_module_directory3)) + + self.assertEqual(expected1, result1) + self.assertEqual(expected2, result2) + self.assertEqual(expected3, result3)