From 9a313742af3c99f0d103f8c1c669f590d0294287 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Tue, 4 Jul 2023 01:16:52 +0530 Subject: [PATCH 1/5] Map unmatched `node_modules` files Signed-off-by: Keshav Priyadarshi --- scanpipe/pipelines/deploy_to_develop.py | 9 +++ scanpipe/pipes/d2d.py | 98 +++++++++++++++++++++++++ scanpipe/pipes/flag.py | 1 + scanpipe/pipes/js.py | 20 +++++ scanpipe/pipes/purldb.py | 10 +++ 5 files changed, 138 insertions(+) diff --git a/scanpipe/pipelines/deploy_to_develop.py b/scanpipe/pipelines/deploy_to_develop.py index 9a19ac7ef9..b6ebcaff8f 100644 --- a/scanpipe/pipelines/deploy_to_develop.py +++ b/scanpipe/pipelines/deploy_to_develop.py @@ -54,6 +54,7 @@ def steps(cls): cls.map_javascript, cls.match_purldb, cls.map_javascript_post_purldb_match, + cls.map_javascript_npm_lookup, cls.map_javascript_path, cls.map_path, cls.flag_mapped_resources_and_ignored_directories, @@ -160,6 +161,14 @@ def map_javascript_post_purldb_match(self): """Map minified javascript file based on existing PurlDB match.""" d2d.map_javascript_post_purldb_match(project=self.project, logger=self.log) + def map_javascript_npm_lookup(self): + """Map unmatched ``node_modules`` files.""" + if not purldb.is_available(): + self.log("PurlDB is not available. Skipping.") + return + + d2d.map_javascript_npm_lookup(project=self.project, logger=self.log) + def map_javascript_path(self): """Map javascript file based on path.""" d2d.map_javascript_path(project=self.project, logger=self.log) diff --git a/scanpipe/pipes/d2d.py b/scanpipe/pipes/d2d.py index a762ad7f36..adf63c98cf 100644 --- a/scanpipe/pipes/d2d.py +++ b/scanpipe/pipes/d2d.py @@ -29,6 +29,7 @@ from scanpipe import pipes from scanpipe.models import CodebaseRelation from scanpipe.models import CodebaseResource +from scanpipe.models import DiscoveredPackage from scanpipe.pipes import flag from scanpipe.pipes import get_resource_diff_ratio from scanpipe.pipes import js @@ -840,3 +841,100 @@ def _map_javascript_path_resource( extra_data=extra_data, ) return len(transpiled) + + +def map_javascript_npm_lookup(project, logger=None): + """Map unmatched ``node_modules`` files.""" + project_directories = project.codebaseresources.directories().only("path") + project_files = project.codebaseresources.files().only("path") + + to_directories_key = ( + project_directories.to_codebase() + .no_status() + .filter(path__regex=r"^.*\/node_modules\/(?!.*\/)") + .distinct() + ) + + to_resources = ( + project_files.to_codebase() + .no_status() + .filter(path__regex=r"^.*\/node_modules\/.*$") + ) + + if not to_directories_key: + logger("No unmatched ``node_modules`` file is available. Skipping.") + return + + resource_count = to_resources.count() + + if logger: + logger( + f"Mapping {resource_count:,d} to/ resources using javascript map " + f"against from/ codebase" + ) + + to_resources_index = pathmap.build_index( + to_resources.values_list("id", "path"), with_subpaths=True + ) + + resource_iterator = to_directories_key.iterator(chunk_size=2000) + last_percent = 0 + map_count = 0 + start_time = timer() + + for resource_index, to_directory in enumerate(resource_iterator): + last_percent = pipes.log_progress( + logger, + resource_index, + resource_count, + last_percent, + increment_percent=10, + start_time=start_time, + ) + map_count += _map_javascript_npm_lookup_resource( + to_directory, + to_resources, + to_resources_index, + project, + ) + + logger(f"{map_count:,d} resource(s) mapped") + + +def _map_javascript_npm_lookup_resource( + to_directory, + to_resources, + to_resources_index, + project, +): + """Map unmatched ``node_modules`` files.""" + purl = js.get_purl_from_node_module(to_directory) + matched = to_resources.filter(path__startswith=to_directory.path) + if not matched: + return 0 + + try: + package = project.discoveredpackages.get( + type=purl.type, + namespace="" if not purl.namespace else purl.namespace, + name=purl.name, + version=purl.version, + ) + package.add_resources(matched) + + except DiscoveredPackage.DoesNotExist: + if results := purldb.fetch_package(purl=str(purl)): + package_data = results[0] + package_data.pop("uuid", None) + package_data.pop("dependencies", None) + + package = pipes.update_or_create_package( + project=project, + package_data=package_data, + codebase_resources=matched, + ) + else: + return 0 + + matched.update(status=flag.NPM_LOOKUP) + return len(matched) diff --git a/scanpipe/pipes/flag.py b/scanpipe/pipes/flag.py index f16cf12e90..60123ea47f 100644 --- a/scanpipe/pipes/flag.py +++ b/scanpipe/pipes/flag.py @@ -49,6 +49,7 @@ MATCHED_TO_PURLDB = "matched-to-purldb" TOO_MANY_MAPS = "too-many-maps" NO_JAVA_SOURCE = "no-java-source" +NPM_LOOKUP = "npm-lookup" def flag_empty_files(project): diff --git a/scanpipe/pipes/js.py b/scanpipe/pipes/js.py index e5b1ee873e..397852338c 100644 --- a/scanpipe/pipes/js.py +++ b/scanpipe/pipes/js.py @@ -24,6 +24,8 @@ import json from pathlib import Path +from packageurl import PackageURL + from scanpipe.pipes import get_text_str_diff_ratio from scanpipe.pipes import pathmap @@ -204,3 +206,21 @@ def get_js_map_basename_and_extension(filename): if filename.endswith(ext): basename = filename[: -len(ext)] return basename, ext + + +def get_purl_from_node_module(node_module_directory): + """Return PURL for given a `node_modules` package directory.""" + path = Path(node_module_directory.path) + path_parts = path.parts + + npm_package = path_parts[-1] + + if "$" in npm_package: + _, npm_package = npm_package.split("$") + + # Handle the scoped pacakage. + if "%2F" in npm_package: + npm_package = npm_package.replace("%2F", "/") + npm_package = f"%40{npm_package}" + + return PackageURL.from_string(f"pkg:npm/{npm_package}") diff --git a/scanpipe/pipes/purldb.py b/scanpipe/pipes/purldb.py index 31dc1ec241..e78ef88203 100644 --- a/scanpipe/pipes/purldb.py +++ b/scanpipe/pipes/purldb.py @@ -127,3 +127,13 @@ def submit_purls(purls, timeout=None, api_url=PURLDB_API_URL): ) return response + + +def fetch_package(purl, timeout=None, api_url=PURLDB_API_URL): + """Fetch package data for the PURL.""" + payload = {"purl": purl} + response = request_get(url=f"{api_url}packages/", payload=payload, timeout=timeout) + + if response and response.get("count"): + results = response["results"] + return results From f29a406262f30e6348fc95be7b2233d0e104e545 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Tue, 4 Jul 2023 02:57:20 +0530 Subject: [PATCH 2/5] Add test for `map_javascript_npm_lookup` Signed-off-by: Keshav Priyadarshi --- scanpipe/tests/__init__.py | 49 ++++++++++++++++++++++++++++++++ scanpipe/tests/pipes/test_d2d.py | 45 +++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+) diff --git a/scanpipe/tests/__init__.py b/scanpipe/tests/__init__.py index 644d0cc37a..68b3ef2eaf 100644 --- a/scanpipe/tests/__init__.py +++ b/scanpipe/tests/__init__.py @@ -145,6 +145,55 @@ def make_resource_file(project, path, **extra): "version": "3.119", } +package_data3 = { + "type": "npm", + "namespace": "", + "name": "luxon", + "version": "1.27.0", + "qualifiers": "", + "subpath": "", + "filename": "luxon-1.27.0.tgz", + "package_content": "source_archive", + "purl": "pkg:npm/luxon@1.27.0", + "primary_language": "JavaScript", + "description": "Immutable date wrapper", + "release_date": None, + "parties": [ + { + "type": "person", + "role": "author", + "name": "Isaac Cambron", + "email": None, + "url": None, + }, + { + "type": "person", + "role": "maintainer", + "name": "icambron", + "email": "icambron@gmail.com", + "url": None, + }, + ], + "keywords": ["date", "immutable"], + "homepage_url": "https://github.com/moment/luxon#readme", + "download_url": "https://registry.npmjs.org/luxon/-/luxon-1.27.0.tgz", + "bug_tracking_url": "https://github.com/moment/luxon/issues", + "code_view_url": None, + "vcs_url": "https://packages.vcs.url", + "repository_homepage_url": None, + "repository_download_url": None, + "api_data_url": None, + "size": None, + "md5": None, + "sha1": "ae10c69113d85dab8f15f5e8390d0cbeddf4f00f", + "sha256": None, + "sha512": None, + "copyright": "Copyright (c) JS Foundation and other contributors", + "holder": None, + "declared_license_expression": "mit", + "declared_license_expression_spdx": "MIT", +} + for_package_uid = "pkg:deb/debian/adduser@3.118?uuid=610bed29-ce39-40e7-92d6-fd8b" dependency_data1 = { diff --git a/scanpipe/tests/pipes/test_d2d.py b/scanpipe/tests/pipes/test_d2d.py index 263d48e215..93b2a8a72d 100644 --- a/scanpipe/tests/pipes/test_d2d.py +++ b/scanpipe/tests/pipes/test_d2d.py @@ -31,10 +31,12 @@ from scanpipe.models import CodebaseResource from scanpipe.models import Project from scanpipe.pipes import d2d +from scanpipe.pipes import flag from scanpipe.pipes.input import copy_input from scanpipe.pipes.input import copy_inputs from scanpipe.tests import make_resource_file from scanpipe.tests import package_data1 +from scanpipe.tests import package_data3 class ScanPipeD2DPipesTest(TestCase): @@ -662,3 +664,46 @@ def test_scanpipe_pipes_d2d_map_javascript_path(self): relation = self.project1.codebaserelations.all() self.assertEqual(from_resource, relation[0].from_resource) self.assertEqual(from_resource, relation[1].from_resource) + + def test_scanpipe_pipes_d2d_map_javascript_npm_lookup(self): + to_map = self.data_location / "d2d-javascript" / "to" / "main.js.map" + to_mini = self.data_location / "d2d-javascript" / "to" / "main.js" + to_dir = ( + self.project1.codebase_path + / "to/project.tar.zst/modules/apps/adaptive-media/" + "adaptive-media-web/src/main/resources/META-INF/resources/" + "node_modules/@adaptive-media-web$luxon@1.27.0" + ) + + to_dir.mkdir(parents=True) + copy_input(to_mini, to_dir) + copy_input(to_map, to_dir) + + d2d.collect_and_create_codebase_resources(self.project1) + + to_map_resource = self.project1.codebaseresources.get( + path=( + "to/project.tar.zst/modules/apps/adaptive-media/" + "adaptive-media-web/src/main/resources/META-INF/resources/" + "node_modules/@adaptive-media-web$luxon@1.27.0/main.js.map" + ) + ) + + package_data = package_data3.copy() + package_data["uuid"] = uuid.uuid4() + + d2d.create_package_from_purldb_data( + self.project1, to_map_resource, package_data + ) + + buffer = io.StringIO() + d2d.map_javascript_npm_lookup( + self.project1, + logger=buffer.write, + ) + expected = "Mapping 1 to/ resources using javascript map against from/ codebase" + self.assertIn(expected, buffer.getvalue()) + + result = self.project1.codebaseresources.filter(status=flag.NPM_LOOKUP).count() + + self.assertEqual(1, result) From 745140b18fce43932d19256ef7231ca2611cb172 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Mon, 31 Jul 2023 18:37:37 +0530 Subject: [PATCH 3/5] Add test for get_purl_from_node_module Signed-off-by: Keshav Priyadarshi --- scanpipe/pipes/d2d.py | 2 +- scanpipe/pipes/js.py | 2 +- scanpipe/tests/pipes/test_js.py | 17 +++++++++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/scanpipe/pipes/d2d.py b/scanpipe/pipes/d2d.py index adf63c98cf..1d7245b868 100644 --- a/scanpipe/pipes/d2d.py +++ b/scanpipe/pipes/d2d.py @@ -908,7 +908,7 @@ def _map_javascript_npm_lookup_resource( project, ): """Map unmatched ``node_modules`` files.""" - purl = js.get_purl_from_node_module(to_directory) + purl = js.get_purl_from_node_module(to_directory.path) matched = to_resources.filter(path__startswith=to_directory.path) if not matched: return 0 diff --git a/scanpipe/pipes/js.py b/scanpipe/pipes/js.py index 397852338c..4581e79da5 100644 --- a/scanpipe/pipes/js.py +++ b/scanpipe/pipes/js.py @@ -210,7 +210,7 @@ def get_js_map_basename_and_extension(filename): def get_purl_from_node_module(node_module_directory): """Return PURL for given a `node_modules` package directory.""" - path = Path(node_module_directory.path) + path = Path(node_module_directory) path_parts = path.parts npm_package = path_parts[-1] diff --git a/scanpipe/tests/pipes/test_js.py b/scanpipe/tests/pipes/test_js.py index 4367f45dd9..7f06c6579c 100644 --- a/scanpipe/tests/pipes/test_js.py +++ b/scanpipe/tests/pipes/test_js.py @@ -319,3 +319,20 @@ def test_scanpipe_pipes_js_get_basename_and_extension(self): for ext in js._js_extensions: basename, extension = js.get_js_map_basename_and_extension(f"file{ext}") self.assertEqual(("file", ext), (basename, extension)) + + def test_scanpipe_pipes_js_get_purl_from_node_module(self): + node_module_directory1 = "atls!template-util$codemirror@5.65.2/" + expected1 = "pkg:npm/codemirror@5.65.2" + result1 = str(js.get_purl_from_node_module(node_module_directory1)) + + node_module_directory2 = "@atls!sig-local$babel%2Fruntime@7.17.9/" + expected2 = "pkg:npm/%40babel/runtime@7.17.9" + result2 = str(js.get_purl_from_node_module(node_module_directory2)) + + node_module_directory3 = "codemirror@5.65.2/" + expected3 = "pkg:npm/codemirror@5.65.2" + result3 = str(js.get_purl_from_node_module(node_module_directory3)) + + self.assertEqual(expected1, result1) + self.assertEqual(expected2, result2) + self.assertEqual(expected3, result3) From adfe55a6afe21fb70195e151474bb393658d1959 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Wed, 2 Aug 2023 20:17:35 +0530 Subject: [PATCH 4/5] Handle multiple discoveredpackages with the same PURL Signed-off-by: Keshav Priyadarshi --- scanpipe/pipes/d2d.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/scanpipe/pipes/d2d.py b/scanpipe/pipes/d2d.py index 1d7245b868..d3aea493a6 100644 --- a/scanpipe/pipes/d2d.py +++ b/scanpipe/pipes/d2d.py @@ -29,7 +29,6 @@ from scanpipe import pipes from scanpipe.models import CodebaseRelation from scanpipe.models import CodebaseResource -from scanpipe.models import DiscoveredPackage from scanpipe.pipes import flag from scanpipe.pipes import get_resource_diff_ratio from scanpipe.pipes import js @@ -910,19 +909,21 @@ def _map_javascript_npm_lookup_resource( """Map unmatched ``node_modules`` files.""" purl = js.get_purl_from_node_module(to_directory.path) matched = to_resources.filter(path__startswith=to_directory.path) + matched_count = matched.count() + if not matched: return 0 - try: - package = project.discoveredpackages.get( - type=purl.type, - namespace="" if not purl.namespace else purl.namespace, - name=purl.name, - version=purl.version, - ) - package.add_resources(matched) + package = project.discoveredpackages.filter( + type=purl.type, + namespace="" if not purl.namespace else purl.namespace, + name=purl.name, + version=purl.version, + ).first() - except DiscoveredPackage.DoesNotExist: + if package: + package.add_resources(matched) + else: if results := purldb.fetch_package(purl=str(purl)): package_data = results[0] package_data.pop("uuid", None) @@ -937,4 +938,4 @@ def _map_javascript_npm_lookup_resource( return 0 matched.update(status=flag.NPM_LOOKUP) - return len(matched) + return matched_count From 0a950c9e95e9fdf5df59ab1a2c85eebd864632e3 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Tue, 8 Aug 2023 14:50:26 +0530 Subject: [PATCH 5/5] Avoid using `only` Signed-off-by: Keshav Priyadarshi --- scanpipe/pipes/d2d.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/scanpipe/pipes/d2d.py b/scanpipe/pipes/d2d.py index ea870573ca..1f06a6df23 100644 --- a/scanpipe/pipes/d2d.py +++ b/scanpipe/pipes/d2d.py @@ -954,20 +954,11 @@ def _map_javascript_colocation_resource( map_type="js_colocation", ) - for match in transpiled: - pipes.make_relation( - from_resource=from_resource, - to_resource=match, - map_type="js_path", - extra_data=extra_data, - ) - return len(transpiled) - def map_javascript_npm_lookup(project, logger=None): """Map unmatched ``node_modules`` files.""" - project_directories = project.codebaseresources.directories().only("path") - project_files = project.codebaseresources.files().only("path") + project_directories = project.codebaseresources.directories() + project_files = project.codebaseresources.files() to_directories_key = ( project_directories.to_codebase()