Skip to content

Commit 04fc0e1

Browse files
AyanSinhaMahapatraAyanSinhaMahapatra
committed
Address feedback and add improvements
Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>
1 parent 01770f0 commit 04fc0e1

6 files changed

Lines changed: 69 additions & 52 deletions

File tree

‎scanpipe/filters.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -684,6 +684,8 @@ class PackageFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
684684
declared_license_expression = django_filters.filters.CharFilter(
685685
widget=HasValueDropdownWidget
686686
)
687+
is_private = StrictBooleanFilter()
688+
is_virtual = StrictBooleanFilter()
687689

688690
class Meta:
689691
model = DiscoveredPackage
@@ -717,6 +719,8 @@ class Meta:
717719
"is_vulnerable",
718720
"compliance_alert",
719721
"tag",
722+
"is_private",
723+
"is_virtual",
720724
]
721725

722726

‎scanpipe/migrations/0061_dependency_resolver_update.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,4 +31,16 @@ class Migration(migrations.Migration):
3131
fields=["is_direct"], name="scanpipe_di_is_dire_6dc594_idx"
3232
),
3333
),
34+
migrations.AddIndex(
35+
model_name="discoveredpackage",
36+
index=models.Index(
37+
fields=["is_private"], name="scanpipe_di_is_priv_9ffd1a_idx"
38+
),
39+
),
40+
migrations.AddIndex(
41+
model_name="discoveredpackage",
42+
index=models.Index(
43+
fields=["is_virtual"], name="scanpipe_di_is_virt_c5c176_idx"
44+
),
45+
),
3446
]

‎scanpipe/models.py‎

Lines changed: 23 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -3099,6 +3099,8 @@ class Meta:
30993099
models.Index(fields=["sha512"]),
31003100
models.Index(fields=["compliance_alert"]),
31013101
models.Index(fields=["tag"]),
3102+
models.Index(fields=["is_private"]),
3103+
models.Index(fields=["is_virtual"]),
31023104
]
31033105
constraints = [
31043106
models.UniqueConstraint(
@@ -3126,15 +3128,7 @@ def purl(self):
31263128

31273129
@classmethod
31283130
def extract_purl_data(cls, package_data):
3129-
purl_data = {}
3130-
3131-
for field_name in PURL_FIELDS:
3132-
value = package_data.get(field_name)
3133-
if field_name == "qualifiers":
3134-
value = normalize_qualifiers(value, encode=True)
3135-
purl_data[field_name] = value or ""
3136-
3137-
return purl_data
3131+
return normalize_package_url_data(package_data)
31383132

31393133
@classmethod
31403134
def create_from_data(cls, project, package_data):
@@ -3619,19 +3613,8 @@ def extract_purl_data(cls, dependency_data, ignore_nulls=False):
36193613
purl_mapping = PackageURL.from_string(
36203614
purl=dependency_data.get("purl"),
36213615
).to_dict()
3622-
purl_data = {}
3623-
3624-
for field_name in PURL_FIELDS:
3625-
value = purl_mapping.get(field_name)
3626-
if field_name == "qualifiers":
3627-
value = normalize_qualifiers(value, encode=True)
3628-
if not ignore_nulls:
3629-
purl_data[field_name] = value or ""
3630-
else:
3631-
if value:
3632-
purl_data[field_name] = value or ""
36333616

3634-
return purl_data
3617+
return normalize_package_url_data(purl_mapping, ignore_nulls)
36353618

36363619
@classmethod
36373620
def populate_dependency_uuid(cls, dependency_data):
@@ -3666,6 +3649,25 @@ def as_spdx(self):
36663649
)
36673650

36683651

3652+
def normalize_package_url_data(purl_mapping, ignore_nulls=False):
3653+
"""
3654+
Normalize a mapping of purl data so database queries with
3655+
purl data can be executed.
3656+
"""
3657+
normalized_purl_mapping = {}
3658+
for field_name in PURL_FIELDS:
3659+
value = purl_mapping.get(field_name)
3660+
if field_name == "qualifiers":
3661+
value = normalize_qualifiers(value, encode=True)
3662+
if not ignore_nulls:
3663+
normalized_purl_mapping[field_name] = value or ""
3664+
else:
3665+
if value:
3666+
normalized_purl_mapping[field_name] = value or ""
3667+
3668+
return normalized_purl_mapping
3669+
3670+
36693671
class WebhookSubscription(UUIDPKModel, ProjectRelatedModel):
36703672
target_url = models.URLField(_("Target URL"), max_length=1024)
36713673
created_date = models.DateTimeField(auto_now_add=True, editable=False)

‎scanpipe/pipes/__init__.py‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -199,9 +199,6 @@ def update_or_create_package(
199199
package = DiscoveredPackage.create_from_data(project, package_data)
200200

201201
if package:
202-
if is_virtual:
203-
package.update(is_virtual=is_virtual)
204-
205202
if datasource_id and datasource_id not in package.datasource_ids:
206203
datasource_ids = package.datasource_ids.copy()
207204
datasource_ids.append(datasource_id)
@@ -291,10 +288,6 @@ def update_or_create_dependency(
291288
if resolved_to_package and not dependency.resolved_to_package:
292289
dependency.update(resolved_to_package=resolved_to_package)
293290
else:
294-
is_direct = dependency_data.get("is_direct")
295-
if not is_direct:
296-
pass
297-
298291
DiscoveredDependency.populate_dependency_uuid(dependency_data)
299292
dependency = DiscoveredDependency.create_from_data(
300293
project,

‎scanpipe/tests/data/resolved_dependencies_inspect_packages.json‎

Lines changed: 22 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@
6565
"notice_text": "",
6666
"source_packages": [],
6767
"extra_data": {},
68-
"package_uid": "",
68+
"package_uid": "pkg:npm/athena-express@6.0.4?uuid=fixed-uid-done-for-testing-5642512d1758",
6969
"is_private": false,
7070
"is_virtual": true,
7171
"datasource_ids": [
@@ -114,7 +114,7 @@
114114
"notice_text": "",
115115
"source_packages": [],
116116
"extra_data": {},
117-
"package_uid": "",
117+
"package_uid": "pkg:npm/bluebird@3.7.2?uuid=fixed-uid-done-for-testing-5642512d1758",
118118
"is_private": false,
119119
"is_virtual": true,
120120
"datasource_ids": [
@@ -237,7 +237,7 @@
237237
"notice_text": "",
238238
"source_packages": [],
239239
"extra_data": {},
240-
"package_uid": "",
240+
"package_uid": "pkg:npm/csvtojson@2.0.10?uuid=fixed-uid-done-for-testing-5642512d1758",
241241
"is_private": false,
242242
"is_virtual": true,
243243
"datasource_ids": [
@@ -286,7 +286,7 @@
286286
"notice_text": "",
287287
"source_packages": [],
288288
"extra_data": {},
289-
"package_uid": "",
289+
"package_uid": "pkg:npm/is-utf8@0.2.1?uuid=fixed-uid-done-for-testing-5642512d1758",
290290
"is_private": false,
291291
"is_virtual": true,
292292
"datasource_ids": [
@@ -335,7 +335,7 @@
335335
"notice_text": "",
336336
"source_packages": [],
337337
"extra_data": {},
338-
"package_uid": "",
338+
"package_uid": "pkg:npm/lodash@4.17.21?uuid=fixed-uid-done-for-testing-5642512d1758",
339339
"is_private": false,
340340
"is_virtual": true,
341341
"datasource_ids": [
@@ -384,7 +384,7 @@
384384
"notice_text": "",
385385
"source_packages": [],
386386
"extra_data": {},
387-
"package_uid": "",
387+
"package_uid": "pkg:npm/strip-bom@2.0.0?uuid=fixed-uid-done-for-testing-5642512d1758",
388388
"is_private": false,
389389
"is_virtual": true,
390390
"datasource_ids": [
@@ -408,7 +408,7 @@
408408
"is_direct": true,
409409
"dependency_uid": "pkg:npm/athena-express?uuid=fixed-uid-done-for-testing-5642512d1758",
410410
"for_package_uid": "pkg:npm/create-athena-partition@1.0.0?uuid=fixed-uid-done-for-testing-5642512d1758",
411-
"resolved_to_package_uid": "",
411+
"resolved_to_package_uid": "pkg:npm/athena-express@6.0.4?uuid=fixed-uid-done-for-testing-5642512d1758",
412412
"datafile_path": "resolved_dependencies.zip-extract/package.json",
413413
"datasource_id": "npm_package_json",
414414
"package_type": "npm",
@@ -423,8 +423,8 @@
423423
"is_resolved": true,
424424
"is_direct": true,
425425
"dependency_uid": "pkg:npm/bluebird?uuid=fixed-uid-done-for-testing-5642512d1758",
426-
"for_package_uid": "",
427-
"resolved_to_package_uid": "",
426+
"for_package_uid": "pkg:npm/csvtojson@2.0.10?uuid=fixed-uid-done-for-testing-5642512d1758",
427+
"resolved_to_package_uid": "pkg:npm/bluebird@3.7.2?uuid=fixed-uid-done-for-testing-5642512d1758",
428428
"datafile_path": "resolved_dependencies.zip-extract/yarn.lock",
429429
"datasource_id": "yarn_lock_v1",
430430
"package_type": "npm",
@@ -439,8 +439,8 @@
439439
"is_resolved": true,
440440
"is_direct": true,
441441
"dependency_uid": "pkg:npm/csvtojson?uuid=fixed-uid-done-for-testing-5642512d1758",
442-
"for_package_uid": "",
443-
"resolved_to_package_uid": "",
442+
"for_package_uid": "pkg:npm/athena-express@6.0.4?uuid=fixed-uid-done-for-testing-5642512d1758",
443+
"resolved_to_package_uid": "pkg:npm/csvtojson@2.0.10?uuid=fixed-uid-done-for-testing-5642512d1758",
444444
"datafile_path": "resolved_dependencies.zip-extract/yarn.lock",
445445
"datasource_id": "yarn_lock_v1",
446446
"package_type": "npm",
@@ -455,8 +455,8 @@
455455
"is_resolved": true,
456456
"is_direct": true,
457457
"dependency_uid": "pkg:npm/is-utf8?uuid=fixed-uid-done-for-testing-5642512d1758",
458-
"for_package_uid": "",
459-
"resolved_to_package_uid": "",
458+
"for_package_uid": "pkg:npm/strip-bom@2.0.0?uuid=fixed-uid-done-for-testing-5642512d1758",
459+
"resolved_to_package_uid": "pkg:npm/is-utf8@0.2.1?uuid=fixed-uid-done-for-testing-5642512d1758",
460460
"datafile_path": "resolved_dependencies.zip-extract/yarn.lock",
461461
"datasource_id": "yarn_lock_v1",
462462
"package_type": "npm",
@@ -471,8 +471,8 @@
471471
"is_resolved": true,
472472
"is_direct": true,
473473
"dependency_uid": "pkg:npm/lodash?uuid=fixed-uid-done-for-testing-5642512d1758",
474-
"for_package_uid": "",
475-
"resolved_to_package_uid": "",
474+
"for_package_uid": "pkg:npm/csvtojson@2.0.10?uuid=fixed-uid-done-for-testing-5642512d1758",
475+
"resolved_to_package_uid": "pkg:npm/lodash@4.17.21?uuid=fixed-uid-done-for-testing-5642512d1758",
476476
"datafile_path": "resolved_dependencies.zip-extract/yarn.lock",
477477
"datasource_id": "yarn_lock_v1",
478478
"package_type": "npm",
@@ -487,8 +487,8 @@
487487
"is_resolved": true,
488488
"is_direct": true,
489489
"dependency_uid": "pkg:npm/strip-bom?uuid=fixed-uid-done-for-testing-5642512d1758",
490-
"for_package_uid": "",
491-
"resolved_to_package_uid": "",
490+
"for_package_uid": "pkg:npm/csvtojson@2.0.10?uuid=fixed-uid-done-for-testing-5642512d1758",
491+
"resolved_to_package_uid": "pkg:npm/strip-bom@2.0.0?uuid=fixed-uid-done-for-testing-5642512d1758",
492492
"datafile_path": "resolved_dependencies.zip-extract/yarn.lock",
493493
"datasource_id": "yarn_lock_v1",
494494
"package_type": "npm",
@@ -1110,13 +1110,13 @@
11101110
}
11111111
],
11121112
"for_packages": [
1113-
"pkg:npm/bluebird@3.7.2?uuid=fixed-uid-done-for-testing-5642512d1758",
1114-
"pkg:npm/lodash@4.17.21?uuid=fixed-uid-done-for-testing-5642512d1758",
1115-
"pkg:npm/strip-bom@2.0.0?uuid=fixed-uid-done-for-testing-5642512d1758",
1116-
"pkg:npm/is-utf8@0.2.1?uuid=fixed-uid-done-for-testing-5642512d1758",
11171113
"pkg:npm/create-athena-partition@1.0.0?uuid=fixed-uid-done-for-testing-5642512d1758",
1114+
"pkg:npm/bluebird@3.7.2?uuid=fixed-uid-done-for-testing-5642512d1758",
1115+
"pkg:npm/csvtojson@2.0.10?uuid=fixed-uid-done-for-testing-5642512d1758",
11181116
"pkg:npm/athena-express@6.0.4?uuid=fixed-uid-done-for-testing-5642512d1758",
1119-
"pkg:npm/csvtojson@2.0.10?uuid=fixed-uid-done-for-testing-5642512d1758"
1117+
"pkg:npm/strip-bom@2.0.0?uuid=fixed-uid-done-for-testing-5642512d1758",
1118+
"pkg:npm/lodash@4.17.21?uuid=fixed-uid-done-for-testing-5642512d1758",
1119+
"pkg:npm/is-utf8@0.2.1?uuid=fixed-uid-done-for-testing-5642512d1758"
11201120
],
11211121
"emails": [],
11221122
"urls": [],

‎scanpipe/tests/test_pipelines.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -470,7 +470,13 @@ def _normalize_package_uids(self, data):
470470
if isinstance(value, (list, dict)):
471471
value = self._normalize_package_uids(value)
472472
if (
473-
key in ("package_uid", "dependency_uid", "for_package_uid")
473+
key
474+
in (
475+
"package_uid",
476+
"dependency_uid",
477+
"for_package_uid",
478+
"resolved_to_package_uid",
479+
)
474480
and value
475481
):
476482
value = purl_with_fake_uuid(value)
@@ -692,7 +698,7 @@ def test_scanpipe_inspect_packages_with_resolved_dependencies(self):
692698
expected_file = (
693699
self.data_location / "resolved_dependencies_inspect_packages.json"
694700
)
695-
self.assertPipelineResultEqual(expected_file, result_file, regen=True)
701+
self.assertPipelineResultEqual(expected_file, result_file)
696702

697703
def test_scanpipe_scan_codebase_can_process_wheel(self):
698704
pipeline_name = "scan_codebase"

0 commit comments

Comments
 (0)