Skip to content

Commit 657c1af

Browse files
authored
Upgrade ScanCode-toolkit to latest v32 #569 (#715)
Signed-off-by: Thomas Druez <tdruez@nexb.com>
1 parent e4488e8 commit 657c1af

61 files changed

Lines changed: 16692 additions & 7378 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

CHANGELOG.rst

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,14 @@ Changelog
44
v33.0.0 (unreleased)
55
--------------------
66

7+
- Upgrade ScanCode-toolkit to latest v32.0.0
8+
Warning: This upgrade requires schema and data migrations (both included).
9+
It is recommended to reset and re-run the pipelines to benefit from the latest
10+
ScanCode detection improvements.
11+
Refer to https://github.com/nexB/scancode-toolkit/blob/develop/CHANGELOG.rst#v3200-next-roadmap
12+
for the full list of changes.
13+
https://github.com/nexB/scancode.io/issues/569
14+
715
- Add a new ``deploy_to_develop`` pipeline specialized in creating relations between
816
the development source code and binaries or deployed code.
917
This pipeline is expecting 2 archive files with "from-" and "to-" filename prefixes

docs/custom-pipelines.rst

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -141,20 +141,20 @@ the file's directory in the :ref:`scancodeio_settings_pipelines_dirs`.
141141
142142
def report_licenses_with_resources(self):
143143
"""
144-
Retrieves codebase resources filtered by license categories,
145-
Generates a licenses report file from a template.
144+
Retrieves codebase resources and generates a licenses report file using
145+
a Jinja template.
146146
"""
147-
categories = ["Commercial", "Copyleft"]
148-
resources = self.project.codebaseresources.licenses_categories(categories)
147+
resources = self.project.codebaseresources.has_license_detections()
149148
150-
resources_by_licenses = defaultdict(list)
149+
resources_by_matched_text = defaultdict(list)
151150
for resource in resources:
152-
for license_data in resource.licenses:
153-
matched_text = license_data.get("matched_text")
154-
resources_by_licenses[matched_text].append(resource.path)
151+
for detection_data in resource.license_detections:
152+
for match in detection_data.get("matches", []):
153+
matched_text = match.get("matched_text")
154+
resources_by_matched_text[matched_text].append(resource.path)
155155
156156
template = Template(self.report_template, lstrip_blocks=True, trim_blocks=True)
157-
report_stream = template.stream(resources=resources_by_licenses)
157+
report_stream = template.stream(resources=resources_by_matched_text)
158158
report_file = self.project.get_output_file_path("license-report", "txt")
159159
report_stream.dump(str(report_file))
160160

docs/output-files.rst

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -183,8 +183,6 @@ similar to the following:
183183
"license_expression": "mit",
184184
"declared_license": "MIT",
185185
"notice_text": "",
186-
"manifest_path": "",
187-
"contains_source_code": null,
188186
"missing_resources": [
189187
"/lib/libc.musl-x86_64.so.1"
190188
],

scancodeio/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
import warnings
2626
from pathlib import Path
2727

28-
__version__ = "32.2.0"
28+
__version__ = "33.0.0-dev"
2929

3030
SCAN_NOTICE = Path(__file__).resolve().parent.joinpath("scan.NOTICE").read_text()
3131

scanpipe/api/serializers.py

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -251,8 +251,11 @@ class Meta:
251251
"is_archive",
252252
"is_media",
253253
"is_key_file",
254-
"licenses",
255-
"license_expressions",
254+
"detected_license_expression",
255+
"detected_license_expression_spdx",
256+
"license_detections",
257+
"license_clues",
258+
"percentage_of_license_text",
256259
"compliance_alert",
257260
"copyrights",
258261
"holders",
@@ -297,14 +300,18 @@ class Meta:
297300
"sha256",
298301
"sha512",
299302
"copyright",
300-
"license_expression",
301-
"declared_license",
303+
"holder",
304+
"declared_license_expression",
305+
"declared_license_expression_spdx",
306+
"license_detections",
307+
"other_license_expression",
308+
"other_license_expression_spdx",
309+
"other_license_detections",
310+
"extracted_license_statement",
302311
"notice_text",
303312
"source_packages",
304313
"extra_data",
305314
"package_uid",
306-
"manifest_path",
307-
"contains_source_code",
308315
"datasource_id",
309316
"file_references",
310317
"missing_resources",

scanpipe/filters.py

Lines changed: 12 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -322,6 +322,7 @@ class ResourceFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
322322
"type",
323323
"size",
324324
"name",
325+
"detected_license_expression",
325326
"extension",
326327
"programming_language",
327328
"mime_type",
@@ -331,16 +332,8 @@ class ResourceFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
331332
"related_from__from_resource__path",
332333
],
333334
)
334-
license_key = JSONContainsFilter(
335-
label="License key",
336-
field_name="licenses",
337-
)
338-
license_category = JSONContainsFilter(
339-
label="License category",
340-
field_name="licenses",
341-
)
342335
compliance_alert = django_filters.ChoiceFilter(
343-
choices=CodebaseResource.Compliance.choices + [("EMPTY", "EMPTY")]
336+
choices=CodebaseResource.Compliance.choices + [("_EMPTY_", "EMPTY")]
344337
)
345338
in_package = InPackageFilter(label="In a Package")
346339
status = StatusFilter(empty_label="All")
@@ -373,9 +366,11 @@ class Meta:
373366
"copyrights",
374367
"holders",
375368
"authors",
376-
"licenses",
377-
"license_category",
378-
"license_expressions",
369+
"detected_license_expression",
370+
"detected_license_expression_spdx",
371+
"license_detections",
372+
"license_clues",
373+
"percentage_of_license_text",
379374
"emails",
380375
"urls",
381376
"in_package",
@@ -414,7 +409,8 @@ class PackageFilterSet(FilterSetUtilsMixin, django_filters.FilterSet):
414409
sort = django_filters.OrderingFilter(
415410
label="Sort",
416411
fields=[
417-
"license_expression",
412+
"declared_license_expression",
413+
"other_license_expression",
418414
"copyright",
419415
"primary_language",
420416
],
@@ -444,11 +440,10 @@ class Meta:
444440
"code_view_url",
445441
"vcs_url",
446442
"type",
447-
"license_expression",
448-
"declared_license",
443+
"declared_license_expression",
444+
"other_license_expression",
445+
"extracted_license_statement",
449446
"copyright",
450-
"manifest_path",
451-
"contains_source_code",
452447
]
453448

454449

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
# Generated by Django 4.2 on 2023-05-05 06:52
2+
3+
from django.db import migrations, models
4+
5+
6+
class Migration(migrations.Migration):
7+
dependencies = [
8+
("scanpipe", "0029_codebaseresource_scanpipe_co_type_ea1dd7_idx_and_more"),
9+
]
10+
11+
operations = [
12+
migrations.RemoveIndex(
13+
model_name="discoveredpackage",
14+
name="scanpipe_di_license_e8ce32_idx",
15+
),
16+
migrations.RenameField(
17+
model_name="discoveredpackage",
18+
old_name="license_expression",
19+
new_name="declared_license_expression",
20+
),
21+
migrations.AlterField(
22+
model_name="discoveredpackage",
23+
name="declared_license_expression",
24+
field=models.TextField(
25+
blank=True,
26+
help_text="The license expression for this package typically derived from its extracted_license_statement or from some other type-specific routine or convention.",
27+
),
28+
),
29+
migrations.RenameField(
30+
model_name="discoveredpackage",
31+
old_name="declared_license",
32+
new_name="extracted_license_statement",
33+
),
34+
migrations.AlterField(
35+
model_name="discoveredpackage",
36+
name="extracted_license_statement",
37+
field=models.TextField(
38+
blank=True,
39+
help_text="The license statement mention, tag or text as found in a package manifest and extracted. This can be a string, a list or dict of strings possibly nested, as found originally in the manifest.",
40+
),
41+
),
42+
migrations.AddField(
43+
model_name="discoveredpackage",
44+
name="declared_license_expression_spdx",
45+
field=models.TextField(
46+
blank=True,
47+
help_text="The SPDX license expression for this package converted from its declared_license_expression.",
48+
),
49+
),
50+
migrations.AddField(
51+
model_name="discoveredpackage",
52+
name="holder",
53+
field=models.TextField(
54+
blank=True,
55+
help_text="Holders for this package. Typically one per line.",
56+
),
57+
),
58+
migrations.AddField(
59+
model_name="discoveredpackage",
60+
name="license_detections",
61+
field=models.JSONField(
62+
blank=True,
63+
default=list,
64+
help_text="A list of LicenseDetection mappings typically derived from its extracted_license_statement or from some other type-specific routine or convention.",
65+
),
66+
),
67+
migrations.AddField(
68+
model_name="discoveredpackage",
69+
name="other_license_detections",
70+
field=models.JSONField(
71+
blank=True,
72+
default=list,
73+
help_text="A list of LicenseDetection mappings which is different from the declared_license_expression, (i.e. not the primary license) These are detections for the detection for the license expressions in other_license_expression. ",
74+
),
75+
),
76+
migrations.AddField(
77+
model_name="discoveredpackage",
78+
name="other_license_expression",
79+
field=models.TextField(
80+
blank=True,
81+
help_text="The license expression for this package which is different from the declared_license_expression, (i.e. not the primary license) routine or convention.",
82+
),
83+
),
84+
migrations.AddField(
85+
model_name="discoveredpackage",
86+
name="other_license_expression_spdx",
87+
field=models.TextField(
88+
blank=True,
89+
help_text="The other SPDX license expression for this package converted from its other_license_expression.",
90+
),
91+
),
92+
migrations.AddField(
93+
model_name="codebaseresource",
94+
name="detected_license_expression",
95+
field=models.TextField(blank=True, help_text="The license expression summarizing the license info for this resource, combined from all the license detections"),
96+
),
97+
migrations.AddField(
98+
model_name="codebaseresource",
99+
name="detected_license_expression_spdx",
100+
field=models.TextField(blank=True, help_text="The detected license expression for this file, with SPDX license keys"),
101+
),
102+
migrations.AddField(
103+
model_name="codebaseresource",
104+
name="license_detections",
105+
field=models.JSONField(
106+
blank=True, default=list, help_text="List of license detection details."
107+
),
108+
),
109+
migrations.AddField(
110+
model_name="codebaseresource",
111+
name="license_clues",
112+
field=models.JSONField(
113+
blank=True, default=list, help_text="List of license matches that are not proper detections and potentially just clues to licenses or likely false positives. Those are not included in computing the detected license expression for the resource."
114+
),
115+
),
116+
migrations.AddField(
117+
model_name="codebaseresource",
118+
name="percentage_of_license_text",
119+
field=models.FloatField(blank=True, help_text="Percentage of file words detected as license text or notice.", null=True),
120+
),
121+
]

0 commit comments

Comments
 (0)