Skip to content

Commit aea1175

Browse files
JonoYangJonoYang
committed
Update Consolidation attributes
 * Remove plugin specific gitignore from project level gitignore * Update tests Signed-off-by: Jono Yang <jyang@nexb.com>
1 parent bb1d846 commit aea1175

11 files changed

Lines changed: 126 additions & 90 deletions

‎.gitignore‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,3 @@ pyvenv.cfg
7878
/.pytest_cache/
7979
lib64
8080
tcl
81-
82-
# Nested ScanCode plugins
83-
*build*
84-
*dist*

‎plugins/scancode-consolidate-scan/.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
/build/
2+
/dist/

‎plugins/scancode-consolidate-scan/src/plugin_consolidate/__init__.py‎

Lines changed: 24 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -72,36 +72,38 @@ class Consolidation(object):
7272
A grouping of files that share the same origin. Other clues found in the
7373
codebase are stored in `other_license_expression` and `other_holders`
7474
"""
75-
# TODO: Add file counts for proper accounting
7675
identifier = attr.ib(default=None)
76+
consolidated_license_expression = attr.ib(default=None)
77+
consolidated_holders = attr.ib(default=attr.Factory(list))
78+
consolidated_copyright = attr.ib(default=None)
7779
core_license_expression = attr.ib(default=None)
78-
other_license_expression = attr.ib(default=None)
79-
consolidated_core_copyright = attr.ib(default=None)
80-
consolidated_other_copyright = attr.ib(default=None)
8180
core_holders = attr.ib(default=attr.Factory(list))
81+
other_license_expression = attr.ib(default=None)
8282
other_holders = attr.ib(default=attr.Factory(list))
83+
resources_count = attr.ib(default=None)
8384
resources = attr.ib(default=attr.Factory(list))
8485

8586
def to_dict(self, **kwargs):
8687
def dict_fields(attr, value):
8788
if attr.name in ('resources', ):
8889
return False
8990
return True
90-
self.consolidated_core_copyright = self.consolidate_core_copyright()
91-
self.consolidated_other_copyright = self.consolidate_other_copyright()
91+
license_expressions_to_consolidate = []
92+
if self.core_license_expression:
93+
license_expressions_to_consolidate.append(self.core_license_expression)
94+
if self.other_license_expression:
95+
license_expressions_to_consolidate.append(self.other_license_expression)
96+
if license_expressions_to_consolidate:
97+
self.consolidated_license_expression = combine_expressions(license_expressions_to_consolidate)
98+
self.consolidated_copyright = self.consolidate_copyright()
9299
return attr.asdict(self, filter=dict_fields, dict_factory=OrderedDict)
93100

94-
def consolidate_core_copyright(self):
95-
# TODO: Verify and test that we are generating detectable copyrights
96-
holders = list(self.core_holders) + list(self.other_holders)
97-
if holders:
98-
return 'Copyright (c) {}'.format(', '.join(holders))
99-
100-
def consolidate_other_copyright(self):
101+
def consolidate_copyright(self):
101102
# TODO: Verify and test that we are generating detectable copyrights
102-
other_holders = list(self.other_holders)
103-
if other_holders:
104-
return 'Copyright (c) {}'.format(', '.join(other_holders))
103+
if not self.consolidated_holders:
104+
self.consolidated_holders = sorted(set(list(self.core_holders) + list(self.other_holders)))
105+
else:
106+
return 'Copyright (c) {}'.format(', '.join(self.consolidated_holders))
105107

106108

107109
@attr.s
@@ -259,9 +261,10 @@ def get_consolidated_packages(codebase):
259261

260262
c = Consolidation(
261263
core_license_expression=package_license_expression,
262-
other_license_expression=simplified_discovered_license_expression,
263264
core_holders=sorted(set(package_holders)),
265+
other_license_expression=simplified_discovered_license_expression,
264266
other_holders=sorted(set(discovered_holders)),
267+
resources_count=len(package_resources),
265268
resources=package_resources,
266269
)
267270
yield ConsolidatedPackage(
@@ -295,6 +298,8 @@ def get_license_holders_consolidated_components(codebase):
295298

296299
# Collect license expression and holders count for stat-based summarization
297300
origin_count = Counter()
301+
# TODO: Consider license match coverage and license score when consolidating things on licenses
302+
# We may consolidate things by a weakly-matched license
298303
for child in children:
299304
if child.extra_data.get('in_package_component'):
300305
continue
@@ -375,6 +380,7 @@ def create_license_holders_consolidated_component(resource, codebase):
375380
c = Consolidation(
376381
core_license_expression=license_expression,
377382
core_holders=holders,
383+
resources_count=len(component_resources),
378384
resources=component_resources,
379385
)
380386
return ConsolidatedComponent(
@@ -427,6 +433,7 @@ def combine_license_holders_consolidated_components(components):
427433
c = Consolidation(
428434
core_license_expression=component_license_expression,
429435
core_holders=component_holders,
436+
resources_count=len(component_resources),
430437
resources=component_resources,
431438
)
432439
yield ConsolidatedComponent(

‎plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/clear-summary-expected.json‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,14 +23,18 @@
2323
{
2424
"type": "license-holders",
2525
"identifier": "the_apache_software_foundation_1",
26+
"consolidated_license_expression": "apache-2.0",
27+
"consolidated_holders": [
28+
"The Apache Software Foundation"
29+
],
30+
"consolidated_copyright": null,
2631
"core_license_expression": "apache-2.0",
27-
"other_license_expression": null,
28-
"consolidated_core_copyright": "Copyright (c) The Apache Software Foundation",
29-
"consolidated_other_copyright": null,
3032
"core_holders": [
3133
"The Apache Software Foundation"
3234
],
33-
"other_holders": []
35+
"other_license_expression": null,
36+
"other_holders": [],
37+
"resources_count": 4
3438
}
3539
],
3640
"consolidated_packages": [],

‎plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/component-package-expected.json‎

Lines changed: 21 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,11 @@
33
{
44
"tool_name": "scancode-toolkit",
55
"options": {
6-
"input": [
7-
"plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/component-package"
8-
],
6+
"input": "<path>",
7+
"--consolidate": true,
98
"--copyright": true,
109
"--info": true,
11-
"--json-pp": "plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/component-package.json",
10+
"--json": "<file>",
1211
"--license": true,
1312
"--package": true
1413
},
@@ -18,37 +17,24 @@
1817
"extra_data": {
1918
"files_count": 7
2019
}
21-
},
22-
{
23-
"tool_name": "scancode-toolkit",
24-
"options": {
25-
"input": "<path>",
26-
"--consolidate": true,
27-
"--from-json": [
28-
true
29-
],
30-
"--json": "<file>"
31-
},
32-
"notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.",
33-
"message": null,
34-
"errors": [],
35-
"extra_data": {
36-
"files_count": 7
37-
}
3820
}
3921
],
4022
"consolidated_components": [
4123
{
4224
"type": "license-holders",
4325
"identifier": "the_apache_software_foundation_1",
26+
"consolidated_license_expression": "apache-2.0",
27+
"consolidated_holders": [
28+
"The Apache Software Foundation"
29+
],
30+
"consolidated_copyright": null,
4431
"core_license_expression": "apache-2.0",
45-
"other_license_expression": null,
46-
"consolidated_core_copyright": "Copyright (c) The Apache Software Foundation",
47-
"consolidated_other_copyright": null,
4832
"core_holders": [
4933
"The Apache Software Foundation"
5034
],
51-
"other_holders": []
35+
"other_license_expression": null,
36+
"other_holders": [],
37+
"resources_count": 4
5238
}
5339
],
5440
"consolidated_packages": [
@@ -89,16 +75,22 @@
8975
"repository_download_url": "https://registry.npmjs.org/test-package/-/test-package-0.0.1.tgz",
9076
"api_data_url": "https://registry.npmjs.org/test-package/0.0.1",
9177
"identifier": "pkg_npm_test_package_0_0_1_1",
78+
"consolidated_license_expression": "apache-2.0 AND (apache-2.0 AND gpl-1.0-plus AND unknown)",
79+
"consolidated_holders": [
80+
"IBM Corp.",
81+
"The Apache Software",
82+
"The Apache Software Foundation"
83+
],
84+
"consolidated_copyright": null,
9285
"core_license_expression": "apache-2.0",
93-
"other_license_expression": "apache-2.0 AND gpl-1.0-plus AND unknown",
94-
"consolidated_core_copyright": "Copyright (c) IBM Corp., The Apache Software, The Apache Software Foundation",
95-
"consolidated_other_copyright": "Copyright (c) IBM Corp., The Apache Software, The Apache Software Foundation",
9686
"core_holders": [],
87+
"other_license_expression": "apache-2.0 AND gpl-1.0-plus AND unknown",
9788
"other_holders": [
9889
"IBM Corp.",
9990
"The Apache Software",
10091
"The Apache Software Foundation"
101-
]
92+
],
93+
"resources_count": 4
10294
}
10395
],
10496
"files": [

‎plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/license-holder-rollup-expected.json‎

Lines changed: 24 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -23,38 +23,50 @@
2323
{
2424
"type": "license-holders",
2525
"identifier": "ibm_corp__1",
26+
"consolidated_license_expression": "gpl-1.0-plus AND gpl-2.0",
27+
"consolidated_holders": [
28+
"IBM Corp."
29+
],
30+
"consolidated_copyright": null,
2631
"core_license_expression": "gpl-1.0-plus AND gpl-2.0",
27-
"other_license_expression": null,
28-
"consolidated_core_copyright": "Copyright (c) IBM Corp.",
29-
"consolidated_other_copyright": null,
3032
"core_holders": [
3133
"IBM Corp."
3234
],
33-
"other_holders": []
35+
"other_license_expression": null,
36+
"other_holders": [],
37+
"resources_count": 2
3438
},
3539
{
3640
"type": "license-holders",
3741
"identifier": "oracle_corp__2",
42+
"consolidated_license_expression": "unknown AND apache-2.0",
43+
"consolidated_holders": [
44+
"Oracle Corp."
45+
],
46+
"consolidated_copyright": null,
3847
"core_license_expression": "unknown AND apache-2.0",
39-
"other_license_expression": null,
40-
"consolidated_core_copyright": "Copyright (c) Oracle Corp.",
41-
"consolidated_other_copyright": null,
4248
"core_holders": [
4349
"Oracle Corp."
4450
],
45-
"other_holders": []
51+
"other_license_expression": null,
52+
"other_holders": [],
53+
"resources_count": 2
4654
},
4755
{
4856
"type": "license-holders",
4957
"identifier": "omega_com_3",
58+
"consolidated_license_expression": "gpl-1.0-plus AND gpl-2.0",
59+
"consolidated_holders": [
60+
"omega.com"
61+
],
62+
"consolidated_copyright": null,
5063
"core_license_expression": "gpl-1.0-plus AND gpl-2.0",
51-
"other_license_expression": null,
52-
"consolidated_core_copyright": "Copyright (c) omega.com",
53-
"consolidated_other_copyright": null,
5464
"core_holders": [
5565
"omega.com"
5666
],
57-
"other_holders": []
67+
"other_license_expression": null,
68+
"other_holders": [],
69+
"resources_count": 3
5870
}
5971
],
6072
"consolidated_packages": [],

‎plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/multiple-same-holder-and-license-expected.json‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,15 +23,19 @@
2323
{
2424
"type": "license-holders",
2525
"identifier": "ibm_corp__ibm_corp__1",
26+
"consolidated_license_expression": "gpl-2.0",
27+
"consolidated_holders": [
28+
"IBM Corp."
29+
],
30+
"consolidated_copyright": null,
2631
"core_license_expression": "gpl-2.0",
27-
"other_license_expression": null,
28-
"consolidated_core_copyright": "Copyright (c) IBM Corp., IBM Corp.",
29-
"consolidated_other_copyright": null,
3032
"core_holders": [
3133
"IBM Corp.",
3234
"IBM Corp."
3335
],
34-
"other_holders": []
36+
"other_license_expression": null,
37+
"other_holders": [],
38+
"resources_count": 3
3539
}
3640
],
3741
"consolidated_packages": [],

‎plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package-files-not-counted-in-license-holders-expected.json‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -58,15 +58,20 @@
5858
"repository_download_url": "https://registry.npmjs.org/test-package/-/test-package-0.0.1.tgz",
5959
"api_data_url": "https://registry.npmjs.org/test-package/0.0.1",
6060
"identifier": "pkg_npm_test_package_0_0_1_1",
61+
"consolidated_license_expression": "apache-2.0",
62+
"consolidated_holders": [
63+
"The Apache Software",
64+
"The Apache Software Foundation"
65+
],
66+
"consolidated_copyright": null,
6167
"core_license_expression": "apache-2.0",
62-
"other_license_expression": "apache-2.0",
63-
"consolidated_core_copyright": "Copyright (c) The Apache Software, The Apache Software Foundation",
64-
"consolidated_other_copyright": "Copyright (c) The Apache Software, The Apache Software Foundation",
6568
"core_holders": [],
69+
"other_license_expression": "apache-2.0",
6670
"other_holders": [
6771
"The Apache Software",
6872
"The Apache Software Foundation"
69-
]
73+
],
74+
"resources_count": 5
7075
}
7176
],
7277
"files": [

‎plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package-fileset-expected.json‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -58,14 +58,18 @@
5858
"repository_download_url": "https://registry.npmjs.org/test-package/-/test-package-0.0.1.tgz",
5959
"api_data_url": "https://registry.npmjs.org/test-package/0.0.1",
6060
"identifier": "pkg_npm_test_package_0_0_1_1",
61+
"consolidated_license_expression": "apache-2.0",
62+
"consolidated_holders": [
63+
"The Apache Software Foundation"
64+
],
65+
"consolidated_copyright": null,
6166
"core_license_expression": "apache-2.0",
62-
"other_license_expression": "apache-2.0",
63-
"consolidated_core_copyright": "Copyright (c) The Apache Software Foundation",
64-
"consolidated_other_copyright": "Copyright (c) The Apache Software Foundation",
6567
"core_holders": [],
68+
"other_license_expression": "apache-2.0",
6669
"other_holders": [
6770
"The Apache Software Foundation"
68-
]
71+
],
72+
"resources_count": 5
6973
}
7074
],
7175
"files": [

‎plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package-manifest-expected.json‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -58,12 +58,14 @@
5858
"repository_download_url": "https://registry.npmjs.org/test-package/-/test-package-0.0.1.tgz",
5959
"api_data_url": "https://registry.npmjs.org/test-package/0.0.1",
6060
"identifier": "pkg_npm_test_package_0_0_1_1",
61+
"consolidated_license_expression": "apache-2.0",
62+
"consolidated_holders": [],
63+
"consolidated_copyright": null,
6164
"core_license_expression": "apache-2.0",
62-
"other_license_expression": "apache-2.0",
63-
"consolidated_core_copyright": null,
64-
"consolidated_other_copyright": null,
6565
"core_holders": [],
66-
"other_holders": []
66+
"other_license_expression": "apache-2.0",
67+
"other_holders": [],
68+
"resources_count": 2
6769
}
6870
],
6971
"files": [

0 commit comments

Comments
 (0)