Skip to content

Commit 64f6d48

Browse files
JonoYangJonoYang
committed
Track child RIDs
 * This is so we do not have to walk the codebase twice to find children that have been summarized * We can instead just iterate through the list of RIDs and modify them * Update tests due to previous methods of tagging summarized Resources being flawed Signed-off-by: Jono Yang <jyang@nexb.com>
1 parent 1be63ad commit 64f6d48

4 files changed

Lines changed: 34 additions & 32 deletions

File tree

‎src/scancode/plugin_summary.py‎

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ def logger_debug(*args):
6565
class OriginSummary(PostScanPlugin):
6666
"""
6767
Rolls up copyright and license results to the directory level if a copyright or license
68-
is detected in 50% or more of total files in a directory
68+
is detected in 75% or more of total files in a directory
6969
"""
7070
resource_attributes = dict(
7171
origin_summary=attr.ib(default=attr.Factory(OrderedDict)),
@@ -92,7 +92,6 @@ def process_codebase(self, codebase, **kwargs):
9292

9393
for resource in codebase.walk(topdown=False):
9494
# TODO: Consider facets for later
95-
# TODO: Group summarizations by copyright holders and license expressions
9695

9796
if resource.is_file:
9897
continue
@@ -101,8 +100,8 @@ def process_codebase(self, codebase, **kwargs):
101100
if not children:
102101
continue
103102

104-
# TODO: Consider using a list of resource id's to avoid walking a codebase multiple times
105103
origin_count = Counter()
104+
child_rids = []
106105

107106
for child in children:
108107
if child.is_file:
@@ -115,7 +114,10 @@ def process_codebase(self, codebase, **kwargs):
115114
else:
116115
# We are in a subdirectory
117116
child_origin_count = child.extra_data.get('origin_count', {})
117+
if not child_origin_count:
118+
continue
118119
origin_count.update(child_origin_count)
120+
child_rids.append(child.rid)
119121

120122
if origin_count:
121123
resource.extra_data['origin_count'] = origin_count
@@ -128,16 +130,17 @@ def process_codebase(self, codebase, **kwargs):
128130
resource.origin_summary['count'] = top_count
129131
codebase.save_resource(resource)
130132

131-
for descendant in resource.walk(codebase, topdown=True):
132-
if descendant.is_file:
133-
d_license_expression = combine_expressions(descendant.license_expressions)
134-
d_holders = tuple(h['value'] for h in descendant.holders)
133+
for child_rid in child_rids:
134+
child = codebase.get_resource(child_rid)
135+
if child.is_file:
136+
child_license_expression = combine_expressions(child.license_expressions)
137+
child_holders = tuple(h['value'] for h in child.holders)
135138
else:
136-
d_license_expression = descendant.origin_summary.get('license_expression')
137-
d_holders = descendant.origin_summary.get('holders')
138-
if (d_holders, d_license_expression) == (holders, license_expression):
139-
descendant.summarized_to = resource.path
140-
descendant.save(codebase)
139+
child_license_expression = child.origin_summary.get('license_expression')
140+
child_holders = child.origin_summary.get('holders')
141+
if (child_holders, child_license_expression) == (holders, license_expression):
142+
child.summarized_to = resource.path
143+
child.save(codebase)
141144

142145

143146
def is_majority(count, files_count):

‎tests/scancode/data/plugin_origin_summary/clear-summary-expected.json‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,8 @@
1414
"--origin-summary": true
1515
},
1616
"notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.",
17-
"start_timestamp": "2019-06-06T234305.680143",
18-
"end_timestamp": "2019-06-06T234310.595711",
17+
"start_timestamp": "2019-06-07T012733.684306",
18+
"end_timestamp": "2019-06-07T012738.755282",
1919
"message": null,
2020
"errors": [],
2121
"extra_data": {
@@ -248,7 +248,7 @@
248248
],
249249
"authors": [],
250250
"origin_summary": {},
251-
"summarized_to": "clear_summary",
251+
"summarized_to": "clear_summary/src",
252252
"files_count": 0,
253253
"dirs_count": 0,
254254
"size_count": 0,
@@ -326,7 +326,7 @@
326326
],
327327
"authors": [],
328328
"origin_summary": {},
329-
"summarized_to": "clear_summary",
329+
"summarized_to": "clear_summary/src",
330330
"files_count": 0,
331331
"dirs_count": 0,
332332
"size_count": 0,
@@ -404,7 +404,7 @@
404404
],
405405
"authors": [],
406406
"origin_summary": {},
407-
"summarized_to": "clear_summary",
407+
"summarized_to": "clear_summary/src",
408408
"files_count": 0,
409409
"dirs_count": 0,
410410
"size_count": 0,
@@ -482,7 +482,7 @@
482482
],
483483
"authors": [],
484484
"origin_summary": {},
485-
"summarized_to": "clear_summary",
485+
"summarized_to": "clear_summary/src",
486486
"files_count": 0,
487487
"dirs_count": 0,
488488
"size_count": 0,
@@ -560,7 +560,7 @@
560560
],
561561
"authors": [],
562562
"origin_summary": {},
563-
"summarized_to": "clear_summary",
563+
"summarized_to": "clear_summary/src",
564564
"files_count": 0,
565565
"dirs_count": 0,
566566
"size_count": 0,
@@ -638,7 +638,7 @@
638638
],
639639
"authors": [],
640640
"origin_summary": {},
641-
"summarized_to": "clear_summary",
641+
"summarized_to": "clear_summary/src",
642642
"files_count": 0,
643643
"dirs_count": 0,
644644
"size_count": 0,
@@ -716,7 +716,7 @@
716716
],
717717
"authors": [],
718718
"origin_summary": {},
719-
"summarized_to": "clear_summary",
719+
"summarized_to": "clear_summary/src",
720720
"files_count": 0,
721721
"dirs_count": 0,
722722
"size_count": 0,
@@ -794,7 +794,7 @@
794794
],
795795
"authors": [],
796796
"origin_summary": {},
797-
"summarized_to": "clear_summary",
797+
"summarized_to": "clear_summary/src",
798798
"files_count": 0,
799799
"dirs_count": 0,
800800
"size_count": 0,
@@ -872,7 +872,7 @@
872872
],
873873
"authors": [],
874874
"origin_summary": {},
875-
"summarized_to": "clear_summary",
875+
"summarized_to": "clear_summary/src",
876876
"files_count": 0,
877877
"dirs_count": 0,
878878
"size_count": 0,
@@ -1024,7 +1024,7 @@
10241024
],
10251025
"authors": [],
10261026
"origin_summary": {},
1027-
"summarized_to": "clear_summary/third_party",
1027+
"summarized_to": "clear_summary/third_party/module_2",
10281028
"files_count": 0,
10291029
"dirs_count": 0,
10301030
"size_count": 0,
@@ -1102,7 +1102,7 @@
11021102
],
11031103
"authors": [],
11041104
"origin_summary": {},
1105-
"summarized_to": "clear_summary/third_party",
1105+
"summarized_to": "clear_summary/third_party/module_2",
11061106
"files_count": 0,
11071107
"dirs_count": 0,
11081108
"size_count": 0,
@@ -1217,7 +1217,7 @@
12171217
],
12181218
"authors": [],
12191219
"origin_summary": {},
1220-
"summarized_to": "clear_summary/third_party",
1220+
"summarized_to": "clear_summary/third_party/module_3",
12211221
"files_count": 0,
12221222
"dirs_count": 0,
12231223
"size_count": 0,

‎tests/scancode/data/plugin_origin_summary/no-summary-expected.json‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,8 @@
1414
"--origin-summary": true
1515
},
1616
"notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.",
17-
"start_timestamp": "2019-06-06T234614.713175",
18-
"end_timestamp": "2019-06-06T234619.473447",
17+
"start_timestamp": "2019-06-07T012821.074369",
18+
"end_timestamp": "2019-06-07T012825.892004",
1919
"message": null,
2020
"errors": [],
2121
"extra_data": {
@@ -550,7 +550,7 @@
550550
],
551551
"authors": [],
552552
"origin_summary": {},
553-
"summarized_to": "no_summary/third_party",
553+
"summarized_to": "no_summary/third_party/module_2",
554554
"files_count": 0,
555555
"dirs_count": 0,
556556
"size_count": 0,
@@ -628,7 +628,7 @@
628628
],
629629
"authors": [],
630630
"origin_summary": {},
631-
"summarized_to": "no_summary/third_party",
631+
"summarized_to": "no_summary/third_party/module_2",
632632
"files_count": 0,
633633
"dirs_count": 0,
634634
"size_count": 0,
@@ -743,7 +743,7 @@
743743
],
744744
"authors": [],
745745
"origin_summary": {},
746-
"summarized_to": "no_summary/third_party",
746+
"summarized_to": "no_summary/third_party/module_3",
747747
"files_count": 0,
748748
"dirs_count": 0,
749749
"size_count": 0,

‎tests/scancode/test_plugin_summary.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@
3131
import json
3232

3333
from commoncode.testcase import FileDrivenTesting
34-
from scancode.cli_test_utils import check_json_scan
3534
from scancode.cli_test_utils import run_scan_click
3635
from scancode.plugin_summary import is_majority
3736

0 commit comments

Comments
 (0)