diff --git a/plugins/scancode-consolidate-scan/.gitignore b/plugins/scancode-consolidate-scan/.gitignore new file mode 100644 index 00000000000..e2cae228c89 --- /dev/null +++ b/plugins/scancode-consolidate-scan/.gitignore @@ -0,0 +1,2 @@ +/build/ +/dist/ diff --git a/plugins/scancode-consolidate-scan/MANIFEST.in b/plugins/scancode-consolidate-scan/MANIFEST.in new file mode 100644 index 00000000000..c2cc8541f4c --- /dev/null +++ b/plugins/scancode-consolidate-scan/MANIFEST.in @@ -0,0 +1,11 @@ +graft src + +include setup.py +include setup.cfg +include .gitignore +include README.md +include MANIFEST.in +include NOTICE +include apache-2.0.LICENSE + +global-exclude *.py[co] __pycache__ *.*~ diff --git a/plugins/scancode-consolidate-scan/NOTICE b/plugins/scancode-consolidate-scan/NOTICE new file mode 100644 index 00000000000..edd8b6b075b --- /dev/null +++ b/plugins/scancode-consolidate-scan/NOTICE @@ -0,0 +1,25 @@ +Software license +================ + +Copyright (c) 2019 nexB Inc. and others. All rights reserved. +http://nexb.com and https://github.com/nexB/scancode-toolkit/ +The ScanCode software is licensed under the Apache License version 2.0. +Data generated with ScanCode require an acknowledgment. +ScanCode is a trademark of nexB Inc. + +You may not use this software except in compliance with the License. +You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +When you publish or redistribute any data created with ScanCode or any ScanCode +derivative work, you must accompany this data with the following acknowledgment: + + Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES + OR CONDITIONS OF ANY KIND, either express or implied. No content created from + ScanCode should be considered or used as legal advice. Consult an Attorney + for any legal advice. + ScanCode is a free software code scanning tool from nexB Inc. and others. + Visit https://github.com/nexB/scancode-toolkit/ for support and download. diff --git a/plugins/scancode-consolidate-scan/README.md b/plugins/scancode-consolidate-scan/README.md new file mode 100644 index 00000000000..bfb20d3526a --- /dev/null +++ b/plugins/scancode-consolidate-scan/README.md @@ -0,0 +1,15 @@ +A ScanCode post-scan plugin to return consolidated components and consolidated +packages for different types of codebase summarization. + +A consolidated component is a group of Resources that have the same origin. +Currently, consolidated components are created by grouping Resources that have +the same license expression and copyright holders and the files that contain +this license expression and copyright holders combination make up 75% or more of +the files in the directory where they are found. + +A consolidated package is a detected package in the scanned codebase that has +been enhanced with data about other licenses and holders found within it. + +If a Resource is part of a consolidated component or consolidated package, then +the identifier of the consolidated component or consolidated package it is part +of is in the Resource's `consolidated_to` field. diff --git a/plugins/scancode-consolidate-scan/apache-2.0.LICENSE b/plugins/scancode-consolidate-scan/apache-2.0.LICENSE new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/plugins/scancode-consolidate-scan/apache-2.0.LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/scancode-consolidate-scan/setup.cfg b/plugins/scancode-consolidate-scan/setup.cfg new file mode 100644 index 00000000000..917cf1e0f0d --- /dev/null +++ b/plugins/scancode-consolidate-scan/setup.cfg @@ -0,0 +1,8 @@ +[metadata] +license_file = NOTICE + +[bdist_wheel] +universal = 1 + +[aliases] +release = clean --all bdist_wheel diff --git a/plugins/scancode-consolidate-scan/setup.py b/plugins/scancode-consolidate-scan/setup.py new file mode 100644 index 00000000000..5129294a148 --- /dev/null +++ b/plugins/scancode-consolidate-scan/setup.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +from __future__ import absolute_import +from __future__ import print_function + +from glob import glob +from os.path import basename +from os.path import join +from os.path import splitext + +from setuptools import find_packages +from setuptools import setup + + +desc = '''A ScanCode post-scan plugin that consolidates components and packages.''' + +setup( + name='scancode-consolidate-scan', + version='1.0.0', + license='apache-2.0 AND scancode-acknowledgement', + description=desc, + long_description=desc, + author='nexB', + author_email='info@aboutcode.org', + url='https://github.com/nexB/scancode-toolkit/plugins/scancode-consolidate-scan', + packages=find_packages('src'), + package_dir={'': 'src'}, + py_modules=[splitext(basename(path))[0] for path in glob('src/*.py')], + include_package_data=True, + zip_safe=False, + classifiers=[ + # complete classifier list: http://pypi.python.org/pypi?%3Aaction=list_classifiers + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2.7', + 'Topic :: Utilities', + ], + keywords=[ + 'open source', + 'scancode', + 'components', + 'packages', + 'summarization', + 'consolidation' + ], + install_requires=[ + 'scancode-toolkit', + ], + entry_points={ + 'scancode_post_scan': [ + 'consolidate = plugin_consolidate:Consolidator', + ], + } +) diff --git a/plugins/scancode-consolidate-scan/src/plugin_consolidate/__init__.py b/plugins/scancode-consolidate-scan/src/plugin_consolidate/__init__.py new file mode 100644 index 00000000000..b8a4bc89cb6 --- /dev/null +++ b/plugins/scancode-consolidate-scan/src/plugin_consolidate/__init__.py @@ -0,0 +1,443 @@ +# +# Copyright (c) 2019 nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/scancode-toolkit/ +# The ScanCode software is licensed under the Apache License version 2.0. +# Data generated with ScanCode require an acknowledgment. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with ScanCode or any ScanCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# ScanCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode-toolkit/ for support and download. + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from collections import Counter +from collections import defaultdict +from collections import OrderedDict + +import attr + +from cluecode.copyrights import CopyrightDetector +from commoncode.text import python_safe_name +from license_expression import Licensing +from packagedcode import get_package_instance +from packagedcode.utils import combine_expressions +from plugincode.post_scan import PostScanPlugin +from plugincode.post_scan import post_scan_impl +from scancode import CommandLineOption +from scancode import POST_SCAN_GROUP + + +# Tracing flags +TRACE = False + + +def logger_debug(*args): + pass + + +if TRACE: + import logging + import sys + + logger = logging.getLogger(__name__) + # logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) + logging.basicConfig(stream=sys.stdout) + logger.setLevel(logging.DEBUG) + + def logger_debug(*args): + return logger.debug( + ' '.join(isinstance(a, unicode) and a or repr(a) for a in args)) + + +@attr.s +class Consolidation(object): + """ + A grouping of files that share the same origin. Other clues found in the + codebase are stored in `other_license_expression` and `other_holders` + """ + identifier = attr.ib(default=None) + consolidated_license_expression = attr.ib(default=None) + consolidated_holders = attr.ib(default=attr.Factory(list)) + consolidated_copyright = attr.ib(default=None) + core_license_expression = attr.ib(default=None) + core_holders = attr.ib(default=attr.Factory(list)) + other_license_expression = attr.ib(default=None) + other_holders = attr.ib(default=attr.Factory(list)) + files_count = attr.ib(default=None) + resources = attr.ib(default=attr.Factory(list)) + + def to_dict(self, **kwargs): + def dict_fields(attr, value): + if attr.name in ('resources', ): + return False + return True + license_expressions_to_combine = [] + if self.core_license_expression: + license_expressions_to_combine.append(self.core_license_expression) + if self.other_license_expression: + license_expressions_to_combine.append(self.other_license_expression) + if license_expressions_to_combine: + combined_license_expression = combine_expressions(license_expressions_to_combine) + if combined_license_expression: + self.consolidated_license_expression = str(Licensing().parse(combined_license_expression).simplify()) + self.consolidated_copyright = self.consolidate_copyright() + return attr.asdict(self, filter=dict_fields, dict_factory=OrderedDict) + + def consolidate_copyright(self): + # TODO: Verify and test that we are generating detectable copyrights + if not self.consolidated_holders: + self.consolidated_holders = sorted(set(list(self.core_holders) + list(self.other_holders))) + return 'Copyright (c) {}'.format(', '.join(self.consolidated_holders)) + + +@attr.s +class ConsolidatedComponent(object): + # TODO: have an attribute for key files (one that strongly determines origin) + type=attr.ib() + consolidation = attr.ib() + + def to_dict(self, **kwargs): + c = OrderedDict(type=self.type) + c.update(self.consolidation.to_dict()) + return c + + +@attr.s +class ConsolidatedPackage(object): + package = attr.ib() + consolidation = attr.ib() + + def to_dict(self, **kwargs): + package = self.package.to_dict() + package.update(self.consolidation.to_dict()) + return package + + +@post_scan_impl +class Consolidator(PostScanPlugin): + """ + A post-scan plugin that returns Components for consolidated resources. + + This plugin consolidates resources by: + - Packages + - Copyright holders and license expression + """ + codebase_attributes = OrderedDict( + consolidated_components=attr.ib(default=attr.Factory(list)), + consolidated_packages=attr.ib(default=attr.Factory(list)) + ) + + resource_attributes = dict( + consolidated_to=attr.ib(default=attr.Factory(list)) + ) + + sort_order = 8 + + options = [ + CommandLineOption(('--consolidate',), + is_flag=True, default=False, + help='Return a list of consolidated packages and a list of consolidated components', + help_group=POST_SCAN_GROUP + ) + ] + + def is_enabled(self, consolidate, **kwargs): + return consolidate + + def process_codebase(self, codebase, **kwargs): + # Collect ConsolidatedPackages and ConsolidatedComponents + # TODO: Have a "catch-all" Component for the things that we haven't grouped + packages = [] + components = [] + root = codebase.root + if hasattr(root, 'packages') and hasattr(root, 'copyrights') and hasattr(root, 'licenses'): + packages.extend(get_consolidated_packages(codebase)) + if hasattr(root, 'copyrights') and hasattr(root, 'licenses'): + components.extend(get_license_holders_consolidated_components(codebase)) + + if not packages and not components: + return + + # Process ConsolidatedPackages and ConsolidatedComponents (if needed) + # components = combine_license_holders_consolidated_components(components) + + # Add ConsolidatedPackages and ConsolidatedComponents to top-level codebase attributes + codebase.attributes.consolidated_packages = consolidated_packages = [] + for index, package in enumerate(packages, start=1): + if not package.consolidation.resources: + continue + identifier = python_safe_name('{}_{}'.format(package.package.purl, index)) + package.consolidation.identifier = identifier + for resource in package.consolidation.resources: + resource.consolidated_to.append(identifier) + resource.save(codebase) + consolidated_packages.append(package.to_dict()) + + codebase.attributes.consolidated_components = consolidated_components = [] + for index, component in enumerate(components, start=1): + # Skip ConsolidatedComponents that do not have files in them + if not component.consolidation.resources: + continue + # TODO: Consider adding license expression to be part of name + holders = '_'.join(component.consolidation.core_holders) + other_holders = '_'.join(component.consolidation.other_holders) + holders = holders or other_holders + # We do not want the name to be too long + holders = holders[:65] + if holders: + identifier = python_safe_name('{}_{}'.format(holders, index)) + else: + identifier = index + component.consolidation.identifier = identifier + for resource in component.consolidation.resources: + resource.consolidated_to.append(identifier) + resource.save(codebase) + consolidated_components.append(component.to_dict()) + + # Dedupe and sort names in consolidated_to + for resource in codebase.walk(topdown=True): + resource.consolidated_to = sorted(set(resource.consolidated_to)) + resource.save(codebase) + + +def get_consolidated_packages(codebase): + """ + Yield a ConsolidatedPackage for each detected package in the codebase + """ + for resource in codebase.walk(topdown=False): + for package_data in resource.packages: + package = get_package_instance(package_data) + package_resources = list(package.get_package_resources(resource, codebase)) + package_license_expression = package.license_expression + package_copyright = package.copyright + + package_holders = [] + if package_copyright: + numbered_lines = [(0, package_copyright)] + for _, holder, _, _ in CopyrightDetector().detect(numbered_lines, + copyrights=False, holders=True, authors=False, include_years=False): + package_holders.append(holder) + + discovered_license_expressions = [] + discovered_holders = [] + for package_resource in package_resources: + # If a resource is part of a package Component, then it cannot be part of any other type of Component + package_resource.extra_data['in_package_component'] = True + package_resource.save(codebase) + + package_resource_license_expression = combine_expressions(package_resource.license_expressions) + package_resource_holders = package_resource.holders + if not package_resource_license_expression and not package_resource_holders: + continue + discovered_license_expressions.append(package_resource_license_expression) + discovered_holders.extend(h.get('value') for h in package_resource_holders) + + # Remove NoneTypes from discovered licenses + discovered_license_expressions = [lic for lic in discovered_license_expressions if lic] + # Remove NoneTypes from discovered holders + discovered_holders = [holder for holder in discovered_holders if holder] + + combined_discovered_license_expression = combine_expressions(discovered_license_expressions) + if combined_discovered_license_expression: + simplified_discovered_license_expression = str(Licensing().parse(combined_discovered_license_expression).simplify()) + else: + simplified_discovered_license_expression = None + + c = Consolidation( + core_license_expression=package_license_expression, + core_holders=sorted(set(package_holders)), + other_license_expression=simplified_discovered_license_expression, + other_holders=sorted(set(discovered_holders)), + files_count=sum(1 for package_resource in package_resources if package_resource.is_file), + resources=package_resources, + ) + yield ConsolidatedPackage( + package=package, + consolidation=c + ) + + +def get_license_holders_consolidated_components(codebase): + """ + Yield a ConsolidatedComponent for each directory where 75% or more of the files have the + same license expression and copyright holders + """ + # TODO: Create Consolidated Components for the 25% or less of files that + # aren't part of the majority + # TODO: Take license score into account + root = codebase.root + if root.extra_data.get('in_package_component'): + return + + origin_translation_table = {} + for resource in codebase.walk(topdown=False): + # TODO: Consider facets for later + + if resource.is_file or resource.extra_data.get('in_package_component'): + continue + + children = resource.children(codebase) + if not children: + continue + + # Collect license expression and holders count for stat-based summarization + origin_count = Counter() + # TODO: Consider license match coverage and license score when consolidating things on licenses + # We may consolidate things by a weakly-matched license + for child in children: + if child.extra_data.get('in_package_component'): + continue + if child.is_file: + license_expression = combine_expressions(child.license_expressions) + holders = tuple(h['value'] for h in child.holders) + if not license_expression or not holders: + continue + origin = holders, license_expression + origin_key = ''.join(holders) + license_expression + origin_translation_table[origin_key] = origin + origin_count[origin_key] += 1 + else: + # We are in a subdirectory + child_origin_count = child.extra_data.get('origin_count', {}) + if not child_origin_count: + continue + origin_count.update(child_origin_count) + + if origin_count: + resource.extra_data['origin_count'] = origin_count + resource.save(codebase) + + # TODO: When there is a tie, we need to be explicit and consistent about the tiebreaker + # TODO: Consider creating two components instead of tiebreaking + origin_key, top_count = origin_count.most_common(1)[0] + if is_majority(top_count, resource.files_count): + majority_holders, majority_license_expression = origin_translation_table[origin_key] + resource.extra_data['origin_summary_license_expression'] = majority_license_expression + resource.extra_data['origin_summary_holders'] = majority_holders + resource.extra_data['origin_summary_count'] = top_count + resource.save(codebase) + + # Create consolidated components for a child that has a majority + # that is different than the one we have now + for child in children: + origin_summary_license_expression = child.extra_data.get('origin_summary_license_expression') + origin_summary_holders = child.extra_data.get('origin_summary_holders') + if (origin_summary_license_expression and origin_summary_holders + and origin_summary_license_expression != majority_license_expression + and origin_summary_holders != majority_holders): + c = create_license_holders_consolidated_component(child, codebase) + if c: + yield c + else: + # If there is no majority, we see if any of our child directories had majorities + for child in children: + c = create_license_holders_consolidated_component(child, codebase) + if c: + yield c + + # Yield a Component for root if there is a majority + c = create_license_holders_consolidated_component(root, codebase) + if c: + yield c + + +def is_majority(count, files_count): + """ + Return True if `count` divided by `files_count` is greater than or equal to + `threshold` + """ + # TODO: Increase this and test with real codebases + threshold = 0.75 + return count / files_count >= threshold + + +def create_license_holders_consolidated_component(resource, codebase): + """ + Return a ConsolidatedComponent for `resource` if it can be summarized on license + expression and holders + """ + license_expression = resource.extra_data.get('origin_summary_license_expression') + holders = resource.extra_data.get('origin_summary_holders') + if license_expression and holders: + component_resources = get_consolidated_component_resources(resource, codebase) + if component_resources: + c = Consolidation( + core_license_expression=license_expression, + core_holders=holders, + files_count=sum(1 for component_resource in component_resources if component_resource.is_file), + resources=component_resources, + ) + return ConsolidatedComponent( + type='license-holders', + consolidation=c + ) + + +def get_consolidated_component_resources(resource, codebase): + """ + Return a list of resources to be used to create a ConsolidatedComponent from `resource` + """ + license_expression = resource.extra_data.get('origin_summary_license_expression') + holders = resource.extra_data.get('origin_summary_holders') + if not license_expression and holders: + return + resources = [] if resource.extra_data.get('in_package_component') else [resource] + for r in resource.walk(codebase, topdown=False): + if r.extra_data.get('in_package_component'): + continue + resource_holders = tuple(h.get('value') for h in r.holders) + if ((r.is_file + and combine_expressions(r.license_expressions) == license_expression + and resource_holders == holders) + or (r.is_dir + and r.extra_data.get('origin_summary_license_expression', '') == license_expression + and r.extra_data.get('origin_summary_holders', tuple()) == holders)): + resources.append(r) + return resources + + +def combine_license_holders_consolidated_components(components): + """ + Combine ConsolidatedComponents with the same license expression and holders into a + single ConsolidatedComponent + """ + origin_translation_table = {} + components_by_holders_license_expression = defaultdict(list) + for component in components: + origin = component.consolidation.core_holders, component.consolidation.core_license_expression + origin_key = ''.join(component.consolidation.core_holders) + component.consolidation.core_license_expression + origin_translation_table[origin_key] = origin + components_by_holders_license_expression[origin_key].append(component) + + for origin_key, components in components_by_holders_license_expression.items(): + component_resources = [] + for component in components: + component_resources.extend(component.consolidation.resources) + component_holders, component_license_expression = origin_translation_table[origin_key] + c = Consolidation( + core_license_expression=component_license_expression, + core_holders=component_holders, + files_count=sum(1 for component_resource in component_resources if component_resource.is_file), + resources=component_resources, + ) + yield ConsolidatedComponent( + type='license-holders', + consolidation=c, + ) diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/clear-summary-expected.json b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/clear-summary-expected.json new file mode 100644 index 00000000000..3a9f84f8e4c --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/clear-summary-expected.json @@ -0,0 +1,389 @@ +{ + "headers": [ + { + "tool_name": "scancode-toolkit", + "options": { + "input": "", + "--consolidate": true, + "--copyright": true, + "--info": true, + "--json": "", + "--license": true, + "--package": true + }, + "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "message": null, + "errors": [], + "extra_data": { + "files_count": 4 + } + } + ], + "consolidated_components": [ + { + "type": "license-holders", + "identifier": "the_apache_software_foundation_1", + "consolidated_license_expression": "apache-2.0", + "consolidated_holders": [ + "The Apache Software Foundation" + ], + "consolidated_copyright": "Copyright (c) The Apache Software Foundation", + "core_license_expression": "apache-2.0", + "core_holders": [ + "The Apache Software Foundation" + ], + "other_license_expression": null, + "other_holders": [], + "files_count": 3 + } + ], + "consolidated_packages": [], + "files": [ + { + "path": "clear-summary", + "type": "directory", + "name": "clear-summary", + "base_name": "clear-summary", + "extension": "", + "size": 0, + "sha1": null, + "md5": null, + "mime_type": null, + "file_type": null, + "programming_language": null, + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [], + "license_expressions": [], + "holders": [], + "copyrights": [], + "authors": [], + "packages": [], + "consolidated_to": [ + "the_apache_software_foundation_1" + ], + "files_count": 4, + "dirs_count": 0, + "size_count": 200, + "scan_errors": [] + }, + { + "path": "clear-summary/src1", + "type": "file", + "name": "src1", + "base_name": "src1", + "extension": "", + "size": 56, + "sha1": "f3eae084a5690638fd1ada38e653a94179f14263", + "md5": "7e0e8162de50e718e248ccd7a2d2cb65", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "apache-2.0", + "score": 50.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 50.0 + } + } + ], + "license_expressions": [ + "apache-2.0" + ], + "holders": [ + { + "value": "The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "Copyright (c) The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "consolidated_to": [ + "the_apache_software_foundation_1" + ], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "clear-summary/src2", + "type": "file", + "name": "src2", + "base_name": "src2", + "extension": "", + "size": 56, + "sha1": "f3eae084a5690638fd1ada38e653a94179f14263", + "md5": "7e0e8162de50e718e248ccd7a2d2cb65", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "apache-2.0", + "score": 50.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 50.0 + } + } + ], + "license_expressions": [ + "apache-2.0" + ], + "holders": [ + { + "value": "The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "Copyright (c) The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "consolidated_to": [ + "the_apache_software_foundation_1" + ], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "clear-summary/src3", + "type": "file", + "name": "src3", + "base_name": "src3", + "extension": "", + "size": 56, + "sha1": "f3eae084a5690638fd1ada38e653a94179f14263", + "md5": "7e0e8162de50e718e248ccd7a2d2cb65", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "apache-2.0", + "score": 50.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 50.0 + } + } + ], + "license_expressions": [ + "apache-2.0" + ], + "holders": [ + { + "value": "The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "Copyright (c) The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "consolidated_to": [ + "the_apache_software_foundation_1" + ], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "clear-summary/src4", + "type": "file", + "name": "src4", + "base_name": "src4", + "extension": "", + "size": 32, + "sha1": "9d75736fe41651abebb509da45380299d93fbafd", + "md5": "55abb8def9f0ecedeca243349fbb9b62", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "gpl-2.0", + "score": 80.0, + "name": "GNU General Public License 2.0", + "short_name": "GPL 2.0", + "category": "Copyleft", + "is_exception": false, + "owner": "Free Software Foundation (FSF)", + "homepage_url": "http://www.gnu.org/licenses/gpl-2.0.html", + "text_url": "http://www.gnu.org/licenses/gpl-2.0.txt", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:gpl-2.0", + "spdx_license_key": "GPL-2.0-only", + "spdx_url": "https://spdx.org/licenses/GPL-2.0-only", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "gpl-2.0_52.RULE", + "license_expression": "gpl-2.0", + "licenses": [ + "gpl-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 80.0 + } + } + ], + "license_expressions": [ + "gpl-2.0" + ], + "holders": [ + { + "value": "IBM Corp.", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "Copyright (c) IBM Corp.", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "consolidated_to": [], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + } + ] +} \ No newline at end of file diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/clear-summary.json b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/clear-summary.json new file mode 100644 index 00000000000..5095683d34d --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/clear-summary.json @@ -0,0 +1,366 @@ +{ + "headers": [ + { + "tool_name": "scancode-toolkit", + "tool_version": "3.0.2.post1250.904952c36", + "options": { + "input": [ + "plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/clear-summary" + ], + "--copyright": true, + "--info": true, + "--json-pp": "plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/clear-summary.json", + "--license": true, + "--package": true + }, + "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "start_timestamp": "2019-07-30T192541.418011", + "end_timestamp": "2019-07-30T192546.417756", + "message": null, + "errors": [], + "extra_data": { + "files_count": 4 + } + } + ], + "files": [ + { + "path": "clear-summary", + "type": "directory", + "name": "clear-summary", + "base_name": "clear-summary", + "extension": "", + "size": 0, + "date": null, + "sha1": null, + "md5": null, + "mime_type": null, + "file_type": null, + "programming_language": null, + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [], + "license_expressions": [], + "holders": [], + "copyrights": [], + "authors": [], + "packages": [], + "files_count": 4, + "dirs_count": 0, + "size_count": 200, + "scan_errors": [] + }, + { + "path": "clear-summary/src1", + "type": "file", + "name": "src1", + "base_name": "src1", + "extension": "", + "size": 56, + "date": "2019-07-30", + "sha1": "f3eae084a5690638fd1ada38e653a94179f14263", + "md5": "7e0e8162de50e718e248ccd7a2d2cb65", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "apache-2.0", + "score": 50.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 50.0 + } + } + ], + "license_expressions": [ + "apache-2.0" + ], + "holders": [ + { + "value": "The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "Copyright (c) The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "clear-summary/src2", + "type": "file", + "name": "src2", + "base_name": "src2", + "extension": "", + "size": 56, + "date": "2019-07-30", + "sha1": "f3eae084a5690638fd1ada38e653a94179f14263", + "md5": "7e0e8162de50e718e248ccd7a2d2cb65", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "apache-2.0", + "score": 50.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 50.0 + } + } + ], + "license_expressions": [ + "apache-2.0" + ], + "holders": [ + { + "value": "The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "Copyright (c) The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "clear-summary/src3", + "type": "file", + "name": "src3", + "base_name": "src3", + "extension": "", + "size": 56, + "date": "2019-07-30", + "sha1": "f3eae084a5690638fd1ada38e653a94179f14263", + "md5": "7e0e8162de50e718e248ccd7a2d2cb65", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "apache-2.0", + "score": 50.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 50.0 + } + } + ], + "license_expressions": [ + "apache-2.0" + ], + "holders": [ + { + "value": "The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "Copyright (c) The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "clear-summary/src4", + "type": "file", + "name": "src4", + "base_name": "src4", + "extension": "", + "size": 32, + "date": "2019-07-30", + "sha1": "9d75736fe41651abebb509da45380299d93fbafd", + "md5": "55abb8def9f0ecedeca243349fbb9b62", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "gpl-2.0", + "score": 80.0, + "name": "GNU General Public License 2.0", + "short_name": "GPL 2.0", + "category": "Copyleft", + "is_exception": false, + "owner": "Free Software Foundation (FSF)", + "homepage_url": "http://www.gnu.org/licenses/gpl-2.0.html", + "text_url": "http://www.gnu.org/licenses/gpl-2.0.txt", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:gpl-2.0", + "spdx_license_key": "GPL-2.0-only", + "spdx_url": "https://spdx.org/licenses/GPL-2.0-only", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "gpl-2.0_52.RULE", + "license_expression": "gpl-2.0", + "licenses": [ + "gpl-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 80.0 + } + } + ], + "license_expressions": [ + "gpl-2.0" + ], + "holders": [ + { + "value": "IBM Corp.", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "Copyright (c) IBM Corp.", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + } + ] +} diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/clear-summary/src1 b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/clear-summary/src1 new file mode 100644 index 00000000000..429e8931814 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/clear-summary/src1 @@ -0,0 +1,2 @@ +Copyright (c) The Apache Software Foundation +Apache 2.0 diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/clear-summary/src2 b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/clear-summary/src2 new file mode 100644 index 00000000000..429e8931814 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/clear-summary/src2 @@ -0,0 +1,2 @@ +Copyright (c) The Apache Software Foundation +Apache 2.0 diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/clear-summary/src3 b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/clear-summary/src3 new file mode 100644 index 00000000000..429e8931814 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/clear-summary/src3 @@ -0,0 +1,2 @@ +Copyright (c) The Apache Software Foundation +Apache 2.0 diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/clear-summary/src4 b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/clear-summary/src4 new file mode 100644 index 00000000000..5a7558b7210 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/clear-summary/src4 @@ -0,0 +1,2 @@ +Copyright (c) IBM Corp. +GPL 2.0 diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/component-package-expected.json b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/component-package-expected.json new file mode 100644 index 00000000000..9a5aa7e1ac6 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/component-package-expected.json @@ -0,0 +1,810 @@ +{ + "headers": [ + { + "tool_name": "scancode-toolkit", + "options": { + "input": "", + "--consolidate": true, + "--copyright": true, + "--info": true, + "--json": "", + "--license": true, + "--package": true + }, + "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "message": null, + "errors": [], + "extra_data": { + "files_count": 7 + } + } + ], + "consolidated_components": [ + { + "type": "license-holders", + "identifier": "the_apache_software_foundation_1", + "consolidated_license_expression": "apache-2.0", + "consolidated_holders": [ + "The Apache Software Foundation" + ], + "consolidated_copyright": "Copyright (c) The Apache Software Foundation", + "core_license_expression": "apache-2.0", + "core_holders": [ + "The Apache Software Foundation" + ], + "other_license_expression": null, + "other_holders": [], + "files_count": 3 + } + ], + "consolidated_packages": [ + { + "type": "npm", + "namespace": null, + "name": "test-package", + "version": "0.0.1", + "qualifiers": null, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://registry.npmjs.org/test-package/-/test-package-0.0.1.tgz", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": "apache-2.0", + "declared_license": [ + "apache-2.0" + ], + "notice_text": null, + "manifest_path": "package.json", + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:npm/test-package@0.0.1", + "repository_homepage_url": "https://www.npmjs.com/package/test-package", + "repository_download_url": "https://registry.npmjs.org/test-package/-/test-package-0.0.1.tgz", + "api_data_url": "https://registry.npmjs.org/test-package/0.0.1", + "identifier": "pkg_npm_test_package_0_0_1_1", + "consolidated_license_expression": "apache-2.0 AND gpl-1.0-plus AND unknown", + "consolidated_holders": [ + "IBM Corp.", + "The Apache Software", + "The Apache Software Foundation" + ], + "consolidated_copyright": "Copyright (c) IBM Corp., The Apache Software, The Apache Software Foundation", + "core_license_expression": "apache-2.0", + "core_holders": [], + "other_license_expression": "apache-2.0 AND gpl-1.0-plus AND unknown", + "other_holders": [ + "IBM Corp.", + "The Apache Software", + "The Apache Software Foundation" + ], + "files_count": 3 + } + ], + "files": [ + { + "path": "component-package", + "type": "directory", + "name": "component-package", + "base_name": "component-package", + "extension": "", + "size": 0, + "sha1": null, + "md5": null, + "mime_type": null, + "file_type": null, + "programming_language": null, + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [], + "license_expressions": [], + "holders": [], + "copyrights": [], + "authors": [], + "packages": [], + "consolidated_to": [], + "files_count": 7, + "dirs_count": 2, + "size_count": 477, + "scan_errors": [] + }, + { + "path": "component-package/component", + "type": "directory", + "name": "component", + "base_name": "component", + "extension": "", + "size": 0, + "sha1": null, + "md5": null, + "mime_type": null, + "file_type": null, + "programming_language": null, + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [], + "license_expressions": [], + "holders": [], + "copyrights": [], + "authors": [], + "packages": [], + "consolidated_to": [ + "the_apache_software_foundation_1" + ], + "files_count": 4, + "dirs_count": 0, + "size_count": 202, + "scan_errors": [] + }, + { + "path": "component-package/component/src1", + "type": "file", + "name": "src1", + "base_name": "src1", + "extension": "", + "size": 56, + "sha1": "f3eae084a5690638fd1ada38e653a94179f14263", + "md5": "7e0e8162de50e718e248ccd7a2d2cb65", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "apache-2.0", + "score": 50.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 50.0 + } + } + ], + "license_expressions": [ + "apache-2.0" + ], + "holders": [ + { + "value": "The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "Copyright (c) The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "consolidated_to": [ + "the_apache_software_foundation_1" + ], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "component-package/component/src2", + "type": "file", + "name": "src2", + "base_name": "src2", + "extension": "", + "size": 56, + "sha1": "f3eae084a5690638fd1ada38e653a94179f14263", + "md5": "7e0e8162de50e718e248ccd7a2d2cb65", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "apache-2.0", + "score": 50.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 50.0 + } + } + ], + "license_expressions": [ + "apache-2.0" + ], + "holders": [ + { + "value": "The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "Copyright (c) The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "consolidated_to": [ + "the_apache_software_foundation_1" + ], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "component-package/component/src3", + "type": "file", + "name": "src3", + "base_name": "src3", + "extension": "", + "size": 56, + "sha1": "f3eae084a5690638fd1ada38e653a94179f14263", + "md5": "7e0e8162de50e718e248ccd7a2d2cb65", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "apache-2.0", + "score": 50.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 50.0 + } + } + ], + "license_expressions": [ + "apache-2.0" + ], + "holders": [ + { + "value": "The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "Copyright (c) The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "consolidated_to": [ + "the_apache_software_foundation_1" + ], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "component-package/component/src4", + "type": "file", + "name": "src4", + "base_name": "src4", + "extension": "", + "size": 34, + "sha1": "ce82bf2d5d3e79dabf5960159d1e14e46560da63", + "md5": "5817e002d5c6420ebcd4dfa66e500f10", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "lgpl-2.0", + "score": 80.0, + "name": "GNU Library General Public License 2.0", + "short_name": "LGPL 2.0", + "category": "Copyleft Limited", + "is_exception": false, + "owner": "Free Software Foundation (FSF)", + "homepage_url": "http://www.gnu.org/licenses/old-licenses/lgpl-2.0.html", + "text_url": "http://www.gnu.org/licenses/lgpl-2.0.html", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:lgpl-2.0", + "spdx_license_key": "LGPL-2.0-only", + "spdx_url": "https://spdx.org/licenses/LGPL-2.0-only", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "lgpl-2.0_bare_id.RULE", + "license_expression": "lgpl-2.0", + "licenses": [ + "lgpl-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": false, + "is_license_tag": true, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 80.0 + } + } + ], + "license_expressions": [ + "lgpl-2.0" + ], + "holders": [ + { + "value": "nexB, Inc.", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "Copyright (c) nexB, Inc.", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "consolidated_to": [], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "component-package/package", + "type": "directory", + "name": "package", + "base_name": "package", + "extension": "", + "size": 0, + "sha1": null, + "md5": null, + "mime_type": null, + "file_type": null, + "programming_language": null, + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [], + "license_expressions": [], + "holders": [], + "copyrights": [], + "authors": [], + "packages": [ + { + "type": "npm", + "namespace": null, + "name": "test-package", + "version": "0.0.1", + "qualifiers": null, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://registry.npmjs.org/test-package/-/test-package-0.0.1.tgz", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": "apache-2.0", + "declared_license": [ + "apache-2.0" + ], + "notice_text": null, + "manifest_path": "package.json", + "dependencies": [], + "contains_source_code": null, + "source_packages": [] + } + ], + "consolidated_to": [ + "pkg_npm_test_package_0_0_1_1" + ], + "files_count": 3, + "dirs_count": 0, + "size_count": 275, + "scan_errors": [] + }, + { + "path": "component-package/package/package.json", + "type": "file", + "name": "package.json", + "base_name": "package", + "extension": ".json", + "size": 141, + "sha1": "a4a4a28858425dfb6795ff4eb7b397846f03fda9", + "md5": "e10ff141105dc64fbde2dc77c61275db", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "apache-2.0", + "score": 100.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 5, + "end_line": 5, + "matched_rule": { + "identifier": "apache-2.0_65.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": false, + "is_license_tag": true, + "matcher": "2-aho", + "rule_length": 4, + "matched_length": 4, + "match_coverage": 100.0, + "rule_relevance": 100.0 + } + } + ], + "license_expressions": [ + "apache-2.0" + ], + "holders": [ + { + "value": "The Apache Software", + "start_line": 4, + "end_line": 5 + } + ], + "copyrights": [ + { + "value": "Copyright (c) The Apache Software", + "start_line": 4, + "end_line": 5 + } + ], + "authors": [], + "packages": [], + "consolidated_to": [ + "pkg_npm_test_package_0_0_1_1" + ], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "component-package/package/src1", + "type": "file", + "name": "src1", + "base_name": "src1", + "extension": "", + "size": 56, + "sha1": "f3eae084a5690638fd1ada38e653a94179f14263", + "md5": "7e0e8162de50e718e248ccd7a2d2cb65", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "apache-2.0", + "score": 50.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 50.0 + } + } + ], + "license_expressions": [ + "apache-2.0" + ], + "holders": [ + { + "value": "The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "Copyright (c) The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "consolidated_to": [ + "pkg_npm_test_package_0_0_1_1" + ], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "component-package/package/src2", + "type": "file", + "name": "src2", + "base_name": "src2", + "extension": "", + "size": 78, + "sha1": "ddbbd1133bac042f1c1bcbd4aa31337756837271", + "md5": "79cb837ae2b9a134e62b7bf3762b5ddc", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "unknown", + "score": 16.0, + "name": "Unknown license detected but not recognized", + "short_name": "unknown", + "category": "Unstated License", + "is_exception": false, + "owner": "Unspecified", + "homepage_url": null, + "text_url": "", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:unknown", + "spdx_license_key": null, + "spdx_url": "", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "unknown_25.RULE", + "license_expression": "unknown", + "licenses": [ + "unknown" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 16 + } + }, + { + "key": "gpl-1.0-plus", + "score": 100.0, + "name": "GNU General Public License 1.0 or later", + "short_name": "GPL 1.0 or later", + "category": "Copyleft", + "is_exception": false, + "owner": "Free Software Foundation (FSF)", + "homepage_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", + "text_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:gpl-1.0-plus", + "spdx_license_key": "GPL-1.0-or-later", + "spdx_url": "https://spdx.org/licenses/GPL-1.0-or-later", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "gpl-1.0-plus_33.RULE", + "license_expression": "gpl-1.0-plus", + "licenses": [ + "gpl-1.0-plus" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 5, + "matched_length": 5, + "match_coverage": 100.0, + "rule_relevance": 100.0 + } + } + ], + "license_expressions": [ + "unknown", + "gpl-1.0-plus" + ], + "holders": [ + { + "value": "IBM Corp.", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "copyright (c) 2017 IBM Corp.", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "consolidated_to": [ + "pkg_npm_test_package_0_0_1_1" + ], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + } + ] +} \ No newline at end of file diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/component-package.json b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/component-package.json new file mode 100644 index 00000000000..b8649006188 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/component-package.json @@ -0,0 +1,728 @@ +{ + "headers": [ + { + "tool_name": "scancode-toolkit", + "tool_version": "3.0.2.post1250.904952c36", + "options": { + "input": [ + "plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/component-package" + ], + "--copyright": true, + "--info": true, + "--json-pp": "plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/component-package.json", + "--license": true, + "--package": true + }, + "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "start_timestamp": "2019-07-30T193211.267553", + "end_timestamp": "2019-07-30T193215.803227", + "message": null, + "errors": [], + "extra_data": { + "files_count": 7 + } + } + ], + "files": [ + { + "path": "component-package", + "type": "directory", + "name": "component-package", + "base_name": "component-package", + "extension": "", + "size": 0, + "date": null, + "sha1": null, + "md5": null, + "mime_type": null, + "file_type": null, + "programming_language": null, + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [], + "license_expressions": [], + "holders": [], + "copyrights": [], + "authors": [], + "packages": [], + "files_count": 7, + "dirs_count": 2, + "size_count": 477, + "scan_errors": [] + }, + { + "path": "component-package/component", + "type": "directory", + "name": "component", + "base_name": "component", + "extension": "", + "size": 0, + "date": null, + "sha1": null, + "md5": null, + "mime_type": null, + "file_type": null, + "programming_language": null, + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [], + "license_expressions": [], + "holders": [], + "copyrights": [], + "authors": [], + "packages": [], + "files_count": 4, + "dirs_count": 0, + "size_count": 202, + "scan_errors": [] + }, + { + "path": "component-package/component/src1", + "type": "file", + "name": "src1", + "base_name": "src1", + "extension": "", + "size": 56, + "date": "2019-07-30", + "sha1": "f3eae084a5690638fd1ada38e653a94179f14263", + "md5": "7e0e8162de50e718e248ccd7a2d2cb65", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "apache-2.0", + "score": 50.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 50.0 + } + } + ], + "license_expressions": [ + "apache-2.0" + ], + "holders": [ + { + "value": "The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "Copyright (c) The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "component-package/component/src2", + "type": "file", + "name": "src2", + "base_name": "src2", + "extension": "", + "size": 56, + "date": "2019-07-30", + "sha1": "f3eae084a5690638fd1ada38e653a94179f14263", + "md5": "7e0e8162de50e718e248ccd7a2d2cb65", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "apache-2.0", + "score": 50.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 50.0 + } + } + ], + "license_expressions": [ + "apache-2.0" + ], + "holders": [ + { + "value": "The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "Copyright (c) The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "component-package/component/src3", + "type": "file", + "name": "src3", + "base_name": "src3", + "extension": "", + "size": 56, + "date": "2019-07-30", + "sha1": "f3eae084a5690638fd1ada38e653a94179f14263", + "md5": "7e0e8162de50e718e248ccd7a2d2cb65", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "apache-2.0", + "score": 50.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 50.0 + } + } + ], + "license_expressions": [ + "apache-2.0" + ], + "holders": [ + { + "value": "The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "Copyright (c) The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "component-package/component/src4", + "type": "file", + "name": "src4", + "base_name": "src4", + "extension": "", + "size": 34, + "date": "2019-07-30", + "sha1": "ce82bf2d5d3e79dabf5960159d1e14e46560da63", + "md5": "5817e002d5c6420ebcd4dfa66e500f10", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "lgpl-2.0", + "score": 80.0, + "name": "GNU Library General Public License 2.0", + "short_name": "LGPL 2.0", + "category": "Copyleft Limited", + "is_exception": false, + "owner": "Free Software Foundation (FSF)", + "homepage_url": "http://www.gnu.org/licenses/old-licenses/lgpl-2.0.html", + "text_url": "http://www.gnu.org/licenses/lgpl-2.0.html", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:lgpl-2.0", + "spdx_license_key": "LGPL-2.0-only", + "spdx_url": "https://spdx.org/licenses/LGPL-2.0-only", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "lgpl-2.0_bare_id.RULE", + "license_expression": "lgpl-2.0", + "licenses": [ + "lgpl-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": false, + "is_license_tag": true, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 80.0 + } + } + ], + "license_expressions": [ + "lgpl-2.0" + ], + "holders": [ + { + "value": "nexB, Inc.", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "Copyright (c) nexB, Inc.", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "component-package/package", + "type": "directory", + "name": "package", + "base_name": "package", + "extension": "", + "size": 0, + "date": null, + "sha1": null, + "md5": null, + "mime_type": null, + "file_type": null, + "programming_language": null, + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [], + "license_expressions": [], + "holders": [], + "copyrights": [], + "authors": [], + "packages": [ + { + "type": "npm", + "namespace": null, + "name": "test-package", + "version": "0.0.1", + "qualifiers": null, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://registry.npmjs.org/test-package/-/test-package-0.0.1.tgz", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": "apache-2.0", + "declared_license": [ + "apache-2.0" + ], + "notice_text": null, + "manifest_path": "package.json", + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:npm/test-package@0.0.1", + "repository_homepage_url": "https://www.npmjs.com/package/test-package", + "repository_download_url": "https://registry.npmjs.org/test-package/-/test-package-0.0.1.tgz", + "api_data_url": "https://registry.npmjs.org/test-package/0.0.1" + } + ], + "files_count": 3, + "dirs_count": 0, + "size_count": 275, + "scan_errors": [] + }, + { + "path": "component-package/package/package.json", + "type": "file", + "name": "package.json", + "base_name": "package", + "extension": ".json", + "size": 141, + "date": "2019-07-30", + "sha1": "a4a4a28858425dfb6795ff4eb7b397846f03fda9", + "md5": "e10ff141105dc64fbde2dc77c61275db", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "apache-2.0", + "score": 100.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 5, + "end_line": 5, + "matched_rule": { + "identifier": "apache-2.0_65.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": false, + "is_license_tag": true, + "matcher": "2-aho", + "rule_length": 4, + "matched_length": 4, + "match_coverage": 100.0, + "rule_relevance": 100.0 + } + } + ], + "license_expressions": [ + "apache-2.0" + ], + "holders": [ + { + "value": "The Apache Software", + "start_line": 4, + "end_line": 5 + } + ], + "copyrights": [ + { + "value": "Copyright (c) The Apache Software", + "start_line": 4, + "end_line": 5 + } + ], + "authors": [], + "packages": [], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "component-package/package/src1", + "type": "file", + "name": "src1", + "base_name": "src1", + "extension": "", + "size": 56, + "date": "2019-07-30", + "sha1": "f3eae084a5690638fd1ada38e653a94179f14263", + "md5": "7e0e8162de50e718e248ccd7a2d2cb65", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "apache-2.0", + "score": 50.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 50.0 + } + } + ], + "license_expressions": [ + "apache-2.0" + ], + "holders": [ + { + "value": "The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "Copyright (c) The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "component-package/package/src2", + "type": "file", + "name": "src2", + "base_name": "src2", + "extension": "", + "size": 78, + "date": "2019-07-30", + "sha1": "ddbbd1133bac042f1c1bcbd4aa31337756837271", + "md5": "79cb837ae2b9a134e62b7bf3762b5ddc", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "unknown", + "score": 16.0, + "name": "Unknown license detected but not recognized", + "short_name": "unknown", + "category": "Unstated License", + "is_exception": false, + "owner": "Unspecified", + "homepage_url": null, + "text_url": "", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:unknown", + "spdx_license_key": null, + "spdx_url": "", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "unknown_25.RULE", + "license_expression": "unknown", + "licenses": [ + "unknown" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 16 + } + }, + { + "key": "gpl-1.0-plus", + "score": 100.0, + "name": "GNU General Public License 1.0 or later", + "short_name": "GPL 1.0 or later", + "category": "Copyleft", + "is_exception": false, + "owner": "Free Software Foundation (FSF)", + "homepage_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", + "text_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:gpl-1.0-plus", + "spdx_license_key": "GPL-1.0-or-later", + "spdx_url": "https://spdx.org/licenses/GPL-1.0-or-later", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "gpl-1.0-plus_33.RULE", + "license_expression": "gpl-1.0-plus", + "licenses": [ + "gpl-1.0-plus" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 5, + "matched_length": 5, + "match_coverage": 100.0, + "rule_relevance": 100.0 + } + } + ], + "license_expressions": [ + "unknown", + "gpl-1.0-plus" + ], + "holders": [ + { + "value": "IBM Corp.", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "copyright (c) 2017 IBM Corp.", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + } + ] +} diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/component-package/component/src1 b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/component-package/component/src1 new file mode 100644 index 00000000000..429e8931814 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/component-package/component/src1 @@ -0,0 +1,2 @@ +Copyright (c) The Apache Software Foundation +Apache 2.0 diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/component-package/component/src2 b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/component-package/component/src2 new file mode 100644 index 00000000000..429e8931814 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/component-package/component/src2 @@ -0,0 +1,2 @@ +Copyright (c) The Apache Software Foundation +Apache 2.0 diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/component-package/component/src3 b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/component-package/component/src3 new file mode 100644 index 00000000000..429e8931814 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/component-package/component/src3 @@ -0,0 +1,2 @@ +Copyright (c) The Apache Software Foundation +Apache 2.0 diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/component-package/component/src4 b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/component-package/component/src4 new file mode 100644 index 00000000000..42b764dab90 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/component-package/component/src4 @@ -0,0 +1,2 @@ +Copyright (c) nexB, Inc. +LGPL 2.0 diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/component-package/package/package.json b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/component-package/package/package.json new file mode 100644 index 00000000000..a0ccac174b9 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/component-package/package/package.json @@ -0,0 +1,6 @@ +{ + "name": "test-package", + "version": "0.0.1", + "copyright": "Copyright (c) The Apache Software Foundation", + "license": "apache-2.0" +} diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/component-package/package/src1 b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/component-package/package/src1 new file mode 100644 index 00000000000..429e8931814 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/component-package/package/src1 @@ -0,0 +1,2 @@ +Copyright (c) The Apache Software Foundation +Apache 2.0 diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/component-package/package/src2 b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/component-package/package/src2 new file mode 100644 index 00000000000..44691afa929 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/component-package/package/src2 @@ -0,0 +1,2 @@ +copyright (c) 2017 IBM Corp. +Licensed under the GNU General Public License 2.0 \ No newline at end of file diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/license-holder-rollup-expected.json b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/license-holder-rollup-expected.json new file mode 100644 index 00000000000..0cbbc253178 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/license-holder-rollup-expected.json @@ -0,0 +1,599 @@ +{ + "headers": [ + { + "tool_name": "scancode-toolkit", + "options": { + "input": "", + "--consolidate": true, + "--copyright": true, + "--info": true, + "--json": "", + "--license": true, + "--package": true + }, + "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "message": null, + "errors": [], + "extra_data": { + "files_count": 3 + } + } + ], + "consolidated_components": [ + { + "type": "license-holders", + "identifier": "ibm_corp__1", + "consolidated_license_expression": "gpl-1.0-plus AND gpl-2.0", + "consolidated_holders": [ + "IBM Corp." + ], + "consolidated_copyright": "Copyright (c) IBM Corp.", + "core_license_expression": "gpl-1.0-plus AND gpl-2.0", + "core_holders": [ + "IBM Corp." + ], + "other_license_expression": null, + "other_holders": [], + "files_count": 1 + }, + { + "type": "license-holders", + "identifier": "oracle_corp__2", + "consolidated_license_expression": "apache-2.0 AND unknown", + "consolidated_holders": [ + "Oracle Corp." + ], + "consolidated_copyright": "Copyright (c) Oracle Corp.", + "core_license_expression": "unknown AND apache-2.0", + "core_holders": [ + "Oracle Corp." + ], + "other_license_expression": null, + "other_holders": [], + "files_count": 1 + }, + { + "type": "license-holders", + "identifier": "omega_com_3", + "consolidated_license_expression": "gpl-1.0-plus AND gpl-2.0", + "consolidated_holders": [ + "omega.com" + ], + "consolidated_copyright": "Copyright (c) omega.com", + "core_license_expression": "gpl-1.0-plus AND gpl-2.0", + "core_holders": [ + "omega.com" + ], + "other_license_expression": null, + "other_holders": [], + "files_count": 1 + } + ], + "consolidated_packages": [], + "files": [ + { + "path": "license-holder-rollup", + "type": "directory", + "name": "license-holder-rollup", + "base_name": "license-holder-rollup", + "extension": "", + "size": 0, + "sha1": null, + "md5": null, + "mime_type": null, + "file_type": null, + "programming_language": null, + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [], + "license_expressions": [], + "holders": [], + "copyrights": [], + "authors": [], + "packages": [], + "consolidated_to": [], + "files_count": 3, + "dirs_count": 5, + "size_count": 144, + "scan_errors": [] + }, + { + "path": "license-holder-rollup/clear-majority", + "type": "directory", + "name": "clear-majority", + "base_name": "clear-majority", + "extension": "", + "size": 0, + "sha1": null, + "md5": null, + "mime_type": null, + "file_type": null, + "programming_language": null, + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [], + "license_expressions": [], + "holders": [], + "copyrights": [], + "authors": [], + "packages": [], + "consolidated_to": [ + "omega_com_3" + ], + "files_count": 1, + "dirs_count": 1, + "size_count": 46, + "scan_errors": [] + }, + { + "path": "license-holder-rollup/clear-majority/dir1", + "type": "directory", + "name": "dir1", + "base_name": "dir1", + "extension": "", + "size": 0, + "sha1": null, + "md5": null, + "mime_type": null, + "file_type": null, + "programming_language": null, + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [], + "license_expressions": [], + "holders": [], + "copyrights": [], + "authors": [], + "packages": [], + "consolidated_to": [ + "omega_com_3" + ], + "files_count": 1, + "dirs_count": 0, + "size_count": 46, + "scan_errors": [] + }, + { + "path": "license-holder-rollup/clear-majority/dir1/file1", + "type": "file", + "name": "file1", + "base_name": "file1", + "extension": "", + "size": 46, + "sha1": "21aec44ff80c4f5c2957b4360647785ae25c7854", + "md5": "5b896afc8c6958d74694967f60bf4d70", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "gpl-1.0-plus", + "score": 16.0, + "name": "GNU General Public License 1.0 or later", + "short_name": "GPL 1.0 or later", + "category": "Copyleft", + "is_exception": false, + "owner": "Free Software Foundation (FSF)", + "homepage_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", + "text_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:gpl-1.0-plus", + "spdx_license_key": "GPL-1.0-or-later", + "spdx_url": "https://spdx.org/licenses/GPL-1.0-or-later", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "gpl_208.RULE", + "license_expression": "gpl-1.0-plus", + "licenses": [ + "gpl-1.0-plus" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 16 + } + }, + { + "key": "gpl-2.0", + "score": 80.0, + "name": "GNU General Public License 2.0", + "short_name": "GPL 2.0", + "category": "Copyleft", + "is_exception": false, + "owner": "Free Software Foundation (FSF)", + "homepage_url": "http://www.gnu.org/licenses/gpl-2.0.html", + "text_url": "http://www.gnu.org/licenses/gpl-2.0.txt", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:gpl-2.0", + "spdx_license_key": "GPL-2.0-only", + "spdx_url": "https://spdx.org/licenses/GPL-2.0-only", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "gpl-2.0_52.RULE", + "license_expression": "gpl-2.0", + "licenses": [ + "gpl-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 80.0 + } + } + ], + "license_expressions": [ + "gpl-1.0-plus", + "gpl-2.0" + ], + "holders": [ + { + "value": "omega.com", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "Copyright (c) omega.com", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "consolidated_to": [ + "omega_com_3" + ], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "license-holder-rollup/no-majority", + "type": "directory", + "name": "no-majority", + "base_name": "no-majority", + "extension": "", + "size": 0, + "sha1": null, + "md5": null, + "mime_type": null, + "file_type": null, + "programming_language": null, + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [], + "license_expressions": [], + "holders": [], + "copyrights": [], + "authors": [], + "packages": [], + "consolidated_to": [], + "files_count": 2, + "dirs_count": 2, + "size_count": 98, + "scan_errors": [] + }, + { + "path": "license-holder-rollup/no-majority/bar", + "type": "directory", + "name": "bar", + "base_name": "bar", + "extension": "", + "size": 0, + "sha1": null, + "md5": null, + "mime_type": null, + "file_type": null, + "programming_language": null, + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [], + "license_expressions": [], + "holders": [], + "copyrights": [], + "authors": [], + "packages": [], + "consolidated_to": [ + "ibm_corp__1" + ], + "files_count": 1, + "dirs_count": 0, + "size_count": 46, + "scan_errors": [] + }, + { + "path": "license-holder-rollup/no-majority/bar/qux", + "type": "file", + "name": "qux", + "base_name": "qux", + "extension": "", + "size": 46, + "sha1": "8b94498ae812abc4f82304aa2f9766df45ba1873", + "md5": "87ad74664f4ba242df4bedf2a83b2169", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "gpl-1.0-plus", + "score": 16.0, + "name": "GNU General Public License 1.0 or later", + "short_name": "GPL 1.0 or later", + "category": "Copyleft", + "is_exception": false, + "owner": "Free Software Foundation (FSF)", + "homepage_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", + "text_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:gpl-1.0-plus", + "spdx_license_key": "GPL-1.0-or-later", + "spdx_url": "https://spdx.org/licenses/GPL-1.0-or-later", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "gpl_208.RULE", + "license_expression": "gpl-1.0-plus", + "licenses": [ + "gpl-1.0-plus" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 16 + } + }, + { + "key": "gpl-2.0", + "score": 80.0, + "name": "GNU General Public License 2.0", + "short_name": "GPL 2.0", + "category": "Copyleft", + "is_exception": false, + "owner": "Free Software Foundation (FSF)", + "homepage_url": "http://www.gnu.org/licenses/gpl-2.0.html", + "text_url": "http://www.gnu.org/licenses/gpl-2.0.txt", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:gpl-2.0", + "spdx_license_key": "GPL-2.0-only", + "spdx_url": "https://spdx.org/licenses/GPL-2.0-only", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "gpl-2.0_52.RULE", + "license_expression": "gpl-2.0", + "licenses": [ + "gpl-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 80.0 + } + } + ], + "license_expressions": [ + "gpl-1.0-plus", + "gpl-2.0" + ], + "holders": [ + { + "value": "IBM Corp.", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "Copyright (c) IBM Corp.", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "consolidated_to": [ + "ibm_corp__1" + ], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "license-holder-rollup/no-majority/foo", + "type": "directory", + "name": "foo", + "base_name": "foo", + "extension": "", + "size": 0, + "sha1": null, + "md5": null, + "mime_type": null, + "file_type": null, + "programming_language": null, + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [], + "license_expressions": [], + "holders": [], + "copyrights": [], + "authors": [], + "packages": [], + "consolidated_to": [ + "oracle_corp__2" + ], + "files_count": 1, + "dirs_count": 0, + "size_count": 52, + "scan_errors": [] + }, + { + "path": "license-holder-rollup/no-majority/foo/baz", + "type": "file", + "name": "baz", + "base_name": "baz", + "extension": "", + "size": 52, + "sha1": "e77836a77dea5398842c5dea1eb4e117ae4af544", + "md5": "bf1dfc5106c2a5a818bd26aa414cd38d", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "unknown", + "score": 11.0, + "name": "Unknown license detected but not recognized", + "short_name": "unknown", + "category": "Unstated License", + "is_exception": false, + "owner": "Unspecified", + "homepage_url": null, + "text_url": "", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:unknown", + "spdx_license_key": null, + "spdx_url": "", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "unknown_28.RULE", + "license_expression": "unknown", + "licenses": [ + "unknown" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 2, + "matched_length": 2, + "match_coverage": 100.0, + "rule_relevance": 11 + } + }, + { + "key": "apache-2.0", + "score": 50.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 50.0 + } + } + ], + "license_expressions": [ + "unknown", + "apache-2.0" + ], + "holders": [ + { + "value": "Oracle Corp.", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "Copyright (c) Oracle Corp.", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "consolidated_to": [ + "oracle_corp__2" + ], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + } + ] +} \ No newline at end of file diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/license-holder-rollup/clear-majority/dir1/file1 b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/license-holder-rollup/clear-majority/dir1/file1 new file mode 100644 index 00000000000..e0b2a834921 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/license-holder-rollup/clear-majority/dir1/file1 @@ -0,0 +1,2 @@ +Copyright (c) omega.com +Licensed under GPL-2.0 \ No newline at end of file diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/license-holder-rollup/no-majority/bar/qux b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/license-holder-rollup/no-majority/bar/qux new file mode 100644 index 00000000000..98285c2a8f6 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/license-holder-rollup/no-majority/bar/qux @@ -0,0 +1,2 @@ +Copyright (c) IBM Corp. +Licensed under GPL-2.0 \ No newline at end of file diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/license-holder-rollup/no-majority/foo/baz b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/license-holder-rollup/no-majority/foo/baz new file mode 100644 index 00000000000..842ce478b39 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/license-holder-rollup/no-majority/foo/baz @@ -0,0 +1,2 @@ +Copyright (c) Oracle Corp. +Licensed under Apache 2.0 \ No newline at end of file diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/multiple-same-holder-and-license-expected.json b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/multiple-same-holder-and-license-expected.json new file mode 100644 index 00000000000..723db06af41 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/multiple-same-holder-and-license-expected.json @@ -0,0 +1,320 @@ +{ + "headers": [ + { + "tool_name": "scancode-toolkit", + "options": { + "input": "", + "--consolidate": true, + "--copyright": true, + "--info": true, + "--json": "", + "--license": true, + "--package": true + }, + "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "message": null, + "errors": [], + "extra_data": { + "files_count": 2 + } + } + ], + "consolidated_components": [ + { + "type": "license-holders", + "identifier": "ibm_corp__ibm_corp__1", + "consolidated_license_expression": "gpl-2.0", + "consolidated_holders": [ + "IBM Corp." + ], + "consolidated_copyright": "Copyright (c) IBM Corp.", + "core_license_expression": "gpl-2.0", + "core_holders": [ + "IBM Corp.", + "IBM Corp." + ], + "other_license_expression": null, + "other_holders": [], + "files_count": 2 + } + ], + "consolidated_packages": [], + "files": [ + { + "path": "multiple-same-holder-and-license", + "type": "directory", + "name": "multiple-same-holder-and-license", + "base_name": "multiple-same-holder-and-license", + "extension": "", + "size": 0, + "sha1": null, + "md5": null, + "mime_type": null, + "file_type": null, + "programming_language": null, + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [], + "license_expressions": [], + "holders": [], + "copyrights": [], + "authors": [], + "packages": [], + "consolidated_to": [ + "ibm_corp__ibm_corp__1" + ], + "files_count": 2, + "dirs_count": 0, + "size_count": 164, + "scan_errors": [] + }, + { + "path": "multiple-same-holder-and-license/bar", + "type": "file", + "name": "bar", + "base_name": "bar", + "extension": "", + "size": 82, + "sha1": "aa6bd87dbe009f5fbd70b2653a60fc0b63c78b8e", + "md5": "6456c2560504a2b5896168b54e4c02fa", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "gpl-2.0", + "score": 16.0, + "name": "GNU General Public License 2.0", + "short_name": "GPL 2.0", + "category": "Copyleft", + "is_exception": false, + "owner": "Free Software Foundation (FSF)", + "homepage_url": "http://www.gnu.org/licenses/gpl-2.0.html", + "text_url": "http://www.gnu.org/licenses/gpl-2.0.txt", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:gpl-2.0", + "spdx_license_key": "GPL-2.0-only", + "spdx_url": "https://spdx.org/licenses/GPL-2.0-only", + "start_line": 4, + "end_line": 4, + "matched_rule": { + "identifier": "gpl-2.0_561.RULE", + "license_expression": "gpl-2.0", + "licenses": [ + "gpl-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 16 + } + }, + { + "key": "gpl-2.0", + "score": 16.0, + "name": "GNU General Public License 2.0", + "short_name": "GPL 2.0", + "category": "Copyleft", + "is_exception": false, + "owner": "Free Software Foundation (FSF)", + "homepage_url": "http://www.gnu.org/licenses/gpl-2.0.html", + "text_url": "http://www.gnu.org/licenses/gpl-2.0.txt", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:gpl-2.0", + "spdx_license_key": "GPL-2.0-only", + "spdx_url": "https://spdx.org/licenses/GPL-2.0-only", + "start_line": 5, + "end_line": 5, + "matched_rule": { + "identifier": "gpl-2.0_561.RULE", + "license_expression": "gpl-2.0", + "licenses": [ + "gpl-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 16 + } + } + ], + "license_expressions": [ + "gpl-2.0", + "gpl-2.0" + ], + "holders": [ + { + "value": "IBM Corp.", + "start_line": 1, + "end_line": 2 + }, + { + "value": "IBM Corp.", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "Copyright (c) IBM Corp.", + "start_line": 1, + "end_line": 2 + }, + { + "value": "Copyright (c) IBM Corp.", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "consolidated_to": [ + "ibm_corp__ibm_corp__1" + ], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "multiple-same-holder-and-license/foo", + "type": "file", + "name": "foo", + "base_name": "foo", + "extension": "", + "size": 82, + "sha1": "aa6bd87dbe009f5fbd70b2653a60fc0b63c78b8e", + "md5": "6456c2560504a2b5896168b54e4c02fa", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "gpl-2.0", + "score": 16.0, + "name": "GNU General Public License 2.0", + "short_name": "GPL 2.0", + "category": "Copyleft", + "is_exception": false, + "owner": "Free Software Foundation (FSF)", + "homepage_url": "http://www.gnu.org/licenses/gpl-2.0.html", + "text_url": "http://www.gnu.org/licenses/gpl-2.0.txt", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:gpl-2.0", + "spdx_license_key": "GPL-2.0-only", + "spdx_url": "https://spdx.org/licenses/GPL-2.0-only", + "start_line": 4, + "end_line": 4, + "matched_rule": { + "identifier": "gpl-2.0_561.RULE", + "license_expression": "gpl-2.0", + "licenses": [ + "gpl-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 16 + } + }, + { + "key": "gpl-2.0", + "score": 16.0, + "name": "GNU General Public License 2.0", + "short_name": "GPL 2.0", + "category": "Copyleft", + "is_exception": false, + "owner": "Free Software Foundation (FSF)", + "homepage_url": "http://www.gnu.org/licenses/gpl-2.0.html", + "text_url": "http://www.gnu.org/licenses/gpl-2.0.txt", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:gpl-2.0", + "spdx_license_key": "GPL-2.0-only", + "spdx_url": "https://spdx.org/licenses/GPL-2.0-only", + "start_line": 5, + "end_line": 5, + "matched_rule": { + "identifier": "gpl-2.0_561.RULE", + "license_expression": "gpl-2.0", + "licenses": [ + "gpl-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 16 + } + } + ], + "license_expressions": [ + "gpl-2.0", + "gpl-2.0" + ], + "holders": [ + { + "value": "IBM Corp.", + "start_line": 1, + "end_line": 2 + }, + { + "value": "IBM Corp.", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "Copyright (c) IBM Corp.", + "start_line": 1, + "end_line": 2 + }, + { + "value": "Copyright (c) IBM Corp.", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "consolidated_to": [ + "ibm_corp__ibm_corp__1" + ], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + } + ] +} \ No newline at end of file diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/multiple-same-holder-and-license/bar b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/multiple-same-holder-and-license/bar new file mode 100644 index 00000000000..862f47ea8b4 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/multiple-same-holder-and-license/bar @@ -0,0 +1,5 @@ +Copyright (c) IBM Corp. +Copyright (c) IBM Corp. + +License: GPL-2.0 +License: GPL-2.0 \ No newline at end of file diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/multiple-same-holder-and-license/foo b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/multiple-same-holder-and-license/foo new file mode 100644 index 00000000000..862f47ea8b4 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/multiple-same-holder-and-license/foo @@ -0,0 +1,5 @@ +Copyright (c) IBM Corp. +Copyright (c) IBM Corp. + +License: GPL-2.0 +License: GPL-2.0 \ No newline at end of file diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/nested-npm-packages.json b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/nested-npm-packages.json new file mode 100644 index 00000000000..ff345149e49 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/nested-npm-packages.json @@ -0,0 +1,237 @@ +{ + "headers": [ + { + "tool_name": "scancode-toolkit", + "tool_version": "3.0.2.post1250.904952c36", + "options": { + "input": [ + "plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/nested-packages/" + ], + "--json-pp": "plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/nested-npm-packages.json", + "--package": true + }, + "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "start_timestamp": "2019-07-26T200843.397788", + "end_timestamp": "2019-07-26T200845.708729", + "message": null, + "errors": [], + "extra_data": { + "files_count": 4 + } + } + ], + "files": [ + { + "path": "nested-packages", + "type": "directory", + "packages": [ + { + "type": "npm", + "namespace": null, + "name": "test-package", + "version": "0.0.1", + "qualifiers": null, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://registry.npmjs.org/test-package/-/test-package-0.0.1.tgz", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": "apache-2.0", + "declared_license": [ + "apache-2.0" + ], + "notice_text": null, + "manifest_path": "package.json", + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:npm/test-package@0.0.1", + "repository_homepage_url": "https://www.npmjs.com/package/test-package", + "repository_download_url": "https://registry.npmjs.org/test-package/-/test-package-0.0.1.tgz", + "api_data_url": "https://registry.npmjs.org/test-package/0.0.1" + } + ], + "scan_errors": [] + }, + { + "path": "nested-packages/package.json", + "type": "file", + "packages": [], + "scan_errors": [] + }, + { + "path": "nested-packages/node_modules", + "type": "directory", + "packages": [], + "scan_errors": [] + }, + { + "path": "nested-packages/node_modules/bar", + "type": "directory", + "packages": [ + { + "type": "npm", + "namespace": null, + "name": "bar", + "version": "0.0.1", + "qualifiers": null, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://registry.npmjs.org/bar/-/bar-0.0.1.tgz", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": "apache-2.0", + "declared_license": [ + "apache-2.0" + ], + "notice_text": null, + "manifest_path": "package.json", + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:npm/bar@0.0.1", + "repository_homepage_url": "https://www.npmjs.com/package/bar", + "repository_download_url": "https://registry.npmjs.org/bar/-/bar-0.0.1.tgz", + "api_data_url": "https://registry.npmjs.org/bar/0.0.1" + } + ], + "scan_errors": [] + }, + { + "path": "nested-packages/node_modules/bar/package.json", + "type": "file", + "packages": [], + "scan_errors": [] + }, + { + "path": "nested-packages/node_modules/foo", + "type": "directory", + "packages": [ + { + "type": "npm", + "namespace": null, + "name": "foo", + "version": "0.0.1", + "qualifiers": null, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://registry.npmjs.org/foo/-/foo-0.0.1.tgz", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": "apache-2.0", + "declared_license": [ + "apache-2.0" + ], + "notice_text": null, + "manifest_path": "package.json", + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:npm/foo@0.0.1", + "repository_homepage_url": "https://www.npmjs.com/package/foo", + "repository_download_url": "https://registry.npmjs.org/foo/-/foo-0.0.1.tgz", + "api_data_url": "https://registry.npmjs.org/foo/0.0.1" + } + ], + "scan_errors": [] + }, + { + "path": "nested-packages/node_modules/foo/package.json", + "type": "file", + "packages": [], + "scan_errors": [] + }, + { + "path": "nested-packages/node_modules/foo/node_modules", + "type": "directory", + "packages": [], + "scan_errors": [] + }, + { + "path": "nested-packages/node_modules/foo/node_modules/bar", + "type": "directory", + "packages": [ + { + "type": "npm", + "namespace": null, + "name": "bar", + "version": "0.0.1", + "qualifiers": null, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://registry.npmjs.org/bar/-/bar-0.0.1.tgz", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": "apache-2.0", + "declared_license": [ + "apache-2.0" + ], + "notice_text": null, + "manifest_path": "package.json", + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:npm/bar@0.0.1", + "repository_homepage_url": "https://www.npmjs.com/package/bar", + "repository_download_url": "https://registry.npmjs.org/bar/-/bar-0.0.1.tgz", + "api_data_url": "https://registry.npmjs.org/bar/0.0.1" + } + ], + "scan_errors": [] + }, + { + "path": "nested-packages/node_modules/foo/node_modules/bar/package.json", + "type": "file", + "packages": [], + "scan_errors": [] + } + ] +} diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/nested-packages/node_modules/bar/package.json b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/nested-packages/node_modules/bar/package.json new file mode 100644 index 00000000000..fe3c3f1786a --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/nested-packages/node_modules/bar/package.json @@ -0,0 +1,5 @@ +{ + "name": "bar", + "version": "0.0.1", + "license": "apache-2.0" +} diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/nested-packages/node_modules/foo/node_modules/bar/package.json b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/nested-packages/node_modules/foo/node_modules/bar/package.json new file mode 100644 index 00000000000..fe3c3f1786a --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/nested-packages/node_modules/foo/node_modules/bar/package.json @@ -0,0 +1,5 @@ +{ + "name": "bar", + "version": "0.0.1", + "license": "apache-2.0" +} diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/nested-packages/node_modules/foo/package.json b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/nested-packages/node_modules/foo/package.json new file mode 100644 index 00000000000..586ac3ff363 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/nested-packages/node_modules/foo/package.json @@ -0,0 +1,5 @@ +{ + "name": "foo", + "version": "0.0.1", + "license": "apache-2.0" +} diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/nested-packages/package.json b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/nested-packages/package.json new file mode 100644 index 00000000000..5e8b1f74179 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/nested-packages/package.json @@ -0,0 +1,5 @@ +{ + "name": "test-package", + "version": "0.0.1", + "license": "apache-2.0" +} diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/no-summary-expected.json b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/no-summary-expected.json new file mode 100644 index 00000000000..71a0def8913 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/no-summary-expected.json @@ -0,0 +1,364 @@ +{ + "headers": [ + { + "tool_name": "scancode-toolkit", + "options": { + "input": "", + "--consolidate": true, + "--copyright": true, + "--info": true, + "--json": "", + "--license": true, + "--package": true + }, + "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "message": null, + "errors": [], + "extra_data": { + "files_count": 4 + } + } + ], + "consolidated_components": [], + "consolidated_packages": [], + "files": [ + { + "path": "no-summary", + "type": "directory", + "name": "no-summary", + "base_name": "no-summary", + "extension": "", + "size": 0, + "sha1": null, + "md5": null, + "mime_type": null, + "file_type": null, + "programming_language": null, + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [], + "license_expressions": [], + "holders": [], + "copyrights": [], + "authors": [], + "packages": [], + "consolidated_to": [], + "files_count": 4, + "dirs_count": 0, + "size_count": 176, + "scan_errors": [] + }, + { + "path": "no-summary/src1", + "type": "file", + "name": "src1", + "base_name": "src1", + "extension": "", + "size": 56, + "sha1": "f3eae084a5690638fd1ada38e653a94179f14263", + "md5": "7e0e8162de50e718e248ccd7a2d2cb65", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "apache-2.0", + "score": 50.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 50.0 + } + } + ], + "license_expressions": [ + "apache-2.0" + ], + "holders": [ + { + "value": "The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "Copyright (c) The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "consolidated_to": [], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "no-summary/src2", + "type": "file", + "name": "src2", + "base_name": "src2", + "extension": "", + "size": 56, + "sha1": "f3eae084a5690638fd1ada38e653a94179f14263", + "md5": "7e0e8162de50e718e248ccd7a2d2cb65", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "apache-2.0", + "score": 50.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 50.0 + } + } + ], + "license_expressions": [ + "apache-2.0" + ], + "holders": [ + { + "value": "The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "Copyright (c) The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "consolidated_to": [], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "no-summary/src3", + "type": "file", + "name": "src3", + "base_name": "src3", + "extension": "", + "size": 32, + "sha1": "e9baf6f9b92d9cfc1d858bd156470d166661df28", + "md5": "4dcea879c44185b280e399d754518ada", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "gpl-2.0", + "score": 80.0, + "name": "GNU General Public License 2.0", + "short_name": "GPL 2.0", + "category": "Copyleft", + "is_exception": false, + "owner": "Free Software Foundation (FSF)", + "homepage_url": "http://www.gnu.org/licenses/gpl-2.0.html", + "text_url": "http://www.gnu.org/licenses/gpl-2.0.txt", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:gpl-2.0", + "spdx_license_key": "GPL-2.0-only", + "spdx_url": "https://spdx.org/licenses/GPL-2.0-only", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "gpl-2.0_52.RULE", + "license_expression": "gpl-2.0", + "licenses": [ + "gpl-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 80.0 + } + } + ], + "license_expressions": [ + "gpl-2.0" + ], + "holders": [ + { + "value": "IBM Corp.", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "Copyright (c) IBM Corp.", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "consolidated_to": [], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "no-summary/src4", + "type": "file", + "name": "src4", + "base_name": "src4", + "extension": "", + "size": 32, + "sha1": "e9baf6f9b92d9cfc1d858bd156470d166661df28", + "md5": "4dcea879c44185b280e399d754518ada", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "gpl-2.0", + "score": 80.0, + "name": "GNU General Public License 2.0", + "short_name": "GPL 2.0", + "category": "Copyleft", + "is_exception": false, + "owner": "Free Software Foundation (FSF)", + "homepage_url": "http://www.gnu.org/licenses/gpl-2.0.html", + "text_url": "http://www.gnu.org/licenses/gpl-2.0.txt", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:gpl-2.0", + "spdx_license_key": "GPL-2.0-only", + "spdx_url": "https://spdx.org/licenses/GPL-2.0-only", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "gpl-2.0_52.RULE", + "license_expression": "gpl-2.0", + "licenses": [ + "gpl-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 80.0 + } + } + ], + "license_expressions": [ + "gpl-2.0" + ], + "holders": [ + { + "value": "IBM Corp.", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "Copyright (c) IBM Corp.", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "consolidated_to": [], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + } + ] +} \ No newline at end of file diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/no-summary/src1 b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/no-summary/src1 new file mode 100644 index 00000000000..429e8931814 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/no-summary/src1 @@ -0,0 +1,2 @@ +Copyright (c) The Apache Software Foundation +Apache 2.0 diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/no-summary/src2 b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/no-summary/src2 new file mode 100644 index 00000000000..429e8931814 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/no-summary/src2 @@ -0,0 +1,2 @@ +Copyright (c) The Apache Software Foundation +Apache 2.0 diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/no-summary/src3 b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/no-summary/src3 new file mode 100644 index 00000000000..2408de87b83 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/no-summary/src3 @@ -0,0 +1,2 @@ +Copyright (c) IBM Corp. +GPL-2.0 diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/no-summary/src4 b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/no-summary/src4 new file mode 100644 index 00000000000..2408de87b83 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/no-summary/src4 @@ -0,0 +1,2 @@ +Copyright (c) IBM Corp. +GPL-2.0 diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package-files-not-counted-in-license-holders-expected.json b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package-files-not-counted-in-license-holders-expected.json new file mode 100644 index 00000000000..68a16c54950 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package-files-not-counted-in-license-holders-expected.json @@ -0,0 +1,645 @@ +{ + "headers": [ + { + "tool_name": "scancode-toolkit", + "options": { + "input": "", + "--consolidate": true, + "--copyright": true, + "--info": true, + "--json": "", + "--license": true, + "--package": true + }, + "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "message": null, + "errors": [], + "extra_data": { + "files_count": 6 + } + } + ], + "consolidated_components": [], + "consolidated_packages": [ + { + "type": "npm", + "namespace": null, + "name": "test-package", + "version": "0.0.1", + "qualifiers": null, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://registry.npmjs.org/test-package/-/test-package-0.0.1.tgz", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": "apache-2.0", + "declared_license": [ + "apache-2.0" + ], + "notice_text": null, + "manifest_path": "package.json", + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:npm/test-package@0.0.1", + "repository_homepage_url": "https://www.npmjs.com/package/test-package", + "repository_download_url": "https://registry.npmjs.org/test-package/-/test-package-0.0.1.tgz", + "api_data_url": "https://registry.npmjs.org/test-package/0.0.1", + "identifier": "pkg_npm_test_package_0_0_1_1", + "consolidated_license_expression": "apache-2.0", + "consolidated_holders": [ + "The Apache Software", + "The Apache Software Foundation" + ], + "consolidated_copyright": "Copyright (c) The Apache Software, The Apache Software Foundation", + "core_license_expression": "apache-2.0", + "core_holders": [], + "other_license_expression": "apache-2.0", + "other_holders": [ + "The Apache Software", + "The Apache Software Foundation" + ], + "files_count": 4 + } + ], + "files": [ + { + "path": "package-files-not-counted-in-license-holders", + "type": "directory", + "name": "package-files-not-counted-in-license-holders", + "base_name": "package-files-not-counted-in-license-holders", + "extension": "", + "size": 0, + "sha1": null, + "md5": null, + "mime_type": null, + "file_type": null, + "programming_language": null, + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [], + "license_expressions": [], + "holders": [], + "copyrights": [], + "authors": [], + "packages": [], + "consolidated_to": [], + "files_count": 6, + "dirs_count": 1, + "size_count": 421, + "scan_errors": [] + }, + { + "path": "package-files-not-counted-in-license-holders/package", + "type": "directory", + "name": "package", + "base_name": "package", + "extension": "", + "size": 0, + "sha1": null, + "md5": null, + "mime_type": null, + "file_type": null, + "programming_language": null, + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [], + "license_expressions": [], + "holders": [], + "copyrights": [], + "authors": [], + "packages": [ + { + "type": "npm", + "namespace": null, + "name": "test-package", + "version": "0.0.1", + "qualifiers": null, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://registry.npmjs.org/test-package/-/test-package-0.0.1.tgz", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": "apache-2.0", + "declared_license": [ + "apache-2.0" + ], + "notice_text": null, + "manifest_path": "package.json", + "dependencies": [], + "contains_source_code": null, + "source_packages": [] + } + ], + "consolidated_to": [ + "pkg_npm_test_package_0_0_1_1" + ], + "files_count": 4, + "dirs_count": 0, + "size_count": 309, + "scan_errors": [] + }, + { + "path": "package-files-not-counted-in-license-holders/package/package.json", + "type": "file", + "name": "package.json", + "base_name": "package", + "extension": ".json", + "size": 141, + "sha1": "a4a4a28858425dfb6795ff4eb7b397846f03fda9", + "md5": "e10ff141105dc64fbde2dc77c61275db", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "apache-2.0", + "score": 100.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 5, + "end_line": 5, + "matched_rule": { + "identifier": "apache-2.0_65.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": false, + "is_license_tag": true, + "matcher": "2-aho", + "rule_length": 4, + "matched_length": 4, + "match_coverage": 100.0, + "rule_relevance": 100.0 + } + } + ], + "license_expressions": [ + "apache-2.0" + ], + "holders": [ + { + "value": "The Apache Software", + "start_line": 4, + "end_line": 5 + } + ], + "copyrights": [ + { + "value": "Copyright (c) The Apache Software", + "start_line": 4, + "end_line": 5 + } + ], + "authors": [], + "packages": [], + "consolidated_to": [ + "pkg_npm_test_package_0_0_1_1" + ], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "package-files-not-counted-in-license-holders/package/src1", + "type": "file", + "name": "src1", + "base_name": "src1", + "extension": "", + "size": 56, + "sha1": "f3eae084a5690638fd1ada38e653a94179f14263", + "md5": "7e0e8162de50e718e248ccd7a2d2cb65", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "apache-2.0", + "score": 50.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 50.0 + } + } + ], + "license_expressions": [ + "apache-2.0" + ], + "holders": [ + { + "value": "The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "Copyright (c) The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "consolidated_to": [ + "pkg_npm_test_package_0_0_1_1" + ], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "package-files-not-counted-in-license-holders/package/src2", + "type": "file", + "name": "src2", + "base_name": "src2", + "extension": "", + "size": 56, + "sha1": "f3eae084a5690638fd1ada38e653a94179f14263", + "md5": "7e0e8162de50e718e248ccd7a2d2cb65", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "apache-2.0", + "score": 50.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 50.0 + } + } + ], + "license_expressions": [ + "apache-2.0" + ], + "holders": [ + { + "value": "The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "Copyright (c) The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "consolidated_to": [ + "pkg_npm_test_package_0_0_1_1" + ], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "package-files-not-counted-in-license-holders/package/src3", + "type": "file", + "name": "src3", + "base_name": "src3", + "extension": "", + "size": 56, + "sha1": "f3eae084a5690638fd1ada38e653a94179f14263", + "md5": "7e0e8162de50e718e248ccd7a2d2cb65", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "apache-2.0", + "score": 50.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 50.0 + } + } + ], + "license_expressions": [ + "apache-2.0" + ], + "holders": [ + { + "value": "The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "Copyright (c) The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "consolidated_to": [ + "pkg_npm_test_package_0_0_1_1" + ], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "package-files-not-counted-in-license-holders/src1", + "type": "file", + "name": "src1", + "base_name": "src1", + "extension": "", + "size": 56, + "sha1": "f3eae084a5690638fd1ada38e653a94179f14263", + "md5": "7e0e8162de50e718e248ccd7a2d2cb65", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "apache-2.0", + "score": 50.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 50.0 + } + } + ], + "license_expressions": [ + "apache-2.0" + ], + "holders": [ + { + "value": "The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "Copyright (c) The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "consolidated_to": [], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "package-files-not-counted-in-license-holders/src2", + "type": "file", + "name": "src2", + "base_name": "src2", + "extension": "", + "size": 56, + "sha1": "f3eae084a5690638fd1ada38e653a94179f14263", + "md5": "7e0e8162de50e718e248ccd7a2d2cb65", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "apache-2.0", + "score": 50.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 50.0 + } + } + ], + "license_expressions": [ + "apache-2.0" + ], + "holders": [ + { + "value": "The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "Copyright (c) The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "consolidated_to": [], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + } + ] +} \ No newline at end of file diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package-files-not-counted-in-license-holders/package/package.json b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package-files-not-counted-in-license-holders/package/package.json new file mode 100644 index 00000000000..a0ccac174b9 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package-files-not-counted-in-license-holders/package/package.json @@ -0,0 +1,6 @@ +{ + "name": "test-package", + "version": "0.0.1", + "copyright": "Copyright (c) The Apache Software Foundation", + "license": "apache-2.0" +} diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package-files-not-counted-in-license-holders/package/src1 b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package-files-not-counted-in-license-holders/package/src1 new file mode 100644 index 00000000000..429e8931814 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package-files-not-counted-in-license-holders/package/src1 @@ -0,0 +1,2 @@ +Copyright (c) The Apache Software Foundation +Apache 2.0 diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package-files-not-counted-in-license-holders/package/src2 b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package-files-not-counted-in-license-holders/package/src2 new file mode 100644 index 00000000000..429e8931814 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package-files-not-counted-in-license-holders/package/src2 @@ -0,0 +1,2 @@ +Copyright (c) The Apache Software Foundation +Apache 2.0 diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package-files-not-counted-in-license-holders/package/src3 b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package-files-not-counted-in-license-holders/package/src3 new file mode 100644 index 00000000000..429e8931814 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package-files-not-counted-in-license-holders/package/src3 @@ -0,0 +1,2 @@ +Copyright (c) The Apache Software Foundation +Apache 2.0 diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package-files-not-counted-in-license-holders/src1 b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package-files-not-counted-in-license-holders/src1 new file mode 100644 index 00000000000..429e8931814 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package-files-not-counted-in-license-holders/src1 @@ -0,0 +1,2 @@ +Copyright (c) The Apache Software Foundation +Apache 2.0 diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package-files-not-counted-in-license-holders/src2 b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package-files-not-counted-in-license-holders/src2 new file mode 100644 index 00000000000..429e8931814 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package-files-not-counted-in-license-holders/src2 @@ -0,0 +1,2 @@ +Copyright (c) The Apache Software Foundation +Apache 2.0 diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package-fileset-expected.json b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package-fileset-expected.json new file mode 100644 index 00000000000..83bdab9fc2c --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package-fileset-expected.json @@ -0,0 +1,447 @@ +{ + "headers": [ + { + "tool_name": "scancode-toolkit", + "options": { + "input": "", + "--consolidate": true, + "--copyright": true, + "--info": true, + "--json": "", + "--license": true, + "--package": true + }, + "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "message": null, + "errors": [], + "extra_data": { + "files_count": 4 + } + } + ], + "consolidated_components": [], + "consolidated_packages": [ + { + "type": "npm", + "namespace": null, + "name": "test-package", + "version": "0.0.1", + "qualifiers": null, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://registry.npmjs.org/test-package/-/test-package-0.0.1.tgz", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": "apache-2.0", + "declared_license": [ + "apache-2.0" + ], + "notice_text": null, + "manifest_path": "package.json", + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:npm/test-package@0.0.1", + "repository_homepage_url": "https://www.npmjs.com/package/test-package", + "repository_download_url": "https://registry.npmjs.org/test-package/-/test-package-0.0.1.tgz", + "api_data_url": "https://registry.npmjs.org/test-package/0.0.1", + "identifier": "pkg_npm_test_package_0_0_1_1", + "consolidated_license_expression": "apache-2.0", + "consolidated_holders": [ + "The Apache Software Foundation" + ], + "consolidated_copyright": "Copyright (c) The Apache Software Foundation", + "core_license_expression": "apache-2.0", + "core_holders": [], + "other_license_expression": "apache-2.0", + "other_holders": [ + "The Apache Software Foundation" + ], + "files_count": 4 + } + ], + "files": [ + { + "path": "package", + "type": "directory", + "name": "package", + "base_name": "package", + "extension": "", + "size": 0, + "sha1": null, + "md5": null, + "mime_type": null, + "file_type": null, + "programming_language": null, + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [], + "license_expressions": [], + "holders": [], + "copyrights": [], + "authors": [], + "packages": [ + { + "type": "npm", + "namespace": null, + "name": "test-package", + "version": "0.0.1", + "qualifiers": null, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://registry.npmjs.org/test-package/-/test-package-0.0.1.tgz", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": "apache-2.0", + "declared_license": [ + "apache-2.0" + ], + "notice_text": null, + "manifest_path": "package.json", + "dependencies": [], + "contains_source_code": null, + "source_packages": [] + } + ], + "consolidated_to": [ + "pkg_npm_test_package_0_0_1_1" + ], + "files_count": 4, + "dirs_count": 0, + "size_count": 246, + "scan_errors": [] + }, + { + "path": "package/package.json", + "type": "file", + "name": "package.json", + "base_name": "package", + "extension": ".json", + "size": 78, + "sha1": "cb5244fdb5b56f3c80a354c4c12ad0831b6a78df", + "md5": "985f757e56da685ae306c9cc41253f30", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "apache-2.0", + "score": 100.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 4, + "end_line": 4, + "matched_rule": { + "identifier": "apache-2.0_65.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": false, + "is_license_tag": true, + "matcher": "2-aho", + "rule_length": 4, + "matched_length": 4, + "match_coverage": 100.0, + "rule_relevance": 100.0 + } + } + ], + "license_expressions": [ + "apache-2.0" + ], + "holders": [], + "copyrights": [], + "authors": [], + "packages": [], + "consolidated_to": [ + "pkg_npm_test_package_0_0_1_1" + ], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "package/src1", + "type": "file", + "name": "src1", + "base_name": "src1", + "extension": "", + "size": 56, + "sha1": "f3eae084a5690638fd1ada38e653a94179f14263", + "md5": "7e0e8162de50e718e248ccd7a2d2cb65", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "apache-2.0", + "score": 50.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 50.0 + } + } + ], + "license_expressions": [ + "apache-2.0" + ], + "holders": [ + { + "value": "The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "Copyright (c) The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "consolidated_to": [ + "pkg_npm_test_package_0_0_1_1" + ], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "package/src2", + "type": "file", + "name": "src2", + "base_name": "src2", + "extension": "", + "size": 56, + "sha1": "f3eae084a5690638fd1ada38e653a94179f14263", + "md5": "7e0e8162de50e718e248ccd7a2d2cb65", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "apache-2.0", + "score": 50.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 50.0 + } + } + ], + "license_expressions": [ + "apache-2.0" + ], + "holders": [ + { + "value": "The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "Copyright (c) The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "consolidated_to": [ + "pkg_npm_test_package_0_0_1_1" + ], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "package/src3", + "type": "file", + "name": "src3", + "base_name": "src3", + "extension": "", + "size": 56, + "sha1": "f3eae084a5690638fd1ada38e653a94179f14263", + "md5": "7e0e8162de50e718e248ccd7a2d2cb65", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "apache-2.0", + "score": 50.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 50.0 + } + } + ], + "license_expressions": [ + "apache-2.0" + ], + "holders": [ + { + "value": "The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "Copyright (c) The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "consolidated_to": [ + "pkg_npm_test_package_0_0_1_1" + ], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + } + ] +} \ No newline at end of file diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package-manifest-expected.json b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package-manifest-expected.json new file mode 100644 index 00000000000..ae5bf5258c1 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package-manifest-expected.json @@ -0,0 +1,206 @@ +{ + "headers": [ + { + "tool_name": "scancode-toolkit", + "options": { + "input": "", + "--consolidate": true, + "--copyright": true, + "--info": true, + "--json": "", + "--license": true, + "--package": true + }, + "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "message": null, + "errors": [], + "extra_data": { + "files_count": 1 + } + } + ], + "consolidated_components": [], + "consolidated_packages": [ + { + "type": "npm", + "namespace": null, + "name": "test-package", + "version": "0.0.1", + "qualifiers": null, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://registry.npmjs.org/test-package/-/test-package-0.0.1.tgz", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": "apache-2.0", + "declared_license": [ + "apache-2.0" + ], + "notice_text": null, + "manifest_path": "package.json", + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:npm/test-package@0.0.1", + "repository_homepage_url": "https://www.npmjs.com/package/test-package", + "repository_download_url": "https://registry.npmjs.org/test-package/-/test-package-0.0.1.tgz", + "api_data_url": "https://registry.npmjs.org/test-package/0.0.1", + "identifier": "pkg_npm_test_package_0_0_1_1", + "consolidated_license_expression": "apache-2.0", + "consolidated_holders": [], + "consolidated_copyright": "Copyright (c) ", + "core_license_expression": "apache-2.0", + "core_holders": [], + "other_license_expression": "apache-2.0", + "other_holders": [], + "files_count": 1 + } + ], + "files": [ + { + "path": "package-manifest", + "type": "directory", + "name": "package-manifest", + "base_name": "package-manifest", + "extension": "", + "size": 0, + "sha1": null, + "md5": null, + "mime_type": null, + "file_type": null, + "programming_language": null, + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [], + "license_expressions": [], + "holders": [], + "copyrights": [], + "authors": [], + "packages": [ + { + "type": "npm", + "namespace": null, + "name": "test-package", + "version": "0.0.1", + "qualifiers": null, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://registry.npmjs.org/test-package/-/test-package-0.0.1.tgz", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": "apache-2.0", + "declared_license": [ + "apache-2.0" + ], + "notice_text": null, + "manifest_path": "package.json", + "dependencies": [], + "contains_source_code": null, + "source_packages": [] + } + ], + "consolidated_to": [ + "pkg_npm_test_package_0_0_1_1" + ], + "files_count": 1, + "dirs_count": 0, + "size_count": 78, + "scan_errors": [] + }, + { + "path": "package-manifest/package.json", + "type": "file", + "name": "package.json", + "base_name": "package", + "extension": ".json", + "size": 78, + "sha1": "cb5244fdb5b56f3c80a354c4c12ad0831b6a78df", + "md5": "985f757e56da685ae306c9cc41253f30", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "apache-2.0", + "score": 100.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 4, + "end_line": 4, + "matched_rule": { + "identifier": "apache-2.0_65.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": false, + "is_license_tag": true, + "matcher": "2-aho", + "rule_length": 4, + "matched_length": 4, + "match_coverage": 100.0, + "rule_relevance": 100.0 + } + } + ], + "license_expressions": [ + "apache-2.0" + ], + "holders": [], + "copyrights": [], + "authors": [], + "packages": [], + "consolidated_to": [ + "pkg_npm_test_package_0_0_1_1" + ], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + } + ] +} \ No newline at end of file diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package-manifest/package.json b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package-manifest/package.json new file mode 100644 index 00000000000..5e8b1f74179 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package-manifest/package.json @@ -0,0 +1,5 @@ +{ + "name": "test-package", + "version": "0.0.1", + "license": "apache-2.0" +} diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package/package.json b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package/package.json new file mode 100644 index 00000000000..5e8b1f74179 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package/package.json @@ -0,0 +1,5 @@ +{ + "name": "test-package", + "version": "0.0.1", + "license": "apache-2.0" +} diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package/src1 b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package/src1 new file mode 100644 index 00000000000..429e8931814 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package/src1 @@ -0,0 +1,2 @@ +Copyright (c) The Apache Software Foundation +Apache 2.0 diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package/src2 b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package/src2 new file mode 100644 index 00000000000..429e8931814 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package/src2 @@ -0,0 +1,2 @@ +Copyright (c) The Apache Software Foundation +Apache 2.0 diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package/src3 b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package/src3 new file mode 100644 index 00000000000..429e8931814 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/package/src3 @@ -0,0 +1,2 @@ +Copyright (c) The Apache Software Foundation +Apache 2.0 diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/return-nested-local-majority-expected.json b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/return-nested-local-majority-expected.json new file mode 100644 index 00000000000..3c88760769b --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/return-nested-local-majority-expected.json @@ -0,0 +1,603 @@ +{ + "headers": [ + { + "tool_name": "scancode-toolkit", + "options": { + "input": "", + "--consolidate": true, + "--copyright": true, + "--info": true, + "--json": "", + "--license": true, + "--package": true + }, + "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "message": null, + "errors": [], + "extra_data": { + "files_count": 4 + } + } + ], + "consolidated_components": [ + { + "type": "license-holders", + "identifier": "ibm_corp__1", + "consolidated_license_expression": "gpl-1.0-plus AND gpl-2.0", + "consolidated_holders": [ + "IBM Corp." + ], + "consolidated_copyright": "Copyright (c) IBM Corp.", + "core_license_expression": "gpl-1.0-plus AND gpl-2.0", + "core_holders": [ + "IBM Corp." + ], + "other_license_expression": null, + "other_holders": [], + "files_count": 1 + }, + { + "type": "license-holders", + "identifier": "the_apache_software_foundation_2", + "consolidated_license_expression": "apache-2.0 AND unknown", + "consolidated_holders": [ + "The Apache Software Foundation" + ], + "consolidated_copyright": "Copyright (c) The Apache Software Foundation", + "core_license_expression": "unknown AND apache-2.0", + "core_holders": [ + "The Apache Software Foundation" + ], + "other_license_expression": null, + "other_holders": [], + "files_count": 3 + } + ], + "consolidated_packages": [], + "files": [ + { + "path": "return-nested-local-majority", + "type": "directory", + "name": "return-nested-local-majority", + "base_name": "return-nested-local-majority", + "extension": "", + "size": 0, + "sha1": null, + "md5": null, + "mime_type": null, + "file_type": null, + "programming_language": null, + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [], + "license_expressions": [], + "holders": [], + "copyrights": [], + "authors": [], + "packages": [], + "consolidated_to": [ + "the_apache_software_foundation_2" + ], + "files_count": 4, + "dirs_count": 2, + "size_count": 256, + "scan_errors": [] + }, + { + "path": "return-nested-local-majority/foo", + "type": "directory", + "name": "foo", + "base_name": "foo", + "extension": "", + "size": 0, + "sha1": null, + "md5": null, + "mime_type": null, + "file_type": null, + "programming_language": null, + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [], + "license_expressions": [], + "holders": [], + "copyrights": [], + "authors": [], + "packages": [], + "consolidated_to": [ + "the_apache_software_foundation_2" + ], + "files_count": 4, + "dirs_count": 1, + "size_count": 256, + "scan_errors": [] + }, + { + "path": "return-nested-local-majority/foo/bar", + "type": "file", + "name": "bar", + "base_name": "bar", + "extension": "", + "size": 70, + "sha1": "d40f3510342b31094965c538277171ef83c3ff9a", + "md5": "4393b546ad103ae739b1409fefeff08a", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "unknown", + "score": 11.0, + "name": "Unknown license detected but not recognized", + "short_name": "unknown", + "category": "Unstated License", + "is_exception": false, + "owner": "Unspecified", + "homepage_url": null, + "text_url": "", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:unknown", + "spdx_license_key": null, + "spdx_url": "", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "unknown_28.RULE", + "license_expression": "unknown", + "licenses": [ + "unknown" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 2, + "matched_length": 2, + "match_coverage": 100.0, + "rule_relevance": 11 + } + }, + { + "key": "apache-2.0", + "score": 50.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 50.0 + } + } + ], + "license_expressions": [ + "unknown", + "apache-2.0" + ], + "holders": [ + { + "value": "The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "Copyright (c) The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "consolidated_to": [ + "the_apache_software_foundation_2" + ], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "return-nested-local-majority/foo/bax", + "type": "file", + "name": "bax", + "base_name": "bax", + "extension": "", + "size": 70, + "sha1": "d40f3510342b31094965c538277171ef83c3ff9a", + "md5": "4393b546ad103ae739b1409fefeff08a", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "unknown", + "score": 11.0, + "name": "Unknown license detected but not recognized", + "short_name": "unknown", + "category": "Unstated License", + "is_exception": false, + "owner": "Unspecified", + "homepage_url": null, + "text_url": "", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:unknown", + "spdx_license_key": null, + "spdx_url": "", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "unknown_28.RULE", + "license_expression": "unknown", + "licenses": [ + "unknown" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 2, + "matched_length": 2, + "match_coverage": 100.0, + "rule_relevance": 11 + } + }, + { + "key": "apache-2.0", + "score": 50.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 50.0 + } + } + ], + "license_expressions": [ + "unknown", + "apache-2.0" + ], + "holders": [ + { + "value": "The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "Copyright (c) The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "consolidated_to": [ + "the_apache_software_foundation_2" + ], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "return-nested-local-majority/foo/local-majority", + "type": "directory", + "name": "local-majority", + "base_name": "local-majority", + "extension": "", + "size": 0, + "sha1": null, + "md5": null, + "mime_type": null, + "file_type": null, + "programming_language": null, + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [], + "license_expressions": [], + "holders": [], + "copyrights": [], + "authors": [], + "packages": [], + "consolidated_to": [ + "ibm_corp__1" + ], + "files_count": 1, + "dirs_count": 0, + "size_count": 46, + "scan_errors": [] + }, + { + "path": "return-nested-local-majority/foo/local-majority/baz", + "type": "file", + "name": "baz", + "base_name": "baz", + "extension": "", + "size": 46, + "sha1": "8b94498ae812abc4f82304aa2f9766df45ba1873", + "md5": "87ad74664f4ba242df4bedf2a83b2169", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "gpl-1.0-plus", + "score": 16.0, + "name": "GNU General Public License 1.0 or later", + "short_name": "GPL 1.0 or later", + "category": "Copyleft", + "is_exception": false, + "owner": "Free Software Foundation (FSF)", + "homepage_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", + "text_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:gpl-1.0-plus", + "spdx_license_key": "GPL-1.0-or-later", + "spdx_url": "https://spdx.org/licenses/GPL-1.0-or-later", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "gpl_208.RULE", + "license_expression": "gpl-1.0-plus", + "licenses": [ + "gpl-1.0-plus" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 16 + } + }, + { + "key": "gpl-2.0", + "score": 80.0, + "name": "GNU General Public License 2.0", + "short_name": "GPL 2.0", + "category": "Copyleft", + "is_exception": false, + "owner": "Free Software Foundation (FSF)", + "homepage_url": "http://www.gnu.org/licenses/gpl-2.0.html", + "text_url": "http://www.gnu.org/licenses/gpl-2.0.txt", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:gpl-2.0", + "spdx_license_key": "GPL-2.0-only", + "spdx_url": "https://spdx.org/licenses/GPL-2.0-only", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "gpl-2.0_52.RULE", + "license_expression": "gpl-2.0", + "licenses": [ + "gpl-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 80.0 + } + } + ], + "license_expressions": [ + "gpl-1.0-plus", + "gpl-2.0" + ], + "holders": [ + { + "value": "IBM Corp.", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "Copyright (c) IBM Corp.", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "consolidated_to": [ + "ibm_corp__1" + ], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "return-nested-local-majority/foo/qux", + "type": "file", + "name": "qux", + "base_name": "qux", + "extension": "", + "size": 70, + "sha1": "d40f3510342b31094965c538277171ef83c3ff9a", + "md5": "4393b546ad103ae739b1409fefeff08a", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "unknown", + "score": 11.0, + "name": "Unknown license detected but not recognized", + "short_name": "unknown", + "category": "Unstated License", + "is_exception": false, + "owner": "Unspecified", + "homepage_url": null, + "text_url": "", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:unknown", + "spdx_license_key": null, + "spdx_url": "", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "unknown_28.RULE", + "license_expression": "unknown", + "licenses": [ + "unknown" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 2, + "matched_length": 2, + "match_coverage": 100.0, + "rule_relevance": 11 + } + }, + { + "key": "apache-2.0", + "score": 50.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 2, + "end_line": 2, + "matched_rule": { + "identifier": "spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 50.0 + } + } + ], + "license_expressions": [ + "unknown", + "apache-2.0" + ], + "holders": [ + { + "value": "The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "copyrights": [ + { + "value": "Copyright (c) The Apache Software Foundation", + "start_line": 1, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "consolidated_to": [ + "the_apache_software_foundation_2" + ], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + } + ] +} \ No newline at end of file diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/return-nested-local-majority/foo/bar b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/return-nested-local-majority/foo/bar new file mode 100644 index 00000000000..1f9a1396fd5 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/return-nested-local-majority/foo/bar @@ -0,0 +1,2 @@ +Copyright (c) The Apache Software Foundation +Licensed under Apache 2.0 \ No newline at end of file diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/return-nested-local-majority/foo/bax b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/return-nested-local-majority/foo/bax new file mode 100644 index 00000000000..1f9a1396fd5 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/return-nested-local-majority/foo/bax @@ -0,0 +1,2 @@ +Copyright (c) The Apache Software Foundation +Licensed under Apache 2.0 \ No newline at end of file diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/return-nested-local-majority/foo/local-majority/baz b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/return-nested-local-majority/foo/local-majority/baz new file mode 100644 index 00000000000..98285c2a8f6 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/return-nested-local-majority/foo/local-majority/baz @@ -0,0 +1,2 @@ +Copyright (c) IBM Corp. +Licensed under GPL-2.0 \ No newline at end of file diff --git a/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/return-nested-local-majority/foo/qux b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/return-nested-local-majority/foo/qux new file mode 100644 index 00000000000..1f9a1396fd5 --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/return-nested-local-majority/foo/qux @@ -0,0 +1,2 @@ +Copyright (c) The Apache Software Foundation +Licensed under Apache 2.0 \ No newline at end of file diff --git a/plugins/scancode-consolidate-scan/tests/test_plugin_consolidate.py b/plugins/scancode-consolidate-scan/tests/test_plugin_consolidate.py new file mode 100644 index 00000000000..ff89c6f96bc --- /dev/null +++ b/plugins/scancode-consolidate-scan/tests/test_plugin_consolidate.py @@ -0,0 +1,152 @@ +# +# Copyright (c) 2019 nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/scancode-toolkit/ +# The ScanCode software is licensed under the Apache License version 2.0. +# Data generated with ScanCode require an acknowledgment. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with ScanCode or any ScanCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# ScanCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode-toolkit/ for support and download. + +from __future__ import absolute_import +from __future__ import unicode_literals + +from collections import OrderedDict +from os.path import dirname +from os.path import join +import json + +from commoncode.testcase import FileDrivenTesting +from scancode.cli_test_utils import check_json_scan +from scancode.cli_test_utils import run_scan_click +from plugin_consolidate import is_majority + + +class TestConsolidate(FileDrivenTesting): + test_data_dir = join(dirname(__file__), 'data') + + def test_is_majority_above_threshold(self): + files_count = 10 + src_count = 8 + assert is_majority(src_count, files_count) + + def test_is_majority_below_threshold(self): + files_count = 10 + src_count = 7 + assert not is_majority(src_count, files_count) + + def test_consolidate_clear_summary(self): + # 75% of the files have the same license expression and holder, so we should have one consolidated component + scan_loc = self.get_test_loc('plugin_consolidate/clear-summary') + result_file = self.get_temp_file('json') + expected_file = self.get_test_loc('plugin_consolidate/clear-summary-expected.json') + run_scan_click(['-clip', scan_loc, '--consolidate', '--json', result_file]) + check_json_scan(expected_file, result_file, regen=False, remove_file_date=True) + + def test_consolidate_no_summary(self): + # 50% of the files have the same license expression and holder, so no consolidated component should be created + scan_loc = self.get_test_loc('plugin_consolidate/no-summary') + result_file = self.get_temp_file('json') + expected_file = self.get_test_loc('plugin_consolidate/no-summary-expected.json') + run_scan_click(['-clip', scan_loc, '--consolidate', '--json', result_file]) + check_json_scan(expected_file, result_file, regen=False, remove_file_date=True) + + def test_consolidate_package(self): + scan_loc = self.get_test_loc('plugin_consolidate/package') + result_file = self.get_temp_file('json') + expected_file = self.get_test_loc('plugin_consolidate/package-fileset-expected.json') + run_scan_click(['-clip', scan_loc, '--consolidate', '--json', result_file]) + check_json_scan(expected_file, result_file, regen=False, remove_file_date=True) + + def test_consolidate_package_files_should_not_be_considered_in_license_holder_consolidated_component(self): + scan_loc = self.get_test_loc('plugin_consolidate/package-files-not-counted-in-license-holders') + result_file = self.get_temp_file('json') + # There should not be a consolidated component for license-holder, even + # though every single file in this directory contains the same license + # expression and holder + expected_file = self.get_test_loc('plugin_consolidate/package-files-not-counted-in-license-holders-expected.json') + run_scan_click(['-clip', scan_loc, '--consolidate', '--json', result_file]) + check_json_scan(expected_file, result_file, regen=False, remove_file_date=True) + + def test_consolidate_clear_summary_from_json(self): + # 75% of the files have the same license expression and holder, so we should have one consolidated component + scan_loc = self.get_test_loc('plugin_consolidate/clear-summary.json') + result_file = self.get_temp_file('json') + expected_file = self.get_test_loc('plugin_consolidate/clear-summary-expected.json') + run_scan_click(['--from-json', scan_loc, '--consolidate', '--json', result_file]) + check_json_scan(expected_file, result_file, regen=False, remove_file_date=True, ignore_headers=True) + + def test_consolidate_component_package_from_json_can_run_twice(self): + scan_loc = self.get_test_loc('plugin_consolidate/component-package.json') + result_file = self.get_temp_file('json') + expected_file = self.get_test_loc('plugin_consolidate/component-package-expected.json') + run_scan_click(['--from-json', scan_loc, '--consolidate', '--json', result_file]) + check_json_scan(expected_file, result_file, regen=False, remove_file_date=True, ignore_headers=True) + + result_file2 = self.get_temp_file('json') + run_scan_click(['--from-json', result_file, '--consolidate', '--json', result_file2]) + check_json_scan(expected_file, result_file2, regen=False, remove_file_date=True, ignore_headers=True) + + def test_consolidate_component_package_from_live_scan(self): + scan_loc = self.get_test_loc('plugin_consolidate/component-package') + result_file = self.get_temp_file('json') + expected_file = self.get_test_loc('plugin_consolidate/component-package-expected.json') + run_scan_click(['-clip', scan_loc, '--consolidate', '--json', result_file]) + check_json_scan(expected_file, result_file, regen=False, remove_file_date=True, ignore_headers=True) + + def test_consolidate_package_always_include_own_manifest_file(self): + scan_loc = self.get_test_loc('plugin_consolidate/package-manifest') + result_file = self.get_temp_file('json') + expected_file = self.get_test_loc('plugin_consolidate/package-manifest-expected.json') + run_scan_click(['-clip', scan_loc, '--consolidate', '--json', result_file]) + check_json_scan(expected_file, result_file, regen=False, remove_file_date=True) + + def test_get_package_resources_on_nested_packages_should_include_manifest(self): + from packagedcode import get_package_instance + from scancode.resource import VirtualCodebase + scan_loc = self.get_test_loc('plugin_consolidate/nested-npm-packages.json') + codebase = VirtualCodebase(scan_loc) + for resource in codebase.walk(): + for package_data in resource.packages: + package = get_package_instance(package_data) + package_resources = list(package.get_package_resources(resource, codebase)) + assert any(r.name == 'package.json' for r in package_resources), resource.path + + def test_consolidate_multiple_same_holder_and_license(self): + scan_loc = self.get_test_loc('plugin_consolidate/multiple-same-holder-and-license') + result_file = self.get_temp_file('json') + expected_file = self.get_test_loc('plugin_consolidate/multiple-same-holder-and-license-expected.json') + run_scan_click(['-clip', scan_loc, '--consolidate', '--json', result_file]) + check_json_scan(expected_file, result_file, regen=False, remove_file_date=True, ignore_headers=True) + + def test_consolidate_origin_summary_license_holder_rollup(self): + scan_loc = self.get_test_loc('plugin_consolidate/license-holder-rollup') + result_file = self.get_temp_file('json') + expected_file = self.get_test_loc('plugin_consolidate/license-holder-rollup-expected.json') + run_scan_click(['-clip', scan_loc, '--consolidate', '--json', result_file]) + # There should be two consolidated components for things under + # no-majority and one consolidated component for clear-majority + check_json_scan(expected_file, result_file, regen=False, remove_file_date=True) + + def test_consolidate_return_nested_local_majority(self): + scan_loc = self.get_test_loc('plugin_consolidate/return-nested-local-majority') + result_file = self.get_temp_file('json') + expected_file = self.get_test_loc('plugin_consolidate/return-nested-local-majority-expected.json') + run_scan_click(['-clip', scan_loc, '--consolidate', '--json', result_file]) + # The nested majority is just 1 file, but has a different origin than the rest of the files above it + # and should be reported as a separate consolidated component + check_json_scan(expected_file, result_file, regen=False, remove_file_date=True) diff --git a/src/commoncode/fileutils.py b/src/commoncode/fileutils.py index af28902f9f0..4906540a599 100644 --- a/src/commoncode/fileutils.py +++ b/src/commoncode/fileutils.py @@ -342,7 +342,8 @@ def splitext(path, force_posix=False): # -ignore_nothing = lambda _: False +def ignore_nothing(_): + return False def walk(location, ignored=ignore_nothing): diff --git a/src/commoncode/ignore.py b/src/commoncode/ignore.py index 0a0e33b7419..6f6055a73d9 100644 --- a/src/commoncode/ignore.py +++ b/src/commoncode/ignore.py @@ -44,6 +44,10 @@ def is_ignored(location, ignores, unignores=None, skip_special=True): Return a tuple of (pattern , message) if a file at location is ignored or False otherwise. `ignores` and `unignores` are mappings of patterns to a reason. + + If `skip_special` is True, location is checked and ignored if is considered a special file, + e.g. symlink, FIFO, device file, etc. and location is required to be an actual location to a + file rather than a path string. """ if skip_special and filetype.is_special(location): return True diff --git a/src/packagedcode/__init__.py b/src/packagedcode/__init__.py index 8b088a01618..c995e206a60 100644 --- a/src/packagedcode/__init__.py +++ b/src/packagedcode/__init__.py @@ -124,3 +124,12 @@ def get_package_class(scan_data, default=models.Package): return default ptype_class = PACKAGES_BY_TYPE.get(ptype) return ptype_class or default + + +def get_package_instance(scan_data): + scan_data.pop('api_data_url', None) + scan_data.pop('repository_download_url', None) + scan_data.pop('purl', None) + scan_data.pop('repository_homepage_url', None) + klas = get_package_class(scan_data) + return klas(**scan_data) diff --git a/src/packagedcode/about.py b/src/packagedcode/about.py index b80aba408f9..ce2e9c7c439 100644 --- a/src/packagedcode/about.py +++ b/src/packagedcode/about.py @@ -27,6 +27,7 @@ from __future__ import unicode_literals from collections import OrderedDict +from functools import partial import io import logging @@ -34,6 +35,7 @@ from six import string_types from commoncode import filetype +from commoncode import ignore from packagedcode import models import saneyaml @@ -47,6 +49,7 @@ logging.basicConfig(stream=sys.stdout) logger.setLevel(logging.DEBUG) +# TODO: Override get_package_resource so it returns the Resource that the ABOUT file is describing @attr.s() class AboutPackage(models.Package): diff --git a/src/packagedcode/bower.py b/src/packagedcode/bower.py index a23301d2d46..a538cdd0c34 100644 --- a/src/packagedcode/bower.py +++ b/src/packagedcode/bower.py @@ -27,6 +27,7 @@ from __future__ import unicode_literals from collections import OrderedDict +from functools import partial import io import json import logging @@ -36,6 +37,7 @@ from six import string_types from commoncode import filetype +from commoncode import ignore from packagedcode import models from packagedcode.utils import combine_expressions @@ -59,6 +61,10 @@ class BowerPackage(models.Package): def recognize(cls, location): return parse(location) + @classmethod + def get_package_root(cls, manifest_resource, codebase): + return manifest_resource.parent(codebase) + def compute_normalized_license(self): return compute_normalized_license(self.declared_license) diff --git a/src/packagedcode/chef.py b/src/packagedcode/chef.py index e688dc6104b..246e61212de 100644 --- a/src/packagedcode/chef.py +++ b/src/packagedcode/chef.py @@ -26,6 +26,7 @@ from __future__ import unicode_literals from collections import OrderedDict +from functools import partial import io import json import logging @@ -39,6 +40,7 @@ from commoncode import filetype from commoncode import fileutils +from commoncode import ignore from packagedcode import models diff --git a/src/packagedcode/models.py b/src/packagedcode/models.py index b6fa1e6c477..811d1ef759c 100644 --- a/src/packagedcode/models.py +++ b/src/packagedcode/models.py @@ -506,6 +506,28 @@ def get_package_root(cls, manifest_resource, codebase): """ return manifest_resource + @classmethod + def get_package_resources(cls, package_root, codebase): + """ + Yield the Resources of a Package starting from `package_root` + """ + if not Package.is_ignored_package_resource(package_root, codebase): + yield package_root + for resource in package_root.walk(codebase, topdown=True, ignored=Package.is_ignored_package_resource): + yield resource + + @classmethod + def ignore_resource(cls, resource, codebase): + """ + Return True if `resource` should be ignored. + """ + return False + + @staticmethod + def is_ignored_package_resource(resource, codebase): + from packagedcode import PACKAGE_TYPES + return any(pt.ignore_resource(resource, codebase) for pt in PACKAGE_TYPES) + def compute_normalized_license(self): """ Return a normalized license_expression string using the declared_license diff --git a/src/packagedcode/npm.py b/src/packagedcode/npm.py index 56dc865a1f9..ef1ab0d85a6 100644 --- a/src/packagedcode/npm.py +++ b/src/packagedcode/npm.py @@ -39,6 +39,7 @@ from commoncode import filetype from commoncode import fileutils +from commoncode import ignore from packagedcode import models from packagedcode.utils import combine_expressions from packagedcode.utils import parse_repo_url @@ -86,6 +87,10 @@ def recognize(cls, location): def get_package_root(cls, manifest_resource, codebase): return manifest_resource.parent(codebase) + @classmethod + def ignore_resource(cls, resource, codebase): + return resource.is_dir and resource.name == 'node_modules' + def repository_homepage_url(self, baseurl=default_web_baseurl): return npm_homepage_url(self.namespace, self.name, registry=baseurl) diff --git a/src/packagedcode/nuget.py b/src/packagedcode/nuget.py index 3a4e6fe719b..789dac19201 100644 --- a/src/packagedcode/nuget.py +++ b/src/packagedcode/nuget.py @@ -92,7 +92,7 @@ def repository_download_url(self, baseurl=default_download_baseurl): name=self.name, version=self.version) def api_data_url(self, baseurl=default_api_baseurl): - # the name is lowercased + # the name is lowercased # https://api.nuget.org/v3/registration3/newtonsoft.json/10.0.1.json return baseurl + '{name}/{version}.json'.format( name=self.name.lower(), version=self.version) diff --git a/src/packagedcode/plugin_package.py b/src/packagedcode/plugin_package.py index 34f4ebd6eec..44d6b16aa23 100644 --- a/src/packagedcode/plugin_package.py +++ b/src/packagedcode/plugin_package.py @@ -27,8 +27,11 @@ from __future__ import print_function from __future__ import unicode_literals +import os + import attr +from commoncode.fileutils import parent_directory from plugincode.scan import ScanPlugin from plugincode.scan import scan_impl from scancode import CommandLineOption @@ -96,6 +99,11 @@ def process_codebase(self, codebase, **kwargs): # arches/platforms for a gempsec or multiple sub package (with # "%package") in an RPM .spec file. for package_info in packages_info: + # Break if this Resource has already been set as a new Package root + manifest_path = package_info.get('manifest_path') + if manifest_path and manifest_path != resource.path: + break + package_class = get_package_class(package_info) new_package_root = package_class.get_package_root(resource, codebase) @@ -120,13 +128,19 @@ def process_codebase(self, codebase, **kwargs): # 3. set the manifest_path if needed. # 4. save. - # here we have a relocated Resource and we compute the manifest path - # relative to the new package root - new_package_root_path = new_package_root.path and new_package_root.path.strip('/') - if new_package_root_path: - _, _, manifest_path = resource.path.partition(new_package_root_path) - # note we could have also deserialized and serialized again instead - package_info['manifest_path'] = manifest_path.lstrip('/') + # TODO: this is a hack for the ABOUT file Package parser, ABOUT files are kept alongside + # the resource its for + if new_package_root.is_file and resource.path.endswith('.ABOUT'): + new_manifest_path = os.path.join(parent_directory(new_package_root.path), resource.name) + else: + # here we have a relocated Resource and we compute the manifest path + # relative to the new package root + # TODO: We should have codebase relative paths for manifests + new_package_root_path = new_package_root.path and new_package_root.path.strip('/') + if new_package_root_path: + _, _, new_manifest_path = resource.path.partition(new_package_root_path) + # note we could have also deserialized and serialized again instead + package_info['manifest_path'] = new_manifest_path.lstrip('/') new_package_root.packages.append(package_info) codebase.save_resource(new_package_root) diff --git a/src/packagedcode/pypi.py b/src/packagedcode/pypi.py index 046394f2c27..b51ceeae658 100644 --- a/src/packagedcode/pypi.py +++ b/src/packagedcode/pypi.py @@ -86,7 +86,7 @@ class PythonPackage(models.Package): default_web_baseurl = None default_download_baseurl = None default_api_baseurl = None - + def compute_normalized_license(self): return compute_normalized_license(self.declared_license) @@ -114,7 +114,7 @@ def compute_normalized_license(declared_license): detected_license = models.compute_normalized_license(declared) if detected_license: detected_licenses.append(detected_license) - + if detected_licenses: return combine_expressions(detected_licenses) @@ -291,7 +291,7 @@ def parse_setup_py(location): classifiers = get_classifiers(setup_text) license_classifiers = [c for c in classifiers if c.startswith('License')] declared_license['classifiers'] = license_classifiers - + other_classifiers = [c for c in classifiers if not c.startswith('License')] package = PythonPackage( @@ -372,8 +372,8 @@ def parse_dependencies(location, package): dependencies = parse_with_dparse(resource_location) if dependencies: package.dependencies = dependencies - - + + def parse_source_distribution(location): """ @@ -434,7 +434,7 @@ def parse_with_pkginfo(pkginfo): if pkginfo.license: #TODO: We should make the declared license as it is, this should be updated in scancode to parse a pure string package.declared_license = {'license': pkginfo.license} - + if pkginfo.maintainer: common_data['parties'] = [] common_data['parties'].append(models.Party( @@ -523,7 +523,7 @@ def build_package(package_data): lic = info.get('license') if lic and lic != 'UNKNOWN': setuptext_licenses.append(lic) - declared_license['license'] = setuptext_licenses + declared_license['license'] = setuptext_licenses classifiers_licenses = [] classifiers = info.get('classifiers') @@ -531,13 +531,13 @@ def build_package(package_data): licenses = [lic for lic in classifiers if lic.lower().startswith('license')] for lic in licenses: classifiers_licenses.append(lic) - declared_license['classifiers'] = classifiers_licenses - + declared_license['classifiers'] = classifiers_licenses + common_data['declared_license'] = declared_license kw = info.get('keywords') if kw: common_data['keywords'] = [k.strip() for k in kw.split(',') if k.strip()] - + package = PythonPackage(**common_data) return package diff --git a/src/packagedcode/rubygems.py b/src/packagedcode/rubygems.py index 6436de36991..c10f359862b 100644 --- a/src/packagedcode/rubygems.py +++ b/src/packagedcode/rubygems.py @@ -499,7 +499,7 @@ def get_dependencies(dependencies): is_resolved =False if constraints and len(constraints) == 1: is_resolved = constraint == '=' - + version_constraint = ', '.join(constraints) dep = models.DependentPackage( diff --git a/src/scancode/cli_test_utils.py b/src/scancode/cli_test_utils.py index 84d694f9f3f..a8454bd731e 100644 --- a/src/scancode/cli_test_utils.py +++ b/src/scancode/cli_test_utils.py @@ -133,7 +133,7 @@ def remove_windows_extra_timeout(scancode_options, timeout=WINDOWS_CI_TIMEOUT): del scancode_options['--timeout'] -def check_json_scan(expected_file, result_file, regen=False, remove_file_date=False): +def check_json_scan(expected_file, result_file, regen=False, remove_file_date=False, ignore_headers=False): """ Check the scan `result_file` JSON results against the `expected_file` expected JSON results. @@ -151,6 +151,10 @@ def check_json_scan(expected_file, result_file, regen=False, remove_file_date=Fa expected = load_json_result(expected_file, remove_file_date) + if ignore_headers: + results.pop('headers', None) + expected.pop('headers', None) + # NOTE we redump the JSON as a string for a more efficient display of the # failures comparison/diff # TODO: remove sort, this should no longer be needed diff --git a/src/scancode/resource.py b/src/scancode/resource.py index e1a93f06524..731253723cd 100644 --- a/src/scancode/resource.py +++ b/src/scancode/resource.py @@ -164,6 +164,16 @@ def from_dict(cls, **kwargs): return cls(**kwargs) +def ignore_nothing(resource, codebase): + """ + Return True if `resource` should be ignored. + + This function is used as a callable for `ignored` argument in Codebase and + Resource walk. + """ + return False + + class Codebase(object): """ Represent a codebase being scanned. A Codebase is a tree of Resources. @@ -698,6 +708,8 @@ def _load_resource(self, rid): # TODO: consider messagepack or protobuf for compact/faster processing try: with open(cache_location, 'rb') as cached: + # TODO: Use custom json encoder to encode JSON list as a tuple + # TODO: Consider using simplejson data = json.load(cached, object_pairs_hook=OrderedDict, encoding='utf-8') return self.resource_class(**data) except Exception: @@ -758,7 +770,7 @@ def remove_resource(self, resource): return removed_rids - def walk(self, topdown=True, skip_root=False): + def walk(self, topdown=True, skip_root=False, ignored=ignore_nothing): """ Yield all resources for this Codebase walking its resource tree. Walk the tree top-down, depth-first if `topdown` is True, otherwise walk @@ -769,8 +781,17 @@ def walk(self, topdown=True, skip_root=False): If `skip_root` is True, the root resource is not returned unless this is a codebase with a single resource. + + `ignored` is a callable that accepts two arguments, `resource` and `codebase`, + and returns True if `resource` should be ignored. """ root = self.root + + if ignored(root, self): + return + + root = attr.evolve(root) + # include root if no children (e.g. codebase with a single resource) if skip_root and not root.has_children(): skip_root = False @@ -779,7 +800,7 @@ def walk(self, topdown=True, skip_root=False): if topdown and not skip_root: yield root - for res in root.walk(self, topdown): + for res in root.walk(self, topdown=topdown, ignored=ignored): yield res if not topdown and not skip_root: @@ -1095,7 +1116,7 @@ def _compute_children_counts(self, codebase, skip_filtered=False): return files_count, dirs_count, size_count - def walk(self, codebase, topdown=True,): + def walk(self, codebase, topdown=True, ignored=ignore_nothing): """ Yield all descendant Resources of this Resource. Does not include self. @@ -1104,16 +1125,21 @@ def walk(self, codebase, topdown=True,): Each level is sorted by children sort order (e.g. without-children, then with-children and each group by case-insensitive name) + + `ignored` is a callable that accepts two arguments, `resource` and `codebase`, + and returns True if `resource` should be ignored. """ for child in self.children(codebase): - child = attr.evolve(child) - if topdown: - yield child - for subchild in child.walk(codebase, topdown): - yield subchild - if not topdown: - yield child + if not ignored(child, codebase): + child = attr.evolve(child) + if topdown: + yield child + for subchild in child.walk(codebase, topdown=topdown, ignored=ignored): + if not ignored(subchild, codebase): + yield subchild + if not topdown: + yield child def has_children(self): """ diff --git a/tests/commoncode/test_ignore.py b/tests/commoncode/test_ignore.py index 37d19992300..154a6c2f0a5 100644 --- a/tests/commoncode/test_ignore.py +++ b/tests/commoncode/test_ignore.py @@ -32,6 +32,7 @@ from commoncode import fileutils from commoncode import ignore from commoncode.system import on_mac +from commoncode.system import on_windows import pytest pytestmark = pytest.mark.scanpy3 # NOQA @@ -186,3 +187,32 @@ def test_fileset_is_included_with_default_ignore_does_not_skip_one_char_names(se ] assert expected == result + + @skipIf(on_mac or on_windows, 'We are only testing on posix for now') + def test_is_ignored_path_string_skip_special(self): + test_path = '/test/path' + assert ignore.is_ignored(test_path, {'asdf': 'skip'}, {}, skip_special=True) + assert not ignore.is_ignored(test_path, {'asdf': 'skip'}, {}, skip_special=False) + + @skipIf(on_mac or on_windows, 'We are only testing on posix for now') + def test_is_ignored_special_files_skip_special(self): + test_fifo = self.get_temp_file() + os.mkfifo(test_fifo) + assert ignore.is_ignored(test_fifo, {'asdf': 'skip'}, {}, skip_special=True) + assert not ignore.is_ignored(test_fifo, {'asdf': 'skip'}, {}, skip_special=False) + + test_symlink = self.get_temp_file() + test_file_location = self.get_test_loc('ignore/vcs.tgz') + os.symlink(test_file_location, test_symlink) + assert ignore.is_ignored(test_symlink, {'asdf': 'skip'}, {}, skip_special=True) + assert not ignore.is_ignored(test_symlink, {'asdf': 'skip'}, {}, skip_special=False) + + @skipIf(on_mac or on_windows, 'We are only testing on posix for now') + def test_is_ignored_real_location_skip_special(self): + test_file_location = self.get_test_loc('ignore/vcs.tgz') + assert not ignore.is_ignored(test_file_location, {'asdf': 'skip'}, {}, skip_special=True) + assert not ignore.is_ignored(test_file_location, {'asdf': 'skip'}, {}, skip_special=False) + + test_dir_location = self.get_test_loc('ignore') + assert not ignore.is_ignored(test_file_location, {'asdf': 'skip'}, {}, skip_special=True) + assert not ignore.is_ignored(test_file_location, {'asdf': 'skip'}, {}, skip_special=False) diff --git a/tests/packagedcode/data/models/nested-npm-packages.json b/tests/packagedcode/data/models/nested-npm-packages.json new file mode 100644 index 00000000000..ff345149e49 --- /dev/null +++ b/tests/packagedcode/data/models/nested-npm-packages.json @@ -0,0 +1,237 @@ +{ + "headers": [ + { + "tool_name": "scancode-toolkit", + "tool_version": "3.0.2.post1250.904952c36", + "options": { + "input": [ + "plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/nested-packages/" + ], + "--json-pp": "plugins/scancode-consolidate-scan/tests/data/plugin_consolidate/nested-npm-packages.json", + "--package": true + }, + "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "start_timestamp": "2019-07-26T200843.397788", + "end_timestamp": "2019-07-26T200845.708729", + "message": null, + "errors": [], + "extra_data": { + "files_count": 4 + } + } + ], + "files": [ + { + "path": "nested-packages", + "type": "directory", + "packages": [ + { + "type": "npm", + "namespace": null, + "name": "test-package", + "version": "0.0.1", + "qualifiers": null, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://registry.npmjs.org/test-package/-/test-package-0.0.1.tgz", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": "apache-2.0", + "declared_license": [ + "apache-2.0" + ], + "notice_text": null, + "manifest_path": "package.json", + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:npm/test-package@0.0.1", + "repository_homepage_url": "https://www.npmjs.com/package/test-package", + "repository_download_url": "https://registry.npmjs.org/test-package/-/test-package-0.0.1.tgz", + "api_data_url": "https://registry.npmjs.org/test-package/0.0.1" + } + ], + "scan_errors": [] + }, + { + "path": "nested-packages/package.json", + "type": "file", + "packages": [], + "scan_errors": [] + }, + { + "path": "nested-packages/node_modules", + "type": "directory", + "packages": [], + "scan_errors": [] + }, + { + "path": "nested-packages/node_modules/bar", + "type": "directory", + "packages": [ + { + "type": "npm", + "namespace": null, + "name": "bar", + "version": "0.0.1", + "qualifiers": null, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://registry.npmjs.org/bar/-/bar-0.0.1.tgz", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": "apache-2.0", + "declared_license": [ + "apache-2.0" + ], + "notice_text": null, + "manifest_path": "package.json", + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:npm/bar@0.0.1", + "repository_homepage_url": "https://www.npmjs.com/package/bar", + "repository_download_url": "https://registry.npmjs.org/bar/-/bar-0.0.1.tgz", + "api_data_url": "https://registry.npmjs.org/bar/0.0.1" + } + ], + "scan_errors": [] + }, + { + "path": "nested-packages/node_modules/bar/package.json", + "type": "file", + "packages": [], + "scan_errors": [] + }, + { + "path": "nested-packages/node_modules/foo", + "type": "directory", + "packages": [ + { + "type": "npm", + "namespace": null, + "name": "foo", + "version": "0.0.1", + "qualifiers": null, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://registry.npmjs.org/foo/-/foo-0.0.1.tgz", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": "apache-2.0", + "declared_license": [ + "apache-2.0" + ], + "notice_text": null, + "manifest_path": "package.json", + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:npm/foo@0.0.1", + "repository_homepage_url": "https://www.npmjs.com/package/foo", + "repository_download_url": "https://registry.npmjs.org/foo/-/foo-0.0.1.tgz", + "api_data_url": "https://registry.npmjs.org/foo/0.0.1" + } + ], + "scan_errors": [] + }, + { + "path": "nested-packages/node_modules/foo/package.json", + "type": "file", + "packages": [], + "scan_errors": [] + }, + { + "path": "nested-packages/node_modules/foo/node_modules", + "type": "directory", + "packages": [], + "scan_errors": [] + }, + { + "path": "nested-packages/node_modules/foo/node_modules/bar", + "type": "directory", + "packages": [ + { + "type": "npm", + "namespace": null, + "name": "bar", + "version": "0.0.1", + "qualifiers": null, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://registry.npmjs.org/bar/-/bar-0.0.1.tgz", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": "apache-2.0", + "declared_license": [ + "apache-2.0" + ], + "notice_text": null, + "manifest_path": "package.json", + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:npm/bar@0.0.1", + "repository_homepage_url": "https://www.npmjs.com/package/bar", + "repository_download_url": "https://registry.npmjs.org/bar/-/bar-0.0.1.tgz", + "api_data_url": "https://registry.npmjs.org/bar/0.0.1" + } + ], + "scan_errors": [] + }, + { + "path": "nested-packages/node_modules/foo/node_modules/bar/package.json", + "type": "file", + "packages": [], + "scan_errors": [] + } + ] +} diff --git a/tests/packagedcode/data/npm/get_package_resources/node_modules/this-should-not-be-returned b/tests/packagedcode/data/npm/get_package_resources/node_modules/this-should-not-be-returned new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/packagedcode/data/npm/get_package_resources/package.json b/tests/packagedcode/data/npm/get_package_resources/package.json new file mode 100644 index 00000000000..ece916c0212 --- /dev/null +++ b/tests/packagedcode/data/npm/get_package_resources/package.json @@ -0,0 +1,5 @@ +{ + "name": "test", + "license": "MIT", + "version": "0.1.0" +} diff --git a/tests/packagedcode/data/npm/get_package_resources/this-should-be-returned b/tests/packagedcode/data/npm/get_package_resources/this-should-be-returned new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/packagedcode/test_npm.py b/tests/packagedcode/test_npm.py index 1ed353eafe5..7091136f706 100644 --- a/tests/packagedcode/test_npm.py +++ b/tests/packagedcode/test_npm.py @@ -29,6 +29,7 @@ import os.path from packagedcode import npm +from scancode.resource import Codebase from packages_test_utils import PackageTester @@ -266,5 +267,17 @@ def test_vcs_repository_mapper_handles_version_on_gh(self): result = npm.vcs_repository_mapper(repo, package, rev) assert expected == result.vcs_url + def test_npm_get_package_resources(self): + test_loc= self.get_test_loc('npm/get_package_resources') + codebase = Codebase(test_loc) + root = codebase.root + expected = [ + 'get_package_resources', + 'get_package_resources/package.json', + 'get_package_resources/this-should-be-returned' + ] + results = [r.path for r in npm.NpmPackage.get_package_resources(root, codebase)] + assert expected == results + class MockPackage(object): pass diff --git a/tests/packagedcode/test_package_models.py b/tests/packagedcode/test_package_models.py index 3f4f82221e5..cbaf5418c73 100644 --- a/tests/packagedcode/test_package_models.py +++ b/tests/packagedcode/test_package_models.py @@ -164,3 +164,16 @@ def test_Package_full(self): ) expected_loc = 'models/full-expected.json' self.check_package(package, expected_loc, regen=False) + + def test_Package_get_package_resource_works_with_nested_packages_and_ignores(self): + from packagedcode import get_package_instance + from packagedcode import npm + from scancode.resource import VirtualCodebase + scan_loc = self.get_test_loc('models/nested-npm-packages.json') + codebase = VirtualCodebase(scan_loc) + for resource in codebase.walk(): + for package_data in resource.packages: + package = get_package_instance(package_data) + assert isinstance(package, npm.NpmPackage) + package_resources = list(package.get_package_resources(resource, codebase)) + assert any(r.name == 'package.json' for r in package_resources), resource.path diff --git a/tests/scancode/data/resource/skip_directories_during_walk/skip-this-directory/this-should-not-be-returned b/tests/scancode/data/resource/skip_directories_during_walk/skip-this-directory/this-should-not-be-returned new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/scancode/data/resource/skip_directories_during_walk/this-should-be-returned b/tests/scancode/data/resource/skip_directories_during_walk/this-should-be-returned new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/scancode/test_resource.py b/tests/scancode/test_resource.py index 0f00724bdb5..8717b60b918 100644 --- a/tests/scancode/test_resource.py +++ b/tests/scancode/test_resource.py @@ -28,12 +28,14 @@ from __future__ import unicode_literals from collections import OrderedDict +from functools import partial from os.path import dirname from os.path import exists from os.path import join -from commoncode.testcase import FileBasedTesting +from commoncode import ignore from commoncode.fileutils import parent_directory +from commoncode.testcase import FileBasedTesting from scancode.cli_test_utils import load_json_result from scancode.cli_test_utils import run_scan_click @@ -319,6 +321,18 @@ def test_walk_skip_root_single_dir(self): ] assert expected == [(r.name, r.is_file) for r in results] + def test_walk_skipped_directories_should_not_be_yielded(self): + # Resources that we continue past should not be added to the result list + test_codebase = self.get_test_loc('resource/skip_directories_during_walk') + codebase = Codebase(test_codebase) + result = [] + def _ignored(resource, codebase): + return resource.is_dir and resource.name == 'skip-this-directory' + for resource in codebase.walk(topdown=True, ignored=_ignored,): + result.append(resource.name) + expected = ['skip_directories_during_walk', 'this-should-be-returned'] + assert expected == result + def test__create_resource_can_add_child_to_file(self): test_codebase = self.get_test_loc('resource/codebase/et131x.h') codebase = Codebase(test_codebase)