diff --git a/etc/scripts/licenses/report_license_rules.py b/etc/scripts/licenses/report_license_rules.py new file mode 100644 index 00000000000..f25f257de41 --- /dev/null +++ b/etc/scripts/licenses/report_license_rules.py @@ -0,0 +1,195 @@ +# -*- coding: utf-8 -*- +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import io + +import click +import csv + +from commoncode.cliutils import PluggableCommandLineOption +from licensedcode.models import load_licenses +from licensedcode.models import load_rules + + +LICENSES_FIELDNAMES = [ + 'key', 'short_name', 'name', 'category', 'owner', 'text', 'words_count', 'notes', + 'minimum_coverage', 'homepage_url', 'is_exception', 'language', 'is_unknown', + 'spdx_license_key', 'reference_url', 'text_urls', 'other_urls', 'standard_notice', + 'license_filename', 'faq_url', 'ignorable_authors', + 'ignorable_copyrights', 'ignorable_holders', 'ignorable_urls', 'ignorable_emails', + 'osi_license_key', 'osi_url', 'other_spdx_license_keys', +] + + +RULES_FIELDNAMES = [ + 'identifier', 'license_expression', 'relevance', 'text', 'words_count', 'category', + 'is_false_positive', 'is_license_text', 'is_license_notice', 'is_license_tag', + 'is_license_reference', 'is_license_intro', 'has_unknown', 'only_known_words', + 'notes', 'referenced_filenames', 'minimum_coverage', 'ignorable_copyrights', + 'ignorable_holders', 'ignorable_authors', 'ignorable_urls', 'ignorable_emails', +] + + +SCANCODE_LICENSEDB_URL = 'https://scancode-licensedb.aboutcode.org/{}' + + +def write_data_to_csv(data, output_csv, fieldnames): + + with open(output_csv, 'w') as f: + w = csv.DictWriter(f, fieldnames=fieldnames) + w.writeheader() + for entry in data: + w.writerow(entry) + + +def filter_by_attribute(data, attribute, required_key): + """ + Filters by attribute, if value is required_key. + Example `attribute`: `category`. Example `required_key`: `Permissive`. + """ + return [entry for entry in data if entry.get(attribute, 'None') == required_key] + +def flatten_output(data): + + assert isinstance(data, list) + + output = [] + for entry in data: + assert isinstance(entry, dict) + + output_entry = {} + for key, value in entry.items(): + if value is None: + continue + + if isinstance(value, list): + value = ' '.join(value) + elif not isinstance(value, str): + value = repr(value) + + output_entry[key] = value + + output.append(output_entry) + + return output + +@click.command() +@click.option('-l', '--licenses', + type=click.Path(dir_okay=False, writable=True, readable=False), + default=None, + metavar='FILE', + help='Write all Licenses data to the csv FILE.', + cls=PluggableCommandLineOption +) +@click.option('-r', '--rules', + type=click.Path(dir_okay=False, writable=True, readable=False), + default=None, + metavar='FILE', + help='Write all Rules data to the csv FILE.', + cls=PluggableCommandLineOption, +) +@click.option('-c', '--category', + type=str, + default=None, + metavar='STRING', + help='An optional filter to only output licenses/rules of this category. ' + 'Example STRING: `permissive`.', + cls=PluggableCommandLineOption, +) +@click.option('-k', '--license-key', + type=str, + default=None, + metavar='STRING', + help='An optional filter to only output licenses/rules which has this license key. ' + 'Example STRING: `mit`.', + cls=PluggableCommandLineOption, +) +@click.option('-t', '--with-text', + is_flag=True, + default=False, + help='Also include the license/rules texts (First 200 characters). ' + 'Note that this increases the file size significantly.', + cls=PluggableCommandLineOption, +) +@click.help_option('-h', '--help') +def cli(licenses, rules, category, license_key, with_text): + """ + Write Licenses/Rules from scancode into a CSV file with all details. + Output can be optionally filtered by category/license-key. + """ + licenses_output = [] + rules_output = [] + + licenses_data = load_licenses() + + if licenses: + for license in licenses_data.values(): + license_data = license.to_dict() + if with_text: + license_data['text'] = license.text[:200] + license_data['is_unknown'] = license.is_unknown + license_data['words_count'] = len(license.text) + license_data['reference_url'] = SCANCODE_LICENSEDB_URL.format(license.key) + licenses_output.append(license_data) + + if category: + licenses_output = filter_by_attribute( + data=licenses_output, + attribute='category', + required_key=category + ) + + if license_key: + licenses_output = filter_by_attribute( + data=licenses_output, + attribute='key', + required_key=license_key, + ) + + licenses_output = flatten_output(data=licenses_output) + write_data_to_csv(data=licenses_output, output_csv=licenses, fieldnames=LICENSES_FIELDNAMES) + + + if rules: + rules_data = list(load_rules()) + for rule in rules_data: + rule_data = rule.to_dict() + rule_data['identifier'] = rule.identifier + rule_data['referenced_filenames'] = rule.referenced_filenames + if with_text: + rule_data['text'] = rule.text()[:200] + rule_data['has_unknown'] = rule.has_unknown + rule_data['words_count'] = len(rule.text()) + try: + rule_data['category'] = licenses_data[rule_data['license_expression']].category + except KeyError: + pass + rules_output.append(rule_data) + + if category: + rules_output = filter_by_attribute( + data=rules_output, + attribute='category', + required_key=category, + ) + + if license_key: + rules_output = filter_by_attribute( + data=rules_output, + attribute='license_expression', + required_key=license_key, + ) + + rules_output = flatten_output(rules_output) + write_data_to_csv(data=rules_output, output_csv=rules, fieldnames=RULES_FIELDNAMES) + + +if __name__ == '__main__': + cli()