diff --git a/src/attributecode/cmd.py b/src/attributecode/cmd.py index 0b0f784d..9c9e5abb 100644 --- a/src/attributecode/cmd.py +++ b/src/attributecode/cmd.py @@ -494,6 +494,84 @@ def check(location, verbose): sys.exit(severe_errors_count) +###################################################################### +# transform subcommand +###################################################################### + +def print_config_help(ctx, param, value): + if not value or ctx.resilient_parsing: + return + from attributecode.transform import tranformer_config_help + click.echo(tranformer_config_help) + ctx.exit() + + +@about.command(cls=AboutCommand, + short_help='Transform a CSV by applying renamings, filters and checks.') + +@click.argument('location', + required=True, + callback=partial(validate_extensions, extensions=('.csv',)), + metavar='LOCATION', + type=click.Path(exists=True, dir_okay=False, readable=True, resolve_path=True)) + +@click.argument('output', + required=True, + callback=partial(validate_extensions, extensions=('.csv',)), + metavar='OUTPUT', + type=click.Path(exists=False, dir_okay=False, writable=True, resolve_path=True)) + +@click.option('-c', '--configuration', + metavar='FILE', + type=click.Path(exists=True, dir_okay=False, readable=True, resolve_path=True), + help='Path to an optional YAML configuration file. See --help-format for ' + 'format help.') + +@click.option('--help-format', + is_flag=True, is_eager=True, expose_value=False, + callback=print_config_help, + help='Show configuration file format help and exit.') + +@click.option('-q', '--quiet', + is_flag=True, + help='Do not print error or warning messages.') + +@click.option('--verbose', + is_flag=True, + help='Show all error and warning messages.') + +@click.help_option('-h', '--help') + +def transform(location, output, configuration, quiet, verbose): # NOQA + """ +Transform the CSV file at LOCATION by applying renamings, filters and checks +and write a new CSV to OUTPUT. + +LOCATION: Path to a CSV file. + +OUTPUT: Path to CSV inventory file to create. + """ + from attributecode.transform import transform_csv_to_csv + from attributecode.transform import Transformer + + if not quiet: + print_version() + click.echo('Transforming CSV...') + + if not configuration: + transformer = Transformer.default() + else: + transformer = Transformer.from_file(configuration) + + errors = transform_csv_to_csv(location, output, transformer) + + errors_count = report_errors(errors, quiet, verbose) + if not quiet: + msg = 'Transformed CSV written to {output}.'.format(**locals()) + click.echo(msg) + sys.exit(errors_count) + + ###################################################################### # Error management ###################################################################### diff --git a/src/attributecode/transform.py b/src/attributecode/transform.py new file mode 100644 index 00000000..5afe0ef4 --- /dev/null +++ b/src/attributecode/transform.py @@ -0,0 +1,304 @@ +#!/usr/bin/env python +# -*- coding: utf8 -*- +# ============================================================================ +# Copyright (c) 2013-2018 nexB Inc. http://www.nexb.com/ - All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +from collections import Counter +from collections import OrderedDict +import io + +import attr + +from attributecode import CRITICAL +from attributecode import Error +from attributecode import saneyaml +from attributecode.util import python2 + +if python2: # pragma: nocover + from itertools import izip_longest as zip_longest # NOQA + + import backports.csv as csv # NOQA +else: # pragma: nocover + from itertools import zip_longest # NOQA + + import csv # NOQA + + +def transform_csv_to_csv(location, output, transformer): + """ + Read a CSV file at `location` and write a new CSV file at `output`. Apply + transformations using the `transformer` Tranformer. + Return a list of Error objects. + """ + if not transformer: + raise ValueError('Cannot transform without Transformer') + + rows = read_csv_rows(location) + + column_names, data, errors = transform_data(rows, transformer) + + if errors: + return errors + else: + write_csv(output, data, column_names) + return [] + + +def transform_data(rows, transformer): + """ + Read a list of list of CSV-like data `rows` and apply transformations using the + `transformer` Tranformer. + Return a tuple of: + ([column names...], [transformed ordered mappings...], [Error objects..]) + """ + + if not transformer: + return rows + + errors = [] + rows = iter(rows) + column_names = next(rows) + column_names = transformer.clean_columns(column_names) + + dupes = check_duplicate_columns(column_names) + + if dupes: + msg = 'Duplicated column name: {name}' + errors.extend(Error(CRITICAL, msg.format(name)) for name in dupes) + return column_names, [], errors + + column_names = transformer.apply_renamings(column_names) + + # convert to mappings using the renamed columns + data = [OrderedDict(zip_longest(column_names, row)) for row in rows] + + if transformer.column_filters: + data = list(transformer.filter_columns(data)) + column_names = [c for c in column_names if c in transformer.column_filters] + + errors = transformer.check_required_columns(data) + if errors: + return column_names, data, errors + + if transformer.row_filters: + data = list(transformer.filter_rows(data)) + + return column_names, data, errors + + +tranformer_config_help = ''' +A transform configuration file is used to describe which transformations and +validations to apply to a source CSV file. This is a simple text file using YAML +format, using the same format as an .ABOUT file. + +The attributes that can be set in a configuration file are: + +* column_renamings: +An optional mapping of source CSV column name to target CSV new column name that +is used to rename CSV columns. + +For instance with this configuration the columns "Directory/Location" will be +renamed to "about_resource" and "foo" to "bar": + renamings: + 'Directory/Location' : about_resource + foo : bar + +The renaming is always applied first before other transforms and checks. All +other column names referenced below are these that exist AFTER the renamings +have been applied to the existing column names. + +* required_columns: +An optional list of required column names that must have a value, beyond the +standard columns names. If a source CSV does not have such a column or a row is +missing a value for a required column, an error is reported. + +For instance with this configuration an error will be reported if the columns +"name" and "version" are missing or if any row does not have a value set for +these columns: + required_columns: + - name + - version + +* column_filters: +An optional list of column names that should be kept in the transformed CSV. If +this list is provided, all the columns from the source CSV that should be kept +in the target CSV must be listed be even if they are standard or required +columns. If this list is not provided, all source CSV columns are kept in the +transformed target CSV. + +For instance with this configuration the target CSV will only contains the "name" +and "version" columns and no other column: + column_filters: + - name + - version + +* row_filters: +An optional list of mappings of : that a source CSV row +should match to be added to the transformed target CSV. If any column value of a +row matches any such filter it is kept. Otherwise it is skipped. Filters are +applied last after all renamings, checks and tranforms and can therefore onlu +use remaining column names. + +For instance with this configuration the target CSV will only contain rows that +have a "path" equal to "/root/user/lib": + row_filters: + path : /root/user/lib +''' + + +@attr.attributes +class Transformer(object): + __doc__ = tranformer_config_help + + column_renamings = attr.attrib(default=attr.Factory(dict)) + required_columns = attr.attrib(default=attr.Factory(list)) + column_filters = attr.attrib(default=attr.Factory(list)) + row_filters = attr.attrib(default=attr.Factory(list)) + + # TODO: populate these! + # a list of all the standard columns from AboutCode toolkit + standard_columns = attr.attrib(default=attr.Factory(list), init=False) + # a list of the subset of standard columns that are essential and MUST be + # present for AboutCode toolkit to work + essential_columns = attr.attrib(default=attr.Factory(list), init=False) + + @classmethod + def default(cls): + """ + Return a default Transformer with built-in transforms. + """ + return cls( + column_renamings={}, + required_columns=[], + column_filters=[], + row_filters=[], + ) + + @classmethod + def from_file(cls, location): + """ + Load and return a Transformer instance from a YAML configuration file at + `location`. + """ + with io.open(location, encoding='utf-8') as conf: + data = saneyaml.load(conf.read()) + return cls( + column_renamings=data.get('column_renamings', {}), + required_columns=data.get('required_columns', []), + column_filters=data.get('column_filters', []), + row_filters=data.get('row_filters', []), + ) + + def check_required_columns(self, data): + """ + Return a list of Error for a `data` list of ordered mappings where a + mapping is missing a value for a required column name. + """ + errors = [] + required = set(self.essential_columns + self.required_columns) + if not required: + return [] + + for rn, item in enumerate(data): + missings = [rk for rk in required if not item.get(rk)] + if not missings: + continue + + missings = ', '.join(missings) + msg = 'Row {rn} is missing required values for columns: {missings}' + errors.append(Error(CRITICAL, msg.format(**locals()))) + return errors + + def apply_renamings(self, column_names): + """ + Return a tranformed list of `column_names` where columns are renamed + based on this Transformer configuration. + """ + renamings = self.column_renamings + if not renamings: + return column_names + renamings = {n.lower(): rn.lower() for n, rn in renamings.items()} + + renamed = [] + for name in column_names: + name = name.lower() + new_name = renamings.get(name, name) + renamed.append(new_name) + return renamed + + def clean_columns(self, column_names): + """ + Apply standard cleanups to a list of columns and return these. + """ + if not column_names: + return column_names + return [c.strip().lower() for c in column_names] + + def filter_columns(self, data): + """ + Yield transformed mappings from a `data` list of mappings keeping only + columns with a name in the `column_filters`of this Transformer. + Return the data unchanged if no `column_filters` exists. + """ + column_filters = set(self.clean_columns(self.column_filters)) + for entry in data: + items = ((k, v) for k, v in entry.items() if k in column_filters) + yield OrderedDict(items) + + def filter_rows(self, data): + """ + Yield a filtered list of mappings from a `data` list of mappings keeping + only items that match any one of the `row_filters` of this Transformer. + Return the data unchanged if no `row_filters` is avilable in this + Transformer. + """ + filters = self.row_filters + for entry in data: + for filt in filters: + for filtered_column_name, filtered_column_value in filt.items(): + if entry.get(filtered_column_name) == filtered_column_value: + yield entry + + +def check_duplicate_columns(column_names): + """ + Check that there are no duplicate in the `column_names` list of column name + strings, ignoring case. Return a list of unique duplicated column names. + """ + counted = Counter(c.lower() for c in column_names) + return [column for column, count in sorted(counted.items()) if count > 1] + + +def read_csv_rows(location): + """ + Yield rows (as a list of values) from a CSV file at `location`. + """ + with io.open(location, encoding='utf-8') as csvfile: + reader = csv.reader(csvfile) + for row in reader: + yield row + + +def write_csv(location, data, column_names): # NOQA + """ + Write a CSV file at `location` the `data` list of ordered mappings using the + `column_names`. + """ + with io.open(location, 'w', encoding='utf-8') as csvfile: + writer = csv.DictWriter(csvfile, fieldnames=column_names) + writer.writeheader() + writer.writerows(data) diff --git a/tests/test_cmd.py b/tests/test_cmd.py index dcc42795..36ae8c88 100644 --- a/tests/test_cmd.py +++ b/tests/test_cmd.py @@ -408,3 +408,21 @@ def test_about_attrib_command_can_run_minimally_without_error(): test_dir = get_test_loc('test_cmd/repository-mini') result = get_temp_file() run_about_command_test_click(['attrib', test_dir, result]) + + +def test_about_transform_command_can_run_minimally_without_error(): + test_file = get_test_loc('test_cmd/transform.csv') + result = get_temp_file('file_name.csv') + run_about_command_test_click(['transform', test_file, result]) + + +def test_about_transform_help_text(): + check_about_stdout( + ['transform', '--help'], + 'test_cmd/help/about_transform_help.txt', regen=False) + + +def test_about_transform_expanded_help_text(): + check_about_stdout( + ['transform', '--help-format'], + 'test_cmd/help/about_transform_config_help.txt', regen=False) diff --git a/tests/testdata/test_cmd/help/about_help.txt b/tests/testdata/test_cmd/help/about_help.txt index e903fb17..173b1c44 100644 --- a/tests/testdata/test_cmd/help/about_help.txt +++ b/tests/testdata/test_cmd/help/about_help.txt @@ -18,3 +18,4 @@ Commands: errors and warnings. gen Generate .ABOUT files from an inventory as CSV or JSON. inventory Collect the inventory of .ABOUT files to a CSV or JSON file. + transform Transform a CSV by applying renamings, filters and checks. diff --git a/tests/testdata/test_cmd/help/about_transform_config_help.txt b/tests/testdata/test_cmd/help/about_transform_config_help.txt new file mode 100644 index 00000000..28e28528 --- /dev/null +++ b/tests/testdata/test_cmd/help/about_transform_config_help.txt @@ -0,0 +1,58 @@ + +A transform configuration file is used to describe which transformations and +validations to apply to a source CSV file. This is a simple text file using YAML +format, using the same format as an .ABOUT file. + +The attributes that can be set in a configuration file are: + +* column_renamings: +An optional mapping of source CSV column name to target CSV new column name that +is used to rename CSV columns. + +For instance with this configuration the columns "Directory/Location" will be +renamed to "about_resource" and "foo" to "bar": + renamings: + 'Directory/Location' : about_resource + foo : bar + +The renaming is always applied first before other transforms and checks. All +other column names referenced below are these that exist AFTER the renamings +have been applied to the existing column names. + +* required_columns: +An optional list of required column names that must have a value, beyond the +standard columns names. If a source CSV does not have such a column or a row is +missing a value for a required column, an error is reported. + +For instance with this configuration an error will be reported if the columns +"name" and "version" are missing or if any row does not have a value set for +these columns: + required_columns: + - name + - version + +* column_filters: +An optional list of column names that should be kept in the transformed CSV. If +this list is provided, all the columns from the source CSV that should be kept +in the target CSV must be listed be even if they are standard or required +columns. If this list is not provided, all source CSV columns are kept in the +transformed target CSV. + +For instance with this configuration the target CSV will only contains the "name" +and "version" columns and no other column: + column_filters: + - name + - version + +* row_filters: +An optional list of mappings of : that a source CSV row +should match to be added to the transformed target CSV. If any column value of a +row matches any such filter it is kept. Otherwise it is skipped. Filters are +applied last after all renamings, checks and tranforms and can therefore onlu +use remaining column names. + +For instance with this configuration the target CSV will only contain rows that +have a "path" equal to "/root/user/lib": + row_filters: + path : /root/user/lib + diff --git a/tests/testdata/test_cmd/help/about_transform_help.txt b/tests/testdata/test_cmd/help/about_transform_help.txt new file mode 100644 index 00000000..0aad484e --- /dev/null +++ b/tests/testdata/test_cmd/help/about_transform_help.txt @@ -0,0 +1,16 @@ +Usage: about transform [OPTIONS] LOCATION OUTPUT + + Transform the CSV file at LOCATION by applying renamings, filters and checks + and write a new CSV to OUTPUT. + + LOCATION: Path to a CSV file. + + OUTPUT: Path to CSV inventory file to create. + +Options: + -c, --configuration FILE Path to an optional YAML configuration file. See + --help-format for format help. + --help-format Show configuration file format help and exit. + -q, --quiet Do not print error or warning messages. + --verbose Show all error and warning messages. + -h, --help Show this message and exit. diff --git a/tests/testdata/test_cmd/transform.csv b/tests/testdata/test_cmd/transform.csv new file mode 100644 index 00000000..653d190a --- /dev/null +++ b/tests/testdata/test_cmd/transform.csv @@ -0,0 +1,3 @@ +"About_file_path","about_resource","name","version","download_url","description","homepage_url","notes","license_expression","license_key","license_name","license_file","license_url","copyright","notice_file","notice_url","redistribute","attribute","track_changes","modified","internal_use_only","changelog_file","owner","owner_url","contact","author","author_file","vcs_tool","vcs_repository","vcs_path","vcs_tag","vcs_branch","vcs_revision","checksum_md5","checksum_sha1","checksum_sha256","spec_version" +"/about/about.ABOUT",0,"AboutCode","0.11.0",,"AboutCode is a tool to process ABOUT files. An ABOUT file is a file.","http://dejacode.org",,"apache-2.0","apache-2.0",,"apache-2.0.LICENSE",,"Copyright (c) 2013-2014 nexB Inc.","NOTICE",,,,,,,,"nexB Inc.",,,"Jillian Daguil, Chin Yeung Li, Philippe Ombredanne, Thomas Druez",,"git","https://github.com/dejacode/about-code-tool.git",,,,,,,, +"/about/about.ABOUT2",0,"AboutCode","0.11.0",,"AboutCode is a tool to process ABOUT files. An ABOUT file is a file.","http://dejacode.org",,"apache-2.0","apache-2.0",,"apache-2.0.LICENSE",,"Copyright (c) 2013-2014 nexB Inc.","NOTICE",,,,,,,,"nexB Inc.",,,"Jillian Daguil, Chin Yeung Li, Philippe Ombredanne, Thomas Druez",,"git","https://github.com/dejacode/about-code-tool.git",,,,,,,,