diff --git a/src/attributecode/attrib.py b/src/attributecode/attrib.py index 8a078f0e..06350521 100644 --- a/src/attributecode/attrib.py +++ b/src/attributecode/attrib.py @@ -31,7 +31,6 @@ from attributecode.licenses import COMMON_LICENSES from attributecode.model import parse_license_expression from attributecode.util import add_unc -from attributecode.util import get_about_file_path # FIXME: the template dir should be outside the code tree @@ -167,7 +166,7 @@ def generate_from_file(abouts, template_loc=DEFAULT_TEMPLATE_FILE, variables=Non def generate_and_save(abouts, output_location, template_loc=None, variables=None, - mapping_file=None, inventory_location=None): + mapping_file=None): """ Generate an attribution text from an `abouts` list of About objects, a `template_loc` template file location and a `variables` optional @@ -175,85 +174,18 @@ def generate_and_save(abouts, output_location, template_loc=None, variables=None `output_location` file. Return a list of Error objects if any. - FIXME: these three argument are too complex: - Optionally use the `mapping_file` mapping config if provided. - Optionally filter `abouts` object based on the inventory JSON or CSV at `inventory_location`. """ updated_abouts = [] - lstrip_afp = [] - afp_list = [] - not_match_path = [] errors = [] - if not inventory_location: - updated_abouts = abouts - - # FIXME: this is too complex - # Do the following if a filter list (inventory_location) is provided - else: - if not os.path.exists(inventory_location): - # FIXME: this message does not make sense - msg = (u'"INVENTORY_LOCATION" does not exist. Generation halted.') - errors.append(Error(ERROR, msg)) - return errors - - if inventory_location.endswith('.csv') or inventory_location.endswith('.json'): - # FIXME: we should use the same inventory loading that we use everywhere - - try: - # Return a list which contains only the about file path - about_list = get_about_file_path(inventory_location, mapping_file=mapping_file) - # FIXME: why catching all exceptions? - except Exception: - # 'about_file_path' key/column doesn't exist - - msg = u"The required key: 'about_file_path' does not exist. Generation halted." - errors.append(Error(ERROR, msg)) - return errors - else: - # FIXME: this message does not make sense - msg = u'Only .csv and .json are supported for the "INVENTORY_LOCATION". Generation halted.' - errors.append(Error(ERROR, msg)) - return errors - - for afp in about_list: - lstrip_afp.append(afp.lstrip('/')) - - # return a list of paths that point all to .ABOUT files - about_files_list = as_about_paths(lstrip_afp) - - # Collect all the about_file_path - for about in abouts: - afp_list.append(about.about_file_path) - - # Get the not matching list if any - for fp in about_files_list: - if not fp in afp_list: - not_match_path.append(fp) - - if not_match_path: - if len(not_match_path) == len(about_files_list): - msg = "None of the paths in the provided 'inventory_location' match with the 'LOCATION'." - errors.append(Error(ERROR, msg)) - return errors - else: - for path in not_match_path: - msg = 'Path: ' + path + ' cannot be found.' - errors.append(Error(ERROR, msg)) - - for about in abouts: - for fp in about_files_list: - if about.about_file_path == fp: - updated_abouts.append(about) - # Parse license_expression and save to the license list for about in updated_abouts: if not about.license_expression.value: continue special_char_in_expression, lic_list = parse_license_expression(about.license_expression.value) if special_char_in_expression: - msg = (u"The following character(s) cannot be in the licesne_expression: " + + msg = (u"The following character(s) cannot be in the license_expression: " + str(special_char_in_expression)) errors.append(Error(ERROR, msg)) else: @@ -274,24 +206,3 @@ def generate_and_save(abouts, output_location, template_loc=None, variables=None of.write(rendered) return errors - - -# FIXME: this function purpose needs to be explained. -def as_about_paths(paths): - """ - Return a list of paths to .ABOUT files from a list of `paths` - strings. - """ - from posixpath import basename - from posixpath import dirname - - about_paths = [] - for path in paths: - if path.endswith('.ABOUT'): - about_paths.append(path) - else: - # FIXME: this is not the way to check that a path is a directory, too weak - if path.endswith('/'): - path += basename(dirname(path)) - about_paths.append(path + '.ABOUT') - return about_paths diff --git a/src/attributecode/cmd.py b/src/attributecode/cmd.py index 7f2eda2a..055d4b88 100644 --- a/src/attributecode/cmd.py +++ b/src/attributecode/cmd.py @@ -43,7 +43,6 @@ from attributecode.model import collect_inventory from attributecode.model import write_output from attributecode.util import extract_zip -from attributecode.util import inventory_filter __copyright__ = """ @@ -165,13 +164,6 @@ def validate_extensions(ctx, param, value, extensions=tuple(('.csv', '.json',))) metavar='OUTPUT', type=click.Path(exists=False, dir_okay=False, writable=True, resolve_path=True)) -# fIXME: this is too complex and should be removed -@click.option('--filter', - multiple=True, - metavar='=', - callback=validate_key_values, - help='Filter the inventory to ABOUT matching these key=value e.g. "license_expression=gpl-2.0') - @click.option('-f', '--format', is_flag=False, default='csv', @@ -203,7 +195,7 @@ def validate_extensions(ctx, param, value, extensions=tuple(('.csv', '.json',))) @click.help_option('-h', '--help') def inventory(location, output, mapping, mapping_file, - format, filter, quiet, verbose): # NOQA + format, quiet, verbose): # NOQA """ Collect the inventory of .ABOUT file data as CSV or JSON. @@ -224,10 +216,6 @@ def inventory(location, output, mapping, mapping_file, errors, abouts = collect_inventory(location, mapping_file=mapping_file) - # FIXME: this is too complex - if filter: - abouts = inventory_filter(abouts, filter) - # Do not write the output if one of the ABOUT files has duplicated keys # TODO: why do this check here?? Also if this is the place, we should list what the errors are. dup_error_msg = u'Duplicated keys' @@ -447,7 +435,6 @@ def attrib(location, output, template, vartext, template_loc=template, variables=vartext, mapping_file=mapping_file, - inventory_location=inventory, ) errors.extend(attrib_errors) diff --git a/src/attributecode/transform.py b/src/attributecode/transform.py index 8c9365e3..5d0cf6f3 100644 --- a/src/attributecode/transform.py +++ b/src/attributecode/transform.py @@ -92,9 +92,6 @@ def transform_data(rows, transformer): if errors: return column_names, data, errors - if transformer.row_filters: - data = list(transformer.filter_rows(data)) - return column_names, data, errors @@ -143,18 +140,6 @@ def transform_data(rows, transformer): column_filters: - name - version - -* row_filters: -An optional list of mappings of : that a source CSV row -should match to be added to the transformed target CSV. If any column value of a -row matches any such filter it is kept. Otherwise it is skipped. Filters are -applied last after all renamings, checks and tranforms and can therefore onlu -use remaining column names. - -For instance with this configuration the target CSV will only contain rows that -have a "path" equal to "/root/user/lib": - row_filters: - path : /root/user/lib ''' @@ -165,15 +150,20 @@ class Transformer(object): column_renamings = attr.attrib(default=attr.Factory(dict)) required_columns = attr.attrib(default=attr.Factory(list)) column_filters = attr.attrib(default=attr.Factory(list)) - row_filters = attr.attrib(default=attr.Factory(list)) - # TODO: populate these! # a list of all the standard columns from AboutCode toolkit standard_columns = attr.attrib(default=attr.Factory(list), init=False) # a list of the subset of standard columns that are essential and MUST be # present for AboutCode toolkit to work essential_columns = attr.attrib(default=attr.Factory(list), init=False) + # called by attr after the __init__() + def __attrs_post_init__(self, *args, **kwargs): + from attributecode.model import About + about = About() + self.essential_columns = list(about.required_fields) + self.standard_columns = [f.name for f in about.all_fields()] + @classmethod def default(cls): """ @@ -183,7 +173,6 @@ def default(cls): column_renamings={}, required_columns=[], column_filters=[], - row_filters=[], ) @classmethod @@ -198,7 +187,6 @@ def from_file(cls, location): column_renamings=data.get('column_renamings', {}), required_columns=data.get('required_columns', []), column_filters=data.get('column_filters', []), - row_filters=data.get('row_filters', []), ) def check_required_columns(self, data): @@ -257,20 +245,6 @@ def filter_columns(self, data): items = ((k, v) for k, v in entry.items() if k in column_filters) yield OrderedDict(items) - def filter_rows(self, data): - """ - Yield a filtered list of mappings from a `data` list of mappings keeping - only items that match any one of the `row_filters` of this Transformer. - Return the data unchanged if no `row_filters` is avilable in this - Transformer. - """ - filters = self.row_filters - for entry in data: - for filt in filters: - for filtered_column_name, filtered_column_value in filt.items(): - if entry.get(filtered_column_name) == filtered_column_value: - yield entry - def check_duplicate_columns(column_names): """ diff --git a/src/attributecode/util.py b/src/attributecode/util.py index 1b44f105..92c01c68 100644 --- a/src/attributecode/util.py +++ b/src/attributecode/util.py @@ -348,21 +348,6 @@ def format_output(about_data, mapping_file=None): order_dict[other_key] = about_data[other_key] return order_dict -# FIXME: why is this used for -def get_about_file_path(location, mapping_file=None): - """ - Read file at location, return a list of about_file_path. - """ - afp_list = [] - if location.endswith('.csv'): - about_data = load_csv(location, mapping_file=mapping_file) - else: - about_data = load_json(location) - - for about in about_data: - afp_list.append(about['about_file_path']) - return afp_list - def load_csv(location, mapping_file=None): """ @@ -567,34 +552,6 @@ def copy_license_notice_files(fields, base_dir, reference_dir, afp): print('Cannot copy file at %(from_lic_path)r.' % locals()) -# FIXME: this is NOT a util but something to move with inventories or a method -# from About objects -def inventory_filter(abouts, filters): - """ - Return a list of filtered About objects from an `abouts` list of About - object using the `filters` mapping of: - {field_name: [acceptable_values, ....]} - - ... such that only the About object that have a field_name with a value that - matches one of the acceptable values is returned. Other About object are - filtered out. - """ - matching_abouts = [] - for about in abouts: - for field_name, acceptable_values in filters.items(): - # Check if the about object has the filtered attribute and if the - # attributed value is the same as the defined in the filter - actual_value = getattr(about, field_name, None) - if actual_value in acceptable_values and not about in matching_abouts: - matching_abouts.append(about) - # FIXME: if it matches once it matches always which is probably not right - break - - return matching_abouts - - - - # FIXME: we should use a license object instead def ungroup_licenses(licenses): """ diff --git a/tests/test_util.py b/tests/test_util.py index 07ee2762..fc15a535 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -397,13 +397,6 @@ def test_load_csv_with_mapping(self): result = util.load_csv(test_file, mapping_file=DEFAULT_MAPPING) assert expected == result - def test_get_about_file_path_from_csv_using_mapping(self): - test_file = get_test_loc('test_util/csv/about.csv') - expected = ['about.ABOUT'] - result = util.get_about_file_path( - test_file, mapping_file=DEFAULT_MAPPING) - assert expected == result - def test_load_csv_does_convert_column_names_to_lowercase(self): test_file = get_test_loc('test_util/csv/about_key_with_upper_case.csv') expected = [OrderedDict( @@ -474,13 +467,6 @@ def test_load_non_list_json(self): result = util.load_json(test_file) assert expected == result - # FIXME: mappings are a CSV-only feature!!!!! - def test_get_about_file_path_from_json_using_mapping(self): - test_file = get_test_loc('test_util/json/expected.json') - expected = ['/load/this.ABOUT'] - result = util.get_about_file_path(test_file, mapping_file=DEFAULT_MAPPING) - assert expected == result - def test_load_non_list_json2(self): test_file = get_test_loc('test_util/json/not_a_list.json') expected = [OrderedDict([ @@ -632,17 +618,6 @@ def test_load_yaml_about_file_with_multiline(self): # notes: exceptio is rasied only for the first dupe assert 'Duplicate key in YAML source: owner' == str(e) - def test_inventory_filter(self): - test_loc = get_test_loc('test_util/inventory_filter') - _errors, abouts = model.collect_inventory(test_loc) - - filter_dict = {'name': ['simple']} - # The test loc has 2 .about files, only the simple.about is taken after - # the filtering - updated_abouts = util.inventory_filter(abouts, filter_dict) - for about in updated_abouts: - assert about.name.value == 'simple' - def test_ungroup_licenses(self): about = [ OrderedDict([ diff --git a/tests/testdata/test_cmd/help/about_inventory_help.txt b/tests/testdata/test_cmd/help/about_inventory_help.txt index 416c1f29..bbfbf501 100644 --- a/tests/testdata/test_cmd/help/about_inventory_help.txt +++ b/tests/testdata/test_cmd/help/about_inventory_help.txt @@ -7,8 +7,6 @@ Usage: about inventory [OPTIONS] LOCATION OUTPUT OUTPUT: Path to the JSON or CSV inventory file to create. Options: - --filter = Filter the inventory to ABOUT matching these - key=value e.g. "license_expression=gpl-2.0 -f, --format [json|csv] Set OUTPUT inventory file format. [default: csv] --mapping Use the default built-in "mapping.config" file with mapping between input keys and .ABOUT field diff --git a/tests/testdata/test_cmd/help/about_transform_config_help.txt b/tests/testdata/test_cmd/help/about_transform_config_help.txt index 28e28528..090a1ac0 100644 --- a/tests/testdata/test_cmd/help/about_transform_config_help.txt +++ b/tests/testdata/test_cmd/help/about_transform_config_help.txt @@ -44,15 +44,3 @@ and "version" columns and no other column: - name - version -* row_filters: -An optional list of mappings of : that a source CSV row -should match to be added to the transformed target CSV. If any column value of a -row matches any such filter it is kept. Otherwise it is skipped. Filters are -applied last after all renamings, checks and tranforms and can therefore onlu -use remaining column names. - -For instance with this configuration the target CSV will only contain rows that -have a "path" equal to "/root/user/lib": - row_filters: - path : /root/user/lib - diff --git a/tests/testdata/test_util/inventory_filter/basic.about b/tests/testdata/test_util/inventory_filter/basic.about deleted file mode 100644 index e822a131..00000000 --- a/tests/testdata/test_util/inventory_filter/basic.about +++ /dev/null @@ -1,34 +0,0 @@ -about_resource: . -name: optional -version: 2.2 -about_format: 2.0 -date: 2013-1-1 -description: A description for this component. -homepage_url: http://msn.com -download_url: http://msn.com/download -readme: README STUFF -install: HOW TO INSTALLInstallation information for this component. You may use install_file when this is a long text or a pre-existing file as INSTALL files are commonly found in code archives. -changelog: Changelog text for this component. You may use changelog_file when this is a long text or a pre-existing file as CHANGELOG files are commonly found in code archives. -news: News text for this component. You may use news_file when this is a long text or a pre-existing file as NEWS files are commonly found in code archives. You may use news_url to point to an internet news feed for this component. -notes: SOME NOTES -usage: Describe the intended usage -contact: Davide Berti dberti@nexb.com -organization: NexB -copyright: Copyright Davide Berti 2013 statement for this component. You may use copyright_file when this is a long text or to reference a pre-existing file. For instance COPYRIGHT files are commonly found in code archives. -notice: Text containing a legal notice for this component. You may use notice_file when this is a long text or to reference a pre-existing file. For instance NOTICE files are commonly found in code archives. -notice_url: http://msn.com/notice -license_text: License text for this component. You may use license_text_file when this is a long text or to reference a pre-existing file. For instance LICENSE or COPYING files are commonly found in code archives. -license_url: http://msn.com/license -license_spdx: The SPDX license short form identifiers for the license of this component. See http://spdx.org/licenses/ for details. You can separate each identifier with an " or " and " and " as defined in the SPDX specification 1.1 to document the relationship between multiple license identifiers, such as a choice of license. -redistribute_sources: yes -scm_tool: SCM tool such as git, svn, cvs, etc. -scm_repository: Typically a URL or some other identifier used by the tool to point to a repository, folder or file, such as an SVN or Git repository URL. -scm_path: Path to a file, folder or module used by certain SCM pointing inside a repository. -scm_tag: tag name or path used by certain SCM. -scm_branch: branch name or path used by certain SCM. -scm_rev: revision identifier such as a revision hash or version number. -signature_gpg_file: linux-3.1.7.tar.sign -checksum_sha1: 87aaf7bc7e8715f0455997bb8c6791aa -dje_component: The DejaCode Enterprise URN for this component. -dje_license: The DejaCode Enterprise URNs of the licenses for this component. -dje_organization: The DejaCode Enterprise URN for this component organization. \ No newline at end of file diff --git a/tests/testdata/test_util/inventory_filter/simple.about b/tests/testdata/test_util/inventory_filter/simple.about deleted file mode 100644 index cad2118e..00000000 --- a/tests/testdata/test_util/inventory_filter/simple.about +++ /dev/null @@ -1,6 +0,0 @@ -about_resource: . -name: simple -version: 2.2 - -dje_license: apache-2.0 -dje_license_name: Apache License 2.0 \ No newline at end of file