diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 0b4254e3..00000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "python.pythonPath": "/home/sarthak/gsoc/aboutcode-toolkit/local/bin/python2.7" -} \ No newline at end of file diff --git a/docs/UsingAboutCodetoDocumentYourSoftwareAssets.md b/docs/UsingAboutCodetoDocumentYourSoftwareAssets.md index 688105c4..33830a3f 100644 --- a/docs/UsingAboutCodetoDocumentYourSoftwareAssets.md +++ b/docs/UsingAboutCodetoDocumentYourSoftwareAssets.md @@ -287,6 +287,19 @@ and "version" columns and no other column: - name - version +* exclude_fields: +An optional list of field names that should be excluded in the transformed CSV/JSON. If +this list is provided, all the fields from the source CSV/JSON that should be excluded +in the target CSV/JSON must be listed. Excluding standard or required fields will cause +an error. If this list is not provided, all source CSV/JSON fields are kept in the +transformed target CSV/JSON. + +For instance with this configuration the target CSV/JSON will not contain the "type" +and "temp" fields: + exclude_fields: + - type + - temp + ## Run gen to Generate AboutCode Toolkit Files diff --git a/etc/scripts/irc-notify.py b/etc/scripts/irc-notify.py index 8b8376f2..9da17184 100644 --- a/etc/scripts/irc-notify.py +++ b/etc/scripts/irc-notify.py @@ -139,7 +139,7 @@ def appveyor_vars(): response = line.split() if response[0] == 'PING': - irc_file.send('PONG {}\r\n'.format(reponse[1]).encode()) + irc_file.send('PONG {}\r\n'.format(response[1]).encode()) elif response[1] == '433': irc_sock.send('NICK {}\r\n'.format(irc_nick).encode()) diff --git a/src/attributecode/transform.py b/src/attributecode/transform.py index e6104119..9caf7617 100644 --- a/src/attributecode/transform.py +++ b/src/attributecode/transform.py @@ -109,6 +109,10 @@ def transform_csv(rows, transformer): data = list(transformer.filter_fields(data)) field_names = [c for c in field_names if c in transformer.field_filters] + if transformer.exclude_fields: + data = list(transformer.filter_excluded(data)) + field_names = [c for c in field_names if c not in transformer.exclude_fields] + errors = transformer.check_required_fields(data) return field_names, data, errors @@ -132,9 +136,6 @@ def transform_json(data, transformer): if(data["headers"][0]["tool_name"] == "scancode-toolkit"): #only takes data inside "files" data = data["files"] - #automatically renames path to about_resource - if("path" not in renamings.keys()): - renamings["path"] = "about_resource" except: pass if isinstance(data, list): @@ -165,6 +166,11 @@ def process_json_keys(data, renamings, transformer): new_data = list(transformer.filter_fields(new_data)) else: new_data = list(new_data) + + if transformer.exclude_fields: + new_data = list(transformer.filter_excluded(new_data)) + else: + new_data = list(new_data) errors = transformer.check_required_fields(new_data) return new_data, errors @@ -215,6 +221,19 @@ def process_json_keys(data, renamings, transformer): field_filters: - name - version + +* exclude_fields: +An optional list of field names that should be excluded in the transformed CSV/JSON. If +this list is provided, all the fields from the source CSV/JSON that should be excluded +in the target CSV/JSON must be listed. Excluding standard or required fields will cause +an error. If this list is not provided, all source CSV/JSON fields are kept in the +transformed target CSV/JSON. + +For instance with this configuration the target CSV/JSON will not contain the "type" +and "temp" fields: + exclude_fields: + - type + - temp ''' @@ -225,6 +244,7 @@ class Transformer(object): field_renamings = attr.attrib(default=attr.Factory(dict)) required_fields = attr.attrib(default=attr.Factory(list)) field_filters = attr.attrib(default=attr.Factory(list)) + exclude_fields = attr.attrib(default=attr.Factory(list)) # a list of all the standard fields from AboutCode toolkit standard_fields = attr.attrib(default=attr.Factory(list), init=False) @@ -248,6 +268,7 @@ def default(cls): field_renamings={}, required_fields=[], field_filters=[], + exclude_fields=[], ) @classmethod @@ -262,6 +283,7 @@ def from_file(cls, location): field_renamings=data.get('field_renamings', {}), required_fields=data.get('required_fields', []), field_filters=data.get('field_filters', []), + exclude_fields=data.get('exclude_fields', []), ) def check_required_fields(self, data): @@ -320,6 +342,17 @@ def filter_fields(self, data): items = ((k, v) for k, v in entry.items() if k in field_filters) yield OrderedDict(items) + def filter_excluded(self, data): + """ + Yield transformed dicts from a `data` list of dicts excluding + fields with names in the `exclude_fields`of this Transformer. + Return the data unchanged if no `exclude_fields` exists. + """ + exclude_fields = set(self.clean_fields(self.exclude_fields)) + for entry in data: + items = ((k, v) for k, v in entry.items() if k not in exclude_fields) + yield OrderedDict(items) + def check_duplicate_fields(field_names): """ diff --git a/tests/testdata/test_cmd/help/about_transform_config_help.txt b/tests/testdata/test_cmd/help/about_transform_config_help.txt index 5b896b35..44f5fd89 100644 --- a/tests/testdata/test_cmd/help/about_transform_config_help.txt +++ b/tests/testdata/test_cmd/help/about_transform_config_help.txt @@ -6,41 +6,54 @@ format, using the same format as an .ABOUT file. The attributes that can be set in a configuration file are: * field_renamings: -An optional map of source CSV column name to target CSV new column name that -is used to rename CSV columns. +An optional map of source CSV or JSON field name to target CSV/JSON new field name that +is used to rename CSV fields. -For instance with this configuration the columns "Directory/Location" will be +For instance with this configuration the fields "Directory/Location" will be renamed to "about_resource" and "foo" to "bar": field_renamings: 'Directory/Location' : about_resource foo : bar The renaming is always applied first before other transforms and checks. All -other column names referenced below are these that exist AFTER the renamings -have been applied to the existing column names. +other field names referenced below are these that exist AFTER the renamings +have been applied to the existing field names. * required_fields: -An optional list of required column names that must have a value, beyond the -standard columns names. If a source CSV does not have such a column or a row is -missing a value for a required column, an error is reported. +An optional list of required field names that must have a value, beyond the +standard fields names. If a source CSV/JSON does not have such a field or a row is +missing a value for a required field, an error is reported. -For instance with this configuration an error will be reported if the columns +For instance with this configuration an error will be reported if the fields "name" and "version" are missing or if any row does not have a value set for -these columns: +these fields: required_fields: - name - version * field_filters: -An optional list of column names that should be kept in the transformed CSV. If -this list is provided, all the columns from the source CSV that should be kept -in the target CSV must be listed be even if they are standard or required -columns. If this list is not provided, all source CSV columns are kept in the -transformed target CSV. - -For instance with this configuration the target CSV will only contains the "name" -and "version" columns and no other column: +An optional list of field names that should be kept in the transformed CSV/JSON. If +this list is provided, all the fields from the source CSV/JSON that should be kept +in the target CSV/JSON must be listed be even if they are standard or required +fields. If this list is not provided, all source CSV/JSON fields are kept in the +transformed target CSV/JSON. + +For instance with this configuration the target CSV/JSON will only contains the "name" +and "version" fields and no other field: field_filters: - name - version +* exclude_fields: +An optional list of field names that should be excluded in the transformed CSV/JSON. If +this list is provided, all the fields from the source CSV/JSON that should be excluded +in the target CSV/JSON must be listed. Excluding standard or required fields will cause +an error. If this list is not provided, all source CSV/JSON fields are kept in the +transformed target CSV/JSON. + +For instance with this configuration the target CSV/JSON will not contain the "type" +and "temp" fields: + exclude_fields: + - type + - temp + diff --git a/tests/testdata/test_transform/configuration b/tests/testdata/test_transform/configuration index 77ebb91a..89d1e8b3 100644 --- a/tests/testdata/test_transform/configuration +++ b/tests/testdata/test_transform/configuration @@ -5,6 +5,9 @@ field_filters: - about_resource - name - version + - temp required_fields: - name - - version \ No newline at end of file + - version +exclude_fields: + - temp \ No newline at end of file diff --git a/tests/testdata/test_transform/configuration_scancode b/tests/testdata/test_transform/configuration_scancode index 10988579..ecc5095b 100644 --- a/tests/testdata/test_transform/configuration_scancode +++ b/tests/testdata/test_transform/configuration_scancode @@ -1,10 +1,13 @@ field_renamings: extension : new_extension + path : about_resource field_filters: - name - new_extension - about_resource + - type required_fields: - name - +exclude_fields: + - type diff --git a/tests/testdata/test_transform/input.csv b/tests/testdata/test_transform/input.csv index 7f863d86..a9f18446 100644 --- a/tests/testdata/test_transform/input.csv +++ b/tests/testdata/test_transform/input.csv @@ -1,2 +1,2 @@ -Directory/Filename,Component,version,notes -/tmp/test.c, test.c,1,test +Directory/Filename,Component,version,notes,temp +/tmp/test.c, test.c,1,test,foo diff --git a/tests/testdata/test_transform/input.json b/tests/testdata/test_transform/input.json index 73981241..f088ea2a 100644 --- a/tests/testdata/test_transform/input.json +++ b/tests/testdata/test_transform/input.json @@ -2,5 +2,6 @@ "Directory/Filename": "/aboutcode-toolkit/", "Component": "AboutCode-toolkit", "version": "1.2.3", - "note": "test" + "note": "test", + "temp": "foo" } \ No newline at end of file diff --git a/tests/testdata/test_transform/input_as_array.json b/tests/testdata/test_transform/input_as_array.json index f6c07108..0bfa970c 100644 --- a/tests/testdata/test_transform/input_as_array.json +++ b/tests/testdata/test_transform/input_as_array.json @@ -2,11 +2,13 @@ { "Directory/Filename": "/aboutcode-toolkit/", "Component": "AboutCode-toolkit", - "version": "1.0" + "version": "1.0", + "temp": "fpp" }, { "Directory/Filename": "/aboutcode-toolkit1/", "Component": "AboutCode-toolkit1", - "version": "1.1" + "version": "1.1", + "temp": "foo" } ] \ No newline at end of file