420: added exclude_fields to transform

docs/UsingAboutCodetoDocumentYourSoftwareAssets.md

@@ -287,6 +287,19 @@ and "version" columns and no other column:
         - name
         - version
 
+* exclude_fields:
+An optional list of field names that should be excluded in the transformed CSV/JSON. If
+this list is provided, all the fields from the source CSV/JSON that should be excluded
+in the target CSV/JSON must be listed. Excluding standard or required fields will cause
+an error. If this list is not provided, all source CSV/JSON fields are kept in the
+transformed target CSV/JSON.
+
+For instance with this configuration the target CSV/JSON will not contain the "type"
+and "temp" fields:
+    exclude_fields:
+        - type
+        - temp
+
 
 ## <a name="RungentoGenerateAboutCodeToolkitFiles">Run gen to Generate AboutCode Toolkit Files</a>
 

etc/scripts/irc-notify.py

@@ -139,7 +139,7 @@ def appveyor_vars():
         response = line.split()
 
         if response[0] == 'PING':
-            irc_file.send('PONG {}\r\n'.format(reponse[1]).encode())
+            irc_file.send('PONG {}\r\n'.format(response[1]).encode())
 
         elif response[1] == '433':
             irc_sock.send('NICK {}\r\n'.format(irc_nick).encode())

src/attributecode/transform.py

@@ -109,6 +109,10 @@ def transform_csv(rows, transformer):
         data = list(transformer.filter_fields(data))
         field_names = [c for c in field_names if c in transformer.field_filters]
 
+    if transformer.exclude_fields:
+        data = list(transformer.filter_excluded(data))
+        field_names = [c for c in field_names if c not in transformer.exclude_fields]
+
     errors = transformer.check_required_fields(data)
 
     return field_names, data, errors
@@ -132,9 +136,6 @@ def transform_json(data, transformer):
         if(data["headers"][0]["tool_name"] == "scancode-toolkit"):
             #only takes data inside "files"
             data = data["files"]
-            #automatically renames path to about_resource
-            if("path" not in renamings.keys()):
-                renamings["path"] = "about_resource"
     except:
         pass
     if isinstance(data, list):
@@ -165,6 +166,11 @@ def process_json_keys(data, renamings, transformer):
         new_data = list(transformer.filter_fields(new_data))
     else:
         new_data = list(new_data)
+
+    if transformer.exclude_fields:
+        new_data = list(transformer.filter_excluded(new_data))
+    else:
+        new_data = list(new_data)
 
     errors = transformer.check_required_fields(new_data)
     return new_data, errors
@@ -215,6 +221,19 @@ def process_json_keys(data, renamings, transformer):
     field_filters:
         - name
         - version
+
+* exclude_fields:
+An optional list of field names that should be excluded in the transformed CSV/JSON. If
+this list is provided, all the fields from the source CSV/JSON that should be excluded
+in the target CSV/JSON must be listed. Excluding standard or required fields will cause
+an error. If this list is not provided, all source CSV/JSON fields are kept in the
+transformed target CSV/JSON.
+
+For instance with this configuration the target CSV/JSON will not contain the "type"
+and "temp" fields:
+    exclude_fields:
+        - type
+        - temp
 '''
 
 
@@ -225,6 +244,7 @@ class Transformer(object):
     field_renamings = attr.attrib(default=attr.Factory(dict))
     required_fields = attr.attrib(default=attr.Factory(list))
     field_filters = attr.attrib(default=attr.Factory(list))
+    exclude_fields = attr.attrib(default=attr.Factory(list))
 
     # a list of all the standard fields from AboutCode toolkit
     standard_fields = attr.attrib(default=attr.Factory(list), init=False)
@@ -248,6 +268,7 @@ def default(cls):
             field_renamings={},
             required_fields=[],
             field_filters=[],
+            exclude_fields=[],
         )
 
     @classmethod
@@ -262,6 +283,7 @@ def from_file(cls, location):
             field_renamings=data.get('field_renamings', {}),
             required_fields=data.get('required_fields', []),
             field_filters=data.get('field_filters', []),
+            exclude_fields=data.get('exclude_fields', []),
         )
 
     def check_required_fields(self, data):
@@ -320,6 +342,17 @@ def filter_fields(self, data):
             items = ((k, v) for k, v in entry.items() if k in field_filters)
             yield OrderedDict(items)
 
+    def filter_excluded(self, data):
+        """
+        Yield transformed dicts from a `data` list of dicts excluding 
+        fields with names in the `exclude_fields`of this Transformer.
+        Return the data unchanged if no `exclude_fields` exists.
+        """
+        exclude_fields = set(self.clean_fields(self.exclude_fields))
+        for entry in data:
+            items = ((k, v) for k, v in entry.items() if k not in exclude_fields)
+            yield OrderedDict(items)
+
 
 def check_duplicate_fields(field_names):
     """

tests/testdata/test_cmd/help/about_transform_config_help.txt

@@ -6,41 +6,54 @@ format, using the same format as an .ABOUT file.
 The attributes that can be set in a configuration file are:
 
 * field_renamings:
-An optional map of source CSV column name to target CSV new column name that
-is used to rename CSV columns.
+An optional map of source CSV or JSON field name to target CSV/JSON new field name that
+is used to rename CSV fields.
 
-For instance with this configuration the columns "Directory/Location" will be
+For instance with this configuration the fields "Directory/Location" will be
 renamed to "about_resource" and "foo" to "bar":
     field_renamings:
         'Directory/Location' : about_resource
         foo : bar
 
 The renaming is always applied first before other transforms and checks. All
-other column names referenced below are these that exist AFTER the renamings
-have been applied to the existing column names.
+other field names referenced below are these that exist AFTER the renamings
+have been applied to the existing field names.
 
 * required_fields:
-An optional list of required column names that must have a value, beyond the
-standard columns names. If a source CSV does not have such a column or a row is
-missing a value for a required column, an error is reported.
+An optional list of required field names that must have a value, beyond the
+standard fields names. If a source CSV/JSON does not have such a field or a row is
+missing a value for a required field, an error is reported.
 
-For instance with this configuration an error will be reported if the columns
+For instance with this configuration an error will be reported if the fields
 "name" and "version" are missing or if any row does not have a value set for
-these columns:
+these fields:
     required_fields:
         - name
         - version
 
 * field_filters:
-An optional list of column names that should be kept in the transformed CSV. If
-this list is provided, all the columns from the source CSV that should be kept
-in the target CSV must be listed be even if they are standard or required
-columns. If this list is not provided, all source CSV columns are kept in the
-transformed target CSV.
-
-For instance with this configuration the target CSV will only contains the "name"
-and "version" columns and no other column:
+An optional list of field names that should be kept in the transformed CSV/JSON. If
+this list is provided, all the fields from the source CSV/JSON that should be kept
+in the target CSV/JSON must be listed be even if they are standard or required
+fields. If this list is not provided, all source CSV/JSON fields are kept in the
+transformed target CSV/JSON.
+
+For instance with this configuration the target CSV/JSON will only contains the "name"
+and "version" fields and no other field:
     field_filters:
         - name
         - version
 
+* exclude_fields:
+An optional list of field names that should be excluded in the transformed CSV/JSON. If
+this list is provided, all the fields from the source CSV/JSON that should be excluded
+in the target CSV/JSON must be listed. Excluding standard or required fields will cause
+an error. If this list is not provided, all source CSV/JSON fields are kept in the
+transformed target CSV/JSON.
+
+For instance with this configuration the target CSV/JSON will not contain the "type"
+and "temp" fields:
+    exclude_fields:
+        - type
+        - temp
+

tests/testdata/test_transform/configuration

@@ -5,6 +5,9 @@ field_filters:
     - about_resource
     - name
     - version
+    - temp
 required_fields:
 	- name
-	- version
+	- version
+exclude_fields:
+    - temp

tests/testdata/test_transform/configuration_scancode

@@ -1,10 +1,13 @@
 field_renamings:
 	extension : new_extension
+	path : about_resource
 field_filters:
 	- name
 	- new_extension
 	- about_resource
+	- type
 required_fields:
 	- name
-
+exclude_fields:
+	- type
 

tests/testdata/test_transform/input.csv

@@ -1,2 +1,2 @@
-Directory/Filename,Component,version,notes
-/tmp/test.c, test.c,1,test
+Directory/Filename,Component,version,notes,temp
+/tmp/test.c, test.c,1,test,foo

tests/testdata/test_transform/input.json

@@ -2,5 +2,6 @@
     "Directory/Filename": "/aboutcode-toolkit/",
 	"Component": "AboutCode-toolkit",
 	"version": "1.2.3",
-	"note": "test"
+	"note": "test",
+	"temp": "foo"
 }

tests/testdata/test_transform/input_as_array.json

@@ -2,11 +2,13 @@
   {
     "Directory/Filename": "/aboutcode-toolkit/", 
     "Component": "AboutCode-toolkit",
-	"version": "1.0"
+  "version": "1.0",
+  "temp": "fpp"
   },
   {
     "Directory/Filename": "/aboutcode-toolkit1/", 
     "Component": "AboutCode-toolkit1",
-	"version": "1.1"
+  "version": "1.1",
+  "temp": "foo"
   }  
 ]