417, 418: modified transform to work with json from scancode and renamed configuration key for transform

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "python.pythonPath": "/home/sarthak/gsoc/aboutcode-toolkit/local/bin/python2.7"
+}

src/attributecode/transform.py

@@ -129,77 +129,24 @@ def transform_json(data, transformer):
     renamings = transformer.column_renamings
     #if json is output of scancode-toolkit
     if(data["headers"][0]["tool_name"] == "scancode-toolkit"):
-        new_data, errors = process_json_keys_scancode(data, renamings, transformer)
-
-    elif isinstance(data, list):
-            for item in data:
-                element, err = process_json_keys(item, renamings, transformer)
-                for e in element:
-                    new_data.append(e)
-                for e in err:
-                    errors.append(e)
+        #only takes data inside "files"
+        data = data["files"]
+        #automatically renames path to about_resource
+        if("path" not in renamings.keys()):
+            renamings["path"] = "about_resource"
+
+    if isinstance(data, list):
+        for item in data:
+            element, err = process_json_keys(item, renamings, transformer)
+            for e in element:
+                new_data.append(e)
+            for e in err:
+                errors.append(e)
     else: 
         new_data, errors = process_json_keys(data, renamings, transformer)
 
     return new_data, errors
 
-def process_json_keys_scancode(data, renamings, transformer):
-    o_dict = OrderedDict()
-    o_dict_headers_list = []
-    o_dict_files_list = []
-    new_data = []
-
-    for item in data["headers"]:
-        o_dict_headers = OrderedDict()
-        for k in item.keys():
-            if k in renamings.keys():
-                for r_key in renamings.keys():
-                    if k == r_key:
-                        o_dict_headers[renamings[r_key]] = item[k]
-            else:
-                o_dict_headers[k] = item[k]
-        o_dict_headers_list.append(o_dict_headers)
-
-
-    for item in data["files"]:
-        o_dict_files = OrderedDict()
-        for k in item.keys():
-            if k in renamings.keys():
-                for r_key in renamings.keys():
-                    if k == r_key:
-                        o_dict_files[renamings[r_key]] = item[k]
-            else:
-                o_dict_files[k] = item[k]
-        o_dict_files_list.append(o_dict_files)
-
-
-    for k in data.keys():
-        if k in renamings.keys():
-            for r_key in renamings.keys():
-                if k == r_key:
-                    o_dict[renamings[r_key]] = data[k]
-        else:
-            o_dict[k] = data[k]
-
-    if("files" in renamings.keys()):
-        o_dict[renamings["files"]] = o_dict_files_list
-    else:
-        o_dict["files"] = o_dict_files_list
-    if("headers" in renamings.keys()):
-        o_dict[renamings["headers"]] = o_dict_headers_list
-    else:
-        o_dict["headers"] = o_dict_headers_list
-    new_data = [o_dict]
-
-    if transformer.column_filters:
-        new_data = list(transformer.filter_columns(new_data))
-    else:
-        new_data = list(new_data)
-
-    errors = transformer.check_required_columns(new_data, isFromScancode=True)
-
-    return new_data, errors
-
 
 def process_json_keys(data, renamings, transformer):
     o_dict = OrderedDict()
@@ -315,16 +262,13 @@ def from_file(cls, location):
             column_filters=data.get('column_filters', []),
         )
 
-    def check_required_columns(self, data, isFromScancode=False):
+    def check_required_columns(self, data):
         """
         Return a list of Error for a `data` list of ordered dict where a
         dict is missing a value for a required column name.
         """
         errors = []
-        if(isFromScancode):
-            required = set(self.required_columns)
-        else:
-            required = set(self.essential_columns + self.required_columns)
+        required = set(self.essential_columns + self.required_columns)
         if not required:
             return []
 

tests/test_transform.py

@@ -68,4 +68,17 @@ def test_transform_data_json_as_array(self):
         for item in data:
             keys = list(item.keys())
         expect = [u'about_resource', u'name', u'version']
-        assert keys == expect
+        assert keys == expect
+
+    def test_transform_data_json_scancode(self):
+        test_file = get_test_loc('test_transform/input_scancode.json')
+        configuration = get_test_loc('test_transform/configuration_scancode')
+        json_data = read_json(test_file)
+        transformer = Transformer.from_file(configuration)
+        data, err = transform_json(json_data, transformer)
+        keys = []
+        for item in data:
+            keys = list(item.keys())
+        expect = [u'about_resource', u'name', u'new_extension']
+        assert keys == expect
+

tests/testdata/test_transform/configuration_scancode

@@ -0,0 +1,11 @@
+column_renamings:
+	extension : new_extension
+column_filters:
+	- name
+	- new_extension
+	- about_resource
+required_columns:
+	- name
+	- type
+
+