Merge pull request #1543 from nexB/1439-load-multiple-scans

1439 load multiple scans

Author: pombredanne

src/scancode/cli.py

@@ -320,6 +320,7 @@ def print_options(ctx, param, value):
 
 @click.option('--from-json',
     is_flag=True,
+    multiple=True,
     help='Load codebase from an existing JSON scan',
     help_group=CORE_GROUP, sort_order=25, cls=CommandLineOption)
 
@@ -543,7 +544,12 @@ def echo_func(*_args, **_kwargs):
     elif len(input) == 1:
         # we received a single input path, so we treat this as a single path
         input = input[0]  # NOQA
-    else:
+
+    # This is the case where we have a list of inputs, but the list of inputs are not from the
+    # `from_json` option. If the `from_json` option is available and we have a list of inputs
+    # from it, we can pass `input` just fine when we create a VirtualCodebase, otherwise we have to
+    # process `input` below.
+    elif not from_json:
         # we received a several input paths: we can handle this IFF they share
         # a common root directory and none is an absolute path
 

src/scancode/resource.py

@@ -1374,19 +1374,15 @@ def __init__(self, location,
         self.resource_attributes = resource_attributes or OrderedDict()
         self.resource_class = None
         self.has_single_resource = False
+        self.location = location
 
         scan_data = self._get_scan_data(location)
         self._populate(scan_data)
 
-    def _get_scan_data(self, location):
+    def _get_scan_data_helper(self, location):
         """
-        Return scan data loaded from `location` that is either:
-        - a path string
-        - a JSON string
-        - a Python mapping
+        Return scan data loaded from `location`, which is a path string
         """
-        if isinstance(location, dict):
-            return location
         try:
             return json.loads(location, object_pairs_hook=OrderedDict)
         except:
@@ -1398,6 +1394,33 @@ def _get_scan_data(self, location):
                     f, object_pairs_hook=OrderedDict, encoding='utf-8')
             return scan_data
 
+    def _get_scan_data(self, location):
+        """
+        Return scan data loaded from `location` that is either:
+        - a path string
+        - a JSON string
+        - a Python mapping
+
+        or `location` is a List or a Tuple that contains multiple paths to scans that are to be joined together.
+        """
+        if isinstance(location, dict):
+            return location
+        if isinstance(location, (list, tuple,)):
+            combined_scan_data = OrderedDict(headers=[], files=[])
+            for loc in location:
+                scan_data = self._get_scan_data_helper(loc)
+                headers = scan_data.get('headers')
+                if headers:
+                    combined_scan_data['headers'].extend(headers)
+                files = scan_data.get('files')
+                if files:
+                    combined_scan_data['files'].extend(files)
+                else:
+                    raise Exception('Input file does not have Resources to import: {}'.format(loc))
+            combined_scan_data['headers'] = sorted(combined_scan_data['headers'], key=lambda x: x['start_timestamp'])
+            return combined_scan_data
+        return self._get_scan_data_helper(location)
+
     def _create_empty_resource_data(self):
         """
         Return a dictionary of Resource fields and their default values.
@@ -1527,9 +1550,15 @@ def _populate(self, scan_data):
         ##########################################################
         # Create root resource without setting root data just yet. If we run into the root data
         # while we iterate through `resources_data`, we fill in the data then.
-        sample_resource_path = sample_resource_data['path']
-        sample_resource_path = sample_resource_path.strip('/')
-        root_path = sample_resource_path.split('/')[0]
+
+        # Create a virtual root if we are merging multiple scans together
+        multiple_input = isinstance(self.location, (list, tuple,)) and len(self.location) > 1
+        if multiple_input:
+            root_path = 'virtual_root'
+        else:
+            sample_resource_path = sample_resource_data['path']
+            sample_resource_path = sample_resource_path.strip('/')
+            root_path = sample_resource_path.split('/')[0]
         root_name = root_path
         root_is_file = False
         root_data = self._create_empty_resource_data()
@@ -1541,6 +1570,9 @@ def _populate(self, scan_data):
 
         for resource_data in resources_data:
             path = resource_data.get('path')
+            # Append virtual_root path to imported Resource path if we are merging multiple scans
+            if multiple_input:
+                path = os.path.join(root_path, path)
             name = resource_data.get('name', None)
             if not name:
                 name = file_name(path)

tests/scancode/data/resource/virtual_codebase/combine-1.json

@@ -0,0 +1,15 @@
+{
+  "files_count": 2,
+  "files": [
+    {
+      "path": "samples",
+      "type": "directory",
+      "summary": []
+    },
+    {
+      "path": "samples/NOTICE",
+      "type": "file",
+      "summary": []
+    }
+  ]
+}

tests/scancode/data/resource/virtual_codebase/combine-2.json

@@ -0,0 +1,15 @@
+{
+  "files_count": 2,
+  "files": [
+    {
+      "path": "thirdparty",
+      "type": "directory",
+      "summary": []
+    },
+    {
+      "path": "thirdparty/example.zip",
+      "type": "file",
+      "summary": []
+    }
+  ]
+}