Merge pull request #1423 from nexB/1362-resource-not-loaded-before-removal

Load resource from disk before returning None in get_resource()

Author: pombredanne

src/scancode/plugin_ignore.py

@@ -56,7 +56,6 @@ def logger_debug(*args):
             ' '.join(isinstance(a, unicode) and a or repr(a) for a in args))
 
 
-
 @pre_scan_impl
 class ProcessIgnore(PreScanPlugin):
     """
@@ -125,7 +124,7 @@ def process_codebase(self, codebase, ignore=(), include=(), **kwargs):
                 logger_debug(codebase.get_resource(rid))
 
         remove_resource = codebase.remove_resource
-        
+
         # Then, walk bottom-up and remove the non-included Resources from the
         # Codebase if the Resource's rid is in our list of rid's to remove.
         for resource in codebase.walk(topdown=False):

src/scancode/resource.py

@@ -632,10 +632,10 @@ def get_resource(self, rid):
 
         if rid == 0:
             res = attr.evolve(self.root)
-        elif not rid or rid not in self.resource_ids:
-            res = None
         elif self._use_disk_cache_for_resource(rid):
             res = self._load_resource(rid)
+        elif not rid or rid not in self.resource_ids:
+            res = None
         else:
             res = self.resources.get(rid)
             res = attr.evolve(res)

tests/scancode/test_plugin_ignore.py

@@ -128,6 +128,12 @@ def test_ProcessIgnore_with_multiple_ignores(self):
         ]
         self.check_ProcessIgnore(test_dir, expected, ignore)
 
+    def test_ProcessIgnore_process_codebase_does_not_fail_to_access_an_ignored_resourced_cached_to_disk(self):
+        test_dir = self.extract_test_tar('plugin_ignore/user.tgz')
+        codebase = Codebase(test_dir, max_in_memory=1)
+        test_plugin = ProcessIgnore()
+        ignore = ['test']
+        test_plugin.process_codebase(codebase, ignore=ignore)
 
 
 class TestScanPluginIgnoreFiles(FileDrivenTesting):
@@ -235,3 +241,19 @@ def test_scancode_multiple_ignores(self):
         assert 0 == scan_result['headers'][0]['extra_data']['files_count']
         scan_locs = [x['path'] for x in scan_result['files']]
         assert [u'user', u'user/src'] == scan_locs
+
+    def test_scancode_codebase_attempt_to_access_an_ignored_resourced_cached_to_disk(self):
+        test_dir = self.extract_test_tar('plugin_ignore/user.tgz')
+        result_file = self.get_temp_file('json')
+        args = ['--copyright', '--strip-root', '--ignore', 'test', test_dir, '--max-in-memory', '1', '--json', result_file]
+        run_scan_click(args)
+        scan_result = load_json_result(result_file)
+        assert 2 == scan_result['headers'][0]['extra_data']['files_count']
+        scan_locs = [x['path'] for x in scan_result['files']]
+        expected = [
+            u'user',
+            u'user/ignore.doc',
+            u'user/src',
+            u'user/src/ignore.doc',
+        ]
+        assert expected == scan_locs