Update package model with ignore_resource

    * Package parsers can define what resources to ignore when returning package files
    * Resource and Codebase have been updated to support using this callable
    * Update tests

Signed-off-by: Jono Yang <jyang@nexb.com>

Author: JonoYang

src/packagedcode/about.py

@@ -49,16 +49,12 @@
     logging.basicConfig(stream=sys.stdout)
     logger.setLevel(logging.DEBUG)
 
+# TODO: Override get_package_resource so it returns the Resource that the ABOUT file is describing
 
 @attr.s()
 class AboutPackage(models.Package):
     metafiles = ('*.ABOUT',)
     default_type = 'about'
-    # This is a mapping containing path patterns that we do not want to return
-    # as part of this Package's set of resources
-    ignorable_path_patterns = {
-        'node_modules': 'skip'
-    }
 
     @classmethod
     def recognize(cls, location):
@@ -76,13 +72,6 @@ def get_package_root(cls, manifest_resource, codebase):
                     return child
         return manifest_resource
 
-    @classmethod
-    def get_package_resources(cls, package_root, codebase):
-        _ignored = partial(ignore.is_ignored, ignores=cls.ignorable_path_patterns, skip_special=False)
-        yield package_root
-        for resource in package_root.walk(codebase, topdown=True, ignored=_ignored):
-            yield resource
-
 
 def is_about_file(location):
     return (filetype.is_file(location)

src/packagedcode/bower.py

@@ -56,11 +56,6 @@
 class BowerPackage(models.Package):
     metafiles = ('bower.json', '.bower.json')
     default_type = 'bower'
-    # This is a mapping containing path patterns that we do not want to return
-    # as part of this Package's set of resources
-    ignorable_path_patterns = {
-        'node_modules': 'skip'
-    }
 
     @classmethod
     def recognize(cls, location):
@@ -70,13 +65,6 @@ def recognize(cls, location):
     def get_package_root(cls, manifest_resource, codebase):
         return manifest_resource.parent(codebase)
 
-    @classmethod
-    def get_package_resources(cls, package_root, codebase):
-        _ignored = partial(ignore.is_ignored, ignores=cls.ignorable_path_patterns, skip_special=False)
-        yield package_root
-        for resource in package_root.walk(codebase, topdown=True, ignored=_ignored):
-            yield resource
-
     def compute_normalized_license(self):
         return compute_normalized_license(self.declared_license)
 

src/packagedcode/chef.py

@@ -69,11 +69,6 @@ class ChefPackage(models.Package):
     default_web_baseurl = 'https://supermarket.chef.io/cookbooks'
     default_download_baseurl = 'https://supermarket.chef.io/cookbooks'
     default_api_baseurl = 'https://supermarket.chef.io/api/v1'
-    # This is a mapping containing path patterns that we do not want to return
-    # as part of this Package's set of resources
-    ignorable_path_patterns = {
-        'node_modules': 'skip'
-    }
 
     @classmethod
     def recognize(cls, location):
@@ -83,13 +78,6 @@ def recognize(cls, location):
     def get_package_root(cls, manifest_resource, codebase):
         return manifest_resource.parent(codebase)
 
-    @classmethod
-    def get_package_resources(cls, package_root, codebase):
-        _ignored = partial(ignore.is_ignored, ignores=cls.ignorable_path_patterns, skip_special=False)
-        yield package_root
-        for resource in package_root.walk(codebase, topdown=True, ignored=_ignored):
-            yield resource
-
     def repository_download_url(self, baseurl=default_download_baseurl):
         return chef_download_url(self.name, self.version, registry=baseurl)
 

src/packagedcode/models.py

@@ -511,10 +511,23 @@ def get_package_resources(cls, package_root, codebase):
         """
         Yield the Resources of a Package starting from `package_root`
         """
-        yield package_root
-        for resource in package_root.walk(codebase, topdown=True):
+        if not Package.is_ignored_package_resource(package_root, codebase):
+            yield package_root
+        for resource in package_root.walk(codebase, topdown=True, ignored=Package.is_ignored_package_resource):
             yield resource
 
+    @classmethod
+    def ignore_resource(cls, resource, codebase):
+        """
+        Return True if `resource` should be ignored.
+        """
+        return False
+
+    @staticmethod
+    def is_ignored_package_resource(resource, codebase):
+        from packagedcode import PACKAGE_TYPES
+        return any(pt.ignore_resource(resource, codebase) for pt in PACKAGE_TYPES)
+
     def compute_normalized_license(self):
         """
         Return a normalized license_expression string using the declared_license

src/packagedcode/npm.py

@@ -78,11 +78,6 @@ class NpmPackage(models.Package):
     default_web_baseurl = 'https://www.npmjs.com/package'
     default_download_baseurl = 'https://registry.npmjs.org'
     default_api_baseurl = 'https://registry.npmjs.org'
-    # This is a mapping containing path patterns that we do not want to return
-    # as part of this Package's set of resources
-    ignorable_path_patterns = {
-        'node_modules': 'skip'
-    }
 
     @classmethod
     def recognize(cls, location):
@@ -93,11 +88,8 @@ def get_package_root(cls, manifest_resource, codebase):
         return manifest_resource.parent(codebase)
 
     @classmethod
-    def get_package_resources(cls, package_root, codebase):
-        _ignored = partial(ignore.is_ignored, ignores=cls.ignorable_path_patterns, skip_special=False)
-        yield package_root
-        for resource in package_root.walk(codebase, topdown=True, ignored=_ignored):
-            yield resource
+    def ignore_resource(cls, resource, codebase):
+        return resource.is_dir and resource.name == 'node_modules'
 
     def repository_homepage_url(self, baseurl=default_web_baseurl):
         return npm_homepage_url(self.namespace, self.name, registry=baseurl)

src/scancode/cli_test_utils.py

@@ -133,7 +133,7 @@ def remove_windows_extra_timeout(scancode_options, timeout=WINDOWS_CI_TIMEOUT):
             del scancode_options['--timeout']
 
 
-def check_json_scan(expected_file, result_file, regen=False, remove_file_date=False):
+def check_json_scan(expected_file, result_file, regen=False, remove_file_date=False, ignore_headers=False):
     """
     Check the scan `result_file` JSON results against the `expected_file`
     expected JSON results.
@@ -151,6 +151,10 @@ def check_json_scan(expected_file, result_file, regen=False, remove_file_date=Fa
 
     expected = load_json_result(expected_file, remove_file_date)
 
+    if ignore_headers:
+        results.pop('headers', None)
+        expected.pop('headers', None)
+
     # NOTE we redump the JSON as a string for a more efficient display of the
     # failures comparison/diff
     # TODO: remove sort, this should no longer be needed

src/scancode/resource.py

@@ -164,7 +164,13 @@ def from_dict(cls, **kwargs):
         return cls(**kwargs)
 
 
-def ignore_nothing(_):
+def ignore_nothing(resource, codebase):
+    """
+    Return True if `resource` should be ignored.
+
+    This function is used as a callable for `ignored` argument in Codebase and
+    Resource walk.
+    """
     return False
 
 
@@ -775,10 +781,13 @@ def walk(self, topdown=True, skip_root=False, ignored=ignore_nothing):
 
         If `skip_root` is True, the root resource is not returned unless this is
         a codebase with a single resource.
+
+        `ignored` is a callable that accepts two arguments, `resource` and `codebase`,
+        and returns True if `resource` should be ignored.
         """
         root = self.root
 
-        if ignored(root.path):
+        if ignored(root, self):
             return
 
         root = attr.evolve(root)
@@ -1116,17 +1125,21 @@ def walk(self, codebase, topdown=True, ignored=ignore_nothing):
 
         Each level is sorted by children sort order (e.g. without-children, then
         with-children and each group by case-insensitive name)
+
+        `ignored` is a callable that accepts two arguments, `resource` and `codebase`,
+        and returns True if `resource` should be ignored.
         """
 
         for child in self.children(codebase):
-            child = attr.evolve(child)
-            if topdown and not ignored(child.path):
-                yield child
-            for subchild in child.walk(codebase, topdown=topdown, ignored=ignored):
-                if not ignored(subchild.path):
-                    yield subchild
-            if not topdown and not ignored(child.path):
-                yield child
+            if not ignored(child, codebase):
+                child = attr.evolve(child)
+                if topdown:
+                    yield child
+                for subchild in child.walk(codebase, topdown=topdown, ignored=ignored):
+                    if not ignored(subchild, codebase):
+                        yield subchild
+                if not topdown:
+                    yield child
 
     def has_children(self):
         """