Move --ignore functionality to codebase creation

Deprecate --ingore and --include pre-scan plugins and move the
ignore/include functionality to codebase import stage to get rid
of multiple codebase walks.

Signed-off-by: Ayan Sinha Mahapatra <asmahapatra@aboutcode.org>

Author: AyanSinhaMahapatra

pyproject-scancode-toolkit-mini.toml

@@ -256,7 +256,6 @@ scancode-train-gibberish-model = "textcode.train_gibberish_model:train_gibberish
 # scancode_pre_scan is the entry point for pre_scan plugins executed before the
 # scans. See also plugincode.pre_scan module for details and doc.
 [project.entry-points.scancode_pre_scan]
-ignore = "scancode.plugin_ignore:ProcessIgnore"
 facet = "summarycode.facet:AddFacet"
 
 

pyproject-scancode-toolkit.toml

@@ -257,7 +257,6 @@ scancode-train-gibberish-model = "textcode.train_gibberish_model:train_gibberish
 # scancode_pre_scan is the entry point for pre_scan plugins executed before the
 # scans. See also plugincode.pre_scan module for details and doc.
 [project.entry-points.scancode_pre_scan]
-ignore = "scancode.plugin_ignore:ProcessIgnore"
 facet = "summarycode.facet:AddFacet"
 
 

pyproject.toml

@@ -262,7 +262,6 @@ scancode-train-gibberish-model = "textcode.train_gibberish_model:train_gibberish
 # scancode_pre_scan is the entry point for pre_scan plugins executed before the
 # scans. See also plugincode.pre_scan module for details and doc.
 [project.entry-points.scancode_pre_scan]
-ignore = "scancode.plugin_ignore:ProcessIgnore"
 facet = "summarycode.facet:AddFacet"
 
 

src/commoncode/resource.py

@@ -39,6 +39,7 @@
 from commoncode.datautils import List
 from commoncode.datautils import Mapping
 from commoncode.datautils import String
+from commoncode.fileset import is_included
 from commoncode.filetype import is_file as filetype_is_file
 from commoncode.filetype import is_special
 from commoncode.fileutils import as_posixpath
@@ -62,7 +63,7 @@
 
 # Tracing flags
 TRACE = False
-TRACE_DEEP = False
+TRACE_DEEP = True
 
 
 def logger_debug(*args):
@@ -98,7 +99,7 @@ def skip_ignored(location):
     if TRACE_DEEP:
         logger_debug()
         logger_debug(
-            "Codebase.populate: walk: ignored loc:",
+            "Codebase.populate: walk: skip_ignored:",
             location,
             "ignored:",
             ignored(location),
@@ -109,6 +110,42 @@ def skip_ignored(location):
     return is_special(location) or ignored(location)
 
 
+def is_ignored(location,  includes=None, excludes=None):
+
+    excludes = {
+        pattern: 'User ignore: Supplied by --ignore' for pattern in excludes
+    }
+
+    includes = {
+        pattern: 'User include: Supplied by --include' for pattern in includes
+    }
+
+    included_from_options = is_included(
+        path=location,
+        includes=includes,
+        excludes=excludes,
+    )
+
+    if TRACE_DEEP:
+        logger_debug(
+            "Codebase.populate: walk: is_ignored:",
+            "is_ignored: location:",
+            location,
+            "included_from_options:",
+            included_from_options,
+            "skip_ignored",
+            skip_ignored(location)
+        )
+
+    if skip_ignored(location) or not included_from_options:
+        if TRACE_DEEP:
+            logger_debug("is_ignored: location:", location, "is_skipped",)
+
+        return True
+
+    return False
+
+
 def depth_walk(
     root_location,
     max_depth,
@@ -202,6 +239,8 @@ class Codebase:
     __slots__ = (
         "max_depth",
         "location",
+        "includes",
+        "ignores",
         "has_single_resource",
         "resource_attributes",
         "resource_class",
@@ -236,6 +275,8 @@ def __init__(
         max_in_memory=10000,
         max_depth=0,
         paths=tuple(),
+        ignores=tuple(),
+        includes=tuple(),
         *args,
         **kwargs,
     ):
@@ -298,6 +339,8 @@ def __init__(
 
         # finally populate
         self.paths = self._prepare_clean_paths(paths)
+        self.ignores = ignores
+        self.includes = includes
         self._populate()
 
     def _prepare_clean_paths(self, paths=tuple()):
@@ -461,11 +504,17 @@ def _populate(self):
             return
 
         if self.paths:
-            return self._create_resources_from_paths(root=root, paths=self.paths)
+            # In case of a list of full paths, we create resources without walking
+            return self._create_resources_from_full_paths(root=root, paths=self.paths)
+            # In case we have multiple 
         else:
-            return self._create_resources_from_root(root=root)
+            return self._create_resources_from_root(
+                root=root,
+                includes=self.includes,
+                ignores=self.ignores,
+            )
 
-    def _create_resources_from_paths(self, root, paths):
+    def _create_resources_from_full_paths(self, root, paths):
         # without paths we iterate the provided paths. We report an error
         # if a path is missing on disk.
 
@@ -483,22 +532,21 @@ def _create_resources_from_paths(self, root, paths):
                 msg = f"ERROR: cannot populate codebase: path: {path!r} not found in {res_loc!r}"
                 self.errors.append(msg)
                 raise Exception(path, join(base_location, path))
-                continue
 
             # create all parents. The last parent is the one we want to use
             parent = root
             if TRACE:
-                logger_debug("Codebase._create_resources_from_paths: parent", parent)
+                logger_debug("Codebase._create_resources_from_full_paths: parent", parent)
             for parent_path in get_ancestor_paths(path, include_self=False):
                 if TRACE:
                     logger_debug(
-                        f"  Codebase._create_resources_from_paths: parent_path: {parent_path!r}"
+                        f"  Codebase._create_resources_from_full_paths: parent_path: {parent_path!r}"
                     )
                 if not parent_path:
                     continue
                 newpar = parents_by_path.get(parent_path)
                 if TRACE:
-                    logger_debug("  Codebase._create_resources_from_paths: newpar", repr(newpar))
+                    logger_debug("  Codebase._create_resources_from_full_paths: newpar", repr(newpar))
 
                 if not newpar:
                     newpar = self._get_or_create_resource(
@@ -509,7 +557,7 @@ def _create_resources_from_paths(self, root, paths):
                     )
                     if not newpar:
                         raise Exception(
-                            "ERROR: Codebase._create_resources_from_paths:"
+                            "ERROR: Codebase._create_resources_from_full_paths:"
                             f" cannot create parent for: {parent_path!r}"
                         )
                     parent = newpar
@@ -518,7 +566,7 @@ def _create_resources_from_paths(self, root, paths):
 
                     if TRACE:
                         logger_debug(
-                            f"  Codebase._create_resources_from_paths:",
+                            f"  Codebase._create_resources_from_full_paths:",
                             f"created newpar: {newpar!r}",
                         )
 
@@ -529,10 +577,10 @@ def _create_resources_from_paths(self, root, paths):
                 is_file=isfile(res_loc),
             )
             if TRACE:
-                logger_debug("Codebase._create_resources_from_paths: resource", res)
+                logger_debug("Codebase._create_resources_from_full_paths: resource", res)
 
-    def _create_resources_from_root(self, root):
-        # without paths we walks the root location top-down
+    def _create_resources_from_root(self, root, includes, ignores):
+        # without paths we walk the root location top-down
 
         # track resources parents by location during construction.
         # NOTE: this cannot exhaust memory on a large codebase, because we do
@@ -545,9 +593,15 @@ def err(_error):
                 f"ERROR: cannot populate codebase: {_error}\n{traceback.format_exc()}"
             )
 
+        skip_ignored = partial(is_ignored, includes=includes, excludes=ignores)
+
+        if TRACE_DEEP:
+            logger_debug(f"parents_by_loc: {parents_by_loc}, ignores: {ignores}, includes: {includes}")
+
         # Walk over the directory and build the resource tree
         for top, dirs, files in depth_walk(
             root_location=root.location,
+            skip_ignored=skip_ignored,
             max_depth=self.max_depth,
             error_handler=err,
         ):
@@ -557,6 +611,7 @@ def err(_error):
                 top=top,
                 dirs=dirs,
                 files=files,
+                skip_ignored=skip_ignored,
             ):
                 # on the plain, bare FS, files cannot be parents
                 if not created.is_file:
@@ -574,6 +629,8 @@ def _create_resources(self, parent, top, dirs, files, skip_ignored=skip_ignored)
             for name in names:
                 location = join(top, name)
                 if skip_ignored(location):
+                    if TRACE_DEEP:
+                        logger_debug(f"_create_resources, depth_walk loop: ignored location: {location}")
                     continue
                 res = self._get_or_create_resource(
                     name=name,

src/scancode/cli.py

@@ -221,6 +221,26 @@ def default_processes():
     callback=validate_input_path,
     type=click.Path(exists=True, readable=True, path_type=str))
 
+@click.option('--include',
+    multiple=True,
+    default=None,
+    metavar='<pattern>',
+    help='Include files matching <pattern>.',
+    sort_order=11,
+    help_group=cliutils.CORE_GROUP,
+    cls=PluggableCommandLineOption,
+)
+
+@click.option('--ignore',
+    multiple=True,
+    default=None,
+    metavar='<pattern>',
+    help='Ignore files matching <pattern>.',
+    sort_order=10,
+    help_group=cliutils.CORE_GROUP,
+    cls=PluggableCommandLineOption,
+)
+
 @click.option('--strip-root',
     is_flag=True,
     default=False,
@@ -395,6 +415,8 @@ def default_processes():
 def scancode(
     ctx,
     input,  # NOQA
+    include,
+    ignore,
     strip_root,
     full_root,
     processes,
@@ -505,6 +527,8 @@ def scancode(
         # run proper
         success, _results = run_scan(
             input=input,
+            include=include,
+            ignore=ignore,
             from_json=from_json,
             strip_root=strip_root,
             full_root=full_root,
@@ -545,7 +569,9 @@ def scancode(
 
 
 def run_scan(
-    input,  # NOQA
+    input,  # 
+    include=[],
+    ignore=[],
     from_json=False,
     strip_root=False,
     full_root=False,
@@ -644,12 +670,10 @@ def echo_func(*_args, **_kwargs):
         # and we craft a list of synthetic --include path pattern options from
         # the input list of paths
         included_paths = [as_posixpath(path).rstrip('/') for path in input]
-        # FIXME: this is a hack as this "include" is from an external plugin!!!
-        include = list(requested_options.get('include', []) or [])
         include.extend(included_paths)
-        requested_options['include'] = include
 
         # ... and use the common prefix as our new input
+        # FIXME: we should not walk outside inputs
         input = common_prefix  # NOQA
 
     # build mappings of all options to pass down to plugins
@@ -894,6 +918,8 @@ def echo_func(*_args, **_kwargs):
         try:
             codebase = codebase_class(
                 location=input,
+                includes=include,
+                ignores=ignore,
                 resource_attributes=resource_attributes,
                 codebase_attributes=codebase_attributes,
                 full_root=full_root,