Add get_top_level_resources() to DatafileHandler class

CHANGELOG.rst

@@ -69,6 +69,11 @@ Package detection:
 
   https://github.com/nexB/scancode-toolkit/issues/3290
 
+- DatafileHandlers now have a classmethod named ``get_top_level_resources()``,
+  which is supposed to yield the top-level Resources of a Package codebase,
+  relative to a Package manifest file. ``maven.MavenPomXmlHandler`` is the first
+  DatafileHandler that has this method implemented.
+
 
 License detection:
 ~~~~~~~~~~~~~~~~~~~
@@ -161,10 +166,10 @@ License detection:
 
   - We can now detect licenses using custom license texts and license rules
     stored in a directory or packaged as a plugin for consistent reuse and deployment.
-  
+
   - There is an ``--additional-directory`` option with the ``scancode-reindex-licenses``
     command to add the licenses from a directory.
-    
+
   - There is also a ``--only-builtin`` option to use ony builtin licenses
     ignoring any additional license plugins.
 
@@ -208,7 +213,7 @@ v31.2.5 - 2023-04-21
 
 This is a minor backport release.
 
-This adds license rule changes and was requested here: 
+This adds license rule changes and was requested here:
 https://github.com/nexB/scancode-toolkit/issues/3310
 This was originally merged in #3218 and included in
 the latest release v32.x, and is also being backported
@@ -602,6 +607,10 @@ Changes:
     - `other_holders`
     - `other_languages`
 
+- A new field ``run_order`` has been added to ``BasePlugin`` and set on all
+  ScanCode plugins. Plugin run order and output order are now set independently
+  of one another.
+
 
 Documentation Update
 ~~~~~~~~~~~~~~~~~~~~~~~~

requirements.txt

@@ -48,7 +48,7 @@ pefile==2022.5.30
 pip-requirements-parser==32.0.1
 pkginfo2==30.0.0
 pluggy==1.0.0
-plugincode==31.0.0
+plugincode==32.0.0
 ply==3.11
 publicsuffix2==2.20191221
 pyahocorasick==2.0.0

setup.cfg

@@ -97,7 +97,7 @@ install_requires =
     pkginfo2 >= 30.0.0
     pip-requirements-parser >= 32.0.1
     pluggy >= 1.0.0
-    plugincode >= 31.0.0
+    plugincode >= 32.0.0
     publicsuffix2
     pyahocorasick >= 2.0.0
     pygmars >= 0.7.0
@@ -165,7 +165,6 @@ console_scripts =
 scancode_pre_scan =
     ignore = scancode.plugin_ignore:ProcessIgnore
     facet = summarycode.facet:AddFacet
-    classify = summarycode.classify_plugin:FileClassifier
 
 
 # scancode_scan is the entry point for scan plugins that run a scan after the
@@ -196,6 +195,7 @@ scancode_post_scan =
     filter-clues = cluecode.plugin_filter_clues:RedundantCluesFilter
     consolidate = summarycode.plugin_consolidate:Consolidator
     license-references = licensedcode.licenses_reference:LicenseReference
+    classify = summarycode.classify_plugin:FileClassifier
 
 
 # scancode_output_filter is the entry point for filter plugins executed after

src/cluecode/plugin_copyright.py

@@ -28,6 +28,7 @@ class CopyrightScanner(ScanPlugin):
         ('authors', attr.ib(default=attr.Factory(list))),
     ])
 
+    run_order = 6
     sort_order = 6
 
     options = [

src/cluecode/plugin_email.py

@@ -25,6 +25,7 @@ class EmailScanner(ScanPlugin):
     """
     resource_attributes = dict(emails=attr.ib(default=attr.Factory(list)))
 
+    run_order = 7
     sort_order = 7
 
     options = [

src/cluecode/plugin_filter_clues.py

@@ -41,6 +41,7 @@ class RedundantCluesFilter(PostScanPlugin):
     Filter redundant clues (copyrights, authors, emails, and urls) that are
     already contained in a matched license text.
     """
+    run_order = 1
     sort_order = 1
 
     options = [

src/cluecode/plugin_url.py

@@ -26,6 +26,7 @@ class UrlScanner(ScanPlugin):
 
     resource_attributes = dict(urls=attr.ib(default=attr.Factory(list)))
 
+    run_order = 8
     sort_order = 8
 
     options = [

src/licensedcode/licenses_reference.py

@@ -48,6 +48,7 @@ class LicenseReference(PostScanPlugin):
     )
 
     # TODO: send to the tail of the scan, after files
+    run_order = 1000
     sort_order = 1000
 
     options = [

src/licensedcode/plugin_license.py

@@ -71,6 +71,7 @@ class LicenseScanner(ScanPlugin):
         license_detections=attr.ib(default=attr.Factory(list)),
     )
 
+    run_order = 4
     sort_order = 4
 
     options = [

src/licensedcode/plugin_license_policy.py

@@ -51,6 +51,7 @@ class LicensePolicy(PostScanPlugin):
 
     resource_attributes = dict(license_policy=attr.ib(default=attr.Factory(list)))
 
+    run_order = 9
     sort_order = 9
 
     options = [
@@ -100,8 +101,8 @@ def process_codebase(self, codebase, license_policy, **kwargs):
                     if key == policy.get('license_key'):
                         # Apply the policy to the Resource
                         license_policies.append(policy)
-            
-            resource.license_policy = sorted(license_policies, key=lambda d: d['license_key']) 
+
+            resource.license_policy = sorted(license_policies, key=lambda d: d['license_key'])
             codebase.save_resource(resource)
 
 

src/packagedcode/maven.py

@@ -152,6 +152,20 @@ def assign_package_to_resources(cls, package, resource, codebase, package_adder)
             package_adder=package_adder
         )
 
+    @classmethod
+    def get_top_level_resources(cls, manifest_resource, codebase):
+        """
+        Yield Resources that are top-level based on a JAR's directory structure
+        """
+        if 'META-INF' in manifest_resource.path:
+            path_segments = manifest_resource.path.split('META-INF')
+            leading_segment = path_segments[0].strip()
+            meta_inf_dir_path = f'{leading_segment}/META-INF'
+            meta_inf_resource = codebase.get_resource(meta_inf_dir_path)
+            if meta_inf_resource:
+                yield meta_inf_resource
+                yield from meta_inf_resource.walk(codebase)
+
 
 # TODO: assemble with its pom!!
 class MavenPomPropertiesHandler(models.NonAssemblableDatafileHandler):

src/packagedcode/models.py

@@ -1304,6 +1304,13 @@ def populate_license_fields(cls, package_data):
         if package_data.extracted_license_statement and not isinstance(package_data.extracted_license_statement, str):
             package_data.extracted_license_statement = repr(package_data.extracted_license_statement)
 
+    @classmethod
+    def get_top_level_resources(cls, manifest_resource, codebase):
+        """
+        Yield Resources that are considered top-level for a Package type.
+        """
+        pass
+
 
 class NonAssemblableDatafileHandler(DatafileHandler):
     """

src/packagedcode/plugin_package.py

@@ -134,6 +134,7 @@ class PackageScanner(ScanPlugin):
 
     required_plugins = ['scan:licenses']
 
+    run_order = 3
     sort_order = 3
 
     options = [

src/scancode/cli.py

@@ -1059,9 +1059,12 @@ def run_codebase_plugins(
     if verbose and plugins:
         echo_func(stage_msg % locals(), fg='green')
 
+    # Sort plugins by run_order, from low to high
+    sorted_plugins = sorted(plugins, key=lambda x: x.run_order)
+
     success = True
     # TODO: add progress indicator
-    for plugin in plugins:
+    for plugin in sorted_plugins:
         name = plugin.name
         plugin_start = time()
 

src/scancode/plugin_info.py

@@ -38,6 +38,7 @@ class InfoScanner(ScanPlugin):
         ('is_script', attr.ib(default=False, type=bool, repr=False)),
     ])
 
+    run_order = 0
     sort_order = 0
 
     options = [

src/scancode/plugin_mark_source.py

@@ -25,6 +25,7 @@ class MarkSource(PostScanPlugin):
 
     resource_attributes = dict(source_count=attr.ib(default=0, type=int, repr=False))
 
+    run_order = 8
     sort_order = 8
 
     options = [

src/summarycode/classify_plugin.py

@@ -8,11 +8,13 @@
 #
 
 from commoncode.datautils import Boolean
-from plugincode.pre_scan import PreScanPlugin
-from plugincode.pre_scan import pre_scan_impl
+from plugincode.post_scan import PostScanPlugin
+from plugincode.post_scan import post_scan_impl
 from commoncode.cliutils import PluggableCommandLineOption
-from commoncode.cliutils import PRE_SCAN_GROUP
+from commoncode.cliutils import POST_SCAN_GROUP
 
+from packagedcode import get_package_handler
+from packagedcode.models import PackageData
 from summarycode.classify import set_classification_flags
 
 """
@@ -50,8 +52,8 @@ def logger_debug(*args):
         return logger.debug(' '.join(isinstance(a, str) and a or repr(a) for a in args))
 
 
-@pre_scan_impl
-class FileClassifier(PreScanPlugin):
+@post_scan_impl
+class FileClassifier(PostScanPlugin):
     """
     Classify a file such as a COPYING file or a package manifest with a flag.
     """
@@ -93,14 +95,15 @@ class FileClassifier(PreScanPlugin):
 
     ])
 
-    sort_order = 30
+    run_order = 4
+    sort_order = 4
 
     options = [
         PluggableCommandLineOption(('--classify',),
             is_flag=True, default=False,
-            help='Classify files with flags indicating whether the file is a '
-                 'legal, readme, test or similar file.',
-            help_group=PRE_SCAN_GROUP,
+            help='Classify files with flags telling if the file is a legal, '
+                 'or readme or test file, etc.',
+            help_group=POST_SCAN_GROUP,
             sort_order=50,
         )
     ]
@@ -115,11 +118,31 @@ def process_codebase(self, codebase, classify, **kwargs):
             real_root = codebase.root
         real_root_dist = real_root.distance(codebase)
 
+        seen_resources = set()
         for resource in codebase.walk(topdown=True):
-            real_dist = resource.distance(codebase) - real_root_dist
-            # this means this is either a child of the root dir or the root itself.
-            resource.is_top_level = (real_dist < 2)
-            if resource.is_file:
-                # TODO: should we do something about directories? for now we only consider files
-                set_classification_flags(resource)
-            resource.save(codebase)
+            if resource.path in seen_resources:
+                continue
+
+            has_package_data = bool(getattr(resource, 'package_data', False))
+            if not has_package_data:
+                real_dist = resource.distance(codebase) - real_root_dist
+                # this means this is either a child of the root dir or the root itself.
+                resource.is_top_level = (real_dist < 2)
+                if resource.is_file:
+                    # TODO: should we do something about directories? for now we only consider files
+                    set_classification_flags(resource)
+                resource.save(codebase)
+                seen_resources.add(resource.path)
+            else:
+                for package_data in resource.package_data:
+                    pd = PackageData.from_dict(package_data)
+                    package_handler = get_package_handler(pd)
+                    top_level_resources = package_handler.get_top_level_resources(resource, codebase)
+                    if not top_level_resources:
+                        break
+                    for r in top_level_resources:
+                        r.is_top_level = True
+                        set_classification_flags(r)
+                        r.save(codebase)
+                        seen_resources.add(r.path)
+                seen_resources.add(resource.path)

src/summarycode/facet.py

@@ -107,6 +107,7 @@ class AddFacet(PreScanPlugin):
 
     resource_attributes = dict(facets=attr.ib(default=attr.Factory(list), repr=False))
 
+    run_order = 20
     sort_order = 20
 
     options = [

src/summarycode/generated.py

@@ -49,6 +49,7 @@ class GeneratedCodeDetector(ScanPlugin):
     resource_attributes = dict(is_generated=Boolean(
         help='True if this file is likely an automatically generated file.'))
 
+    run_order = 50
     sort_order = 50
 
     options = [

src/summarycode/plugin_consolidate.py

@@ -139,6 +139,7 @@ class Consolidator(PostScanPlugin):
         consolidated_to=attr.ib(default=attr.Factory(list))
     )
 
+    run_order = 10
     sort_order = 10
 
     options = [

src/summarycode/score.py

@@ -53,7 +53,8 @@ class LicenseClarityScore(PostScanPlugin):
 
     codebase_attributes = dict(summary=attr.ib(default=attr.Factory(dict)))
 
-    sort_order = 5
+    run_order = 5
+    sort_order = 2
 
     options = [
         PluggableCommandLineOption(
@@ -320,22 +321,19 @@ def get_field_values_from_codebase_resources(
     """
     values = []
     for resource in codebase.walk(topdown=True):
-        if not (resource.is_dir and resource.is_top_level):
-            continue
-        for child in resource.walk(codebase):
-            if key_files_only:
-                if not child.is_key_file:
-                    continue
-            else:
-                if child.is_key_file:
-                    continue
-            if is_string:
-                value = getattr(child, field_name, None) or None
-                if value:
-                    values.append(value)
-            else:
-                for value in getattr(child, field_name, []) or []:
-                    values.append(value)
+        if key_files_only:
+            if not resource.is_key_file:
+                continue
+        else:
+            if resource.is_key_file:
+                continue
+        if is_string:
+            value = getattr(resource, field_name, None) or None
+            if value:
+                values.append(value)
+        else:
+            for value in getattr(resource, field_name, []) or []:
+                values.append(value)
     return values
 
 

src/summarycode/summarizer.py

@@ -57,6 +57,7 @@ class ScanSummary(PostScanPlugin):
     Summarize a scan at the codebase level.
     """
 
+    run_order = 6
     sort_order = 2
 
     codebase_attributes = dict(summary=attr.ib(default=attr.Factory(dict)))
@@ -324,8 +325,8 @@ def is_key_package(package, codebase):
 
     datafile_paths = set(package.datafile_paths or [])
     for resource in codebase.walk(topdown=True):
-        if not resource.is_top_level:
-            break
+        if resource.is_dir:
+            continue
         if resource.path in datafile_paths:
             return True