Prepare Release 31.b5

CHANGELOG.rst

@@ -172,61 +172,61 @@ Package detection:
 License Clarity Scoring Update
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
- - We are moving away from the license clarity scoring defined by ClearlyDefined
-   in the license clarity score plugin. The previous license clarity scoring
-   logic produced a score that was misleading when it would return a low score
-   due to the stringent scoring criteria. We are now
-   using more general criteria to get a sense of what provenance information has
-   been provided and whether or not there is a conflict in licensing between
-   what licenses were declared at the top-level key files and what licenses have
-   been detected in the files under the top-level.
+- We are moving away from the license clarity scoring defined by ClearlyDefined
+  in the license clarity score plugin. The previous license clarity scoring
+  logic produced a score that was misleading when it would return a low score
+  due to the stringent scoring criteria. We are now
+  using more general criteria to get a sense of what provenance information has
+  been provided and whether or not there is a conflict in licensing between
+  what licenses were declared at the top-level key files and what licenses have
+  been detected in the files under the top-level.
 
- - The license clarity score is a value from 0-100 calculated by combining the
-   weighted values determined for each of the scoring elements:
+- The license clarity score is a value from 0-100 calculated by combining the
+  weighted values determined for each of the scoring elements:
 
-   - Declared license:
+  - Declared license:
 
-     - When true, indicates that the software package licensing is documented at
-       top-level or well-known locations in the software project, typically in a
-       package manifest, NOTICE, LICENSE, COPYING or README file.
-     - Scoring Weight = 40
+    - When true, indicates that the software package licensing is documented at
+      top-level or well-known locations in the software project, typically in a
+      package manifest, NOTICE, LICENSE, COPYING or README file.
+    - Scoring Weight = 40
 
-   - Identification precision:
+  - Identification precision:
 
-     - Indicates how well the license statement(s) of the software identify known
-       licenses that can be designated by precise keys (identifiers) as provided in
-       a publicly available license list, such as the ScanCode LicenseDB, the SPDX
-       license list, the OSI license list, or a URL pointing to a specific license
-       text in a project or organization website.
-     - Scoring Weight = 40
+    - Indicates how well the license statement(s) of the software identify known
+      licenses that can be designated by precise keys (identifiers) as provided in
+      a publicly available license list, such as the ScanCode LicenseDB, the SPDX
+      license list, the OSI license list, or a URL pointing to a specific license
+      text in a project or organization website.
+    - Scoring Weight = 40
 
-   - License texts:
+  - License texts:
 
-     - License texts are provided to support the declared license expression in
-       files such as a package manifest, NOTICE, LICENSE, COPYING or README.
-     - Scoring Weight = 10
+    - License texts are provided to support the declared license expression in
+      files such as a package manifest, NOTICE, LICENSE, COPYING or README.
+    - Scoring Weight = 10
 
-   - Declared copyright:
+  - Declared copyright:
 
-     - When true, indicates that the software package copyright is documented at
-       top-level or well-known locations in the software project, typically in a
-       package manifest, NOTICE, LICENSE, COPYING or README file.
-     - Scoring Weight = 10
+    - When true, indicates that the software package copyright is documented at
+      top-level or well-known locations in the software project, typically in a
+      package manifest, NOTICE, LICENSE, COPYING or README file.
+    - Scoring Weight = 10
 
-   - Ambiguous compound licensing:
+  - Ambiguous compound licensing:
 
-     - When true, indicates that the software has a license declaration that
-       makes it difficult to construct a reliable license expression, such as in
-       the case of multiple licenses where the conjunctive versus disjunctive
-       relationship is not well defined.
-     - Scoring Weight = -10
+    - When true, indicates that the software has a license declaration that
+      makes it difficult to construct a reliable license expression, such as in
+      the case of multiple licenses where the conjunctive versus disjunctive
+      relationship is not well defined.
+    - Scoring Weight = -10
 
-   - Conflicting license categories:
+  - Conflicting license categories:
 
-     - When true, indicates that the declared license expression of the software is in
-       the permissive category, but that other potentially conflicting categories,
-       such as copyleft and proprietary, have been detected in lower level code.
-     - Scoring Weight = -20
+    - When true, indicates that the declared license expression of the software is in
+      the permissive category, but that other potentially conflicting categories,
+      such as copyleft and proprietary, have been detected in lower level code.
+    - Scoring Weight = -20
 
 
 Summary Plugin Update
@@ -258,7 +258,7 @@ Outputs:
 Output version
 --------------
 
-Scancode Data Output Version is now 3.0.0.
+Scancode Data Output Version is now 1.0.0.
 
 Changes:
 

requirements.txt

@@ -9,7 +9,7 @@ chardet==4.0.0
 charset-normalizer==2.0.12
 click==8.0.4
 colorama==0.4.4
-commoncode==30.2.0
+commoncode==31.0.0b4
 construct==2.10.68
 container-inspector==31.0.0
 cryptography==36.0.2
@@ -49,7 +49,7 @@ pefile==2021.9.3
 pip-requirements-parser==31.2.0
 pkginfo2==30.0.0
 pluggy==1.0.0
-plugincode==30.0.0
+plugincode==31.0.0b1
 ply==3.11
 publicsuffix2==2.20191221
 pyahocorasick==2.0.0b1

setup-mini.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = scancode-toolkit-mini
-version = 31.0.0b4
+version = 31.0.0b5
 license = Apache-2.0 AND CC-BY-4.0 AND LicenseRef-scancode-other-permissive AND LicenseRef-scancode-other-copyleft
 
 # description must be on ONE line https://github.com/pypa/setuptools/issues/1390
@@ -65,11 +65,11 @@ python_requires = >=3.6.*
 install_requires =
     attrs >= 18.1, !=20.1.0
     Beautifulsoup4 >= 4.0.0
-    boolean.py >= 3.5
+    boolean.py >= 4.0
     chardet >= 3.0.0
     click >= 6.7, !=7.0
     colorama >= 0.3.9
-    commoncode >= 30.2.0
+    commoncode==31.0.0b4
     container-inspector >= 31.0.0
     debian-inspector >= 30.0.0
     dparse2 >= 0.6.1
@@ -84,7 +84,7 @@ install_requires =
     javaproperties >= 0.5
     jinja2 >= 2.7.0
     jsonstreams >= 0.5.0
-    license_expression >= 21.6.14
+    license_expression >= 30.0.0
     lxml >= 4.6.5
     MarkupSafe >= 1.0
     packageurl_python >= 0.9.0
@@ -96,7 +96,7 @@ install_requires =
     pkginfo2 >= 30.0.0
     pip-requirements-parser >= 31.2.0
     pluggy >= 1.0.0
-    plugincode >= 30.0.0
+    plugincode==31.0.0b1
     publicsuffix2
     pyahocorasick >= 2.0.0b1
     pygmars >= 0.7.0
@@ -126,7 +126,7 @@ full =
 testing =
     pytest >= 6, != 7.0.0
     pytest-xdist >= 2
-    aboutcode-toolkit >= 6.0.0
+    aboutcode-toolkit >= 7.0.2
     twine
 
 docs =

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = scancode-toolkit
-version = 31.0.0b4
+version = 31.0.0b5
 license = Apache-2.0 AND CC-BY-4.0 AND LicenseRef-scancode-other-permissive AND LicenseRef-scancode-other-copyleft
 
 # description must be on ONE line https://github.com/pypa/setuptools/issues/1390
@@ -69,7 +69,7 @@ install_requires =
     chardet >= 3.0.0
     click >= 6.7, !=7.0
     colorama >= 0.3.9
-    commoncode >= 30.2.0
+    commoncode==31.0.0b4
     container-inspector >= 31.0.0
     debian-inspector >= 30.0.0
     dparse2 >= 0.6.1
@@ -96,7 +96,7 @@ install_requires =
     pkginfo2 >= 30.0.0
     pip-requirements-parser >= 31.2.0
     pluggy >= 1.0.0
-    plugincode >= 30.0.0
+    plugincode==31.0.0b1
     publicsuffix2
     pyahocorasick >= 2.0.0b1
     pygmars >= 0.7.0

src/licensedcode/plugin_license.py

@@ -7,22 +7,23 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
+import posixpath
 from functools import partial
 
 import attr
-
-from plugincode.scan import ScanPlugin
-from plugincode.scan import scan_impl
 from commoncode.cliutils import MISC_GROUP
 from commoncode.cliutils import PluggableCommandLineOption
 from commoncode.cliutils import SCAN_OPTIONS_GROUP
 from commoncode.cliutils import SCAN_GROUP
-from commoncode.fileutils import file_name
-from scancode.api import SCANCODE_LICENSEDB_URL
+from commoncode.resource import clean_path
+from plugincode.scan import ScanPlugin
+from plugincode.scan import scan_impl
 
+from scancode.api import SCANCODE_LICENSEDB_URL
 
 TRACE = False
 
+
 def logger_debug(*args): pass
 
 
@@ -274,8 +275,16 @@ def find_referenced_resource(referenced_filename, resource, codebase, **kwargs):
     ``resource``. ``referenced_filename`` is the path or filename referenced in
     a LicenseMatch of ``resource``,
     """
-    # this can be a path
-    ref_filename = file_name(referenced_filename)
-    for child in resource.parent(codebase).children(codebase):
-        if child.name == ref_filename:
-            return child
+    if not resource:
+        return
+
+    parent_path = resource.parent_path()
+    if not parent_path:
+        return
+
+    # this can be a path or a plain name
+    referenced_filename = clean_path(referenced_filename)
+    path = posixpath.join(parent_path, referenced_filename)
+    resource = codebase.get_resource(path=path)
+    if resource:
+        return resource

src/packagedcode/__init__.py

@@ -45,12 +45,11 @@
 
 # Note: the order matters: from the most to the least specific parser.
 # a handler classes MUST be added to this list to be active
-PACKAGE_DATAFILE_HANDLERS = [
+APPLICATION_PACKAGE_DATAFILE_HANDLERS = [
     about.AboutFileHandler,
 
     alpine.AlpineApkArchiveHandler,
     alpine.AlpineApkbuildHandler,
-    alpine.AlpineInstalledDatabaseHandler,
 
     bower.BowerJsonHandler,
 
@@ -78,14 +77,7 @@
     cran.CranDescriptionFileHandler,
 
     debian_copyright.DebianCopyrightFileInPackageHandler,
-    debian_copyright.DebianCopyrightFileInSourceHandler,
-    # TODO: consider activating? debian_copyright.StandaloneDebianCopyrightFileHandler,
-
     debian.DebianDscFileHandler,
-    debian.DebianDistrolessInstalledDatabaseHandler,
-    debian.DebianInstalledFilelistHandler,
-    debian.DebianInstalledMd5sumFilelistHandler,
-    debian.DebianInstalledStatusDatabaseHandler,
 
     debian.DebianControlFileInExtractedDebHandler,
     debian.DebianControlFileInSourceHandler,
@@ -182,9 +174,6 @@
     readme.ReadmeHandler,
 
     rpm.RpmArchiveHandler,
-    rpm.RpmInstalledBdbDatabaseHandler,
-    rpm.RpmInstalledSqliteDatabaseHandler,
-    rpm.RpmInstalledNdbDatabaseHandler,
     rpm.RpmSpecfileHandler,
 
     rubygems.GemMetadataArchiveExtractedHandler,
@@ -206,15 +195,40 @@
     windows.MicrosoftUpdateManifestHandler,
 
     win_pe.WindowsExecutableHandler,
+]
+
+
+SYSTEM_PACKAGE_DATAFILE_HANDLERS = [
+    alpine.AlpineInstalledDatabaseHandler,
+
+    debian_copyright.DebianCopyrightFileInPackageHandler,
+    debian_copyright.DebianCopyrightFileInSourceHandler,
+
+    # TODO: consider activating? debian_copyright.StandaloneDebianCopyrightFileHandler,
+
+    debian.DebianDistrolessInstalledDatabaseHandler,
+
+    debian.DebianInstalledFilelistHandler,
+    debian.DebianInstalledMd5sumFilelistHandler,
+    debian.DebianInstalledStatusDatabaseHandler,
+
+    rpm.RpmInstalledBdbDatabaseHandler,
+    rpm.RpmInstalledSqliteDatabaseHandler,
+    rpm.RpmInstalledNdbDatabaseHandler,
 
     win_reg.InstalledProgramFromDockerSoftwareDeltaHandler,
     win_reg.InstalledProgramFromDockerFilesSoftwareHandler,
     win_reg.InstalledProgramFromDockerUtilityvmSoftwareHandler,
 ]
 
-HANDLER_BY_DATASOURCE_ID = {
-    handler.datasource_id: handler for handler in PACKAGE_DATAFILE_HANDLERS
-}
+ALL_DATAFILE_HANDLERS= (
+    APPLICATION_PACKAGE_DATAFILE_HANDLERS + [
+        p for p in SYSTEM_PACKAGE_DATAFILE_HANDLERS 
+        if p not in APPLICATION_PACKAGE_DATAFILE_HANDLERS
+    ]
+)
+
+HANDLER_BY_DATASOURCE_ID = {handler.datasource_id: handler for handler in ALL_DATAFILE_HANDLERS}
 
 
 class UnknownPackageDatasource(Exception):

src/packagedcode/about.py

@@ -130,7 +130,7 @@ def assemble(cls, package_data, resource, codebase):
 
             yield package
 
-            if resource.pid is not None and package_data.file_references:
+            if resource.has_parent() and package_data.file_references:
                 parent_resource = resource.parent(codebase)
                 if parent_resource and package_data.file_references:
                     root_path = Path(parent_resource.path)

src/packagedcode/alpine.py

@@ -52,7 +52,11 @@ class AlpineInstalledDatabaseHandler(models.DatafileHandler):
 
     @classmethod
     def parse(cls, location):
-        yield from parse_alpine_installed_db(location)
+        yield from parse_alpine_installed_db(
+            location=location,
+            datasource_id=cls.datasource_id,
+            package_type=cls.default_package_type,
+        )
 
     @classmethod
     def assemble(cls, package_data, resource, codebase):

src/packagedcode/debian_copyright.py

@@ -6,10 +6,12 @@
 # See https://github.com/nexB/scancode-toolkit for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
+import fnmatch
 import os
 import sys
 from collections import defaultdict
 from itertools import chain
+from pathlib import Path
 
 import attr
 from debian_inspector.copyright import CatchAllParagraph
@@ -33,8 +35,6 @@
 from packagedcode.licensing import get_license_matches_from_query_string
 from packagedcode.utils import combine_expressions
 from textcode.analysis import unicode_text
-import fnmatch
-from pathlib import Path
 
 """
 Detect licenses and copyright in Debian copyright files. Can handle dep-5