Merge pull request #3430 from nexB/fix-summary-license

Fix licenses data in summary plugin

Author: pombredanne

src/licensedcode/detection.py

@@ -810,6 +810,20 @@ def get_license_detection_object(self):
         )
 
 
+def sort_unique_detections(license_detections):
+    """
+    Return a sorted list of UniqueDetection mappings from a unsorted list of the same.
+    These are sorted in alphabetical order of the license_expression (and same license
+    expressions are sorted by their detection_count in descending order, and then by their
+    UUID).
+    """
+
+    def by_expression_count(detection):
+        return detection["license_expression"], -detection["detection_count"], detection["identifier"]
+
+    return sorted(license_detections, key=by_expression_count)
+
+
 def get_detections_by_id(license_detections):
     """
     Get a dict(hashmap) where each item is: {detection.identifier: all_detections} where

src/licensedcode/plugin_license.py

@@ -27,6 +27,7 @@
 from licensedcode.detection import get_referenced_filenames
 from licensedcode.detection import DetectionCategory
 from licensedcode.detection import LicenseDetectionFromResult
+from licensedcode.detection import sort_unique_detections
 from licensedcode.detection import UniqueDetection
 from packagedcode.utils import combine_expressions
 from scancode.api import SCANCODE_LICENSEDB_URL
@@ -228,11 +229,13 @@ def process_codebase(self, codebase, license_diagnostics, **kwargs):
                 f'unique_license_detections: {unique_license_detections}',
             )
 
-        codebase.attributes.license_detections.extend([
+        unsorted_license_detections = [
             unique_detection.to_dict(license_diagnostics=license_diagnostics)
             for unique_detection in unique_license_detections
-        ])
-
+        ]
+        codebase.attributes.license_detections.extend(
+            sort_unique_detections(unsorted_license_detections)
+        )
 
 
 def add_referenced_filenames_license_matches_for_detections(resource, codebase):

src/summarycode/tallies.py

@@ -159,25 +159,46 @@ def license_tallies(resource, children, keep_details=False):
     sorted by decreasing count.
     """
     LIC_EXP = 'detected_license_expression'
+    LIC_DET = 'license_detections'
+    LIC_CLUE = 'license_clues'
     license_expressions = []
 
     # Collect current data
-    lic_expression = getattr(resource, LIC_EXP, None)
-    if not lic_expression and resource.is_file:
+    detected_expressions = []
+    for detection in getattr(resource, LIC_DET, []):
+        detected_expressions.append(detection["license_expression"])
+    for match in getattr(resource, LIC_CLUE, []):
+        detected_expressions.append(match["license_expression"])
+
+    package_license_detections = []
+    PACKAGE_DATA = 'package_data'
+    package_data = getattr(resource, PACKAGE_DATA, [])
+    if package_data:
+        package_license_detections.extend(
+            [
+                detection
+                for detection in getattr(package_data, LIC_DET, [])
+                if detection
+            ]
+        )
+
+    for detection in package_license_detections:
+        detected_expressions.append(detection["license_expression"])
+
+    if not detected_expressions and resource.is_file:
         # also count files with no detection
         license_expressions.append(None)
     else:
-        license_expressions.append(lic_expression)
+        license_expressions.extend(detected_expressions)
 
     # Collect direct children expression tallies
     for child in children:
         child_tallies = get_resource_tallies(child, key=LIC_EXP, as_attribute=keep_details) or []
         for child_tally in child_tallies:
             # TODO: review this: this feels rather weird
             child_sum_val = child_tally.get('value')
-            if child_sum_val:
-                values = [child_sum_val] * child_tally['count']
-                license_expressions.extend(values)
+            values = [child_sum_val] * child_tally['count']
+            license_expressions.extend(values)
 
     # summarize proper
     licenses_counter = tally_licenses(license_expressions)

tests/cluecode/data/copyrights/openoffice_org_report_builder_bin.copyright.yml

@@ -18,7 +18,7 @@ copyrights:
   - Copyright (c) 2002 France Telecom
   - Copyright (c) 1990-2003 Sleepycat Software
   - Copyright (c) 1990, 1993, 1994, 1995 The Regents of the University of California
-  - Copyright (c) 2003 by Bitstream, Inc. Cppyright
+  - Copyright (c) 2003 by Bitstream, Inc.
   - Copyright (c) 2006 by Tavmjong Bah
   - Copyright (c) 2007 Red Hat, Inc
   - Copyright (c) 2007 Red Hat, Inc.

tests/formattedcode/data/common/manifests-expected.json

@@ -471,39 +471,34 @@
   ],
   "license_detections": [
     {
-      "identifier": "cddl_1_0-c6dbef4d-659c-289f-5ee9-1ca0278edad6",
-      "license_expression": "cddl-1.0",
-      "detection_count": 1
+      "identifier": "apache_2_0-d66ab77d-a5cc-7104-e702-dc7df61fe9e8",
+      "license_expression": "apache-2.0",
+      "detection_count": 3
     },
     {
-      "identifier": "cddl_1_0-9893b55c-3b2b-4ee8-a932-6c6c93a63fc5",
-      "license_expression": "cddl-1.0",
+      "identifier": "apache_2_0-ec759ae0-ea5a-f138-793e-388520e080c0",
+      "license_expression": "apache-2.0",
       "detection_count": 1
     },
     {
-      "identifier": "cddl_1_0-ef82fc8c-50cb-6f35-1814-d2eb0bc13e83",
+      "identifier": "cddl_1_0-9893b55c-3b2b-4ee8-a932-6c6c93a63fc5",
       "license_expression": "cddl-1.0",
       "detection_count": 1
     },
     {
-      "identifier": "apache_2_0-ec759ae0-ea5a-f138-793e-388520e080c0",
-      "license_expression": "apache-2.0",
+      "identifier": "cddl_1_0-c6dbef4d-659c-289f-5ee9-1ca0278edad6",
+      "license_expression": "cddl-1.0",
       "detection_count": 1
     },
     {
-      "identifier": "apache_2_0-d66ab77d-a5cc-7104-e702-dc7df61fe9e8",
-      "license_expression": "apache-2.0",
-      "detection_count": 3
-    },
-    {
-      "identifier": "mit-3fce6ea2-8abd-6c6b-3ede-a37af7c6efee",
-      "license_expression": "mit",
+      "identifier": "cddl_1_0-ef82fc8c-50cb-6f35-1814-d2eb0bc13e83",
+      "license_expression": "cddl-1.0",
       "detection_count": 1
     },
     {
-      "identifier": "mit-a822f434-d61f-f2b1-c792-8b8cb9e7b9bf",
-      "license_expression": "mit",
-      "detection_count": 1
+      "identifier": "lgpl_3_0-272571eb-5e68-95b6-ddb0-71de2d8df321",
+      "license_expression": "lgpl-3.0",
+      "detection_count": 2
     },
     {
       "identifier": "lgpl_3_0-121be3c2-9c80-df84-d3da-8f674e4125c0",
@@ -516,13 +511,18 @@
       "detection_count": 1
     },
     {
-      "identifier": "lgpl_3_0-272571eb-5e68-95b6-ddb0-71de2d8df321",
+      "identifier": "lgpl_3_0-38174920-e8ed-7bda-41ec-94df7380b7d5",
       "license_expression": "lgpl-3.0",
-      "detection_count": 2
+      "detection_count": 1
     },
     {
-      "identifier": "lgpl_3_0-38174920-e8ed-7bda-41ec-94df7380b7d5",
-      "license_expression": "lgpl-3.0",
+      "identifier": "mit-3fce6ea2-8abd-6c6b-3ede-a37af7c6efee",
+      "license_expression": "mit",
+      "detection_count": 1
+    },
+    {
+      "identifier": "mit-a822f434-d61f-f2b1-c792-8b8cb9e7b9bf",
+      "license_expression": "mit",
       "detection_count": 1
     }
   ],

tests/formattedcode/data/common/manifests-expected.jsonlines

@@ -507,39 +507,34 @@
   {
     "license_detections": [
       {
-        "identifier": "cddl_1_0-c6dbef4d-659c-289f-5ee9-1ca0278edad6",
-        "license_expression": "cddl-1.0",
-        "detection_count": 1
+        "identifier": "apache_2_0-d66ab77d-a5cc-7104-e702-dc7df61fe9e8",
+        "license_expression": "apache-2.0",
+        "detection_count": 3
       },
       {
-        "identifier": "cddl_1_0-9893b55c-3b2b-4ee8-a932-6c6c93a63fc5",
-        "license_expression": "cddl-1.0",
+        "identifier": "apache_2_0-ec759ae0-ea5a-f138-793e-388520e080c0",
+        "license_expression": "apache-2.0",
         "detection_count": 1
       },
       {
-        "identifier": "cddl_1_0-ef82fc8c-50cb-6f35-1814-d2eb0bc13e83",
+        "identifier": "cddl_1_0-9893b55c-3b2b-4ee8-a932-6c6c93a63fc5",
         "license_expression": "cddl-1.0",
         "detection_count": 1
       },
       {
-        "identifier": "apache_2_0-ec759ae0-ea5a-f138-793e-388520e080c0",
-        "license_expression": "apache-2.0",
+        "identifier": "cddl_1_0-c6dbef4d-659c-289f-5ee9-1ca0278edad6",
+        "license_expression": "cddl-1.0",
         "detection_count": 1
       },
       {
-        "identifier": "apache_2_0-d66ab77d-a5cc-7104-e702-dc7df61fe9e8",
-        "license_expression": "apache-2.0",
-        "detection_count": 3
-      },
-      {
-        "identifier": "mit-3fce6ea2-8abd-6c6b-3ede-a37af7c6efee",
-        "license_expression": "mit",
+        "identifier": "cddl_1_0-ef82fc8c-50cb-6f35-1814-d2eb0bc13e83",
+        "license_expression": "cddl-1.0",
         "detection_count": 1
       },
       {
-        "identifier": "mit-a822f434-d61f-f2b1-c792-8b8cb9e7b9bf",
-        "license_expression": "mit",
-        "detection_count": 1
+        "identifier": "lgpl_3_0-272571eb-5e68-95b6-ddb0-71de2d8df321",
+        "license_expression": "lgpl-3.0",
+        "detection_count": 2
       },
       {
         "identifier": "lgpl_3_0-121be3c2-9c80-df84-d3da-8f674e4125c0",
@@ -552,13 +547,18 @@
         "detection_count": 1
       },
       {
-        "identifier": "lgpl_3_0-272571eb-5e68-95b6-ddb0-71de2d8df321",
+        "identifier": "lgpl_3_0-38174920-e8ed-7bda-41ec-94df7380b7d5",
         "license_expression": "lgpl-3.0",
-        "detection_count": 2
+        "detection_count": 1
       },
       {
-        "identifier": "lgpl_3_0-38174920-e8ed-7bda-41ec-94df7380b7d5",
-        "license_expression": "lgpl-3.0",
+        "identifier": "mit-3fce6ea2-8abd-6c6b-3ede-a37af7c6efee",
+        "license_expression": "mit",
+        "detection_count": 1
+      },
+      {
+        "identifier": "mit-a822f434-d61f-f2b1-c792-8b8cb9e7b9bf",
+        "license_expression": "mit",
         "detection_count": 1
       }
     ]

tests/formattedcode/data/common/manifests-expected.yaml

@@ -47,14 +47,12 @@ summary:
   declared_holder:
   primary_language: Python
   other_license_expressions:
-    - value:
-      count: 1
+    - value: lgpl-3.0
+      count: 3
     - value: apache-2.0
-      count: 1
+      count: 2
     - value: cddl-1.0
       count: 1
-    - value: lgpl-3.0
-      count: 1
     - value: mit
       count: 1
   other_holders:
@@ -465,39 +463,39 @@ dependencies:
     datafile_path: manifests/npm-license-string/package.json
     datasource_id: npm_package_json
 license_detections:
-  - identifier: cddl_1_0-c6dbef4d-659c-289f-5ee9-1ca0278edad6
-    license_expression: cddl-1.0
+  - identifier: apache_2_0-d66ab77d-a5cc-7104-e702-dc7df61fe9e8
+    license_expression: apache-2.0
+    detection_count: 3
+  - identifier: apache_2_0-ec759ae0-ea5a-f138-793e-388520e080c0
+    license_expression: apache-2.0
     detection_count: 1
   - identifier: cddl_1_0-9893b55c-3b2b-4ee8-a932-6c6c93a63fc5
     license_expression: cddl-1.0
     detection_count: 1
-  - identifier: cddl_1_0-ef82fc8c-50cb-6f35-1814-d2eb0bc13e83
+  - identifier: cddl_1_0-c6dbef4d-659c-289f-5ee9-1ca0278edad6
     license_expression: cddl-1.0
     detection_count: 1
-  - identifier: apache_2_0-ec759ae0-ea5a-f138-793e-388520e080c0
-    license_expression: apache-2.0
-    detection_count: 1
-  - identifier: apache_2_0-d66ab77d-a5cc-7104-e702-dc7df61fe9e8
-    license_expression: apache-2.0
-    detection_count: 3
-  - identifier: mit-3fce6ea2-8abd-6c6b-3ede-a37af7c6efee
-    license_expression: mit
-    detection_count: 1
-  - identifier: mit-a822f434-d61f-f2b1-c792-8b8cb9e7b9bf
-    license_expression: mit
+  - identifier: cddl_1_0-ef82fc8c-50cb-6f35-1814-d2eb0bc13e83
+    license_expression: cddl-1.0
     detection_count: 1
+  - identifier: lgpl_3_0-272571eb-5e68-95b6-ddb0-71de2d8df321
+    license_expression: lgpl-3.0
+    detection_count: 2
   - identifier: lgpl_3_0-121be3c2-9c80-df84-d3da-8f674e4125c0
     license_expression: lgpl-3.0
     detection_count: 1
   - identifier: lgpl_3_0-2db87bcf-56b4-9d7d-7075-2effae31c631
     license_expression: lgpl-3.0
     detection_count: 1
-  - identifier: lgpl_3_0-272571eb-5e68-95b6-ddb0-71de2d8df321
-    license_expression: lgpl-3.0
-    detection_count: 2
   - identifier: lgpl_3_0-38174920-e8ed-7bda-41ec-94df7380b7d5
     license_expression: lgpl-3.0
     detection_count: 1
+  - identifier: mit-3fce6ea2-8abd-6c6b-3ede-a37af7c6efee
+    license_expression: mit
+    detection_count: 1
+  - identifier: mit-a822f434-d61f-f2b1-c792-8b8cb9e7b9bf
+    license_expression: mit
+    detection_count: 1
 license_references:
   - key: apache-2.0
     language: en

tests/formattedcode/data/yaml/package-and-licenses-expected.yaml

@@ -29,9 +29,9 @@ headers:
       system_environment:
         operating_system: linux
         cpu_architecture: 64
-        platform: Linux-5.14.0-1059-oem-x86_64-with-glibc2.29
-        platform_version: '#67-Ubuntu SMP Mon Mar 13 14:22:10 UTC 2023'
-        python_version: "3.8.10 (default, Mar 13 2023, 10:26:41) \n[GCC 9.4.0]"
+        platform: Linux-5.15.0-73-generic-x86_64-with-glibc2.29
+        platform_version: '#80~20.04.1-Ubuntu SMP Wed May 17 14:58:14 UTC 2023'
+        python_version: "3.8.10 (default, May 26 2023, 14:05:08) \n[GCC 9.4.0]"
       spdx_license_list_version: '3.20'
       files_count: 4
 summary:
@@ -47,8 +47,6 @@ summary:
   declared_holder: Example Corp.
   primary_language: Python
   other_license_expressions:
-    - value:
-      count: 1
     - value: apache-2.0 AND (apache-2.0 OR mit)
       count: 1
     - value: mit
@@ -121,18 +119,18 @@ packages:
     purl: pkg:pypi/codebase
 dependencies: []
 license_detections:
+  - identifier: apache_2_0-ec759ae0-ea5a-f138-793e-388520e080c0
+    license_expression: apache-2.0
+    detection_count: 2
   - identifier: apache_2_0-ab23f79b-ec38-9a8a-9b23-85059407f34d
     license_expression: apache-2.0
     detection_count: 1
-  - identifier: mit-cacd5c0c-204a-85c2-affc-e4c125b2492a
-    license_expression: mit
-    detection_count: 1
   - identifier: apache_2_0_and__apache_2_0_or_mit-9b638e72-e872-a67f-3447-eec297ef7b39
     license_expression: apache-2.0 AND (apache-2.0 OR mit)
     detection_count: 1
-  - identifier: apache_2_0-ec759ae0-ea5a-f138-793e-388520e080c0
-    license_expression: apache-2.0
-    detection_count: 2
+  - identifier: mit-cacd5c0c-204a-85c2-affc-e4c125b2492a
+    license_expression: mit
+    detection_count: 1
 license_references:
   - key: apache-2.0
     language: en