Update following referenced licenses and rules

In case of referenced license files, update the logic to not follow
and de-reference licenses when all the license leys in the referenced
files are already present in the main license detection. This preserves
the license conditions in case there are extra license keys in an
expression.

Reference: #3523
Signed-off-by: Ayan Sinha Mahapatra <asmahapatra@aboutcode.org>

Author: AyanSinhaMahapatra

src/licensedcode/data/rules/lgpl-2.1_or_mpl-1.1_10.RULE

@@ -0,0 +1,6 @@
+---
+license_expression: lgpl-2.1 OR mpl-1.1
+is_license_tag: yes
+---
+
+license: LGPL/MPL

src/licensedcode/data/rules/lgpl-2.1_or_mpl-1.1_11.RULE

@@ -0,0 +1,6 @@
+---
+license_expression: lgpl-2.1 OR mpl-1.1
+is_license_notice: yes
+---
+
+dual-licensed under the LGPL and the MPL.

src/licensedcode/data/rules/lgpl-2.1_or_mpl-1.1_12.RULE

@@ -0,0 +1,9 @@
+---
+license_expression: lgpl-2.1 OR mpl-1.1
+is_license_notice: yes
+---
+
+Cairo is free software and is available to be redistributed and/or
+modified under the terms of either the GNU Lesser General Public
+License (LGPL) version 2.1 or the Mozilla Public License (MPL) version
+1.1.

src/licensedcode/data/rules/lgpl-2.1_or_mpl-1.1_13.RULE

@@ -0,0 +1,25 @@
+---
+license_expression: lgpl-2.1 OR mpl-1.1
+is_license_notice: yes
+referenced_filenames:
+    - COPYING-LGPL-2.1
+    - COPYING-MPL-1.1
+---
+
+is free software.
+
+Every source file in the implementation of is available to be
+redistributed and/or modified under the terms of either the GNU Lesser
+General Public License (LGPL) version 2.1 or the Mozilla Public
+License (MPL) version 1.1.  Some files are available under more
+liberal terms, but we believe that in all cases, each file may be used
+under either the LGPL or the MPL.
+
+See the following files in this directory for the precise terms and
+conditions of either license:
+
+	COPYING-LGPL-2.1
+	COPYING-MPL-1.1
+
+Please see each file in the implementation for copyright and licensing
+information, (in the opening comment of each file).

src/licensedcode/data/rules/lgpl-2.1_or_mpl-1.1_14.RULE

@@ -0,0 +1,10 @@
+---
+license_expression: lgpl-2.1 OR mpl-1.1
+is_license_reference: yes
+referenced_filenames:
+    - COPYING
+---
+
+* The code in the patch is copyright under the LGPL, but
+ * has been relicensed under the LGPL/MPL dual license for inclusion
+ * into (see COPYING).

src/licensedcode/data/rules/lgpl-2.1_or_mpl-1.1_7.RULE

@@ -0,0 +1,11 @@
+---
+license_expression: lgpl-2.1 OR mpl-1.1
+is_license_notice: yes
+referenced_filenames:
+    - COPYING
+---
+
+Cairo is released under the terms of either the {{GNU Lesser General Public
+License version 2.1}}, or the terms of the {{Mozilla Public License version 1.1}}.
+
+See the [`COPYING`](./COPYING) document for more information.

src/licensedcode/data/rules/lgpl-2.1_or_mpl-1.1_8.RULE

@@ -0,0 +1,7 @@
+---
+license_expression: lgpl-2.1 OR mpl-1.1
+is_license_notice: yes
+---
+
+Cairo is released under the terms of either the {{GNU Lesser General Public
+License version 2.1}}, or the terms of the {{Mozilla Public License version 1.1}}

src/licensedcode/data/rules/lgpl-2.1_or_mpl-1.1_9.RULE

@@ -0,0 +1,11 @@
+---
+license_expression: lgpl-2.1 OR mpl-1.1
+is_license_reference: yes
+---
+
+New license: LGPL/MPL
+
+The most significant news with this release is that the license of
+cairo has changed. It is now dual-licensed under the LGPL and the
+MPL. For details see the COPYING file as well as COPYING-LGPL-2.1 and
+COPYING-MPL-1.1.

src/licensedcode/detection.py

@@ -60,19 +60,19 @@ def logger_debug(*args):
 
 logger = logging.getLogger(__name__)
 
-if TRACE:
+if (
+    TRACE
+    or TRACE_ANALYSIS
+    or TRACE_REFERENCE
+    or TRACE_IS_FUNCTIONS
+):
 
-    if (
-        TRACE
-        or TRACE_ANALYSIS
-        or TRACE_IS_FUNCTIONS
-    ):
+    logging.basicConfig(stream=sys.stdout)
+    logger.setLevel(logging.DEBUG)
 
-        logging.basicConfig(stream=sys.stdout)
-        logger.setLevel(logging.DEBUG)
+    def logger_debug(*args):
+        return logger.debug(' '.join(isinstance(a, str) and a or repr(a) for a in args))
 
-        def logger_debug(*args):
-            return logger.debug(' '.join(isinstance(a, str) and a or repr(a) for a in args))
 
 MATCHER_UNDETECTED = '5-undetected'
 MATCHER_UNDETECTED_ORDER = 4
@@ -1455,6 +1455,12 @@ def use_referenced_license_expression(referenced_license_expression, license_det
     if same_license_keys and not same_expression:
         return False
 
+    # If all the license keys of the referenced license expression already
+    # present in the actual license expression
+    # note that this preserves AND/OR expressions in the original license
+    if referenced_license_keys.issubset(license_keys):
+        return False
+
     # when there are many license keys in an expression, and there are no
     # unknown or other cases, we cannot safely conclude that we should
     # follow the license in the referenced filenames. This is likely
@@ -2072,6 +2078,10 @@ def update_detection_from_referenced_files(
         relation='AND',
         licensing=get_cache().licensing,
     ))
+    if TRACE_REFERENCE:
+        logger_debug(
+            f'update_detection_from_referenced_files: referenced_license_expression: {referenced_license_expression}',
+        )
 
     if not use_referenced_license_expression(
         referenced_license_expression=referenced_license_expression,
@@ -2080,24 +2090,26 @@ def update_detection_from_referenced_files(
         if TRACE_REFERENCE and referenced_resources:
             paths = [
                 resource.path
-                for resource in referenced_resource
+                for resource in referenced_resources
             ]
             logger_debug(
                 f'use_referenced_license_expression: False for '
                 f'resources: {paths} and '
-                f'license_expression: {referenced_license_expression}',
+                f'referenced_license_expression: {referenced_license_expression}',
+                f'license_detection.expression: {license_detection.license_expression}',
             )
         return False
 
     if TRACE_REFERENCE and referenced_resources:
         paths = [
             resource.path
-            for resource in referenced_resource
+            for resource in referenced_resources
         ]
         logger_debug(
             f'use_referenced_license_expression: True for '
             f'resources: {paths} and '
-            f'license_expression: {referenced_license_expression}',
+            f'referenced_license_expression: {referenced_license_expression}',
+            f'license_detection.expression: {license_detection.license_expression}',
         )
 
     matches_to_extend = get_matches_from_detection_mappings(

src/licensedcode/plugin_license.py

@@ -210,15 +210,15 @@ def process_codebase(self, codebase, license_text=False, license_diagnostics=Fal
         modified = False
         for resource in codebase.walk(topdown=False):
             # follow license references to other files
-            if TRACE:
+            if TRACE_REFERENCE:
                 license_expressions_before = resource.detected_license_expression
 
             try:
                 modified = add_referenced_filenames_license_matches_for_detections(resource, codebase)
             except Exception as e:
                 raise Exception(f"Failed to process resource: {resource!r}") from e
 
-            if TRACE and modified:
+            if TRACE_REFERENCE and modified:
                 license_expressions_after = resource.detected_license_expression
                 logger_debug(
                     f'add_referenced_filenames_license_matches: Modified:',
@@ -291,6 +291,11 @@ def add_referenced_filenames_license_matches_for_detections(resource, codebase):
                 )
             continue
 
+        if TRACE_REFERENCE:
+            logger_debug(
+                f'add_referenced_license_matches: referenced_filenames: {referenced_filenames}',
+            )
+
         is_modified = update_detection_from_referenced_files(
             referenced_filenames=referenced_filenames,
             license_detection_mapping=license_detection_mapping,