improve detection and handling of extra-words and add test for 3-seq

add a new `extra-phrase` for a rule i.e bsd-new

Author: alok1304

src/licensedcode/data/rules/bsd-new_578.RULE

@@ -6,9 +6,7 @@ minimum_coverage: 99
 
 Software License Agreement (BSD License)
 
-[[15]]
-
-Redistribution and use [[4]] in source and binary forms, with or without
+Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions
 are met:
 
@@ -18,7 +16,7 @@ are met:
    copyright notice, this list of conditions and the following
    disclaimer in the documentation and/or other materials provided
    with the distribution.
- * Neither the name of [[6]] nor the names of its
+ * Neither the name of nor the names of its
    contributors may be used to endorse or promote products derived
    from this software without specific prior written permission.
 

src/licensedcode/data/rules/bsd-new_newlib3.RULE

@@ -11,7 +11,7 @@ are permitted provided that the following conditions are met:
     * Redistributions in binary form must reproduce the above copyright notice,
       this list of conditions and the following disclaimer in the documentation
       and/or other materials provided with the distribution.
-    * Neither the name of the University nor the names of its contributors 
+    * Neither the name of the [[3]] University nor the names of its contributors 
       may be used to endorse or promote products derived from this software 
       without specific prior written permission.
 

src/licensedcode/detection.py

@@ -408,8 +408,12 @@ def score(self):
         by the length of a match to the overall detection length.
         """
         length = self.length
-        weighted_scores = (m.score() * (m.len() / length) for m in self.matches)
-        return min([round(sum(weighted_scores), 2), 100])
+        for m in self.matches:
+            # Check whether extra words in the matched text appear in allowed positions,
+            # and do not exceed the maximum allowed word count at those positions.
+            score = 100 if is_extra_words_position_valid(m) else m.score()
+            weighted_scores += score * (m.len() / length)
+        return min([round(weighted_scores, 2), 100])
 
     def append(
         self,
@@ -1165,13 +1169,18 @@ def has_low_rule_relevance(license_matches):
 
 def is_extra_words_at_valid_positions(license_matches):
     """
-    Return True if any of the matches in ``license_matches`` List of LicenseMatch
+    Return True if all the matches in `license_matches List of LicenseMatch
     has extra words are in the correct place.
     """
-    return any(
-        is_extra_words_position_valid(license_match)
-        for license_match in license_matches
-    )
+    for match in license_matches:
+        # check when we have `extra-words` detection
+        # if `query_coverage_coefficient` is positive number then 'extra-words` exit
+        if calculate_query_coverage_coefficient(match) > 0:
+            if not is_extra_words_position_valid(match):
+                return False
+            
+    # at the end return True if all matches have no extra-wors or this extra-words are in the right place
+    return True
 
 def is_false_positive(license_matches, package_license=False):
     """
@@ -1827,14 +1836,14 @@ def analyze_detection(license_matches, package_license=False):
         threshold=IMPERFECT_MATCH_COVERAGE_THR,
     ):
         return DetectionCategory.IMPERFECT_COVERAGE.value
-    
-    # Case where `extra-words` are in the right place
-    elif is_extra_words_at_valid_positions(license_matches=license_matches):
-        return DetectionCategory.EXTRA_WORDS_PERMITTED.value
 
     # Case where at least one of the match have extra words
     elif has_extra_words(license_matches=license_matches):
-        return DetectionCategory.EXTRA_WORDS.value
+        # Case where `extra-words` are in the right place
+        if is_extra_words_at_valid_positions(license_matches=license_matches):
+            return DetectionCategory.EXTRA_WORDS_PERMITTED.value
+        else:
+            return DetectionCategory.EXTRA_WORDS.value
 
     # Cases where Match Coverage is a perfect 100 for all matches
     else:

src/licensedcode/match.py

@@ -597,13 +597,7 @@ def score(self):
         computed from the number of matched tokens, the number of query tokens
         in the matched range (including unknowns and unmatched) and the matched
         rule relevance.
-        """
-
-        # Check whether extra words in the matched text appear in allowed positions,
-        # and do not exceed the maximum allowed word count at those positions.
-        if is_extra_words_position_valid(match=self):
-            return 100
-        
+        """       
         # relevance is a number between 0 and 100. Divide by 100
         relevance = self.rule.relevance / 100
         if not relevance:
@@ -832,7 +826,11 @@ def to_dict(
         result['start_line'] = self.start_line
         result['end_line'] = self.end_line
         result['matcher'] = self.matcher
-        result['score'] = self.score()
+        # update score if `extra-words` are in right place
+        if(is_extra_words_position_valid(match=self)):
+            result['score'] = 100
+        else:
+            result['score'] = self.score()            
         result['matched_length'] = self.len()
         result['match_coverage'] = self.coverage()
         result['rule_relevance'] = self.rule.relevance

tests/licensedcode/data/plugin_license/extra-words/scan-extra-words-2-aho-license.expected.json

@@ -1,7 +1,7 @@
 {
   "license_detections": [
     {
-      "identifier": "bsd_new-f3efb258-e9c3-604d-9bbd-147fe74af22e",
+      "identifier": "bsd_new-fbfc5955-0c63-4c98-2ce9-08e1e1796f50",
       "license_expression": "bsd-new",
       "license_expression_spdx": "BSD-3-Clause",
       "detection_count": 1,
@@ -59,7 +59,7 @@
           "detection_log": [
             "extra-words-permitted-in-rule"
           ],
-          "identifier": "bsd_new-f3efb258-e9c3-604d-9bbd-147fe74af22e"
+          "identifier": "bsd_new-fbfc5955-0c63-4c98-2ce9-08e1e1796f50"
         }
       ],
       "license_clues": [],

tests/licensedcode/data/plugin_license/extra-words/scan-extra-words-3-seq-license.expected.json

@@ -1,31 +1,31 @@
-Software License Agreement (BSD License)
-
-Copyright (c) 2009-2015, Kevin Decker <kpdecker@gmail.com>
-
-All rights reserved.
-
-Redistribution and use of this software in source and binary forms, with or without modification,
-are permitted provided that the following conditions are met:
-
-* Redistributions of source code must retain the above
-  copyright notice, this list of conditions and the
-  following disclaimer.
-
-* Redistributions in binary form must reproduce the above
-  copyright notice, this list of conditions and the
-  following disclaimer in the documentation and/or other
-  materials provided with the distribution.
-
-* Neither the name of Kevin Decker nor the names of its
-  contributors may be used to endorse or promote products
-  derived from this software without specific prior
-  written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
-IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
-FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
-CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
-IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+Software License Agreement (BSD License)
+
+Copyright (c) 2009-2015, Kevin Decker <kpdecker@gmail.com>
+
+All rights reserved.
+
+Redistribution and use of this software in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above
+  copyright notice, this list of conditions and the
+  following disclaimer.
+
+* Redistributions in binary form must reproduce the above
+  copyright notice, this list of conditions and the
+  following disclaimer in the documentation and/or other
+  materials provided with the distribution.
+
+* Neither the name of Kevin Decker nor the names of its
+  contributors may be used to endorse or promote products
+  derived from this software without specific prior
+  written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
 OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

…s/scan-extra-words-3-seq-license/LICENSE …ds-3-seq-license/with-copyrights/LICENSEtests/licensedcode/data/plugin_license/extra-words/scan-extra-words-3-seq-license/LICENSE renamed to tests/licensedcode/data/plugin_license/extra-words/scan-extra-words-3-seq-license/with-copyrights/LICENSE tests/licensedcode/data/plugin_license/extra-words/scan-extra-words-3-seq-license/LICENSE renamed to tests/licensedcode/data/plugin_license/extra-words/scan-extra-words-3-seq-license/with-copyrights/LICENSE

@@ -0,0 +1,32 @@
+Copyright (c) 2007, 2008, 2009 Mutsuo Saito, Makoto Matsumoto
+and Hiroshima University.
+Copyright (c) 2011, 2002 Mutsuo Saito, Makoto Matsumoto, Hiroshima
+University and The University of Tokyo.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+        notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above
+        copyright notice, this list of conditions and the following
+        disclaimer in the documentation and/or other materials provided
+        with the distribution.
+    * Neither the name of the Hiroshima University nor the names of
+        its contributors may be used to endorse or promote products
+        derived from this software without specific prior written
+        permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.