From 7f730f79336788c786504dbfa604c9a6aa4b5b97 Mon Sep 17 00:00:00 2001 From: Alok Kumar Date: Tue, 27 May 2025 20:58:05 +0530 Subject: [PATCH 1/6] Add test for extra-words and improve detection_log Reference: https://github.com/aboutcode-org/scancode-toolkit/issues/4400 Signed-off-by: Alok Kumar --- src/licensedcode/detection.py | 8 +++ ...an-extra-words-3-seq-license.expected.json | 70 +++++++++++++++++++ .../scan-extra-words-3-seq-license/LICENSE | 31 ++++++++ .../test_plugin_license_detection.py | 18 +++++ 4 files changed, 127 insertions(+) create mode 100644 tests/licensedcode/data/plugin_license/extra-words/scan-extra-words-3-seq-license.expected.json create mode 100644 tests/licensedcode/data/plugin_license/extra-words/scan-extra-words-3-seq-license/LICENSE diff --git a/src/licensedcode/detection.py b/src/licensedcode/detection.py index 92d0908e018..1b594647c13 100644 --- a/src/licensedcode/detection.py +++ b/src/licensedcode/detection.py @@ -124,6 +124,7 @@ class DetectionRule(Enum): These are logged in LicenseDetection.detection_log for verbosity. """ UNKNOWN_MATCH = 'unknown-match' + EXTRA_WORDS = 'extra-words' LICENSE_CLUES = 'license-clues' LOW_QUALITY_MATCH_FRAGMENTS = 'low-quality-matches' FALSE_POSITIVE = 'possible-false-positive' @@ -1545,6 +1546,13 @@ def get_detected_license_expression( # in detections but ideally we should return synthetic unknowns for these detection_log.append(DetectionRule.LOW_QUALITY_MATCH_FRAGMENTS.value) return detection_log, combined_expression + + elif analysis == DetectionCategory.EXTRA_WORDS.value: + if TRACE_ANALYSIS: + logger_debug(f'analysis {DetectionCategory.EXTRA_WORDS.value}') + # Apply filtering or handling logic if needed + matches_for_expression = license_matches + detection_log.append(DetectionRule.EXTRA_WORDS.value) else: if TRACE_ANALYSIS: diff --git a/tests/licensedcode/data/plugin_license/extra-words/scan-extra-words-3-seq-license.expected.json b/tests/licensedcode/data/plugin_license/extra-words/scan-extra-words-3-seq-license.expected.json new file mode 100644 index 00000000000..da87dd331a8 --- /dev/null +++ b/tests/licensedcode/data/plugin_license/extra-words/scan-extra-words-3-seq-license.expected.json @@ -0,0 +1,70 @@ +{ + "license_detections": [ + { + "identifier": "bsd_new-95249a8d-f533-e7c7-159a-9b6e173cba42", + "license_expression": "bsd-new", + "license_expression_spdx": "BSD-3-Clause", + "detection_count": 1, + "detection_log": [ + "extra-words" + ], + "reference_matches": [ + { + "license_expression": "bsd-new", + "license_expression_spdx": "BSD-3-Clause", + "from_file": "scan-extra-words-3-seq-license/LICENSE", + "start_line": 1, + "end_line": 31, + "matcher": "3-seq", + "score": 93.89, + "matched_length": 215, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "bsd-new_578.RULE", + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/bsd-new_578.RULE", + "matched_text": "Software License Agreement (BSD License)\n\nCopyright (c) 2009-2015, Kevin Decker \n\nAll rights reserved.\n\nRedistribution and use of this software in source and binary forms, with or without modification,\nare permitted provided that the following conditions are met:\n\n* Redistributions of source code must retain the above\n copyright notice, this list of conditions and the\n following disclaimer.\n\n* Redistributions in binary form must reproduce the above\n copyright notice, this list of conditions and the\n following disclaimer in the documentation and/or other\n materials provided with the distribution.\n\n* Neither the name of Kevin Decker nor the names of its\n contributors may be used to endorse or promote products\n derived from this software without specific prior\n written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR\nIMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND\nFITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\nCONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\nDAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\nDATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER\nIN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT\nOF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.", + "matched_text_diagnostics": "Software License Agreement (BSD License)\n\n[Copyright] ([c]) [2009]-[2015], [Kevin] [Decker] <[kpdecker]@[gmail].[com]>\n\n[All] [rights] [reserved].\n\nRedistribution and use [of] [this] [software] in source and binary forms, with or without modification,\nare permitted provided that the following conditions are met:\n\n* Redistributions of source code must retain the above\n copyright notice, this list of conditions and the\n following disclaimer.\n\n* Redistributions in binary form must reproduce the above\n copyright notice, this list of conditions and the\n following disclaimer in the documentation and/or other\n materials provided with the distribution.\n\n* Neither the name of [Kevin] [Decker] nor the names of its\n contributors may be used to endorse or promote products\n derived from this software without specific prior\n written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR\nIMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND\nFITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\nCONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\nDAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\nDATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER\nIN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT\nOF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE." + } + ] + } + ], + "files": [ + { + "path": "LICENSE", + "type": "file", + "detected_license_expression": "bsd-new", + "detected_license_expression_spdx": "BSD-3-Clause", + "license_detections": [ + { + "license_expression": "bsd-new", + "license_expression_spdx": "BSD-3-Clause", + "matches": [ + { + "license_expression": "bsd-new", + "license_expression_spdx": "BSD-3-Clause", + "from_file": "scan-extra-words-3-seq-license/LICENSE", + "start_line": 1, + "end_line": 31, + "matcher": "3-seq", + "score": 93.89, + "matched_length": 215, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "bsd-new_578.RULE", + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/bsd-new_578.RULE", + "matched_text": "Software License Agreement (BSD License)\n\nCopyright (c) 2009-2015, Kevin Decker \n\nAll rights reserved.\n\nRedistribution and use of this software in source and binary forms, with or without modification,\nare permitted provided that the following conditions are met:\n\n* Redistributions of source code must retain the above\n copyright notice, this list of conditions and the\n following disclaimer.\n\n* Redistributions in binary form must reproduce the above\n copyright notice, this list of conditions and the\n following disclaimer in the documentation and/or other\n materials provided with the distribution.\n\n* Neither the name of Kevin Decker nor the names of its\n contributors may be used to endorse or promote products\n derived from this software without specific prior\n written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR\nIMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND\nFITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\nCONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\nDAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\nDATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER\nIN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT\nOF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.", + "matched_text_diagnostics": "Software License Agreement (BSD License)\n\n[Copyright] ([c]) [2009]-[2015], [Kevin] [Decker] <[kpdecker]@[gmail].[com]>\n\n[All] [rights] [reserved].\n\nRedistribution and use [of] [this] [software] in source and binary forms, with or without modification,\nare permitted provided that the following conditions are met:\n\n* Redistributions of source code must retain the above\n copyright notice, this list of conditions and the\n following disclaimer.\n\n* Redistributions in binary form must reproduce the above\n copyright notice, this list of conditions and the\n following disclaimer in the documentation and/or other\n materials provided with the distribution.\n\n* Neither the name of [Kevin] [Decker] nor the names of its\n contributors may be used to endorse or promote products\n derived from this software without specific prior\n written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR\nIMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND\nFITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR\nCONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\nDAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\nDATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER\nIN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT\nOF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE." + } + ], + "detection_log": [ + "extra-words" + ], + "identifier": "bsd_new-95249a8d-f533-e7c7-159a-9b6e173cba42" + } + ], + "license_clues": [], + "percentage_of_license_text": 92.67, + "scan_errors": [] + } + ] +} \ No newline at end of file diff --git a/tests/licensedcode/data/plugin_license/extra-words/scan-extra-words-3-seq-license/LICENSE b/tests/licensedcode/data/plugin_license/extra-words/scan-extra-words-3-seq-license/LICENSE new file mode 100644 index 00000000000..4e7146ed78a --- /dev/null +++ b/tests/licensedcode/data/plugin_license/extra-words/scan-extra-words-3-seq-license/LICENSE @@ -0,0 +1,31 @@ +Software License Agreement (BSD License) + +Copyright (c) 2009-2015, Kevin Decker + +All rights reserved. + +Redistribution and use of this software in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above + copyright notice, this list of conditions and the + following disclaimer. + +* Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the + following disclaimer in the documentation and/or other + materials provided with the distribution. + +* Neither the name of Kevin Decker nor the names of its + contributors may be used to endorse or promote products + derived from this software without specific prior + written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER +IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/tests/licensedcode/test_plugin_license_detection.py b/tests/licensedcode/test_plugin_license_detection.py index ed91090acb6..7477ef82a8e 100644 --- a/tests/licensedcode/test_plugin_license_detection.py +++ b/tests/licensedcode/test_plugin_license_detection.py @@ -93,6 +93,24 @@ def test_license_match_unknown_clues_is_not_in_expression(): check_json_scan(test_loc, result_file, regen=REGEN_TEST_FIXTURES) +def test_license_match_extra_words_3_seq(): + test_dir = test_env.get_test_loc('plugin_license/extra-words/scan-extra-words-3-seq-license/', copy=True) + result_file = test_env.get_temp_file('json') + args = [ + '--license', + '--license-text', + '--license-text-diagnostics', + '--license-diagnostics', + '--strip-root', + '--verbose', + '--json', result_file, + test_dir, + ] + run_scan_click(args) + test_loc = test_env.get_test_loc('plugin_license/extra-words/scan-extra-words-3-seq-license.expected.json') + check_json_scan(test_loc, result_file, regen=REGEN_TEST_FIXTURES) + + def test_license_match_unknown_license_intro_eclipse_foundation(): test_dir = test_env.get_test_loc('plugin_license/unknown_intro/scan-unknown-intro-eclipse-foundation/', copy=True) result_file = test_env.get_temp_file('json') From 3db9bac7de3b465b5894f447d5929c765ce7c4db Mon Sep 17 00:00:00 2001 From: Alok Kumar Date: Thu, 29 May 2025 16:50:20 +0530 Subject: [PATCH 2/6] add test for `2-aho` Signed-off-by: Alok Kumar --- ...an-extra-words-2-aho-license.expected.json | 66 +++++++++++++++++++ .../scan-extra-words-2-aho-license/LICENSE | 28 ++++++++ .../test_plugin_license_detection.py | 18 +++++ 3 files changed, 112 insertions(+) create mode 100644 tests/licensedcode/data/plugin_license/extra-words/scan-extra-words-2-aho-license.expected.json create mode 100644 tests/licensedcode/data/plugin_license/extra-words/scan-extra-words-2-aho-license/LICENSE diff --git a/tests/licensedcode/data/plugin_license/extra-words/scan-extra-words-2-aho-license.expected.json b/tests/licensedcode/data/plugin_license/extra-words/scan-extra-words-2-aho-license.expected.json new file mode 100644 index 00000000000..d6271fccce6 --- /dev/null +++ b/tests/licensedcode/data/plugin_license/extra-words/scan-extra-words-2-aho-license.expected.json @@ -0,0 +1,66 @@ +{ + "license_detections": [ + { + "identifier": "bsd_new-fbfc5955-0c63-4c98-2ce9-08e1e1796f50", + "license_expression": "bsd-new", + "license_expression_spdx": "BSD-3-Clause", + "detection_count": 1, + "detection_log": [], + "reference_matches": [ + { + "license_expression": "bsd-new", + "license_expression_spdx": "BSD-3-Clause", + "from_file": "scan-extra-words-2-aho-license/LICENSE", + "start_line": 4, + "end_line": 27, + "matcher": "2-aho", + "score": 99.53, + "matched_length": 210, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "bsd-new_158.RULE", + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/bsd-new_158.RULE", + "matched_text": "Redistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n* Redistributions of source code must retain the above copyright notice, this\r\n list of conditions and the following disclaimer.\r\n\r\n* Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n* Neither the name of filesize nor the names of its\r\n contributors may be used to endorse or promote products derived from\r\n this software without specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\r\nAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\r\nIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\r\nFOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\r\nDAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\r\nSERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\r\nCAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\r\nOR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\r\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.", + "matched_text_diagnostics": "Redistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n* Redistributions of source code must retain the above copyright notice, this\r\n list of conditions and the following disclaimer.\r\n\r\n* Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n* Neither the name of [filesize] nor the names of its\r\n contributors may be used to endorse or promote products derived from\r\n this software without specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\r\nAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\r\nIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\r\nFOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\r\nDAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\r\nSERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\r\nCAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\r\nOR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\r\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE." + } + ] + } + ], + "files": [ + { + "path": "LICENSE", + "type": "file", + "detected_license_expression": "bsd-new", + "detected_license_expression_spdx": "BSD-3-Clause", + "license_detections": [ + { + "license_expression": "bsd-new", + "license_expression_spdx": "BSD-3-Clause", + "matches": [ + { + "license_expression": "bsd-new", + "license_expression_spdx": "BSD-3-Clause", + "from_file": "scan-extra-words-2-aho-license/LICENSE", + "start_line": 4, + "end_line": 27, + "matcher": "2-aho", + "score": 99.53, + "matched_length": 210, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "bsd-new_158.RULE", + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/bsd-new_158.RULE", + "matched_text": "Redistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n* Redistributions of source code must retain the above copyright notice, this\r\n list of conditions and the following disclaimer.\r\n\r\n* Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n* Neither the name of filesize nor the names of its\r\n contributors may be used to endorse or promote products derived from\r\n this software without specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\r\nAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\r\nIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\r\nFOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\r\nDAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\r\nSERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\r\nCAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\r\nOR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\r\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.", + "matched_text_diagnostics": "Redistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n* Redistributions of source code must retain the above copyright notice, this\r\n list of conditions and the following disclaimer.\r\n\r\n* Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n* Neither the name of [filesize] nor the names of its\r\n contributors may be used to endorse or promote products derived from\r\n this software without specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\r\nAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\r\nIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\r\nFOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\r\nDAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\r\nSERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\r\nCAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\r\nOR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\r\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE." + } + ], + "detection_log": [], + "identifier": "bsd_new-fbfc5955-0c63-4c98-2ce9-08e1e1796f50" + } + ], + "license_clues": [], + "percentage_of_license_text": 95.89, + "scan_errors": [] + } + ] +} \ No newline at end of file diff --git a/tests/licensedcode/data/plugin_license/extra-words/scan-extra-words-2-aho-license/LICENSE b/tests/licensedcode/data/plugin_license/extra-words/scan-extra-words-2-aho-license/LICENSE new file mode 100644 index 00000000000..919daaf4d51 --- /dev/null +++ b/tests/licensedcode/data/plugin_license/extra-words/scan-extra-words-2-aho-license/LICENSE @@ -0,0 +1,28 @@ +Copyright (c) 2022, Jason Mulligan +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of filesize nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/tests/licensedcode/test_plugin_license_detection.py b/tests/licensedcode/test_plugin_license_detection.py index 7477ef82a8e..cc4d3a15fba 100644 --- a/tests/licensedcode/test_plugin_license_detection.py +++ b/tests/licensedcode/test_plugin_license_detection.py @@ -111,6 +111,24 @@ def test_license_match_extra_words_3_seq(): check_json_scan(test_loc, result_file, regen=REGEN_TEST_FIXTURES) +def test_license_match_extra_words_2_aho(): + test_dir = test_env.get_test_loc('plugin_license/extra-words/scan-extra-words-2-aho-license/', copy=True) + result_file = test_env.get_temp_file('json') + args = [ + '--license', + '--license-text', + '--license-text-diagnostics', + '--license-diagnostics', + '--strip-root', + '--verbose', + '--json', result_file, + test_dir, + ] + run_scan_click(args) + test_loc = test_env.get_test_loc('plugin_license/extra-words/scan-extra-words-2-aho-license.expected.json') + check_json_scan(test_loc, result_file, regen=REGEN_TEST_FIXTURES) + + def test_license_match_unknown_license_intro_eclipse_foundation(): test_dir = test_env.get_test_loc('plugin_license/unknown_intro/scan-unknown-intro-eclipse-foundation/', copy=True) result_file = test_env.get_temp_file('json') From 1f5a1942036167d09ef67aad62abb26530f8945c Mon Sep 17 00:00:00 2001 From: Alok Kumar Date: Sat, 31 May 2025 16:42:59 +0530 Subject: [PATCH 3/6] write function such that we get `extra-words` for `2-aho` Signed-off-by: Alok Kumar --- src/licensedcode/detection.py | 21 +++++++++++++++++-- ...an-extra-words-2-aho-license.expected.json | 8 +++++-- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/licensedcode/detection.py b/src/licensedcode/detection.py index 1b594647c13..c227b13a1ee 100644 --- a/src/licensedcode/detection.py +++ b/src/licensedcode/detection.py @@ -1055,10 +1055,27 @@ def is_correct_detection_non_unknown(license_matches): are correct/perfect license detections and also there aren't any unknowns. """ return ( - is_correct_detection(license_matches) + is_correct_detection_2(license_matches) and not has_unknown_matches(license_matches) ) +def is_correct_detection_2(license_matches): + """ + Return True if all the matches in ``license_matches`` List of LicenseMatch + are perfect license detections, and the matcher is always either `1-hash` + or `1-spdx-id`. + """ + matchers = (license_match.matcher for license_match in license_matches) + is_match_coverage_perfect = [ + license_match.coverage() == 100 + for license_match in license_matches + ] + + return ( + all(matcher in ("1-hash", "1-spdx-id") for matcher in matchers) + and all(is_match_coverage_perfect) + ) + def is_correct_detection(license_matches): """ @@ -1734,7 +1751,7 @@ def analyze_detection(license_matches, package_license=False): ): return DetectionCategory.LICENSE_CLUES.value - # Case where all matches have `matcher` as `1-hash` or `4-spdx-id` + # Case where all matches have `matcher` as `1-hash` or `1-spdx-id` elif is_correct_detection_non_unknown(license_matches=license_matches): return DetectionCategory.PERFECT_DETECTION.value diff --git a/tests/licensedcode/data/plugin_license/extra-words/scan-extra-words-2-aho-license.expected.json b/tests/licensedcode/data/plugin_license/extra-words/scan-extra-words-2-aho-license.expected.json index d6271fccce6..fcdb8639ddb 100644 --- a/tests/licensedcode/data/plugin_license/extra-words/scan-extra-words-2-aho-license.expected.json +++ b/tests/licensedcode/data/plugin_license/extra-words/scan-extra-words-2-aho-license.expected.json @@ -5,7 +5,9 @@ "license_expression": "bsd-new", "license_expression_spdx": "BSD-3-Clause", "detection_count": 1, - "detection_log": [], + "detection_log": [ + "extra-words" + ], "reference_matches": [ { "license_expression": "bsd-new", @@ -54,7 +56,9 @@ "matched_text_diagnostics": "Redistribution and use in source and binary forms, with or without\r\nmodification, are permitted provided that the following conditions are met:\r\n\r\n* Redistributions of source code must retain the above copyright notice, this\r\n list of conditions and the following disclaimer.\r\n\r\n* Redistributions in binary form must reproduce the above copyright notice,\r\n this list of conditions and the following disclaimer in the documentation\r\n and/or other materials provided with the distribution.\r\n\r\n* Neither the name of [filesize] nor the names of its\r\n contributors may be used to endorse or promote products derived from\r\n this software without specific prior written permission.\r\n\r\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\r\nAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\r\nIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\r\nDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\r\nFOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\r\nDAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\r\nSERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\r\nCAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\r\nOR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\r\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE." } ], - "detection_log": [], + "detection_log": [ + "extra-words" + ], "identifier": "bsd_new-fbfc5955-0c63-4c98-2ce9-08e1e1796f50" } ], From 9ed912fc4a5b57e21475ac88a5dcef2919f6be86 Mon Sep 17 00:00:00 2001 From: Alok Kumar Date: Sat, 31 May 2025 17:21:46 +0530 Subject: [PATCH 4/6] fix: regenerate and update failing test cases Signed-off-by: Alok Kumar --- .../reference-to-package/paddlenlp.expected.json | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/packagedcode/data/license_detection/reference-to-package/paddlenlp.expected.json b/tests/packagedcode/data/license_detection/reference-to-package/paddlenlp.expected.json index 665ed51816d..35b2cd04102 100644 --- a/tests/packagedcode/data/license_detection/reference-to-package/paddlenlp.expected.json +++ b/tests/packagedcode/data/license_detection/reference-to-package/paddlenlp.expected.json @@ -750,7 +750,9 @@ "license_expression": "apache-2.0", "license_expression_spdx": "Apache-2.0", "detection_count": 2, - "detection_log": [], + "detection_log": [ + "extra-words" + ], "reference_matches": [ { "license_expression": "apache-2.0", @@ -981,7 +983,9 @@ "matched_text_diagnostics": "License\n\n[PaddleNLP\u9075\u5faa][Apache-2.[0\u5f00\u6e90\u534f\u8bae]](./LICENSE)\u3002" } ], - "detection_log": [], + "detection_log": [ + "extra-words" + ], "identifier": "apache_2_0-999670be-3d5e-ebf8-ae18-b555c26c5e80" } ], @@ -1547,7 +1551,9 @@ "matched_text_diagnostics": "License\n\n[PaddleNLP\u9075\u5faa][Apache-2.[0\u5f00\u6e90\u534f\u8bae]](./LICENSE)\u3002" } ], - "detection_log": [], + "detection_log": [ + "extra-words" + ], "identifier": "apache_2_0-999670be-3d5e-ebf8-ae18-b555c26c5e80" }, { From 193aeb84dfe3c50e69f78586f64ecdfa4c2cbf2d Mon Sep 17 00:00:00 2001 From: Alok Kumar Date: Mon, 2 Jun 2025 21:59:18 +0530 Subject: [PATCH 5/6] fix extra-words detection Signed-off-by: Alok Kumar --- src/licensedcode/detection.py | 22 +++------------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/src/licensedcode/detection.py b/src/licensedcode/detection.py index c227b13a1ee..8e5c87f043d 100644 --- a/src/licensedcode/detection.py +++ b/src/licensedcode/detection.py @@ -1055,26 +1055,10 @@ def is_correct_detection_non_unknown(license_matches): are correct/perfect license detections and also there aren't any unknowns. """ return ( - is_correct_detection_2(license_matches) + is_correct_detection(license_matches) and not has_unknown_matches(license_matches) - ) - -def is_correct_detection_2(license_matches): - """ - Return True if all the matches in ``license_matches`` List of LicenseMatch - are perfect license detections, and the matcher is always either `1-hash` - or `1-spdx-id`. - """ - matchers = (license_match.matcher for license_match in license_matches) - is_match_coverage_perfect = [ - license_match.coverage() == 100 - for license_match in license_matches - ] - - return ( - all(matcher in ("1-hash", "1-spdx-id") for matcher in matchers) - and all(is_match_coverage_perfect) - ) + and not has_extra_words(license_matches) + ) def is_correct_detection(license_matches): From 7eb8db79e407ad4fff187f2983724b1fed5eea9f Mon Sep 17 00:00:00 2001 From: Alok Kumar Date: Tue, 3 Jun 2025 11:11:08 +0530 Subject: [PATCH 6/6] improve `extra-words` detection Also i regenerate the tests for https://github.com/aboutcode-org/scancode-toolkit/blob/develop/tests/packagedcode/test_license_detection.py#L265 previously there is not detection of `extra-words` due to `referenced_filenames` tag in license rule Signed-off-by: Alok Kumar --- src/licensedcode/detection.py | 2 +- .../reference-to-package/paddlenlp.expected.json | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/licensedcode/detection.py b/src/licensedcode/detection.py index 8e5c87f043d..e88ca5f82da 100644 --- a/src/licensedcode/detection.py +++ b/src/licensedcode/detection.py @@ -1382,7 +1382,7 @@ def has_references_to_local_files(license_matches): Return True if any of the matched Rule for the ``license_matches`` has a non empty `referenced_filenames`, otherwise return False. """ - return any( + return not has_extra_words(license_matches) and any( bool(match.rule.referenced_filenames) for match in license_matches ) diff --git a/tests/packagedcode/data/license_detection/reference-to-package/paddlenlp.expected.json b/tests/packagedcode/data/license_detection/reference-to-package/paddlenlp.expected.json index 35b2cd04102..6da2af096b7 100644 --- a/tests/packagedcode/data/license_detection/reference-to-package/paddlenlp.expected.json +++ b/tests/packagedcode/data/license_detection/reference-to-package/paddlenlp.expected.json @@ -725,7 +725,9 @@ "license_expression": "apache-2.0", "license_expression_spdx": "Apache-2.0", "detection_count": 2, - "detection_log": [], + "detection_log": [ + "extra-words" + ], "reference_matches": [ { "license_expression": "apache-2.0", @@ -1024,7 +1026,9 @@ "matched_text_diagnostics": "License\n\n[PaddleNLP] is provided under the [Apache-2.0 License](./LICENSE)." } ], - "detection_log": [], + "detection_log": [ + "extra-words" + ], "identifier": "apache_2_0-4571361c-d5af-4e7e-c015-6cc10c1b8174" } ], @@ -1577,7 +1581,9 @@ "matched_text_diagnostics": "License\n\n[PaddleNLP] is provided under the [Apache-2.0 License](./LICENSE)." } ], - "detection_log": [], + "detection_log": [ + "extra-words" + ], "identifier": "apache_2_0-4571361c-d5af-4e7e-c015-6cc10c1b8174" } ],