diff --git a/src/licensedcode/__init__.py b/src/licensedcode/__init__.py index ea9d1c537ad..bdbaa96476c 100644 --- a/src/licensedcode/__init__.py +++ b/src/licensedcode/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2015 nexB Inc. and others. All rights reserved. +# Copyright (c) 2016 nexB Inc. and others. All rights reserved. # http://nexb.com and https://github.com/nexB/scancode-toolkit/ # The ScanCode software is licensed under the Apache License version 2.0. # Data generated with ScanCode require an acknowledgment. @@ -52,9 +52,6 @@ MIN_MATCH_LENGTH = 4 MIN_MATCH_HIGH_LENGTH = 3 -# maximum number of non-matching tokens that can be skipped -MAX_GAP_SKIP = 15 - # maximum distance between two matches to merge MAX_DIST = 120 diff --git a/src/licensedcode/cache.py b/src/licensedcode/cache.py index 780207361cd..a480674f618 100644 --- a/src/licensedcode/cache.py +++ b/src/licensedcode/cache.py @@ -181,8 +181,7 @@ def get(self, query_run): qrs = query_run.start qre = query_run.end - qlbp = query_run.line_by_pos - return [lm.rebase(qrs, qre, qlbp, MATCH_CACHE) for lm in cached] + return [lm.rebase(qrs, qre, MATCH_CACHE) for lm in cached] def put(self, query_run, matches): """ diff --git a/src/licensedcode/data/licenses/day-spec.LICENSE b/src/licensedcode/data/licenses/day-spec.LICENSE index b53399feafc..9a39351f462 100644 --- a/src/licensedcode/data/licenses/day-spec.LICENSE +++ b/src/licensedcode/data/licenses/day-spec.LICENSE @@ -1,3 +1,10 @@ +[Day Specification License] + +Day Management AG ("Licensor") is willing to license this specification +to you ONLY UPON THE CONDITION THAT YOU ACCEPT ALL OF THE TERMS CONTAINED +IN THIS LICENSE AGREEMENT ("Agreement"). Please read the terms and +conditions of this Agreement carefully. + Content Repository for JavaTM Technology API Specification ("Specification") Version: 1.0 Status: FCS diff --git a/src/licensedcode/data/licenses/tatu-ylonen.LICENSE b/src/licensedcode/data/licenses/tatu-ylonen.LICENSE index a165f7bd543..12e57fba136 100644 --- a/src/licensedcode/data/licenses/tatu-ylonen.LICENSE +++ b/src/licensedcode/data/licenses/tatu-ylonen.LICENSE @@ -1,11 +1,5 @@ Tatu Ylonen License -* Author: Tatu Ylonen - * Copyright (c) 1995 Tatu Ylonen , Espoo, Finland - * All rights reserved - * Versions of malloc and friends that check their results, and never return - * failure (they call fatal if they encounter an error). - * * As far as I am concerned, the code I have written for this software * can be used freely for any purpose. Any derived versions of this * software must be clearly marked as such, and if the derived work is diff --git a/src/licensedcode/data/licenses/zlib.SPDX b/src/licensedcode/data/licenses/zlib.SPDX deleted file mode 100644 index c839fc499d8..00000000000 --- a/src/licensedcode/data/licenses/zlib.SPDX +++ /dev/null @@ -1,20 +0,0 @@ -zlib License - - -This software is provided 'as-is', without any express or implied warranty. In no -event will the authors be held liable for any damages arising from the use of this -software. - -Permission is granted to anyone to use this software for any purpose, including -commercial applications, and to alter it and redistribute it freely, subject to the -following restrictions: - -1. The origin of this software must not be misrepresented; you must not claim that - you wrote the original software. If you use this software in a product, an - acknowledgment in the product documentation would be appreciated but is not - required. - -2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - -3. This notice may not be removed or altered from any source distribution. diff --git a/src/licensedcode/data/rules/3pp-proprietary-08_3.yml b/src/licensedcode/data/rules/3pp-proprietary-08_3.yml index d7de9f9dd26..13d574e6119 100644 --- a/src/licensedcode/data/rules/3pp-proprietary-08_3.yml +++ b/src/licensedcode/data/rules/3pp-proprietary-08_3.yml @@ -1,3 +1,3 @@ licenses: - 3pp-proprietary-08 -solid: yes \ No newline at end of file +minimum_score: 100 \ No newline at end of file diff --git a/src/licensedcode/data/rules/adobe-scl_2.yml b/src/licensedcode/data/rules/adobe-scl_2.yml index 048244476e5..d565481e996 100644 --- a/src/licensedcode/data/rules/adobe-scl_2.yml +++ b/src/licensedcode/data/rules/adobe-scl_2.yml @@ -1,3 +1,3 @@ licenses: - adobe-scl -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/afl-2.1_or_bsd-new_4.yml b/src/licensedcode/data/rules/afl-2.1_or_bsd-new_4.yml index 69d6f9f9be8..83493bac913 100644 --- a/src/licensedcode/data/rules/afl-2.1_or_bsd-new_4.yml +++ b/src/licensedcode/data/rules/afl-2.1_or_bsd-new_4.yml @@ -1,5 +1,5 @@ licenses: - afl-2.1 - bsd-new -solid: yes +minimum_score: 100 license_choice: yes diff --git a/src/licensedcode/data/rules/apache-1.1_31.yml b/src/licensedcode/data/rules/apache-1.1_31.yml index b9094bbcf3f..2766893887d 100644 --- a/src/licensedcode/data/rules/apache-1.1_31.yml +++ b/src/licensedcode/data/rules/apache-1.1_31.yml @@ -1,6 +1,6 @@ licenses: - apache-1.1 -solid: yes +minimum_score: 100 notes: > Notice found in http://apache.org/licenses/LICENSE-1.1 but it is really specific to an Apache Software Foundation component. diff --git a/src/licensedcode/data/rules/apache-1.1_36.yml b/src/licensedcode/data/rules/apache-1.1_36.yml index a1ebeb04d29..2b91d8213ab 100644 --- a/src/licensedcode/data/rules/apache-1.1_36.yml +++ b/src/licensedcode/data/rules/apache-1.1_36.yml @@ -1,4 +1,4 @@ licenses: - apache-1.1 -solid: yes +minimum_score: 100 notes: Old, original Apache 1.1 URL diff --git a/src/licensedcode/data/rules/apache-1.1_49.yml b/src/licensedcode/data/rules/apache-1.1_49.yml index ec0956094f7..75e65b17ea3 100644 --- a/src/licensedcode/data/rules/apache-1.1_49.yml +++ b/src/licensedcode/data/rules/apache-1.1_49.yml @@ -1,3 +1,3 @@ licenses: - apache-1.1 -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/apache-2.0_10.yml b/src/licensedcode/data/rules/apache-2.0_10.yml index ef997b1b6db..da2cee2228a 100644 --- a/src/licensedcode/data/rules/apache-2.0_10.yml +++ b/src/licensedcode/data/rules/apache-2.0_10.yml @@ -1,3 +1,3 @@ licenses: - apache-2.0 -solid: yes \ No newline at end of file +minimum_score: 100 \ No newline at end of file diff --git a/src/licensedcode/data/rules/apache-2.0_35.yml b/src/licensedcode/data/rules/apache-2.0_35.yml index c2609b0f059..56486b43f46 100644 --- a/src/licensedcode/data/rules/apache-2.0_35.yml +++ b/src/licensedcode/data/rules/apache-2.0_35.yml @@ -1,3 +1,3 @@ licenses: - apache-2.0 -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/apache-2.0_39.yml b/src/licensedcode/data/rules/apache-2.0_39.yml index c2609b0f059..56486b43f46 100644 --- a/src/licensedcode/data/rules/apache-2.0_39.yml +++ b/src/licensedcode/data/rules/apache-2.0_39.yml @@ -1,3 +1,3 @@ licenses: - apache-2.0 -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/apache-2.0_51.yml b/src/licensedcode/data/rules/apache-2.0_51.yml index c2609b0f059..56486b43f46 100644 --- a/src/licensedcode/data/rules/apache-2.0_51.yml +++ b/src/licensedcode/data/rules/apache-2.0_51.yml @@ -1,3 +1,3 @@ licenses: - apache-2.0 -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/apache.yml b/src/licensedcode/data/rules/apache.yml index c2609b0f059..56486b43f46 100644 --- a/src/licensedcode/data/rules/apache.yml +++ b/src/licensedcode/data/rules/apache.yml @@ -1,3 +1,3 @@ licenses: - apache-2.0 -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/apache_3.yml b/src/licensedcode/data/rules/apache_3.yml index ef997b1b6db..da2cee2228a 100644 --- a/src/licensedcode/data/rules/apache_3.yml +++ b/src/licensedcode/data/rules/apache_3.yml @@ -1,3 +1,3 @@ licenses: - apache-2.0 -solid: yes \ No newline at end of file +minimum_score: 100 \ No newline at end of file diff --git a/src/licensedcode/data/rules/bitzi-pd.yml b/src/licensedcode/data/rules/bitzi-pd.yml index 96f1566e622..962fdc01f63 100644 --- a/src/licensedcode/data/rules/bitzi-pd.yml +++ b/src/licensedcode/data/rules/bitzi-pd.yml @@ -1,3 +1,3 @@ licenses: - bitzi-pd -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/boost-original.yml b/src/licensedcode/data/rules/boost-original.yml index 40549d1892c..d406b75cf7c 100644 --- a/src/licensedcode/data/rules/boost-original.yml +++ b/src/licensedcode/data/rules/boost-original.yml @@ -1,3 +1,3 @@ licenses: - boost-original -solid: yes \ No newline at end of file +minimum_score: 100 \ No newline at end of file diff --git a/src/licensedcode/data/rules/bsd-2-clause-freebsd_5.yml b/src/licensedcode/data/rules/bsd-2-clause-freebsd_5.yml index b4e3bd13567..41edcf6fd29 100644 --- a/src/licensedcode/data/rules/bsd-2-clause-freebsd_5.yml +++ b/src/licensedcode/data/rules/bsd-2-clause-freebsd_5.yml @@ -1,4 +1,4 @@ licenses: - bsd-2-clause-freebsd -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/bsd-new_106.yml b/src/licensedcode/data/rules/bsd-new_106.yml index cf2e45bc286..a9042f5376d 100644 --- a/src/licensedcode/data/rules/bsd-new_106.yml +++ b/src/licensedcode/data/rules/bsd-new_106.yml @@ -1,3 +1,3 @@ licenses: - bsd-new -solid: yes \ No newline at end of file +minimum_score: 100 \ No newline at end of file diff --git a/src/licensedcode/data/rules/bsd-new_108.yml b/src/licensedcode/data/rules/bsd-new_108.yml index cf2e45bc286..a9042f5376d 100644 --- a/src/licensedcode/data/rules/bsd-new_108.yml +++ b/src/licensedcode/data/rules/bsd-new_108.yml @@ -1,3 +1,3 @@ licenses: - bsd-new -solid: yes \ No newline at end of file +minimum_score: 100 \ No newline at end of file diff --git a/src/licensedcode/data/rules/bsd-new_109.yml b/src/licensedcode/data/rules/bsd-new_109.yml index cf2e45bc286..a9042f5376d 100644 --- a/src/licensedcode/data/rules/bsd-new_109.yml +++ b/src/licensedcode/data/rules/bsd-new_109.yml @@ -1,3 +1,3 @@ licenses: - bsd-new -solid: yes \ No newline at end of file +minimum_score: 100 \ No newline at end of file diff --git a/src/licensedcode/data/rules/bsd-new_111.yml b/src/licensedcode/data/rules/bsd-new_111.yml index cb5d7fc7341..d1b31ea5251 100644 --- a/src/licensedcode/data/rules/bsd-new_111.yml +++ b/src/licensedcode/data/rules/bsd-new_111.yml @@ -1,3 +1,3 @@ licenses: - bsd-new -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/bsd-new_116.yml b/src/licensedcode/data/rules/bsd-new_116.yml index cf2e45bc286..a9042f5376d 100644 --- a/src/licensedcode/data/rules/bsd-new_116.yml +++ b/src/licensedcode/data/rules/bsd-new_116.yml @@ -1,3 +1,3 @@ licenses: - bsd-new -solid: yes \ No newline at end of file +minimum_score: 100 \ No newline at end of file diff --git a/src/licensedcode/data/rules/bsd-new_119.yml b/src/licensedcode/data/rules/bsd-new_119.yml index cf2e45bc286..a9042f5376d 100644 --- a/src/licensedcode/data/rules/bsd-new_119.yml +++ b/src/licensedcode/data/rules/bsd-new_119.yml @@ -1,3 +1,3 @@ licenses: - bsd-new -solid: yes \ No newline at end of file +minimum_score: 100 \ No newline at end of file diff --git a/src/licensedcode/data/rules/bsd-new_or_gpl-2.0_3.yml b/src/licensedcode/data/rules/bsd-new_or_gpl-2.0_3.yml index dd189015a9f..23cbffe5c0a 100644 --- a/src/licensedcode/data/rules/bsd-new_or_gpl-2.0_3.yml +++ b/src/licensedcode/data/rules/bsd-new_or_gpl-2.0_3.yml @@ -1,7 +1,7 @@ licenses: - bsd-new - gpl-2.0 -solid: yes +minimum_score: 100 license: bsd-new or gpl-2.0 license_choice: yes notes: LKM with Dual BSD/GPL diff --git a/src/licensedcode/data/rules/bsd-new_or_gpl-2.0_4.yml b/src/licensedcode/data/rules/bsd-new_or_gpl-2.0_4.yml index dd189015a9f..23cbffe5c0a 100644 --- a/src/licensedcode/data/rules/bsd-new_or_gpl-2.0_4.yml +++ b/src/licensedcode/data/rules/bsd-new_or_gpl-2.0_4.yml @@ -1,7 +1,7 @@ licenses: - bsd-new - gpl-2.0 -solid: yes +minimum_score: 100 license: bsd-new or gpl-2.0 license_choice: yes notes: LKM with Dual BSD/GPL diff --git a/src/licensedcode/data/rules/bsd-new_zlib.yml b/src/licensedcode/data/rules/bsd-new_zlib.yml index a2bf27879c5..1b74b59feb5 100644 --- a/src/licensedcode/data/rules/bsd-new_zlib.yml +++ b/src/licensedcode/data/rules/bsd-new_zlib.yml @@ -1,4 +1,4 @@ licenses: - bsd-new -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/bsd-original-uc_6.yml b/src/licensedcode/data/rules/bsd-original-uc_6.yml index d1f5e3ecb09..a9ec6b9e90e 100644 --- a/src/licensedcode/data/rules/bsd-original-uc_6.yml +++ b/src/licensedcode/data/rules/bsd-original-uc_6.yml @@ -1,3 +1,3 @@ licenses: - bsd-original-uc -solid: yes \ No newline at end of file +minimum_score: 100 \ No newline at end of file diff --git a/src/licensedcode/data/rules/false_positive14.RULE b/src/licensedcode/data/rules/false_positive14.RULE deleted file mode 100644 index 7557ff52741..00000000000 --- a/src/licensedcode/data/rules/false_positive14.RULE +++ /dev/null @@ -1 +0,0 @@ -mechanism generally accepted in the software development community for the electronic transfer of data \ No newline at end of file diff --git a/src/licensedcode/data/rules/false_positive14.yml b/src/licensedcode/data/rules/false_positive14.yml deleted file mode 100644 index 7be15eec285..00000000000 --- a/src/licensedcode/data/rules/false_positive14.yml +++ /dev/null @@ -1,2 +0,0 @@ -false_positive: yes - diff --git a/src/licensedcode/data/rules/gfdl-1.1_14.yml b/src/licensedcode/data/rules/gfdl-1.1_14.yml index 90f2caebb86..a54312a0bd1 100644 --- a/src/licensedcode/data/rules/gfdl-1.1_14.yml +++ b/src/licensedcode/data/rules/gfdl-1.1_14.yml @@ -1,3 +1,3 @@ licenses: - gfdl-1.1 -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/gpl-2.0-plus_67.yml b/src/licensedcode/data/rules/gpl-2.0-plus_67.yml index 1915bf86ca3..e4bde416fd3 100644 --- a/src/licensedcode/data/rules/gpl-2.0-plus_67.yml +++ b/src/licensedcode/data/rules/gpl-2.0-plus_67.yml @@ -1,3 +1,3 @@ licenses: - gpl-2.0-plus -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/gpl-2.0-plus_74.yml b/src/licensedcode/data/rules/gpl-2.0-plus_74.yml index 1915bf86ca3..e4bde416fd3 100644 --- a/src/licensedcode/data/rules/gpl-2.0-plus_74.yml +++ b/src/licensedcode/data/rules/gpl-2.0-plus_74.yml @@ -1,3 +1,3 @@ licenses: - gpl-2.0-plus -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/gpl-2.0-plus_75.yml b/src/licensedcode/data/rules/gpl-2.0-plus_75.yml index 1915bf86ca3..e4bde416fd3 100644 --- a/src/licensedcode/data/rules/gpl-2.0-plus_75.yml +++ b/src/licensedcode/data/rules/gpl-2.0-plus_75.yml @@ -1,3 +1,3 @@ licenses: - gpl-2.0-plus -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/gpl-2.0-plus_77.yml b/src/licensedcode/data/rules/gpl-2.0-plus_77.yml index d625788d26c..a0fd63189b8 100644 --- a/src/licensedcode/data/rules/gpl-2.0-plus_77.yml +++ b/src/licensedcode/data/rules/gpl-2.0-plus_77.yml @@ -1,3 +1,3 @@ licenses: - gpl-2.0-plus -solid: yes \ No newline at end of file +minimum_score: 100 \ No newline at end of file diff --git a/src/licensedcode/data/rules/gpl-2.0_37.yml b/src/licensedcode/data/rules/gpl-2.0_37.yml index 7599ac39785..c1955cd16b9 100644 --- a/src/licensedcode/data/rules/gpl-2.0_37.yml +++ b/src/licensedcode/data/rules/gpl-2.0_37.yml @@ -1,3 +1,3 @@ licenses: - gpl-2.0 -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/gpl-2.0_39.yml b/src/licensedcode/data/rules/gpl-2.0_39.yml index 0c29a2e481a..c1955cd16b9 100644 --- a/src/licensedcode/data/rules/gpl-2.0_39.yml +++ b/src/licensedcode/data/rules/gpl-2.0_39.yml @@ -1,4 +1,3 @@ licenses: - gpl-2.0 -solid: yes - +minimum_score: 100 diff --git a/src/licensedcode/data/rules/gpl-2.0_43.yml b/src/licensedcode/data/rules/gpl-2.0_43.yml index cb8132cf574..748df6d23d4 100644 --- a/src/licensedcode/data/rules/gpl-2.0_43.yml +++ b/src/licensedcode/data/rules/gpl-2.0_43.yml @@ -1,4 +1,4 @@ licenses: - gpl-2.0 notes: MODULE_LICENSE in LKMs. FIXME this should be a plain GPL no version -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/gpl-2.0_83.yml b/src/licensedcode/data/rules/gpl-2.0_83.yml index 570a25a74a1..1972964a4a2 100644 --- a/src/licensedcode/data/rules/gpl-2.0_83.yml +++ b/src/licensedcode/data/rules/gpl-2.0_83.yml @@ -1,4 +1,4 @@ licenses: - gpl-2.0 notes: Lizartech gpl notice, rather complex -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/gpl-2.0_86.yml b/src/licensedcode/data/rules/gpl-2.0_86.yml index 684599a191c..8c24c098e33 100644 --- a/src/licensedcode/data/rules/gpl-2.0_86.yml +++ b/src/licensedcode/data/rules/gpl-2.0_86.yml @@ -1,3 +1,3 @@ licenses: - gpl-2.0 -solid: yes \ No newline at end of file +minimum_score: 100 \ No newline at end of file diff --git a/src/licensedcode/data/rules/gpl-3.0_1.yml b/src/licensedcode/data/rules/gpl-3.0_1.yml index e91f4d5a264..86ef846e508 100644 --- a/src/licensedcode/data/rules/gpl-3.0_1.yml +++ b/src/licensedcode/data/rules/gpl-3.0_1.yml @@ -1,3 +1,3 @@ licenses: - gpl-3.0 -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/gpl-3.0_19.yml b/src/licensedcode/data/rules/gpl-3.0_19.yml index e91f4d5a264..86ef846e508 100644 --- a/src/licensedcode/data/rules/gpl-3.0_19.yml +++ b/src/licensedcode/data/rules/gpl-3.0_19.yml @@ -1,3 +1,3 @@ licenses: - gpl-3.0 -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/gpl-3.0_22.yml b/src/licensedcode/data/rules/gpl-3.0_22.yml index c139fda13c7..d514423e15c 100644 --- a/src/licensedcode/data/rules/gpl-3.0_22.yml +++ b/src/licensedcode/data/rules/gpl-3.0_22.yml @@ -1,4 +1,4 @@ licenses: - gpl-3.0 -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/gpl-3.0_24.yml b/src/licensedcode/data/rules/gpl-3.0_24.yml index e91f4d5a264..86ef846e508 100644 --- a/src/licensedcode/data/rules/gpl-3.0_24.yml +++ b/src/licensedcode/data/rules/gpl-3.0_24.yml @@ -1,3 +1,3 @@ licenses: - gpl-3.0 -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/gpl.yml b/src/licensedcode/data/rules/gpl.yml index c1bb8d5571e..a90505b3cae 100644 --- a/src/licensedcode/data/rules/gpl.yml +++ b/src/licensedcode/data/rules/gpl.yml @@ -1,4 +1,4 @@ licenses: - gpl -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/gpl_42.yml b/src/licensedcode/data/rules/gpl_42.yml index 893d925ba13..c3a7fdc9688 100644 --- a/src/licensedcode/data/rules/gpl_42.yml +++ b/src/licensedcode/data/rules/gpl_42.yml @@ -1,3 +1,3 @@ licenses: - gpl -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/gpl_63.yml b/src/licensedcode/data/rules/gpl_63.yml index fcd8f53ad83..e078b661d8c 100644 --- a/src/licensedcode/data/rules/gpl_63.yml +++ b/src/licensedcode/data/rules/gpl_63.yml @@ -1,4 +1,4 @@ licenses: - gpl -solid: yes +minimum_score: 100 notes: No version gpl declaration diff --git a/src/licensedcode/data/rules/gpl_73.yml b/src/licensedcode/data/rules/gpl_73.yml index ea03681b375..318e198c93e 100644 --- a/src/licensedcode/data/rules/gpl_73.yml +++ b/src/licensedcode/data/rules/gpl_73.yml @@ -1,3 +1,3 @@ licenses: - gpl -solid: yes \ No newline at end of file +minimum_score: 100 \ No newline at end of file diff --git a/src/licensedcode/data/rules/gpl_78.yml b/src/licensedcode/data/rules/gpl_78.yml index cfeb7064495..c3a7fdc9688 100644 --- a/src/licensedcode/data/rules/gpl_78.yml +++ b/src/licensedcode/data/rules/gpl_78.yml @@ -1,3 +1,3 @@ licenses: - gpl -solid: +minimum_score: 100 diff --git a/src/licensedcode/data/rules/ietf-trust.yml b/src/licensedcode/data/rules/ietf-trust.yml index 63fdf8b6062..e055c8d9679 100644 --- a/src/licensedcode/data/rules/ietf-trust.yml +++ b/src/licensedcode/data/rules/ietf-trust.yml @@ -1,3 +1,3 @@ licenses: - ietf-trust -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/ietf-trust_3.yml b/src/licensedcode/data/rules/ietf-trust_3.yml index 63fdf8b6062..e055c8d9679 100644 --- a/src/licensedcode/data/rules/ietf-trust_3.yml +++ b/src/licensedcode/data/rules/ietf-trust_3.yml @@ -1,3 +1,3 @@ licenses: - ietf-trust -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/ijg_1.yml b/src/licensedcode/data/rules/ijg_1.yml index 4de478903f0..a1bc0c54385 100644 --- a/src/licensedcode/data/rules/ijg_1.yml +++ b/src/licensedcode/data/rules/ijg_1.yml @@ -1,3 +1,3 @@ licenses: - ijg -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/ijg_2.yml b/src/licensedcode/data/rules/ijg_2.yml index 4de478903f0..a1bc0c54385 100644 --- a/src/licensedcode/data/rules/ijg_2.yml +++ b/src/licensedcode/data/rules/ijg_2.yml @@ -1,3 +1,3 @@ licenses: - ijg -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/indiana-extreme.yml b/src/licensedcode/data/rules/indiana-extreme.yml index 9afd4dbca9a..0d56c27beca 100644 --- a/src/licensedcode/data/rules/indiana-extreme.yml +++ b/src/licensedcode/data/rules/indiana-extreme.yml @@ -1,3 +1,3 @@ licenses: - indiana-extreme -solid: yes \ No newline at end of file +minimum_score: 100 \ No newline at end of file diff --git a/src/licensedcode/data/rules/lgpl-2.0_10.yml b/src/licensedcode/data/rules/lgpl-2.0_10.yml index 87ad50e7fff..597559ee5cc 100644 --- a/src/licensedcode/data/rules/lgpl-2.0_10.yml +++ b/src/licensedcode/data/rules/lgpl-2.0_10.yml @@ -1,3 +1,3 @@ licenses: - lgpl-2.0 -solid: yes \ No newline at end of file +minimum_score: 100 \ No newline at end of file diff --git a/src/licensedcode/data/rules/lgpl-2.1-plus_26.yml b/src/licensedcode/data/rules/lgpl-2.1-plus_26.yml index 04857122f7e..5454cffa94e 100644 --- a/src/licensedcode/data/rules/lgpl-2.1-plus_26.yml +++ b/src/licensedcode/data/rules/lgpl-2.1-plus_26.yml @@ -1,4 +1,4 @@ licenses: - lgpl-2.1 -solid: yes +minimum_score: 100 notes: http://creativecommons.org/licenses/LGPL/2.1/ diff --git a/src/licensedcode/data/rules/lgpl-2.1_15.yml b/src/licensedcode/data/rules/lgpl-2.1_15.yml index bf7be83ed03..a7ed38c39a5 100644 --- a/src/licensedcode/data/rules/lgpl-2.1_15.yml +++ b/src/licensedcode/data/rules/lgpl-2.1_15.yml @@ -1,4 +1,4 @@ licenses: - lgpl-2.1 -solid: yes +minimum_score: 100 notes: TinyMCE license url diff --git a/src/licensedcode/data/rules/lgpl-2.1_36.yml b/src/licensedcode/data/rules/lgpl-2.1_36.yml index 4958111e9e3..50ab2d3c2fc 100644 --- a/src/licensedcode/data/rules/lgpl-2.1_36.yml +++ b/src/licensedcode/data/rules/lgpl-2.1_36.yml @@ -1,3 +1,3 @@ licenses: - lgpl-2.1 -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/lgpl-2.1_41.yml b/src/licensedcode/data/rules/lgpl-2.1_41.yml index b7bf4d0b8f8..515a1df94f4 100644 --- a/src/licensedcode/data/rules/lgpl-2.1_41.yml +++ b/src/licensedcode/data/rules/lgpl-2.1_41.yml @@ -1,3 +1,3 @@ licenses: - lgpl-2.1 -solid: yes \ No newline at end of file +minimum_score: 100 \ No newline at end of file diff --git a/src/licensedcode/data/rules/lgpl_11.yml b/src/licensedcode/data/rules/lgpl_11.yml index 2212e640787..34e3e9ac262 100644 --- a/src/licensedcode/data/rules/lgpl_11.yml +++ b/src/licensedcode/data/rules/lgpl_11.yml @@ -1,4 +1,4 @@ licenses: - lgpl notes: LGPL libgomp -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/lgpl_24.yml b/src/licensedcode/data/rules/lgpl_24.yml index f3785ac8085..36def2daf34 100644 --- a/src/licensedcode/data/rules/lgpl_24.yml +++ b/src/licensedcode/data/rules/lgpl_24.yml @@ -1,3 +1,3 @@ licenses: - lgpl -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/lgpl_27.yml b/src/licensedcode/data/rules/lgpl_27.yml index f3785ac8085..36def2daf34 100644 --- a/src/licensedcode/data/rules/lgpl_27.yml +++ b/src/licensedcode/data/rules/lgpl_27.yml @@ -1,3 +1,3 @@ licenses: - lgpl -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/lgpl_28.yml b/src/licensedcode/data/rules/lgpl_28.yml index f3785ac8085..36def2daf34 100644 --- a/src/licensedcode/data/rules/lgpl_28.yml +++ b/src/licensedcode/data/rules/lgpl_28.yml @@ -1,3 +1,3 @@ licenses: - lgpl -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/lgpl_29.yml b/src/licensedcode/data/rules/lgpl_29.yml index f3785ac8085..36def2daf34 100644 --- a/src/licensedcode/data/rules/lgpl_29.yml +++ b/src/licensedcode/data/rules/lgpl_29.yml @@ -1,3 +1,3 @@ licenses: - lgpl -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/lilo_4.yml b/src/licensedcode/data/rules/lilo_4.yml index 6cefd20515c..c4e5de7c0b7 100644 --- a/src/licensedcode/data/rules/lilo_4.yml +++ b/src/licensedcode/data/rules/lilo_4.yml @@ -1,3 +1,3 @@ licenses: - lilo -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/mit_14.yml b/src/licensedcode/data/rules/mit_14.yml index 7b32dbd05a2..2ef0ccb182e 100644 --- a/src/licensedcode/data/rules/mit_14.yml +++ b/src/licensedcode/data/rules/mit_14.yml @@ -1,3 +1,3 @@ licenses: - mit -solid: yes \ No newline at end of file +minimum_score: 100 \ No newline at end of file diff --git a/src/licensedcode/data/rules/mit_34.yml b/src/licensedcode/data/rules/mit_34.yml index 7b32dbd05a2..2ef0ccb182e 100644 --- a/src/licensedcode/data/rules/mit_34.yml +++ b/src/licensedcode/data/rules/mit_34.yml @@ -1,3 +1,3 @@ licenses: - mit -solid: yes \ No newline at end of file +minimum_score: 100 \ No newline at end of file diff --git a/src/licensedcode/data/rules/mit_35.yml b/src/licensedcode/data/rules/mit_35.yml index 7b32dbd05a2..2ef0ccb182e 100644 --- a/src/licensedcode/data/rules/mit_35.yml +++ b/src/licensedcode/data/rules/mit_35.yml @@ -1,3 +1,3 @@ licenses: - mit -solid: yes \ No newline at end of file +minimum_score: 100 \ No newline at end of file diff --git a/src/licensedcode/data/rules/mit_36.yml b/src/licensedcode/data/rules/mit_36.yml index 7b32dbd05a2..2ef0ccb182e 100644 --- a/src/licensedcode/data/rules/mit_36.yml +++ b/src/licensedcode/data/rules/mit_36.yml @@ -1,3 +1,3 @@ licenses: - mit -solid: yes \ No newline at end of file +minimum_score: 100 \ No newline at end of file diff --git a/src/licensedcode/data/rules/mit_37.yml b/src/licensedcode/data/rules/mit_37.yml index 7b32dbd05a2..2ef0ccb182e 100644 --- a/src/licensedcode/data/rules/mit_37.yml +++ b/src/licensedcode/data/rules/mit_37.yml @@ -1,3 +1,3 @@ licenses: - mit -solid: yes \ No newline at end of file +minimum_score: 100 \ No newline at end of file diff --git a/src/licensedcode/data/rules/mit_38.yml b/src/licensedcode/data/rules/mit_38.yml index 7b32dbd05a2..2ef0ccb182e 100644 --- a/src/licensedcode/data/rules/mit_38.yml +++ b/src/licensedcode/data/rules/mit_38.yml @@ -1,3 +1,3 @@ licenses: - mit -solid: yes \ No newline at end of file +minimum_score: 100 \ No newline at end of file diff --git a/src/licensedcode/data/rules/mit_39.yml b/src/licensedcode/data/rules/mit_39.yml index 7b32dbd05a2..2ef0ccb182e 100644 --- a/src/licensedcode/data/rules/mit_39.yml +++ b/src/licensedcode/data/rules/mit_39.yml @@ -1,3 +1,3 @@ licenses: - mit -solid: yes \ No newline at end of file +minimum_score: 100 \ No newline at end of file diff --git a/src/licensedcode/data/rules/mit_40.yml b/src/licensedcode/data/rules/mit_40.yml index 7b32dbd05a2..2ef0ccb182e 100644 --- a/src/licensedcode/data/rules/mit_40.yml +++ b/src/licensedcode/data/rules/mit_40.yml @@ -1,3 +1,3 @@ licenses: - mit -solid: yes \ No newline at end of file +minimum_score: 100 \ No newline at end of file diff --git a/src/licensedcode/data/rules/mit_42.yml b/src/licensedcode/data/rules/mit_42.yml index a1f94ea6ba2..ee96b583fdf 100644 --- a/src/licensedcode/data/rules/mit_42.yml +++ b/src/licensedcode/data/rules/mit_42.yml @@ -1,3 +1,3 @@ licenses: - mit -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/mit_5.yml b/src/licensedcode/data/rules/mit_5.yml index a1f94ea6ba2..ee96b583fdf 100644 --- a/src/licensedcode/data/rules/mit_5.yml +++ b/src/licensedcode/data/rules/mit_5.yml @@ -1,3 +1,3 @@ licenses: - mit -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/mod-dav-1.1.yml b/src/licensedcode/data/rules/mod-dav-1.1.yml index 24035436e6d..23ae6149ad0 100644 --- a/src/licensedcode/data/rules/mod-dav-1.1.yml +++ b/src/licensedcode/data/rules/mod-dav-1.1.yml @@ -1,2 +1,3 @@ licenses: - mod-dav-1.0 +minimum_score: 80 diff --git a/src/licensedcode/data/rules/mozilla-gc_or_stlport-4.5.yml b/src/licensedcode/data/rules/mozilla-gc_or_stlport-4.5.yml index e1665fa41f9..f017fc72692 100644 --- a/src/licensedcode/data/rules/mozilla-gc_or_stlport-4.5.yml +++ b/src/licensedcode/data/rules/mozilla-gc_or_stlport-4.5.yml @@ -1,4 +1,4 @@ licenses: - mozilla-gc notes: this is an ambiguous text used uniquely in two very similar licenses stlport-4.5 and mozilla-gc -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/mpl-1.1_16.yml b/src/licensedcode/data/rules/mpl-1.1_16.yml index e4bc1c1aee0..509e08d6446 100644 --- a/src/licensedcode/data/rules/mpl-1.1_16.yml +++ b/src/licensedcode/data/rules/mpl-1.1_16.yml @@ -1,3 +1,3 @@ licenses: - mpl-1.1 -solid: yes \ No newline at end of file +minimum_score: 100 \ No newline at end of file diff --git a/src/licensedcode/data/rules/non-commercial.yml b/src/licensedcode/data/rules/non-commercial.yml index 16a416c66a9..cd8631c9d14 100644 --- a/src/licensedcode/data/rules/non-commercial.yml +++ b/src/licensedcode/data/rules/non-commercial.yml @@ -1,3 +1,3 @@ licenses: - non-commercial -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/not-a-license_29.RULE b/src/licensedcode/data/rules/not-a-license_29.RULE new file mode 100644 index 00000000000..73b2b199984 --- /dev/null +++ b/src/licensedcode/data/rules/not-a-license_29.RULE @@ -0,0 +1 @@ +Load gpL data diff --git a/src/licensedcode/data/rules/not-a-license_29.yml b/src/licensedcode/data/rules/not-a-license_29.yml new file mode 100644 index 00000000000..3644ba09fc5 --- /dev/null +++ b/src/licensedcode/data/rules/not-a-license_29.yml @@ -0,0 +1 @@ +notes: Found in some compiled Go binaries diff --git a/src/licensedcode/data/rules/npl-1.1_5.yml b/src/licensedcode/data/rules/npl-1.1_5.yml index eb8fccb0c4e..fda94180e35 100644 --- a/src/licensedcode/data/rules/npl-1.1_5.yml +++ b/src/licensedcode/data/rules/npl-1.1_5.yml @@ -1,3 +1,3 @@ licenses: - npl-1.1 -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/oracle-web-sites-tou.yml b/src/licensedcode/data/rules/oracle-web-sites-tou.yml index 3283701d6bf..92f9244c4cf 100644 --- a/src/licensedcode/data/rules/oracle-web-sites-tou.yml +++ b/src/licensedcode/data/rules/oracle-web-sites-tou.yml @@ -1,3 +1,3 @@ licenses: - oracle-web-sites-tou -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/proprietary.yml b/src/licensedcode/data/rules/proprietary.yml index f33c8b7d884..c5ad4507228 100644 --- a/src/licensedcode/data/rules/proprietary.yml +++ b/src/licensedcode/data/rules/proprietary.yml @@ -1,4 +1,4 @@ licenses: - proprietary -solid: yes +minimum_score: 100 notes: Found in some old debian specs diff --git a/src/licensedcode/data/rules/proprietary_13.yml b/src/licensedcode/data/rules/proprietary_13.yml index d05306e338b..856e5f7f9b6 100644 --- a/src/licensedcode/data/rules/proprietary_13.yml +++ b/src/licensedcode/data/rules/proprietary_13.yml @@ -1,3 +1,3 @@ licenses: - proprietary -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/public-domain.yml b/src/licensedcode/data/rules/public-domain.yml index d0794ade1a3..12162486ac6 100644 --- a/src/licensedcode/data/rules/public-domain.yml +++ b/src/licensedcode/data/rules/public-domain.yml @@ -1,3 +1,3 @@ licenses: - public-domain -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/public-domain_14.yml b/src/licensedcode/data/rules/public-domain_14.yml index d0794ade1a3..12162486ac6 100644 --- a/src/licensedcode/data/rules/public-domain_14.yml +++ b/src/licensedcode/data/rules/public-domain_14.yml @@ -1,3 +1,3 @@ licenses: - public-domain -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/public-domain_15.yml b/src/licensedcode/data/rules/public-domain_15.yml index d0794ade1a3..12162486ac6 100644 --- a/src/licensedcode/data/rules/public-domain_15.yml +++ b/src/licensedcode/data/rules/public-domain_15.yml @@ -1,3 +1,3 @@ licenses: - public-domain -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/public-domain_16.yml b/src/licensedcode/data/rules/public-domain_16.yml index d0794ade1a3..12162486ac6 100644 --- a/src/licensedcode/data/rules/public-domain_16.yml +++ b/src/licensedcode/data/rules/public-domain_16.yml @@ -1,3 +1,3 @@ licenses: - public-domain -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/public-domain_17.yml b/src/licensedcode/data/rules/public-domain_17.yml index c2ed6743322..6575fee6fee 100644 --- a/src/licensedcode/data/rules/public-domain_17.yml +++ b/src/licensedcode/data/rules/public-domain_17.yml @@ -1,4 +1,4 @@ licenses: - public-domain notes: Moby declaration -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/public-domain_18.yml b/src/licensedcode/data/rules/public-domain_18.yml index d0794ade1a3..12162486ac6 100644 --- a/src/licensedcode/data/rules/public-domain_18.yml +++ b/src/licensedcode/data/rules/public-domain_18.yml @@ -1,3 +1,3 @@ licenses: - public-domain -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/public-domain_2.yml b/src/licensedcode/data/rules/public-domain_2.yml index d0794ade1a3..12162486ac6 100644 --- a/src/licensedcode/data/rules/public-domain_2.yml +++ b/src/licensedcode/data/rules/public-domain_2.yml @@ -1,3 +1,3 @@ licenses: - public-domain -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/public-domain_20.yml b/src/licensedcode/data/rules/public-domain_20.yml index d0794ade1a3..12162486ac6 100644 --- a/src/licensedcode/data/rules/public-domain_20.yml +++ b/src/licensedcode/data/rules/public-domain_20.yml @@ -1,3 +1,3 @@ licenses: - public-domain -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/public-domain_21.yml b/src/licensedcode/data/rules/public-domain_21.yml index d0794ade1a3..12162486ac6 100644 --- a/src/licensedcode/data/rules/public-domain_21.yml +++ b/src/licensedcode/data/rules/public-domain_21.yml @@ -1,3 +1,3 @@ licenses: - public-domain -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/public-domain_22.yml b/src/licensedcode/data/rules/public-domain_22.yml index d0794ade1a3..12162486ac6 100644 --- a/src/licensedcode/data/rules/public-domain_22.yml +++ b/src/licensedcode/data/rules/public-domain_22.yml @@ -1,3 +1,3 @@ licenses: - public-domain -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/public-domain_23.yml b/src/licensedcode/data/rules/public-domain_23.yml index d0794ade1a3..12162486ac6 100644 --- a/src/licensedcode/data/rules/public-domain_23.yml +++ b/src/licensedcode/data/rules/public-domain_23.yml @@ -1,3 +1,3 @@ licenses: - public-domain -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/public-domain_25.yml b/src/licensedcode/data/rules/public-domain_25.yml index 123b87fc232..ca4a209a0d1 100644 --- a/src/licensedcode/data/rules/public-domain_25.yml +++ b/src/licensedcode/data/rules/public-domain_25.yml @@ -1,3 +1,3 @@ licenses: - public-domain -solid: yes \ No newline at end of file +minimum_score: 100 \ No newline at end of file diff --git a/src/licensedcode/data/rules/public-domain_26.yml b/src/licensedcode/data/rules/public-domain_26.yml index d0794ade1a3..12162486ac6 100644 --- a/src/licensedcode/data/rules/public-domain_26.yml +++ b/src/licensedcode/data/rules/public-domain_26.yml @@ -1,3 +1,3 @@ licenses: - public-domain -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/public-domain_27.yml b/src/licensedcode/data/rules/public-domain_27.yml index d0794ade1a3..12162486ac6 100644 --- a/src/licensedcode/data/rules/public-domain_27.yml +++ b/src/licensedcode/data/rules/public-domain_27.yml @@ -1,3 +1,3 @@ licenses: - public-domain -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/public-domain_29.yml b/src/licensedcode/data/rules/public-domain_29.yml index d0794ade1a3..12162486ac6 100644 --- a/src/licensedcode/data/rules/public-domain_29.yml +++ b/src/licensedcode/data/rules/public-domain_29.yml @@ -1,3 +1,3 @@ licenses: - public-domain -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/public-domain_3.yml b/src/licensedcode/data/rules/public-domain_3.yml index d0794ade1a3..12162486ac6 100644 --- a/src/licensedcode/data/rules/public-domain_3.yml +++ b/src/licensedcode/data/rules/public-domain_3.yml @@ -1,3 +1,3 @@ licenses: - public-domain -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/public-domain_32.yml b/src/licensedcode/data/rules/public-domain_32.yml index d0794ade1a3..12162486ac6 100644 --- a/src/licensedcode/data/rules/public-domain_32.yml +++ b/src/licensedcode/data/rules/public-domain_32.yml @@ -1,3 +1,3 @@ licenses: - public-domain -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/public-domain_33.yml b/src/licensedcode/data/rules/public-domain_33.yml index d0794ade1a3..12162486ac6 100644 --- a/src/licensedcode/data/rules/public-domain_33.yml +++ b/src/licensedcode/data/rules/public-domain_33.yml @@ -1,3 +1,3 @@ licenses: - public-domain -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/public-domain_5.yml b/src/licensedcode/data/rules/public-domain_5.yml index d0794ade1a3..12162486ac6 100644 --- a/src/licensedcode/data/rules/public-domain_5.yml +++ b/src/licensedcode/data/rules/public-domain_5.yml @@ -1,3 +1,3 @@ licenses: - public-domain -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/public-domain_6.yml b/src/licensedcode/data/rules/public-domain_6.yml index d0794ade1a3..12162486ac6 100644 --- a/src/licensedcode/data/rules/public-domain_6.yml +++ b/src/licensedcode/data/rules/public-domain_6.yml @@ -1,3 +1,3 @@ licenses: - public-domain -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/public-domain_7.yml b/src/licensedcode/data/rules/public-domain_7.yml index d0794ade1a3..12162486ac6 100644 --- a/src/licensedcode/data/rules/public-domain_7.yml +++ b/src/licensedcode/data/rules/public-domain_7.yml @@ -1,3 +1,3 @@ licenses: - public-domain -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/public-domain_8.yml b/src/licensedcode/data/rules/public-domain_8.yml index d0794ade1a3..12162486ac6 100644 --- a/src/licensedcode/data/rules/public-domain_8.yml +++ b/src/licensedcode/data/rules/public-domain_8.yml @@ -1,3 +1,3 @@ licenses: - public-domain -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/public-domain_9.yml b/src/licensedcode/data/rules/public-domain_9.yml index d0794ade1a3..12162486ac6 100644 --- a/src/licensedcode/data/rules/public-domain_9.yml +++ b/src/licensedcode/data/rules/public-domain_9.yml @@ -1,3 +1,3 @@ licenses: - public-domain -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/ruby_4.yml b/src/licensedcode/data/rules/ruby_4.yml index 02600d499be..1f5f47428bd 100644 --- a/src/licensedcode/data/rules/ruby_4.yml +++ b/src/licensedcode/data/rules/ruby_4.yml @@ -1,4 +1,4 @@ licenses: - ruby -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/sleepycat_1.yml b/src/licensedcode/data/rules/sleepycat_1.yml index 53093da2ee2..b9cc668b3a9 100644 --- a/src/licensedcode/data/rules/sleepycat_1.yml +++ b/src/licensedcode/data/rules/sleepycat_1.yml @@ -1,3 +1,3 @@ licenses: - sleepycat -solid: yes +minimum_score: 100 diff --git a/src/licensedcode/data/rules/zlib_1.yml b/src/licensedcode/data/rules/zlib_1.yml index 3ed97e5d891..7717731ae92 100644 --- a/src/licensedcode/data/rules/zlib_1.yml +++ b/src/licensedcode/data/rules/zlib_1.yml @@ -1,3 +1,3 @@ licenses: - zlib -solid: yes \ No newline at end of file +minimum_score: 100 \ No newline at end of file diff --git a/src/licensedcode/data/rules/zlib_8.yml b/src/licensedcode/data/rules/zlib_8.yml index 3ed97e5d891..7717731ae92 100644 --- a/src/licensedcode/data/rules/zlib_8.yml +++ b/src/licensedcode/data/rules/zlib_8.yml @@ -1,3 +1,3 @@ licenses: - zlib -solid: yes \ No newline at end of file +minimum_score: 100 \ No newline at end of file diff --git a/src/licensedcode/frequent_tokens.py b/src/licensedcode/frequent_tokens.py index 894cadd48da..8f706663904 100644 --- a/src/licensedcode/frequent_tokens.py +++ b/src/licensedcode/frequent_tokens.py @@ -3918,6 +3918,7 @@ def global_tokens_by_ranks(): engaged falling basics +basic montreal carpet rv @@ -9445,6 +9446,32 @@ def frequent_names_and_misc(): cases case precedence +com +github +invalidate +compile +containing +actual +incompatible +4d +includes +kind +download +sources +application +source +applications +service +services +allowed +modified +extend +extends +exchange +permitted +medium +offered +released preceding determine paragraph @@ -9480,6 +9507,19 @@ def frequent_names_and_misc(): indicated components component +possible +invalid +normal +resides +reside +unicode +clearly +marked +called +derived +derive +version +versions program programs developers @@ -9779,4 +9819,5 @@ def frequent_names_and_misc(): acpi ubuntu proc +divx '''.lower().split() diff --git a/src/licensedcode/index.py b/src/licensedcode/index.py index 5f975302516..c36d1052df5 100644 --- a/src/licensedcode/index.py +++ b/src/licensedcode/index.py @@ -50,6 +50,7 @@ from licensedcode.match import get_texts from licensedcode.match import merge_matches from licensedcode.match import refine_matches +from licensedcode.match import set_lines from licensedcode import match_aho from licensedcode.match_aho import exact_match @@ -405,7 +406,7 @@ def _add_rules(self, rules, _ranked_tokens=global_tokens_by_ranks): # ... or with the whole rule tokens sequence rules_automaton_add(tids=rule_token_ids, rid=rid) # ... and ngrams: compute ngrams and populate the automaton with ngrams - if USE_AHO_FRAGMENTS and not rule.is_url and not rule.solid and len(rule_token_ids) > NGRAM_LEN: + if USE_AHO_FRAGMENTS and not rule.is_url and rule.minimum_score < 100 and len(rule_token_ids) > NGRAM_LEN: all_ngrams = ngrams(rule_token_ids, ngram_length=NGRAM_LEN) selected_ngrams = select_ngrams(all_ngrams, with_pos=True) for pos, ngram in selected_ngrams: @@ -507,6 +508,7 @@ def match(self, location=None, query_string=None, min_score=0, detect_negative=T hash_matches = match_hash(self, whole_query_run) if hash_matches: self.debug_matches(hash_matches, '#match FINAL Hash matched', location, query_string) + set_lines(hash_matches, qry.line_by_pos) return hash_matches # negative rules exact matching @@ -525,7 +527,7 @@ def match(self, location=None, query_string=None, min_score=0, detect_negative=T exact_matches = exact_match(self, whole_query_run, self.rules_automaton) if TRACE_EXACT: self.debug_matches(exact_matches, ' #match: EXACT matches#:', location, query_string) - exact_matches, exact_discarded = refine_matches(exact_matches, self) + exact_matches, exact_discarded = refine_matches(exact_matches, self, query=qry) if TRACE_EXACT: self.debug_matches(exact_matches, ' #match: ===> exact matches refined') if TRACE_EXACT: self.debug_matches(exact_discarded, ' #match: ===> exact matches discarded') @@ -536,28 +538,29 @@ def match(self, location=None, query_string=None, min_score=0, detect_negative=T ####################################################################### # Per query run matching. ####################################################################### - logger_debug('#match: #QUERY RUNS:', len(qry.query_runs)) + if TRACE: logger_debug('#match: #QUERY RUNS:', len(qry.query_runs)) # check if we have some matchable left # collect qspans matched exactly e.g. with score 100% - matched_qspans = [m.qspan for m in exact_matches if m.score()==100] + # this score check is because we have provision to match fragments (unused for now) + matched_qspans = [m.qspan for m in exact_matches if m.score() == 100] # do not match futher if we do not need to if whole_query_run.is_matchable(include_low=True, qspans=matched_qspans): rules_subset = (self.regular_rids | self.small_rids) for qrnum, query_run in enumerate(qry.query_runs, 1): - logger_debug('#match: ===> processing query run #:', qrnum) + if TRACE: logger_debug('#match: ===> processing query run #:', qrnum) if not query_run.is_matchable(include_low=True): - logger_debug('#match: query_run NOT MATCHABLE') + if TRACE: logger_debug('#match: query_run NOT MATCHABLE') continue # hash match ######################### hash_matches = match_hash(self, query_run) if hash_matches: - self.debug_matches(hash_matches, ' #match Query run matches (hash)', location, query_string) + if TRACE: self.debug_matches(hash_matches, ' #match Query run matches (hash)', location, query_string) matches.extend(hash_matches) # note that we do not cache hash matches continue @@ -581,8 +584,8 @@ def match(self, location=None, query_string=None, min_score=0, detect_negative=T if TRACE_QUERY_RUN: logger_debug(' #match: query_run: number of candidates for seq match #', len(candidates)) - for candidate in candidates: - if TRACE_QUERY_RUN: logger_debug(' #match: query_run: seq matching candidate:', candidate[1]) + for candidate_num, candidate in enumerate(candidates): + if TRACE_QUERY_RUN: logger_debug(' #match: query_run: seq matching candidate#:', candidate_num, 'candidate:', candidate) start_offset = 0 while True: rule_matches = match_sequence(self, candidate, query_run, start_offset=start_offset) @@ -590,10 +593,9 @@ def match(self, location=None, query_string=None, min_score=0, detect_negative=T if not rule_matches: break else: - if TRACE_QUERY_RUN: self.debug_matches(rule_matches, ' #match: query_run: merged seq matches for candidate') + matches_end = max(m.qend for m in rule_matches) run_matches.extend(rule_matches) - matches_end = max(m.qend for m in rule_matches) if matches_end + 1 < query_run.end: start_offset = matches_end + 1 continue @@ -620,10 +622,11 @@ def match(self, location=None, query_string=None, min_score=0, detect_negative=T logger_debug('!!!!!!!!!!!!!!!!!!!!REFINING!!!!!!!!!!!!!!!!!!!!!!!!!!!!') self.debug_matches(matches, '#match: ALL matches from all query runs', location, query_string) - matches, whole_discarded = refine_matches(matches, idx=self, min_score=min_score, max_dist=MAX_DIST // 2) - discarded.extend(whole_discarded) + matches, whole_discarded = refine_matches(matches, idx=self, query=qry, min_score=min_score, max_dist=MAX_DIST // 2) + if TRACE_MATCHES_DISCARD: + discarded.extend(whole_discarded) matches.sort() - + set_lines(matches, qry.line_by_pos) self.debug_matches(matches, '#match: FINAL MERGED', location, query_string) if TRACE_MATCHES_DISCARD: self.debug_matches(discarded, '#match: FINAL DISCARDED', location, query_string) diff --git a/src/licensedcode/match.py b/src/licensedcode/match.py index d89a99c53ca..98a6d99f988 100644 --- a/src/licensedcode/match.py +++ b/src/licensedcode/match.py @@ -48,18 +48,20 @@ TRACE_REFINE = False TRACE_MERGE = False TRACE_REFINE_SMALL = False +TRACE_REFINE_SINGLE = False +TRACE_REFINE_RULE_MIN_SCORE = False TRACE_REFINE_SOLID = False TRACE_SPAN_DETAILS = False TRACE_MERGE_TEXTS = False -# include or not unmatched tokens in debug texts +# include or not unmatched tokens in collect matched texts (usedmostly for debugging) INCLUDE_UNMATCHED_TEXTS = True def logger_debug(*args): pass -if TRACE or TRACE_FILTER_CONTAINS or TRACE_MERGE: +if TRACE or TRACE_FILTER_CONTAINS or TRACE_MERGE or TRACE_REFINE_RULE_MIN_SCORE or TRACE_REFINE_SINGLE: import logging import sys @@ -87,17 +89,15 @@ class LicenseMatch(object): account both the query and index Span. """ - __slots__ = 'rule', 'qspan', 'ispan', 'hispan', 'line_by_pos' , 'query_run_start', 'matcher' + __slots__ = 'rule', 'qspan', 'ispan', 'hispan', 'query_run_start', 'matcher', 'start_line', 'end_line' - def __init__(self, rule, qspan, ispan, hispan=None, line_by_pos=None, query_run_start=0, matcher=''): + def __init__(self, rule, qspan, ispan, hispan=None, query_run_start=0, matcher='', start_line=0, end_line=0): """ Create a new match from: - rule: matched Rule object - qspan: query text matched Span, start at zero which is the absolute query start (not the query_run start). - ispan: rule text matched Span, start at zero which is the rule start. - hispan: rule text matched Span for high tokens, start at zero which is the rule start. Always a subset of ispan. - - line_by_pos: mapping of (query positions -> line numbers). Line numbers start at one. - Optional: if not provided, the `lines` start and end tuple will be (0, 0) and no line information will be available. - matcher: a string indicating which matching procedure this match was created with. Used for debugging and testing only. Note that the relationship between is the qspan and ispan is such that: @@ -110,9 +110,10 @@ def __init__(self, rule, qspan, ispan, hispan=None, line_by_pos=None, query_run_ if hispan is None: hispan = Span() self.hispan = hispan - self.line_by_pos = line_by_pos or {} self.query_run_start = query_run_start self.matcher = matcher + self.start_line = start_line + self.end_line = end_line def __repr__(self, trace=TRACE_SPAN_DETAILS): @@ -166,6 +167,9 @@ def licensing_contains(self, other): """ return self.rule.licensing_contains(other.rule) + def lines(self): + return self.start_line, self.end_line + @property def qstart(self): return self.qspan.start @@ -207,12 +211,6 @@ def hilen(self): """ return len(self.hispan) - def lines(self): - """ - Return a tuple of start and end line for this match. - """ - return self.line_by_pos.get(self.qstart, 0), self.line_by_pos.get(self.qend, 0) - def __contains__(self, other): """ Return True if qspan contains other.qspan and ispan contains other.ispan. @@ -291,14 +289,10 @@ def combine(self, other): else: newmatcher = self.matcher - line_by_pos = dict(self.line_by_pos) - line_by_pos.update(other.line_by_pos) - combined = LicenseMatch(rule=self.rule, qspan=Span(self.qspan | other.qspan), ispan=Span(self.ispan | other.ispan), hispan=Span(self.hispan | other.hispan), - line_by_pos=line_by_pos, query_run_start=min(self.query_run_start, other.query_run_start), matcher=newmatcher) return combined @@ -311,15 +305,14 @@ def update(self, other): self.qspan = combined.qspan self.ispan = combined.ispan self.hispan = combined.hispan - self.line_by_pos = combined.line_by_pos self.matcher = combined.matcher self.query_run_start = min(self.query_run_start, other.query_run_start) return self - def rebase(self, new_query_start, new_query_end, line_by_pos, matcher): + def rebase(self, new_query_start, new_query_end, matcher): """ - Return a copy of this match with a new qspan and new line_by_pos and - updating the matcher of match as needed. + Return a copy of this match with a new qspan updating the matcher of this + copied match as needed. """ offset = new_query_start - self.query_run_start return LicenseMatch( @@ -327,7 +320,6 @@ def rebase(self, new_query_start, new_query_end, line_by_pos, matcher): qspan=self.qspan.rebase(offset), ispan=Span(self.ispan), hispan=Span(self.hispan), - line_by_pos=line_by_pos, query_run_start=new_query_start, matcher=' '.join([self.matcher.replace(cache.MATCH_CACHE, '').strip(), matcher]), ) @@ -342,10 +334,14 @@ def small(self): hilen = self.hilen() ilen = self.ilen() if TRACE_REFINE_SMALL: - logger_debug('LicenseMatch.small(): hilen=%(hilen)r < min_ihigh=%(min_ihigh)r or ilen=%(ilen)r < min_ilen=%(min_ilen)r : thresholds=%(thresholds)r' % locals(),) + logger_debug('LicenseMatch.small(): hilen=%(hilen)r, ilen=%(ilen)r, thresholds=%(thresholds)r' % locals(),) if thresholds.small and self.score() < 50 and (hilen < min_ihigh or ilen < min_ilen): + if TRACE_REFINE_SMALL: + logger_debug('LicenseMatch.small(): CASE 1 thresholds.small and self.score() < 50 and (hilen < min_ihigh or ilen < min_ilen)') return True if hilen < min_ihigh or ilen < min_ilen: + if TRACE_REFINE_SMALL: + logger_debug('LicenseMatch.small(): CASE 2 hilen < min_ihigh or ilen < min_ilen') return True def false_positive(self, idx): @@ -365,6 +361,17 @@ def false_positive(self, idx): return idx.false_positive_rid_by_hash.get(matched_hash) +def set_lines(matches, line_by_pos): + """ + Update a matches sequence with start and end line given a line_by_pos pos->line mapping. + """ + # if there is no line_by_pos, do not bother: the lines will stay to zero. + if line_by_pos: + for match in matches: + match.start_line = line_by_pos[match.qstart] + match.end_line = line_by_pos[match.qend] + + def merge_matches(matches, max_dist=MAX_DIST): """ Merge matches to the same rule in a sequence of matches. Return a new list @@ -372,7 +379,6 @@ def merge_matches(matches, max_dist=MAX_DIST): returned as-is. For being merged two matches must also be in increasing query and index positions. """ - # shortcut for single matches if len(matches) < 2: return matches @@ -380,25 +386,22 @@ def merge_matches(matches, max_dist=MAX_DIST): # only merge matches with the same rule: sort then group by rule # for the same rule, sort on start, longer high, longer match, matcher type sorter = lambda m: (m.rule.identifier, m.qspan.start, -m.hilen(), -m.qlen(), m.matcher) - all_matches = sorted(matches, key=sorter) - - + matches.sort(key=sorter) matches_by_rule = [(rid, list(rule_matches)) for rid, rule_matches - in groupby(all_matches, key=lambda m: m.rule.identifier)] - + in groupby(matches, key=lambda m: m.rule.identifier)] if TRACE_MERGE: print('merge_matches: number of matches to process:', len(matches)) merged = [] - for rid, matches in matches_by_rule: + for rid, rule_matches in matches_by_rule: if TRACE_MERGE: logger_debug('merge_matches: processing rule:', rid) # compare two matches in the sorted sequence: current and next i = 0 - while i < len(matches) - 1: + while i < len(rule_matches) - 1: j = i + 1 - while j < len(matches): - current_match = matches[i] - next_match = matches[j] + while j < len(rule_matches): + current_match = rule_matches[i] + next_match = rule_matches[j] if TRACE_MERGE: logger_debug('---> merge_matches: current:', current_match) if TRACE_MERGE: logger_debug('---> merge_matches: next: ', next_match) @@ -410,7 +413,7 @@ def merge_matches(matches, max_dist=MAX_DIST): # keep one of equal matches if current_match.qspan == next_match.qspan and current_match.ispan == next_match.ispan: if TRACE_MERGE: logger_debug(' ---> ###merge_matches: next EQUALS current, del next') - del matches[j] + del rule_matches[j] continue # if we have two equal ispans and some overlap @@ -420,24 +423,24 @@ def merge_matches(matches, max_dist=MAX_DIST): nqmag = next_match.qspan.magnitude() if cqmag <= nqmag: if TRACE_MERGE: logger_debug(' ---> ###merge_matches: current ispan EQUALS next ispan, current qmagnitude smaller, del next') - del matches[j] + del rule_matches[j] continue else: if TRACE_MERGE: logger_debug(' ---> ###merge_matches: current ispan EQUALS next ispan, next qmagnitude smaller, del current') - del matches[i] + del rule_matches[i] i -= 1 break # remove contained matches if current_match.qcontains(next_match): if TRACE_MERGE: logger_debug(' ---> ###merge_matches: next CONTAINED in current, del next') - del matches[j] + del rule_matches[j] continue # remove contained matches the other way if next_match.qcontains(current_match): if TRACE_MERGE: logger_debug(' ---> ###merge_matches: current CONTAINED in next, del current') - del matches[i] + del rule_matches[i] i -= 1 break @@ -449,7 +452,7 @@ def merge_matches(matches, max_dist=MAX_DIST): # the merged matched is likely aligned current_match.update(next_match) if TRACE_MERGE: logger_debug(' ---> ###merge_matches: current SURROUNDS next, merged as new:', current_match) - del matches[j] + del rule_matches[j] continue # FIXME: qsurround is too weak. We want to check also isurround @@ -460,7 +463,7 @@ def merge_matches(matches, max_dist=MAX_DIST): # the merged matched is likely aligned next_match.update(current_match) if TRACE_MERGE: logger_debug(' ---> ###merge_matches: next SURROUNDS current, merged as new:', current_match) - del matches[i] + del rule_matches[i] i -= 1 break @@ -468,7 +471,7 @@ def merge_matches(matches, max_dist=MAX_DIST): if next_match.is_after(current_match): current_match.update(next_match) if TRACE_MERGE: logger_debug(' ---> ###merge_matches: next follows current, merged as new:', current_match) - del matches[j] + del rule_matches[j] continue # next_match overlaps @@ -484,12 +487,12 @@ def merge_matches(matches, max_dist=MAX_DIST): if qoverlap == ioverlap: current_match.update(next_match) if TRACE_MERGE: logger_debug(' ---> ###merge_matches: next overlaps in sequence current, merged as new:', current_match) - del matches[j] + del rule_matches[j] continue j += 1 i += 1 - merged.extend(matches) + merged.extend(rule_matches) return merged @@ -712,6 +715,23 @@ def filter_contained_matches(matches): return matches, discarded +def filter_rule_min_score(matches): + """ + Return a list of matches scoring at or above a rule-defined minimum score and a + list of matches scoring below. + """ + kept = [] + discarded = [] + for match in matches: + if match.score() < match.rule.minimum_score: + if TRACE_REFINE_RULE_MIN_SCORE: logger_debug(' ==> DISCARDING rule.minimum_score:', match.rule.minimum_score, 'match:', match) + discarded.append(match) + else: + kept.append(match) + + return kept, discarded + + def filter_low_score(matches, min_score=100): """ Return a list of matches scoring above `min_score` and a list of matches scoring below. @@ -731,18 +751,46 @@ def filter_low_score(matches, min_score=100): return kept, discarded -def filter_not_solid(matches): +def filter_spurrious_single_token(matches, query=None, idx=None, unknown_count=5): """ - Return a list of matches to solid rules that are not solid and a list of small matches. + Return a list of matches where matches to a single token are only surrounded by + at least `unknown_count` unknown tokens or short tokens composed of a single + character. """ kept = [] discarded = [] + if not query: + return matches, discarded + + unknowns_by_pos = query.unknowns_by_pos + shorts_pos = query.shorts_pos for match in matches: - if match.rule.solid and match.score() != 100: - if TRACE_REFINE_SOLID: logger_debug(' ==> DISCARDING NOT SOLID:', 'solid:', match.rule.solid, 'score:', match.score(), match) + if not match.qlen() == 1: + kept.append(match) + continue + + qstart = match.qstart + qend = match.qend + + # compute the number of unknown tokens before and after this single matched position + # note: unknowns_by_pos is a defaultdict(int), shorts_pos is a set of integers + before = unknowns_by_pos[qstart - 1] + for p in range(qstart - 1 - unknown_count, qstart): + if p in shorts_pos: + before += 1 + + after = unknowns_by_pos[qstart] + for p in range(qend, qend + 1 + unknown_count): + if p in shorts_pos: + after += 1 + + if before >= unknown_count and after >= unknown_count: + if TRACE_REFINE_SINGLE: logger_debug(' ==> DISCARDING spurrious_single_token:', match) + if TRACE_REFINE_SINGLE: _debug_print_match(match, query, extras=unknown_count) discarded.append(match) else: - if TRACE_REFINE_SOLID: logger_debug(' ===> NOT DISCARDING NOT SOLID:', 'solid:', match.rule.solid, 'score:', match.score(), match) + if TRACE_REFINE_SINGLE: logger_debug(' ==> !!! NOT DISCARDING spurrious_single_token:', match, before, after) + if TRACE_REFINE_SINGLE: _debug_print_match(match, query, extras=unknown_count) kept.append(match) return kept, discarded @@ -807,7 +855,7 @@ def filter_false_positive_matches(idx, matches): return kept, discarded -def refine_matches(matches, idx, min_score=0, max_dist=MAX_DIST): +def refine_matches(matches, idx, query=None, min_score=0, max_dist=MAX_DIST): """ Return two sequences of matches: one contains refined good matches, and the other contains matches that were filtered out. @@ -821,12 +869,22 @@ def refine_matches(matches, idx, min_score=0, max_dist=MAX_DIST): all_discarded = [] - matches, discarded = filter_not_solid(matches) + matches, discarded = filter_rule_min_score(matches) all_discarded.extend(discarded) +# if TRACE: logger_debug(' #####refine_matches: NOT SHORT #', len(matches)) +# if TRACE_REFINE: map(logger_debug, matches) +# if TRACE: logger_debug(' #####refine_matches: SHORT discarded#', len(discarded)) +# if TRACE_REFINE: map(logger_debug, discarded) - matches, discarded = filter_short_matches(matches) + matches, discarded = filter_spurrious_single_token(matches, query, idx) all_discarded.extend(discarded) +# if TRACE: logger_debug(' #####refine_matches: NOT SHORT #', len(matches)) +# if TRACE_REFINE: map(logger_debug, matches) +# if TRACE: logger_debug(' #####refine_matches: SHORT discarded#', len(discarded)) +# if TRACE_REFINE: map(logger_debug, discarded) + matches, discarded = filter_short_matches(matches) + all_discarded.extend(discarded) if TRACE: logger_debug(' #####refine_matches: NOT SHORT #', len(matches)) if TRACE_REFINE: map(logger_debug, matches) if TRACE: logger_debug(' #####refine_matches: SHORT discarded#', len(discarded)) @@ -1007,3 +1065,17 @@ def matched_rule_tokens_str(match, include_unmatched=INCLUDE_UNMATCHED_TEXTS): yield '<%s>' % token else: yield '.' + + +def _debug_print_match(match, query, extras=5): + # create a fake new match with extra unknown left and right + new_match = match.combine(match) + new_qstart = max([0, match.qstart - extras]) + new_qend = min([match.qend + extras, len(query.tokens)]) + new_qspan = Span(new_qstart, new_qend) + new_match.qspan = new_qspan + + logger_debug(new_match) + logger_debug(' MATCHED QUERY TEXT with extras') + qt, _it = get_texts(new_match, location=query.location, query_string=None, idx=query.idx) + print(qt) diff --git a/src/licensedcode/match_aho.py b/src/licensedcode/match_aho.py index 17b16dcadca..4e9ab16c0d5 100644 --- a/src/licensedcode/match_aho.py +++ b/src/licensedcode/match_aho.py @@ -89,7 +89,6 @@ def exact_match(idx, query_run, automaton): qtokens = query_run.tokens qbegin = query_run.start query_run_matchables = query_run.matchables - line_by_pos = query_run.line_by_pos qtokens_as_str = array('h', qtokens).tostring() matches = [] @@ -135,7 +134,7 @@ def exact_match(idx, query_run, automaton): itokens = idx.tids_by_rid[rid] hispan = Span(p for p in ispan if itokens[p] >= len_junk) - match = LicenseMatch(rule, qspan, ispan, hispan, line_by_pos, query_run.start, matcher=matcher) + match = LicenseMatch(rule, qspan, ispan, hispan, query_run.start, matcher=matcher) matches.append(match) if TRACE and matches: diff --git a/src/licensedcode/match_hash.py b/src/licensedcode/match_hash.py index 5e7aeda1ffc..b97f1f2bc7e 100644 --- a/src/licensedcode/match_hash.py +++ b/src/licensedcode/match_hash.py @@ -60,7 +60,7 @@ def logger_debug(*args): def tokens_hash(tokens): """ - Return a digest binary string computed from a sequence of numeric token ids. + Return a digest binary string computed from a sequence of numeric token ids. """ return md5(array('h', tokens).tostring()).digest() @@ -86,12 +86,9 @@ def match_hash(idx, query_run): itokens = idx.tids_by_rid[rid] len_junk = idx.len_junk logger_debug('match_hash: Match:', rule.identifier) - qspan = Span(query_run.start, query_run.end) + qspan = Span(range(query_run.start, query_run.end + 1)) ispan = Span(range(0, rule.length)) hispan = Span(p for p in ispan if itokens[p] >= len_junk) - match = LicenseMatch(rule, qspan, ispan, hispan=hispan, - line_by_pos=query_run.line_by_pos, - query_run_start=query_run.start, - matcher=MATCH_HASH) + match = LicenseMatch(rule, qspan, ispan, hispan=hispan, query_run_start=query_run.start, matcher=MATCH_HASH) matches.append(match) return matches diff --git a/src/licensedcode/match_seq.py b/src/licensedcode/match_seq.py index ce3c0eea4b5..16593e1e948 100644 --- a/src/licensedcode/match_seq.py +++ b/src/licensedcode/match_seq.py @@ -72,7 +72,6 @@ def match_sequence(idx, candidate, query_run, start_offset=0): qbegin = query_run.start + start_offset qfinish = query_run.end qtokens = query_run.query.tokens - line_by_pos = query_run.line_by_pos matches = [] qstart = qbegin @@ -103,7 +102,7 @@ def match_sequence(idx, candidate, query_run, start_offset=0): iposses = range(ipos, ipos + mlen) hispan = Span(p for p in iposses if itokens[p] >= len_junk) ispan = Span(iposses) - match = LicenseMatch(rule, qspan, ispan, hispan, line_by_pos, qbegin, MATCH_SEQ) + match = LicenseMatch(rule, qspan, ispan, hispan, qbegin, MATCH_SEQ) if TRACE2: qt, it = get_texts(match, location=query_run.query.location, query_string=query_run.query.query_string, idx=idx) print('###########################') diff --git a/src/licensedcode/models.py b/src/licensedcode/models.py index 0996be311b2..d22f118dc4f 100644 --- a/src/licensedcode/models.py +++ b/src/licensedcode/models.py @@ -40,7 +40,6 @@ from commoncode.fileutils import file_name from commoncode.fileutils import file_iter -from licensedcode import MAX_GAP_SKIP from licensedcode import MIN_MATCH_LENGTH from licensedcode import MIN_MATCH_HIGH_LENGTH from licensedcode import licenses_data_dir @@ -598,9 +597,7 @@ def load_rules(rule_dir=rules_data_dir): raise Exception(msg % locals()) -Thresholds = namedtuple('Thresholds', - ['high_len', 'low_len', 'length', - 'small', 'min_high', 'min_len', 'max_gap_skip']) +Thresholds = namedtuple('Thresholds', ['high_len', 'low_len', 'length', 'small', 'min_high', 'min_len']) class Rule(object): """ @@ -615,11 +612,11 @@ class Rule(object): 'data_file', 'text_file', '_text', 'length', 'low_length', 'high_length', '_thresholds', 'length_unique', 'low_unique', 'high_unique', '_thresholds_unique', - 'gaps', 'is_url', 'solid' + 'is_url', 'minimum_score' ) def __init__(self, data_file=None, text_file=None, licenses=None, - license_choice=False, notes=None, solid=False, _text=None): + license_choice=False, notes=None, minimum_score=0, _text=None): ########### # FIXME: !!! TWO RULES MAY DIFFER BECAUSE THEY ARE UPDATED BY INDEXING @@ -646,13 +643,12 @@ def __init__(self, data_file=None, text_file=None, licenses=None, # is this rule text a false positive when matched? (filtered out) self.false_positive = False - # is this rule text only to be matched if all tokens are matched? - # not computed but set - self.solid = solid + # is this rule text only to be matched with a minimum score? + self.minimum_score = minimum_score # optional, free text self.notes = notes - + # path to the YAML data file for this rule self.data_file = data_file if data_file: @@ -667,12 +663,12 @@ def __init__(self, data_file=None, text_file=None, licenses=None, # for testing only, when we do not use a file self._text = _text - # These attributes are computed upon text loading or setting the thresholds + # These attributes are computed upon text loading or setting the thresholds ########################################################################### # is this rule text for a bare url? (needs exact matching) self.is_url = False - + # length in number of token strings self.length = 0 @@ -689,14 +685,14 @@ def __init__(self, data_file=None, text_file=None, licenses=None, def tokens(self, lower=True): """ - Return an iterable of token strings for this rule. - Length is recomputed. Tokens inside gaps are skipped and ignored. + Return an iterable of token strings for this rule. Length is recomputed. + Tokens inside double curly braces (eg. {{ignored}}) are skipped and ignored. """ length = 0 text = self.text() text = text.strip() - # FIXME: this is weird: + # FIXME: this is weird: # tag this rule as being a bare URL if it starts with a scheme and is on one line: this is used to determine a matching approach if text.startswith(('http://', 'https://', 'ftp://')) and '\n' not in text[:1000]: self.is_url = True @@ -734,8 +730,8 @@ def __repr__(self): keys = self.licenses choice = self.license_choice fp = self.false_positive - solid = self.solid - return 'Rule(%(idf)r, lics=%(keys)r, fp=%(fp)r, solid=%(solid)r, %(text)r)' % locals() + minimum_score = self.minimum_score + return 'Rule(%(idf)r, lics=%(keys)r, fp=%(fp)r, minimum_score=%(minimum_score)r, %(text)r)' % locals() def same_licensing(self, other): """ @@ -771,35 +767,31 @@ def thresholds(self): Return a Thresholds tuple considering every token occurrence. """ if not self._thresholds: - min_high = MIN_MATCH_HIGH_LENGTH + min_high = min([self.high_length, MIN_MATCH_HIGH_LENGTH]) min_len = MIN_MATCH_LENGTH - max_gap_skip = MAX_GAP_SKIP # note: we cascade ifs from largest to smallest lengths if self.length < 30: - min_high = self.high_length min_len = self.length // 2 - max_gap_skip = 1 if self.length < 10: min_high = self.high_length min_len = self.length - max_gap_skip = 1 + self.minimum_score = 80 if self.length < 3: min_high = self.high_length min_len = self.length - max_gap_skip = 1 - self.solid = True + self.minimum_score = 100 - if self.is_url or self.solid: + if self.is_url or self.minimum_score == 100: min_high = self.high_length min_len = self.length - max_gap_skip = 0 self._thresholds = Thresholds( self.high_length, self.low_length, self.length, - self.small(), min_high, min_len, max_gap_skip) + self.small(), min_high, min_len + ) return self._thresholds def thresholds_unique(self): @@ -807,14 +799,13 @@ def thresholds_unique(self): Return a Thresholds tuple considering only unique token occurrence. """ if not self._thresholds_unique: - min_high = int(min([self.high_unique // 2, MIN_MATCH_HIGH_LENGTH])) + highu = (int(self.high_unique // 2)) or self.high_unique + min_high = min([highu, MIN_MATCH_HIGH_LENGTH]) min_len = MIN_MATCH_LENGTH - max_gap_skip = MAX_GAP_SKIP # note: we cascade IFs from largest to smallest lengths if self.length < 20: min_high = self.high_unique min_len = min_high - max_gap_skip = 1 if self.length < 10: min_high = self.high_unique @@ -822,21 +813,18 @@ def thresholds_unique(self): min_len = self.length_unique else: min_len = self.length_unique - 1 - max_gap_skip = 1 if self.length < 5: min_high = self.high_unique min_len = self.length_unique - max_gap_skip = 1 if self.is_url: min_high = self.high_unique min_len = self.length_unique - max_gap_skip = 0 self._thresholds_unique = Thresholds( self.high_unique, self.low_unique, self.length_unique, - self.small(), min_high, min_len, max_gap_skip) + self.small(), min_high, min_len) return self._thresholds_unique def asdict(self): @@ -853,8 +841,8 @@ def asdict(self): data['license'] = self.license if self.false_positive: data['false_positive'] = self.false_positive - if self.solid: - data['solid'] = self.solid + if self.minimum_score: + data['minimum_score'] = self.minimum_score if self.notes: data['notes'] = self.note return data @@ -894,7 +882,7 @@ def load(self, load_notes=False): self.license_choice = data.get('license_choice', False) self.license = data.get('license') self.false_positive = data.get('false_positive', False) - self.solid = data.get('solid', False) + self.minimum_score = int(data.get('minimum_score', 0)) # these are purely informational and not used at run time if load_notes: diff --git a/src/licensedcode/query.py b/src/licensedcode/query.py index d45a88d5b92..ac584f522b8 100644 --- a/src/licensedcode/query.py +++ b/src/licensedcode/query.py @@ -132,13 +132,16 @@ def __init__(self, location=None, query_string=None, idx=None, # token ids array self.tokens = [] - # index of position -> line number - self.line_by_pos = {} + # index of position -> line number where the pos is the list index + self.line_by_pos = [] # index of known position -> number of unknown tokens after that pos # for unknowns at the start, the pos is -1 self.unknowns_by_pos = defaultdict(int) + # set of known position were we have short, single letter tokens at that pos + self.shorts_pos = set() + self.query_runs = [] if _test_mode: return @@ -193,8 +196,9 @@ def tokens_by_line(self, tokenizer=query_tokenizer): Populate the query `line_by_pos` and `unknowns_by_pos` mappings as a side effect. """ - line_by_pos = self.line_by_pos + line_by_pos_append = self.line_by_pos.append self_unknowns_by_pos = self.unknowns_by_pos + self_shorts_pos_add = self.shorts_pos.add dic_get = self.idx.dictionary.get # note: positions start at zero @@ -215,7 +219,9 @@ def tokens_by_line(self, tokenizer=query_tokenizer): if tid is not None: known_pos += 1 started = True - line_by_pos[known_pos] = lnum + line_by_pos_append(lnum) + if len(token) ==1: + self_shorts_pos_add(known_pos) else: # we have not yet started if not started: @@ -295,6 +301,8 @@ class QueryRun(object): A query run is a slice of whole query tokens identified by a start and end positions inclusive. """ + __slots__ = ('query', 'start', 'end', 'len_junk', '_low_matchables', '_high_matchables') + def __init__(self, query, start, end=None): """ Initialize a query run starting at start and ending at end a parent @@ -308,14 +316,16 @@ def __init__(self, query, start, end=None): self.len_junk = self.query.idx.len_junk - # lines by positions, used for final reporting - self.line_by_pos = query.line_by_pos - # positions with unknown tokens, used for final reporting - self.unknowns_by_pos = query.unknowns_by_pos - self._low_matchables = None self._high_matchables = None + @property + def unknowns_by_pos(self): + """ + Return positions with unknown tokens, used for final reporting. + """ + return self.query.unknowns_by_pos + @property def low_matchables(self): if not self._low_matchables: @@ -372,20 +382,18 @@ def is_matchable(self, include_low=False, qspans=None): Return True if this query run has some matchable high tokens. If a list of qspans is provided, their positions are first subtracted. """ - qs = qspans and [q._set for q in qspans] or [] - if include_low: matchables = self.matchables else: matchables = self.high_matchables - if not qs: + if not qspans: return matchables - matched = intbitset.union(*qs) - high = intbitset(matchables) - high.difference_update(matched) - return high + matched = intbitset.union(*[q._set for q in qspans]) + matchables = intbitset(matchables) + matchables.difference_update(matched) + return matchables @property def matchables(self): diff --git a/src/licensedcode/spans.py b/src/licensedcode/spans.py index cbfadd613af..481169c2449 100644 --- a/src/licensedcode/spans.py +++ b/src/licensedcode/spans.py @@ -1,38 +1,476 @@ -# -*- coding: utf-8 -*- # -# Copyright (c) 2016 nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/scancode-toolkit/ -# The ScanCode software is licensed under the Apache License version 2.0. -# Data generated with ScanCode require an acknowledgment. -# ScanCode is a trademark of nexB Inc. +# Copyright (c) 2010 Matt Chaput. All rights reserved. +# Modifications by nexB Copyright 2016 nexB Inc. All rights reserved. # -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: # -# When you publish or redistribute any data created with ScanCode or any ScanCode -# derivative work, you must accompany this data with the following acknowledgment: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. # -# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# ScanCode should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# ScanCode is a free software code scanning tool from nexB Inc. and others. -# Visit https://github.com/nexB/scancode-toolkit/ for support and download. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY MATT CHAPUT ``AS IS'' AND ANY EXPRESS OR IMPLIED +# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +# EVENT SHALL MATT CHAPUT OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# The views and conclusions contained in the software and documentation are +# those of the authors and should not be interpreted as representing official +# policies, either expressed or implied, of Matt Chaput. -from __future__ import print_function, absolute_import +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function -# this import is exposing which spans implementation is used -from licensedcode.spansintsets import Span +from collections import Set +from itertools import count +from itertools import groupby -""" -See Also -Alternative to sets: +from intbitset import intbitset -https://github.com/MicrosoftGenomics/PySnpTools/blob/master/pysnptools/util/intrangeset.py -http://code.activestate.com/recipes/230113/ -https://github.com/DRMacIver/intset + +""" +Ranges and intervals of integers using bitmaps. +Used as a compact and faster data structure for token and position sets. """ + + +class Span(Set): + """ + Represent ranges of integers (such as tokens positions) as a set of integers. + A Span is hashable and not meant to be modified once created, like a frozenset. + It is equivalent to a sparse closed interval. + Originally derived and heavily modified from Whoosh Span. + """ + def __init__(self, *args): + """ + Create a new Span from a start and end ints or an iterable of ints. + + First form: + Span(start int, end int) : the span is initialized with a range(start, end+1) + + Second form: + Span(iterable of ints) : the span is initialized with the iterable + + Spans are hashable and immutable. + + For example: + >>> s = Span(1) + >>> s.start + 1 + >>> s = Span([1, 2]) + >>> s.start + 1 + >>> s.end + 2 + >>> s + Span(1, 2) + + >>> s = Span(1, 3) + >>> s.start + 1 + >>> s.end + 3 + >>> s + Span(1, 3) + + >>> s = Span([6, 5, 1, 2]) + >>> s.start + 1 + >>> s.end + 6 + >>> s + Span(1, 2)|Span(5, 6) + >>> len(s) + 4 + + >>> Span([5, 6, 7, 8, 9, 10 ,11, 12]) == Span([5, 6, 7, 8, 9, 10 ,11, 12]) + True + >>> hash(Span([5, 6, 7, 8, 9, 10 ,11, 12])) == hash(Span([5, 6, 7, 8, 9, 10 ,11, 12])) + True + >>> hash(Span([5, 6, 7, 8, 9, 10 ,11, 12])) == hash(Span(5, 12)) + True + """ + len_args = len(args) + + if len_args == 0: + self._set = intbitset() + + elif len_args == 1: + # args0 is a single int or an iterable of ints + if isinstance(args[0], (int, long)): + self._set = intbitset(args) + else: + # some sequence or iterable + self._set = intbitset(list(args[0])) + + elif len_args == 2: + # args0 and args1 describe a start and end closed range + self._set = intbitset(range(args[0], args[1] + 1)) + + else: + # args0 is a single int or args is an iterable of ints + # args is an iterable of ints + self._set = intbitset(list(args)) + + @classmethod + def _from_iterable(cls, it): + return cls(list(it)) + + def __len__(self): + return len(self._set) + + def __iter__(self): + return iter(self._set) + + def __hash__(self): + return hash(tuple(self._set)) + + def __eq__(self, other): + return self._set == other._set + + def __and__(self, *others): + return Span(self._set.intersection(*(o._set for o in others))) + + def __or__(self, *others): + return Span(self._set.union(*(o._set for o in others))) + + def union(self, *others): + return self.__or__(*others) + + def difference(self, other): + return self._set.difference(other._set) + + def __repr__(self): + """ + Return a brief representation of this span by only listing contiguous + spans and not all items. + + For example: + >>> Span([1, 2, 3, 4, 5, 7, 8, 9, 10]) + Span(1, 5)|Span(7, 10) + """ + subspans_repr = [] + for subs in self.subspans(): + ls = len(subs) + if not ls: + subspans_repr.append('Span()') + elif ls == 1: + subspans_repr.append('Span(%d)' % subs.start) + else: + subspans_repr.append('Span(%d, %d)' % (subs.start, subs.end)) + return '|'.join(subspans_repr) + + def __contains__(self, other): + """ + Return True if this span contains other span (where other is a Span, an + int or an ints set). + + For example: + >>> Span([5, 7]) in Span(5, 7) + True + >>> Span([5, 8]) in Span([5, 7]) + False + >>> 6 in Span([4, 5, 6, 7, 8]) + True + >>> 2 in Span([4, 5, 6, 7, 8]) + False + >>> 8 in Span([4, 8]) + True + >>> 5 in Span([4, 8]) + False + >>> set([4, 5]) in Span([4, 5, 6, 7, 8]) + True + >>> set([9]) in Span([4, 8]) + False + """ + if isinstance(other, Span): + return self._set.issuperset(other._set) + + if isinstance(other, (int, long)): + return self._set.__contains__(other) + + if isinstance(other, (set, frozenset)): + return self._set.issuperset(intbitset(other)) + + if isinstance(other, intbitset): + return self._set.issuperset(other) + + def issubset(self, other): + return self._set.issubset(other._set) + + def issuperset(self, other): + return self._set.issuperset(other._set) + + @property + def start(self): + if not self._set: + raise TypeError('Empty Span has no start.') + return self._set[0] + + @property + def end(self): + if not self._set: + raise TypeError('Empty Span has no end.') + return self._set[-1] + + @classmethod + def sort(cls, spans): + """ + Return a new sorted sequence of spans given a sequence of spans. + The primary sort is on start. The secondary sort is on length. + If two spans have the same start, the longer span will sort first. + + For example: + >>> spans = [Span([5, 6, 7, 8, 9, 10]), Span([1, 2]), Span([3, 4, 5]), Span([3, 4, 5, 6]), Span([8, 9, 10])] + >>> Span.sort(spans) + [Span(1, 2), Span(3, 6), Span(3, 5), Span(5, 10), Span(8, 10)] + + >>> spans = [Span([1, 2]), Span([3, 4, 5]), Span([3, 4, 5, 6]), Span([8, 9, 10])] + >>> Span.sort(spans) + [Span(1, 2), Span(3, 6), Span(3, 5), Span(8, 10)] + + >>> spans = [Span([1, 2]), Span([4, 5]), Span([7, 8]), Span([11, 12])] + >>> Span.sort(spans) + [Span(1, 2), Span(4, 5), Span(7, 8), Span(11, 12)] + + >>> spans = [Span([1, 2]), Span([7, 8]), Span([5, 6]), Span([12, 13])] + >>> Span.sort(spans) + [Span(1, 2), Span(5, 6), Span(7, 8), Span(12, 13)] + + """ + key = lambda s: (s.start, -len(s),) + return sorted(spans, key=key) + + def magnitude(self): + """ + Return the maximal length represented by this span start and end. The + magnitude is the same as the length for a contiguous span. It will be + greater than the length for a span with non-contiguous int items. + An empty span has a zero magnitude. + + For example: + >>> Span([4, 8]).magnitude() + 5 + >>> len(Span([4, 8])) + 2 + >>> len(Span([4, 5, 6, 7, 8])) + 5 + + >>> Span([4, 5, 6, 14 , 12, 128]).magnitude() + 125 + + >>> Span([4, 5, 6, 7, 8]).magnitude() + 5 + >>> Span([0]).magnitude() + 1 + >>> Span([0]).magnitude() + 1 + """ + if not self._set: + return 0 + return self.end - self.start + 1 + + def density(self): + """ + Return the density of this span as a ratio of its length to its + magnitude, a float between 0 and 1. A dense Span has all its integer + items contiguous and a maximum density of one. A sparse low density span + has some non-contiguous integer items. An empty span has a zero density. + + For example: + >>> Span([4, 8]).density() + 0.4 + >>> Span([4, 5, 6, 7, 8]).density() + 1.0 + >>> Span([0]).density() + 1.0 + >>> Span().density() + 0 + """ + if not self._set: + return 0 + return len(self) / self.magnitude() + + def overlap(self, other): + """ + Return the count of overlapping items between this span and other span. + + For example: + >>> Span([1, 2]).overlap(Span([5, 6])) + 0 + >>> Span([5, 6]).overlap(Span([5, 6])) + 2 + >>> Span([4, 5, 6, 7]).overlap(Span([5, 6])) + 2 + >>> Span([4, 5, 6]).overlap(Span([5, 6, 7])) + 2 + >>> Span([4, 5, 6]).overlap(Span([6])) + 1 + >>> Span([4, 5]).overlap(Span([6, 7])) + 0 + """ + return len(self & other) + + def resemblance(self, other): + """ + Return a resemblance coefficient as a float between 0 and 1. + 0 means the spans are completely different and 1 identical. + """ + if self._set.isdisjoint(other._set): + return 0 + if self._set == other._set: + return 1 + resemblance = self.overlap(other) / len(self | other) + return resemblance + + def containment(self, other): + """ + Return a containment coefficient as a float between 0 and 1. This is an + indication of how much of the other span is contained in this span. + - 1 means the other span is entirely contained in this span. + - 0 means that the other span is not contained at all this span. + """ + if self._set.isdisjoint(other._set): + return 0 + if self._set == other._set: + return 1 + containment = self.overlap(other) / len(other) + return containment + + def surround(self, other): + """ + Return True if this span surrounds other span. + This is different from containment. A span can surround another span region + and have no positions in common with the surrounded. + + For example: + >>> Span([4, 8]).surround(Span([4, 8])) + True + >>> Span([3, 9]).surround(Span([4, 8])) + True + >>> Span([5, 8]).surround(Span([4, 8])) + False + >>> Span([4, 7]).surround(Span([4, 8])) + False + >>> Span([4, 5, 6, 7, 8]).surround(Span([5, 6, 7])) + True + """ + return self.start <= other.start and self.end >= other.end + + def is_before(self, other): + return self.end < other.start + + def is_after(self, other): + return self.start > other.end + + def touch(self, other): + """ + Return True if self sequence is contiguous with other span without overlap. + + For example: + >>> Span([5, 7]).touch(Span([5])) + False + >>> Span([5, 7]).touch(Span([5, 8])) + False + >>> Span([5, 7]).touch(Span([7, 8])) + False + >>> Span([5, 7]).touch(Span([8, 9])) + True + >>> Span([8, 9]).touch(Span([5, 7])) + True + """ + return self.start == other.end + 1 or self.end == other.start - 1 + + def distance_to(self, other): + """ + Return the absolute positive distance from this span to other span. + Touching and overlapping spans have a zero distance. + + For example: + >>> Span([8, 9]).distance_to(Span([5, 7])) + 0 + >>> Span([5, 7]).distance_to(Span([8, 9])) + 0 + >>> Span([5, 6]).distance_to(Span([8, 9])) + 2 + >>> Span([5, 7]).distance_to(Span([5, 7])) + 0 + >>> Span([4, 5, 6]).distance_to(Span([5, 6, 7])) + 0 + >>> Span([5, 7]).distance_to(Span([10, 12])) + 3 + >>> Span([1, 2]).distance_to(Span(range(4, 52))) + 2 + """ + if self.overlap(other) or self.touch(other): + return 0 + elif self.is_before(other): + return other.start - self.end + else: + return self.start - other.end + + def rebase(self, offset): + """ + Return a copy of this span adding `offset` to each item + + For example: + >>> Span([4, 5]).rebase(0) + Span(4, 5) + >>> Span(4, 5).rebase(1) + Span(5, 6) + >>> Span([4, 5]).rebase(3) + Span(7, 8) + >>> Span([1, 4, 5, 8, 9]).rebase(5) + Span(6)|Span(9, 10)|Span(13, 14) + """ + assert self.start + offset >= 0 + return Span([i + offset for i in self._set]) + + @staticmethod + def from_ints(ints): + """ + Return a sequence of Spans from an iterable of ints. A new Span is + created for each group of monotonously increasing int items. + + >>> Span.from_ints([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]) + [Span(1, 12)] + >>> Span.from_ints([1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12]) + [Span(1, 3), Span(5, 12)] + >>> Span.from_ints([0, 2, 3, 5, 6, 7, 8, 9, 10, 11, 13]) + [Span(0), Span(2, 3), Span(5, 11), Span(13)] + """ + ints = sorted(set(ints)) + groups = (group for _, group in groupby(ints, lambda group, c=count(): next(c) - group)) + return map(Span, groups) + + def subspans(self): + """ + Return a list of Spans creating one new Span for each set of contiguous + integer items. + + For example: + >>> span = Span(5, 6, 7, 8, 9, 10) | Span([1, 2]) | Span(3, 5) | Span(3, 6) | Span([8, 9, 10]) + >>> span.subspans() + [Span(1, 10)] + + When subspans are not touching they do not merge : + >>> span = Span([63, 64]) | Span([58, 58]) + >>> span.subspans() + [Span(58), Span(63, 64)] + + Overlapping subspans are merged as needed: + >>> span = Span([12, 17, 24]) | Span([15, 16, 17, 35]) | Span(58) | Span(63, 64) + >>> span.subspans() + [Span(12), Span(15, 17), Span(24), Span(35), Span(58), Span(63, 64)] + """ + return Span.from_ints(self) diff --git a/src/licensedcode/spansintsets.py b/src/licensedcode/spansintsets.py deleted file mode 100644 index 40b14f0a054..00000000000 --- a/src/licensedcode/spansintsets.py +++ /dev/null @@ -1,508 +0,0 @@ -# -# Copyright (c) 2010 Matt Chaput. All rights reserved. -# Modifications by nexB Copyright 2016 nexB Inc. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY MATT CHAPUT ``AS IS'' AND ANY EXPRESS OR IMPLIED -# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO -# EVENT SHALL MATT CHAPUT OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, -# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, -# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# The views and conclusions contained in the software and documentation are -# those of the authors and should not be interpreted as representing official -# policies, either expressed or implied, of Matt Chaput. - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from collections import Set -from itertools import count -from itertools import groupby - -from intbitset import intbitset - - -class Span(Set): - """ - Represent a range of tokens positions as a set of integers. - A Span is hashable and not meant to be modified once created. - It is equivalent to a sparse closed interval. - - Originally derived and heavily modified from Whoosh Span. - """ - def __init__(self, *args): - """ - Create a new Span from a start and end ints or an iterable of ints. - - First form: - Span(start int, end int) : the span is initialized with a range(start, end+1) - - Second form: - Span(iterable of ints) : the span is initialized with the iterable - - Spans are hashable and immutable. - - For example: - >>> s = Span(1) - >>> s.start - 1 - >>> s = Span([1, 2]) - >>> s.start - 1 - >>> s.end - 2 - >>> s - Span(1, 2) - - >>> s = Span(1, 3) - >>> s.start - 1 - >>> s.end - 3 - >>> s - Span(1, 3) - - >>> s = Span([6, 5, 1, 2]) - >>> s.start - 1 - >>> s.end - 6 - >>> s - Span(1, 2)|Span(5, 6) - >>> len(s) - 4 - - >>> Span([5, 6, 7, 8, 9, 10 ,11, 12]) == Span([5, 6, 7, 8, 9, 10 ,11, 12]) - True - >>> hash(Span([5, 6, 7, 8, 9, 10 ,11, 12])) == hash(Span([5, 6, 7, 8, 9, 10 ,11, 12])) - True - >>> hash(Span([5, 6, 7, 8, 9, 10 ,11, 12])) == hash(Span(5, 12)) - True - """ - len_args = len(args) - - if len_args == 0: - self._set = intbitset() - - elif len_args == 1: - # args0 is a single int or an iterable of ints - if isinstance(args[0], (int, long)): - self._set = intbitset(args) - else: - # some sequence or iterable - self._set = intbitset(list(args[0])) - - elif len_args == 2: - # args0 and args1 describe a start and end closed range - self._set = intbitset(range(args[0], args[1] + 1)) - - else: - # args0 is a single int or args is an iterable of ints - # args is an iterable of ints - self._set = intbitset(list(args)) - - @classmethod - def _from_iterable(cls, it): - return cls(list(it)) - - def __len__(self): - return len(self._set) - - def __iter__(self): - return iter(self._set) - - def __hash__(self): - return hash(tuple(self._set)) - - def __eq__(self, other): - return frozenset(self._set) == frozenset(other) - - def __and__(self, *args): - return Span(self._set.intersection(*args)) - - def __or__(self, *args): - return Span(self._set.union(*args)) - - def union(self, *args): - return self.__or__(*args) - - def difference(self, other): - return self._set.difference(other) - - def __repr__(self): - """ - Return a brief representation of this span by only listing contiguous - spans and not all items. - - For example: - >>> Span([1, 2, 3, 4, 5, 7, 8, 9, 10]) - Span(1, 5)|Span(7, 10) - """ - subspans_repr = [] - for subs in self.subspans(): - ls = len(subs) - if not ls: - subspans_repr.append('Span()') - elif ls == 1: - subspans_repr.append('Span(%d)' % subs.start) - else: - subspans_repr.append('Span(%d, %d)' % (subs.start, subs.end)) - return '|'.join(subspans_repr) - - def __contains__(self, other): - """ - Return True if this span contains other span (where other is a Span, an - int or an ints set). - - For example: - >>> Span([5, 7]) in Span(5, 7) - True - >>> Span([5, 8]) in Span([5, 7]) - False - >>> 6 in Span([4, 5, 6, 7, 8]) - True - >>> 2 in Span([4, 5, 6, 7, 8]) - False - >>> 8 in Span([4, 8]) - True - >>> 5 in Span([4, 8]) - False - >>> set([4, 5]) in Span([4, 5, 6, 7, 8]) - True - >>> set([9]) in Span([4, 8]) - False - """ - if isinstance(other, (set, frozenset)): - return self._set.issuperset(intbitset(other)) - - if isinstance(other, intbitset): - return self._set.issuperset(other) - - if isinstance(other, Span): - return self._set.issuperset(other._set) - - if isinstance(other, (int, long)): - return self._set.__contains__(other) - - def issubset(self, other): - return self._set.issubset(other._set) - - def issuperset(self, other): - return self._set.issuperset(other._set) - - @property - def start(self): - if not self._set: - raise TypeError('Empty Span has no start.') - return self._set[0] - - @property - def end(self): - if not self._set: - raise TypeError('Empty Span has no end.') - return self._set[-1] - - @staticmethod - def sort(spans): - """ - Return a new sorted sequence of spans given a sequence of spans. - The primary sort is on start. The secondary sort is on length. - If two spans have the same start, the longer span will sort first. - - For example: - >>> spans = [Span([5, 6, 7, 8, 9, 10]), Span([1, 2]), Span([3, 4, 5]), Span([3, 4, 5, 6]), Span([8, 9, 10])] - >>> Span.sort(spans) - [Span(1, 2), Span(3, 6), Span(3, 5), Span(5, 10), Span(8, 10)] - - >>> spans = [Span([1, 2]), Span([3, 4, 5]), Span([3, 4, 5, 6]), Span([8, 9, 10])] - >>> Span.sort(spans) - [Span(1, 2), Span(3, 6), Span(3, 5), Span(8, 10)] - - >>> spans = [Span([1, 2]), Span([4, 5]), Span([7, 8]), Span([11, 12])] - >>> Span.sort(spans) - [Span(1, 2), Span(4, 5), Span(7, 8), Span(11, 12)] - - >>> spans = [Span([1, 2]), Span([7, 8]), Span([5, 6]), Span([12, 13])] - >>> Span.sort(spans) - [Span(1, 2), Span(5, 6), Span(7, 8), Span(12, 13)] - - """ - key = lambda s: (s.start, -len(s),) - return sorted(spans, key=key) - - def magnitude(self): - """ - Return the maximal length represented by this span start and end. The - magnitude is the same as the length for a contiguous span. It will be - greater than the length for a span with non-contiguous int items. - An empty span has a zero magnitude. - - For example: - >>> Span([4, 8]).magnitude() - 5 - >>> len(Span([4, 8])) - 2 - >>> len(Span([4, 5, 6, 7, 8])) - 5 - - >>> Span([4, 5, 6, 14 , 12, 128]).magnitude() - 125 - - >>> Span([4, 5, 6, 7, 8]).magnitude() - 5 - >>> Span([0]).magnitude() - 1 - >>> Span([0]).magnitude() - 1 - """ - if not self._set: - return 0 - return self.end - self.start + 1 - - def density(self): - """ - Return the density of this span as a ratio of its length to its - magnitude, a float between 0 and 1. A dense Span has all its integer - items contiguous and a maximum density of one. A sparse low density span - has some non-contiguous integer items. An empty span has a zero density. - - For example: - >>> Span([4, 8]).density() - 0.4 - >>> Span([4, 5, 6, 7, 8]).density() - 1.0 - >>> Span([0]).density() - 1.0 - >>> Span().density() - 0 - """ - if not self._set: - return 0 - return len(self) / self.magnitude() - - def overlap(self, other): - """ - Return the count of overlapping items between this span and other span. - - For example: - >>> Span([1, 2]).overlap(Span([5, 6])) - 0 - >>> Span([5, 6]).overlap(Span([5, 6])) - 2 - >>> Span([4, 5, 6, 7]).overlap(Span([5, 6])) - 2 - >>> Span([4, 5, 6]).overlap(Span([5, 6, 7])) - 2 - >>> Span([4, 5, 6]).overlap(Span([6])) - 1 - >>> Span([4, 5]).overlap(Span([6, 7])) - 0 - """ - return len(self & other) - - def resemblance(self, other): - """ - Return a resemblance coefficient as a float between 0 and 1. - 0 means the spans are completely different and 1 identical. - """ - if self._set.isdisjoint(other._set): - return 0 - if self._set == other._set: - return 1 - resemblance = self.overlap(other) / len(self | other) - return resemblance - - def containment(self, other): - """ - Return a containment coefficient as a float between 0 and 1. This is an - indication of how much of the other span is contained in this span. - - 1 means the other span is entirely contained in this span. - - 0 means that the other span is not contained at all this span. - """ - if self._set.isdisjoint(other._set): - return 0 - if self._set == other._set: - return 1 - containment = self.overlap(other) / len(other) - return containment - - def surround(self, other): - """ - Return True if this span surrounds other span. - This is different from containment. A span can surround another span region - and have no positions in common with the surrounded. - - For example: - >>> Span([4, 8]).surround(Span([4, 8])) - True - >>> Span([3, 9]).surround(Span([4, 8])) - True - >>> Span([5, 8]).surround(Span([4, 8])) - False - >>> Span([4, 7]).surround(Span([4, 8])) - False - >>> Span([4, 5, 6, 7, 8]).surround(Span([5, 6, 7])) - True - """ - return self.start <= other.start and self.end >= other.end - - def is_before(self, other): - return self.end < other.start - - def is_after(self, other): - return self.start > other.end - - def touch(self, other): - """ - Return True if self sequence is contiguous with other span without overlap. - - For example: - >>> Span([5, 7]).touch(Span([5])) - False - >>> Span([5, 7]).touch(Span([5, 8])) - False - >>> Span([5, 7]).touch(Span([7, 8])) - False - >>> Span([5, 7]).touch(Span([8, 9])) - True - >>> Span([8, 9]).touch(Span([5, 7])) - True - """ - return self.start == other.end + 1 or self.end == other.start - 1 - - def distance_to(self, other): - """ - Return the absolute positive distance from this span to other span. - Touching and overlapping spans have a zero distance. - - For example: - >>> Span([8, 9]).distance_to(Span([5, 7])) - 0 - >>> Span([5, 7]).distance_to(Span([8, 9])) - 0 - >>> Span([5, 6]).distance_to(Span([8, 9])) - 2 - >>> Span([5, 7]).distance_to(Span([5, 7])) - 0 - >>> Span([4, 5, 6]).distance_to(Span([5, 6, 7])) - 0 - >>> Span([5, 7]).distance_to(Span([10, 12])) - 3 - >>> Span([1, 2]).distance_to(Span(range(4, 52))) - 2 - """ - if self.overlap(other) or self.touch(other): - return 0 - elif self.is_before(other): - return other.start - self.end - else: - return self.start - other.end - - def rebase(self, offset): - """ - Return a copy of this span adding `offset` to each item - - For example: - >>> Span([4, 5]).rebase(0) - Span(4, 5) - >>> Span(4, 5).rebase(1) - Span(5, 6) - >>> Span([4, 5]).rebase(3) - Span(7, 8) - >>> Span([1, 4, 5, 8, 9]).rebase(5) - Span(6)|Span(9, 10)|Span(13, 14) - """ - assert self.start + offset >= 0 - return Span([i + offset for i in self._set]) - - @staticmethod - def from_ints(ints): - """ - Return a sequence of Spans from an iterable of ints. A new Span is - created for each group of monotonously increasing int items. - - >>> Span.from_ints([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]) - [Span(1, 12)] - >>> Span.from_ints([1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12]) - [Span(1, 3), Span(5, 12)] - >>> Span.from_ints([0, 2, 3, 5, 6, 7, 8, 9, 10, 11, 13]) - [Span(0), Span(2, 3), Span(5, 11), Span(13)] - """ - ints = sorted(set(ints)) - groups = (group for _, group in groupby(ints, lambda group, c=count(): next(c) - group)) - return map(Span, groups) - - def subspans(self): - """ - Return a list of Spans creating one new Span for each set of contiguous - integer items. - - For example: - >>> span = Span(5, 6, 7, 8, 9, 10) | Span([1, 2]) | Span(3, 5) | Span(3, 6) | Span([8, 9, 10]) - >>> span.subspans() - [Span(1, 10)] - - When subspans are not touching they do not merge : - >>> span = Span([63, 64]) | Span([58, 58]) - >>> span.subspans() - [Span(58), Span(63, 64)] - - Overlapping subspans are merged as needed: - >>> span = Span([12, 17, 24]) | Span([15, 16, 17, 35]) | Span(58) | Span(63, 64) - >>> span.subspans() - [Span(12), Span(15, 17), Span(24), Span(35), Span(58), Span(63, 64)] - """ - return Span.from_ints(self) - - def gaps(self): - """ - Return a list of Spans representing gaps for this span. A gap starts at the - first missing item in the integer items sequence and ends before the next - contiguous items range starts. - - For example: - >>> Span([4, 8]).gaps() - [Span(5, 7)] - >>> Span([4, 5, 6, 7, 8]).gaps() - [] - >>> Span([0]).gaps() - [] - >>> Span().gaps() - [] - >>> Span([0, 1, 3, 4, 7, 8, 10]).gaps() - [Span(2), Span(5, 6), Span(9)] - - This is idempotent: - >>> s = Span([0, 1, 3, 4, 7, 8, 10]) - >>> s.gaps() - [Span(2), Span(5, 6), Span(9)] - >>> s2 = Span.union(*s.gaps()) - >>> s3 = Span(s.start, s.end) - >>> s == s3.difference(s2) - True - """ - if not self: - return [] - # A gap starts at the end of each sub-span except for the last span. - # Stated differently the gaps are the spans created from grouped ints of - # the difference between an artificial maximum span from this span start - # to end and this span - domain = Span(self.start, self.end) - complement = domain.difference(self) - return Span.from_ints(complement) diff --git a/src/licensedcode/spanssets.py b/src/licensedcode/spanssets.py deleted file mode 100644 index c29a801c4e9..00000000000 --- a/src/licensedcode/spanssets.py +++ /dev/null @@ -1,464 +0,0 @@ -# -# Copyright (c) 2010 Matt Chaput. All rights reserved. -# Modifications by nexB Copyright 2016 nexB Inc. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY MATT CHAPUT ``AS IS'' AND ANY EXPRESS OR IMPLIED -# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO -# EVENT SHALL MATT CHAPUT OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, -# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, -# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# The views and conclusions contained in the software and documentation are -# those of the authors and should not be interpreted as representing official -# policies, either expressed or implied, of Matt Chaput. - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from itertools import count -from itertools import groupby - - -class Span(frozenset): - """ - Represent a range of tokens positions as a set of integers. - A Span is hashable and not meant to be modified once created. - It is equivalent to a sparse closed interval. - - Originally derived and heavily modified from Whoosh Span. - """ - def __new__(cls, *args): - """ - Create a new Span from a start and end ints or an iterable of ints. - - First form: - Span(start int, end int) : the span is initialized with a range(start, end+1) - - Second form: - Span(iterable of ints) : the span is initialized with the iterable - - Spans are hashable and immutable. - - For example: - >>> s = Span([1, 2]) - >>> s.start - 1 - >>> s.end - 2 - >>> s - Span(1, 2) - - >>> s = Span(1, 3) - >>> s.start - 1 - >>> s.end - 3 - >>> s - Span(1, 3) - - >>> s = Span([6, 5, 1, 2]) - >>> s.start - 1 - >>> s.end - 6 - >>> s - Span(1, 2)|Span(5, 6) - >>> len(s) - 4 - - >>> Span([5, 6, 7, 8, 9, 10 ,11, 12]) == Span([5, 6, 7, 8, 9, 10 ,11, 12]) - True - >>> hash(Span([5, 6, 7, 8, 9, 10 ,11, 12])) == hash(Span([5, 6, 7, 8, 9, 10 ,11, 12])) - True - >>> hash(Span([5, 6, 7, 8, 9, 10 ,11, 12])) == hash(Span(5, 12)) - True - """ - len_args = len(args) - - if len_args == 0: - return super(Span, cls).__new__(cls) - - elif len_args == 1: - # args0 is a single int or an iterable of ints - if isinstance(args[0], (int, long)): - return super(Span, cls).__new__(cls, args) - else: - return super(Span, cls).__new__(cls, args[0]) - - elif len_args == 2: - # args0 and args1 describe a start and end closed range - return super(Span, cls).__new__(cls, range(args[0], args[1] + 1)) - - else: - # args0 is a single int or args is an iterable of ints - # args is an iterable of ints - return super(Span, cls).__new__(cls, args) - - def __repr__(self): - """ - Return a brief representation of this span by only listing contiguous - spans and not all items. - - For example: - >>> Span([1, 2, 3, 4, 5, 7, 8, 9, 10]) - Span(1, 5)|Span(7, 10) - """ - subspans_repr = [] - for subs in self.subspans(): - ls = len(subs) - if not ls: - subspans_repr.append('Span()') - elif ls == 1: - subspans_repr.append('Span(%d)' % subs.start) - else: - subspans_repr.append('Span(%d, %d)' % (subs.start, subs.end)) - return '|'.join(subspans_repr) - - def __contains__(self, other): - """ - Return True if this span contains other span (where other is a Span, an - int or an ints set). - - For example: - >>> Span([5, 7]) in Span(5, 7) - True - >>> Span([5, 8]) in Span([5, 7]) - False - >>> 6 in Span([4, 5, 6, 7, 8]) - True - >>> 2 in Span([4, 5, 6, 7, 8]) - False - >>> 8 in Span([4, 8]) - True - >>> 5 in Span([4, 8]) - False - >>> set([4, 5]) in Span([4, 5, 6, 7, 8]) - True - >>> set([9]) in Span([4, 8]) - False - """ - if isinstance(other, (Span, set, frozenset,)): - return self.issuperset(other) - - if isinstance(other, (int, long)): - return super(Span, self).__contains__(other) - - @property - def start(self): - if not self: - raise TypeError('Empty Span has no start.') - return min(self) - - @property - def end(self): - if not self: - raise TypeError('Empty Span has no end.') - return max(self) - - @staticmethod - def sort(spans): - """ - Return a new sorted sequence of spans given a sequence of spans. - The primary sort is on start. The secondary sort is on length. - If two spans have the same start, the longer span will sort first. - - For example: - >>> spans = [Span([5, 6, 7, 8, 9, 10]), Span([1, 2]), Span([3, 4, 5]), Span([3, 4, 5, 6]), Span([8, 9, 10])] - >>> Span.sort(spans) - [Span(1, 2), Span(3, 6), Span(3, 5), Span(5, 10), Span(8, 10)] - - >>> spans = [Span([1, 2]), Span([3, 4, 5]), Span([3, 4, 5, 6]), Span([8, 9, 10])] - >>> Span.sort(spans) - [Span(1, 2), Span(3, 6), Span(3, 5), Span(8, 10)] - - >>> spans = [Span([1, 2]), Span([4, 5]), Span([7, 8]), Span([11, 12])] - >>> Span.sort(spans) - [Span(1, 2), Span(4, 5), Span(7, 8), Span(11, 12)] - - >>> spans = [Span([1, 2]), Span([7, 8]), Span([5, 6]), Span([12, 13])] - >>> Span.sort(spans) - [Span(1, 2), Span(5, 6), Span(7, 8), Span(12, 13)] - - """ - key = lambda s: (s.start, -len(s),) - return sorted(spans, key=key) - - def magnitude(self): - """ - Return the maximal length represented by this span start and end. The - magnitude is the same as the length for a contiguous span. It will be - greater than the length for a span with non-contiguous int items. - An empty span has a zero magnitude. - - For example: - >>> Span([4, 8]).magnitude() - 5 - >>> len(Span([4, 8])) - 2 - >>> len(Span([4, 5, 6, 7, 8])) - 5 - - >>> Span([4, 5, 6, 14 , 12, 128]).magnitude() - 125 - - >>> Span([4, 5, 6, 7, 8]).magnitude() - 5 - >>> Span([0]).magnitude() - 1 - >>> Span([0]).magnitude() - 1 - """ - if not self: - return 0 - return self.end - self.start + 1 - - def density(self): - """ - Return the density of this span as a ratio of its length to its - magnitude, a float between 0 and 1. A dense Span has all its integer - items contiguous and a maximum density of one. A sparse low density span - has some non-contiguous integer items. An empty span has a zero density. - - For example: - >>> Span([4, 8]).density() - 0.4 - >>> Span([4, 5, 6, 7, 8]).density() - 1.0 - >>> Span([0]).density() - 1.0 - >>> Span().density() - 0 - """ - if not self: - return 0 - - return len(self) / self.magnitude() - - def overlap(self, other): - """ - Return the count of overlapping items between this span and other span. - - For example: - >>> Span([1, 2]).overlap(Span([5, 6])) - 0 - >>> Span([5, 6]).overlap(Span([5, 6])) - 2 - >>> Span([4, 5, 6, 7]).overlap(Span([5, 6])) - 2 - >>> Span([4, 5, 6]).overlap(Span([5, 6, 7])) - 2 - >>> Span([4, 5, 6]).overlap(Span([6])) - 1 - >>> Span([4, 5]).overlap(Span([6, 7])) - 0 - """ - return len(self & other) - - def resemblance(self, other): - """ - Return a resemblance coefficient as a float between 0 and 1. - 0 means the spans are completely different and 1 identical. - """ - if self.isdisjoint(other): - return 0 - if self == other: - return 1 - resemblance = self.overlap(other) / len(self | other) - return resemblance - - def containment(self, other): - """ - Return a containment coefficient as a float between 0 and 1. This is an - indication of how much of the other span is contained in this span. - - 1 means the other span is entirely contained in this span. - 0 means that the other span is not contained at all this span. - """ - if self.isdisjoint(other): - return 0 - if self == other: - return 1 - containment = self.overlap(other) / len(other) - return containment - - def surround(self, other): - """ - Return True if this span surrounds other span. - - This is different from containment. A span can surround another span - region and have no positions in common with the surrounded. - - For example: - >>> Span([4, 8]).surround(Span([4, 8])) - True - >>> Span([3, 9]).surround(Span([4, 8])) - True - >>> Span([5, 8]).surround(Span([4, 8])) - False - >>> Span([4, 7]).surround(Span([4, 8])) - False - >>> Span([4, 5, 6, 7, 8]).surround(Span([5, 6, 7])) - True - """ - return self.start <= other.start and self.end >= other.end - - def is_before(self, other): - return self.end < other.start - - def is_after(self, other): - return self.start > other.end - - def touch(self, other): - """ - Return True if self sequence is contiguous with other span without overlap. - - For example: - >>> Span([5, 7]).touch(Span([5])) - False - >>> Span([5, 7]).touch(Span([5, 8])) - False - >>> Span([5, 7]).touch(Span([7, 8])) - False - >>> Span([5, 7]).touch(Span([8, 9])) - True - >>> Span([8, 9]).touch(Span([5, 7])) - True - """ - return self.start == other.end + 1 or self.end == other.start - 1 - - def distance_to(self, other): - """ - Return the absolute positive distance from this span to other span. - Touching and overlapping spans have a zero distance. - - For example: - >>> Span([8, 9]).distance_to(Span([5, 7])) - 0 - >>> Span([5, 7]).distance_to(Span([8, 9])) - 0 - >>> Span([5, 6]).distance_to(Span([8, 9])) - 2 - >>> Span([5, 7]).distance_to(Span([5, 7])) - 0 - >>> Span([4, 5, 6]).distance_to(Span([5, 6, 7])) - 0 - >>> Span([5, 7]).distance_to(Span([10, 12])) - 3 - >>> Span([1, 2]).distance_to(Span(range(4, 52))) - 2 - """ - if self.overlap(other) or self.touch(other): - return 0 - elif self.is_before(other): - return other.start - self.end - else: - return self.start - other.end - - def rebase(self, offset): - """ - Return a copy of this span adding `offset` to each item - - For example: - >>> Span([4, 5]).rebase(0) - Span(4, 5) - >>> Span(4, 5).rebase(1) - Span(5, 6) - >>> Span([4, 5]).rebase(3) - Span(7, 8) - >>> Span([1, 4, 5, 8, 9]).rebase(5) - Span(6)|Span(9, 10)|Span(13, 14) - """ - assert self.start + offset >= 0 - return Span(i + offset for i in self) - - @staticmethod - def from_ints(ints): - """ - Return a sequence of Spans from an iterable of ints. A new Span is - created for each group of monotonously increasing int items. - - >>> Span.from_ints([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]) - [Span(1, 12)] - >>> Span.from_ints([1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12]) - [Span(1, 3), Span(5, 12)] - >>> Span.from_ints([0, 2, 3, 5, 6, 7, 8, 9, 10, 11, 13]) - [Span(0), Span(2, 3), Span(5, 11), Span(13)] - """ - ints = sorted(set(ints)) - groups = (group for _, group in groupby(ints, lambda group, c=count(): next(c) - group)) - return map(Span, groups) - - def subspans(self): - """ - Return a list of Spans creating one new Span for each set of contiguous - integer items. - - For example: - >>> span = Span(5, 6, 7, 8, 9, 10) | Span([1, 2]) | Span(3, 5) | Span(3, 6) | Span([8, 9, 10]) - >>> span.subspans() - [Span(1, 10)] - - When subspans are not touching they do not merge : - >>> span = Span([63, 64]) | Span([58, 58]) - >>> span.subspans() - [Span(58), Span(63, 64)] - - Overlapping subspans are merged as needed: - >>> span = Span([12, 17, 24]) | Span([15, 16, 17, 35]) | Span(58) | Span(63, 64) - >>> span.subspans() - [Span(12), Span(15, 17), Span(24), Span(35), Span(58), Span(63, 64)] - """ - return Span.from_ints(self) - - def gaps(self): - """ - Return a list of Spans representing gaps for this span. A gap starts at - the first missing item in the integer items sequence and ends before the - next contiguous items range starts. - - For example: - >>> Span([4, 8]).gaps() - [Span(5, 7)] - >>> Span([4, 5, 6, 7, 8]).gaps() - [] - >>> Span([0]).gaps() - [] - >>> Span().gaps() - [] - >>> Span([0, 1, 3, 4, 7, 8, 10]).gaps() - [Span(2), Span(5, 6), Span(9)] - - This is idempotent: - >>> s = Span([0, 1, 3, 4, 7, 8, 10]) - >>> s.gaps() - [Span(2), Span(5, 6), Span(9)] - >>> s2 = Span.union(*s.gaps()) - >>> s3 = Span(s.start, s.end) - >>> s == s3.difference(s2) - True - """ - if not self: - return [] - # A gap starts at the end of each sub-span except for the last span. - # Stated differently the gaps are the spans created from grouped ints of - # the difference between an artificial maximum span from this span start - # to end and this span - domain = Span(self.start, self.end) - complement = domain.difference(self) - return Span.from_ints(complement) diff --git a/src/scancode/api.py b/src/scancode/api.py index 1010a25d3af..da33e481e07 100644 --- a/src/scancode/api.py +++ b/src/scancode/api.py @@ -119,10 +119,9 @@ def get_licenses(location, min_score=0): licenses = licenses_getter() # note: we do USE the cache here - for match in idx.match(location=location, min_score=min_score, use_cache=True): + for match in idx.match(location=location, min_score=min_score, use_cache=False): for license_key in match.rule.licenses: lic = licenses.get(license_key) - lines_start, lines_end = match.lines() result = OrderedDict() result['key'] = lic.key result['score'] = match.score() @@ -134,8 +133,8 @@ def get_licenses(location, min_score=0): result['dejacode_url'] = DEJACODE_LICENSE_URL.format(lic.key) result['spdx_license_key'] = lic.spdx_license_key result['spdx_url'] = lic.spdx_url - result['start_line'] = lines_start - result['end_line'] = lines_end + result['start_line'] = match.start_line + result['end_line'] = match.end_line result['matched_rule'] = OrderedDict() result['matched_rule']['identifier'] = match.rule.identifier result['matched_rule']['license_choice'] = match.rule.license_choice diff --git a/src/scancode/cache.py b/src/scancode/cache.py index d8a5fb2fc6b..7aff6001cb0 100644 --- a/src/scancode/cache.py +++ b/src/scancode/cache.py @@ -26,29 +26,40 @@ from collections import OrderedDict from functools import partial +import json +from hashlib import sha1 import os +import posixpath import sys from commoncode import fileutils +from commoncode.fileutils import as_posixpath from commoncode import timeutils from scancode import scans_cache_dir """ -Caching scans on disk: A cache of all the scan results. +Cache scan results for a file or directory disk using a file-based cache. -Each scan results for a file or directory is cached on disk. +The approach is to cache the scan of a file using these files: + - one "global" file contains a log of all the paths scanned. + - for each file being scanned, we store a file that contains the corresponding file + info data as JSON. This file is named after the hash of the path of a scanned file. + - for each unique file being scanned (e.g. based on its content SHA1), we store a + another JSON file that contains the corresponding scan data. This file is named + after the hash of the scanned file content. -The approach is to use to cache: - - the results of a scan, excluding file infos keyed by the hash of a scanned file - - the file infos, keyed by the path of a scanned file - -Once a scan is completed, we iterate the caches to output the scan results using this -procedure: iterate the cached file infos and for each lookup the scan details in the -cached scan results. This iteration is driving the final streaming of results to the -output format (e.g. JSON). +Once a scan is completed, we iterate the cache to output the final scan results: +First iterate the global log file to get the paths, from there collect the cached +file info for that file and from the path and file info collect the cached scanned +result. This iterator is then streamed to the final JSON output. Finally once a scan is completed the cache is destroyed to free up disk space. + +Internally the cache is organized as a tree of directories named after the first few +characters or a path hash or file hash. This is to avoid having having too many files +per directory that can make some filesystems choke as well as having directories that +are too deep or having file paths that are too long which problematic on some OS. """ # Tracing flags @@ -69,123 +80,221 @@ def logger_debug(*args): return logger.debug(' '.join(isinstance(a, basestring) and a or repr(a) for a in args)) -class ScanCache(object): +def get_scans_cache_class(cache_dir=scans_cache_dir): + """ + Return a new persistent cache class configured with a unique storage directory. + """ + # create a unique temp directory in cache_dir + fileutils.create_dir(cache_dir) + cache_dir = fileutils.get_temp_dir(cache_dir, prefix=timeutils.time2tstamp() + '-') + sc = ScanFileCache(cache_dir) + sc.setup() + return partial(ScanFileCache, cache_dir) + + +def info_keys(path): + """ + Return a file info cache keys tripple for a path. + + For example: + >>> sha1('/w421/scancode-toolkit2').hexdigest() + 'fb87db2bb28e9501ac7fdc4812782118f4c94a0f' + >>> info_keys('/w421/scancode-toolkit2') + ('f', 'b', '87db2bb28e9501ac7fdc4812782118f4c94a0f') + """ + return keys_from_hash(sha1(path).hexdigest()) + + +def scan_keys(path, file_info): + """ + Return a scan cache keys tripple for a path and file_info. If the file_info + sha1 is empty (e.g. such as a directory), return a key based on the path instead. + """ + sha1_digest = file_info['sha1'] + if sha1_digest: + return keys_from_hash(sha1_digest) + else: + # we may eventually store directories, in which case we use the path as a key + # with some extra seed + return info_keys(u'empty hash' + path) + + +def keys_from_hash(hexdigest): + """ + Return a cache keys triple for a hash hexdigest string. + + NOTE: since we use the first character and next two characters as directories, we + create at most 16 dir at the first level and 16 dir at the second level for each + first level directory for a maximum total of 16*16 = 256 directories. For a + million files we would have about 4000 files per directory on average with this + scheme which should keep most file systems happy and avoid some performance + issues when there are too many files in a single directory. + + For example: + >>> keys_from_hash('fb87db2bb28e9501ac7fdc4812782118f4c94a0f') + ('f', 'b', '87db2bb28e9501ac7fdc4812782118f4c94a0f') + """ + return hexdigest[0], hexdigest[1], hexdigest[2:] + + +def paths_from_keys(base_path, keys): + """ + Return a tuple of (parent dir path, filename) built from a cache keys triple and + a base_directory. Ensure that the parent directory exist. + """ + dir1, dir2, fname = keys + parent = os.path.join(base_path, dir1, dir2) + fileutils.create_dir(parent) + return parent, fname + + +class ScanFileCache(object): """ - A file-based cache for scan results. - This is NOT thread-safe, but is multi-process safe. + A file-based cache for scan results saving results in files and using no locking. + This is NOT thread-safe and NOT multi-process safe but works OK in our context: + we cache the scan for a given file once and read it only a few times. """ def __init__(self, cache_dir): self.cache_base_dir = cache_dir - # subdirs for infos and scans caches - self.cache_infos_dir = os.path.join(self.cache_base_dir, 'infos') - self.cache_scans_dir = os.path.join(self.cache_base_dir, 'scans') + # subdirs for info and scans caches + self.cache_infos_dir = as_posixpath(os.path.join(self.cache_base_dir, 'infos/')) + self.cache_scans_dir = as_posixpath(os.path.join(self.cache_base_dir, 'scans/')) + self.cache_files_log = as_posixpath(os.path.join(self.cache_base_dir, 'files_log')) - # workaround for https://github.com/grantjenks/python-diskcache/issues/32 - from diskcache import Disk - class DiskWithNoHighPickleProtocol(Disk): - "Subclass of diskcache.Disk that always use the lowest pickle protocol." - def __init__(self, directory, size_threshold, pickle_protocol): - super(DiskWithNoHighPickleProtocol, self).__init__(directory, size_threshold, pickle_protocol) - self._protocol = 0 + def setup(self): + """ + Setup the cache: must be called at least once globally after cache + initialization. + """ + os.makedirs(self.cache_infos_dir) + os.makedirs(self.cache_scans_dir) - # and finally cache instances - from diskcache import Cache - self.infos = Cache(self.cache_infos_dir, disk=DiskWithNoHighPickleProtocol) - self.scans = Cache(self.cache_scans_dir, disk=DiskWithNoHighPickleProtocol) + @classmethod + def log_file_path(cls, logfile_fd, path): + """ + Log file path in the cache logfile_fd **opened** file descriptor. + """ + # we dump the path as JSON, one per line. + # JSON is to avoid any issue with weird file paths/names + logfile_fd.write(json.dumps(path)) + logfile_fd.write('\n') - def scan_key(self, path, file_infos): + def get_cached_info_path(self, path): """ - Return a scan cache key for a path and file_infos. + Return the path where to store a file info in the cache given a path. """ - sha1 = file_infos['sha1'] - # we may eventually store directories, in which case we use the path as a key - return sha1 or path + keys = info_keys(path) + paths = paths_from_keys(self.cache_infos_dir, keys) + return posixpath.join(*paths) - def put_infos(self, path, file_infos): + def put_info(self, path, file_info): """ - Put file_infos for path in the cache and return True if the file referenced - in file_infos has already been scanned or False otherwise. + Put file_info for path in the cache and return True if the file referenced + in file_info has already been scanned or False otherwise. """ - self.infos.set(path, file_infos) - is_scan_cached = self.scan_key(path, file_infos) in self.scans + info_path = self.get_cached_info_path(path) + with open(info_path, 'wb') as cached_infos: + json.dump(file_info, cached_infos, check_circular=False) + scan_path = self.get_cached_scan_path(path, file_info) + is_scan_cached = os.path.exists(scan_path) if TRACE: - logger_debug('put_infos:', 'path:', path, 'is_scan_cached:', is_scan_cached, 'file_infos:', file_infos, '\n') - logger_debug('put_infos:', 'cached_infos:', self.infos[path], '\n') + logger_debug('put_infos:', 'path:', path, 'is_scan_cached:', is_scan_cached, 'file_info:', file_info, '\n') return is_scan_cached - def put_scan(self, path, file_infos, scan_result): + def get_info(self, path): """ - Put scan_result in the cache if not already cached. + Return file info from the cache for a path. + Return None on failure to find the info in the cache. """ - scan_key = self.scan_key(path, file_infos) - self.scans.add(scan_key, scan_result) - if TRACE: - logger_debug('put_scan:', 'scan_key:', scan_key, 'file_infos:', file_infos, 'scan_result:', scan_result, '\n') - logger_debug('put_scan:', 'cached_infos:', self.infos[path], '\n') - logger_debug('put_scan:', 'scan_key:', scan_key, 'cached_scan:', self.scans[scan_key], '\n') + info_path = self.get_cached_info_path(path) + if os.path.exists(info_path): + with open(info_path, 'rb') as ci: + return json.load(ci, object_pairs_hook=OrderedDict) - def iterate(self, with_infos=True): + def get_cached_scan_path(self, path, file_info): """ - Yield scan data for all cached scans e.g. the whole cache. - If a scan is missing for a given info, an error is appended to scan_errors. + Return the path where to store a scan in the cache given a path and file_info. """ - for path in self.infos: - file_infos = self.infos[path] - scan_result = OrderedDict(path=path) - if with_infos: - # infos is always collected but only returnedd if asked: - # we flatten these as direct attributes of a file object - scan_result.update(file_infos.items()) - else: - # always include errors even if empty - scan_result['scan_errors'] = file_infos.get('scan_errors', []) + keys = scan_keys(path, file_info) + paths = paths_from_keys(self.cache_scans_dir, keys) + return posixpath.join(*paths) - no_scan_details = dict(scan_errors=[ - ('ERROR: Requested scan details unavailable in cache.', - 'This is either a bug or processing was aborted with CTRL-C.')] - ) - - scan_key = self.scan_key(path, file_infos) - scan_details = self.scans.get(scan_key, no_scan_details) - if TRACE: - logger_debug('iterate:', 'scan_details:', scan_details, 'for path:', path, 'scan_key:', scan_key, '\n') - - # append errors to other top level errors if any - scan_errors = scan_details.pop('scan_errors', []) - scan_result['scan_errors'].extend(scan_errors) + def put_scan(self, path, file_info, scan_result): + """ + Put scan_result in the cache if not already cached. + """ + scan_path = self.get_cached_scan_path(path, file_info) + if not os.path.exists(scan_path): + with open(scan_path, 'wb') as cached_scan: + json.dump(scan_result, cached_scan, check_circular=False) + if TRACE: + logger_debug('put_scan:', 'scan_path:', scan_path, 'file_info:', file_info, 'scan_result:', scan_result, '\n') - scan_result.update(scan_details) - yield scan_result + def get_scan(self, path, file_info): + """ + Return scan results from the cache for a path and file_info. + Return None on failure to find the scan results in the cache. + """ + scan_path = self.get_cached_scan_path(path, file_info) + if os.path.exists(scan_path): + with open(scan_path, 'rb') as cs: + return json.load(cs, object_pairs_hook=OrderedDict) - def close(self): + def iterate(self, scan_names): """ - Close the underlying caches. + Yield scan data for all cached scans e.g. the whole cache given a list of + scan names. + + The logfile MUST have been closed before calling this method. """ - if self.infos: - self.infos.close() - if self.scans: - self.scans.close() + with open(self.cache_files_log, 'rb') as cached_files: + # iterate the list of (path, (info keys)), one by line + for file_log in cached_files: + path = json.loads(file_log) + file_info = self.get_info(path) + + # rare but possible corner case + if file_info is None: + no_info = ('ERROR: file info unavailable in cache: ' + 'This is either a bug or processing was aborted with CTRL-C.') + scan_result = OrderedDict(path=path) + scan_result['scan_errors'] = [no_info] + if TRACE: + logger_debug('iterate:', 'scan_result:', scan_result, 'for path:', path, '\n') + yield scan_result + continue + + path = file_info.pop('path') + scan_result = OrderedDict(path=path) + + if 'infos' in scan_names: + # infos is always collected but only returned if requested + # we flatten these as direct attributes of a file object + scan_result.update(file_info.items()) + if not scan_result.get('scan_errors'): + scan_result['scan_errors'] = [] + + # check if we have more than just infos + if ['infos'] != scan_names: + errors = scan_result['scan_errors'] + scan_details = self.get_scan(path, file_info) + if scan_details is None: + no_scan_details = ( + 'ERROR: scan details unavailable in cache: ' + 'This is either a bug or processing was aborted with CTRL-C.') + errors.append(no_scan_details) + else: + # append errors to other top level errors if any + scan_errors = scan_details.pop('scan_errors', []) + errors.extend(scan_errors) + scan_result.update(scan_details) + + if TRACE: + logger_debug('iterate:', 'scan_result:', scan_result, 'for path:', path, '\n') + yield scan_result def clear(self, *args): """ Purge the cache by deleting the corresponding cached data files. """ - self.close() fileutils.delete(self.cache_base_dir) - - -def get_scans_cache(cache_dir=scans_cache_dir): - """ - Return a new unique persistent cache instance. - """ - return ScanCache(cache_dir) - - -def get_scans_cache_class(cache_dir=scans_cache_dir): - """ - Return a new unique persistent cache instance. - """ - fileutils.create_dir(cache_dir) - # create a unique temp directory in cache_dir - cache_dir = fileutils.get_temp_dir(cache_dir, prefix=timeutils.time2tstamp() + '-') - return partial(ScanCache, cache_dir) diff --git a/src/scancode/cli.py b/src/scancode/cli.py index 1e9f3e2cf47..6bbc7c40cb3 100644 --- a/src/scancode/cli.py +++ b/src/scancode/cli.py @@ -30,6 +30,7 @@ # FIXME: unknown license ########################################################################### from multiprocessing.pool import IMapIterator, IMapUnorderedIterator +from scancode.cache import ScanFileCache def wrapped(func): # ensure that we do not double wrap @@ -90,7 +91,6 @@ def wrap(self, timeout=None): from scancode.api import get_urls - info_text = ''' ScanCode scans code and other files for origin and license. Visit https://github.com/nexB/scancode-toolkit/ for support and download. @@ -308,11 +308,12 @@ def scancode(ctx, input, output_file, copyright, license, package, files_count, results = scan(input, copyright, license, package, email, url, info, license_score, verbose, quiet, processes, scans_cache_class, to_stdout, diag, timeout, max_memory) + click.secho('Saving results...', err=to_stdout, fg='green') save_results(files_count, results, format, input, output_file) finally: # cleanup cache = scans_cache_class() - cache.clear() + # cache.clear() def scan(input_path, copyright=True, license=True, package=True, @@ -355,10 +356,11 @@ def scan(input_path, copyright=True, license=True, package=True, if license: # build index outside of the main loop # this also ensures that forked processes will get the index on POSIX naturally - click.secho('Building license detection index...', err=to_stdout, fg='green') + click.secho('Building license detection index...', err=to_stdout, fg='green', nl=False) from licensedcode.index import get_index _idx = get_index() indexing_time = time() - scan_start + click.secho('Done.', err=to_stdout, fg='green', nl=True) scan_summary['indexing_time'] = indexing_time @@ -368,38 +370,46 @@ def scan(input_path, copyright=True, license=True, package=True, # maxtasksperchild helps with recycling processes in case of leaks pool = Pool(processes=processes, maxtasksperchild=1000) resources = resource_paths(input_path) - scanit = partial(_scanit, scanners=scanners, scans_cache_class=scans_cache_class, - diag=diag, timeout=timeout, max_memory=max_memory) - # Using chunksize is documented as much more efficient in the Python doc. - # Yet "1" still provides a better and more progressive feedback. - # With imap_unordered, results are returned as soon as ready and out of order. - scanned_files = pool.imap_unordered(scanit, resources, chunksize=1) - pool.close() - - def scan_event(item): - """Progress event displayed each time a file is scanned""" - if item: - _scan_success, _scanned_path = item - _progress_line = verbose and _scanned_path or fileutils.file_name(_scanned_path) - return style('Scanned: ') + style(_progress_line, fg=_scan_success and 'green' or 'red') - - scanning_errors = [] - files_count = 0 - with utils.progressmanager(scanned_files, item_show_func=scan_event, - show_pos=True, verbose=verbose, quiet=quiet) as scanned: - while True: - try: - result = scanned.next() - scan_success, scanned_rel_path = result - if not scan_success: - scanning_errors.append(scanned_rel_path) - files_count += 1 - except StopIteration: - break - except KeyboardInterrupt: - print('\nAborted!') - pool.terminate() - break + logfile_path = scans_cache_class().cache_files_log + with open(logfile_path, 'wb') as logfile_fd: + + logged_resources = _resource_logger(logfile_fd, resources) + + scanit = partial(_scanit, scanners=scanners, scans_cache_class=scans_cache_class, + diag=diag, timeout=timeout, max_memory=max_memory) + # Using chunksize is documented as much more efficient in the Python doc. + # Yet "1" still provides a better and more progressive feedback. + # With imap_unordered, results are returned as soon as ready and out of order. + scanned_files = pool.imap_unordered(scanit, logged_resources, chunksize=1) + pool.close() + + click.secho('Scanning files...', err=to_stdout, fg='green') + + + def scan_event(item): + """Progress event displayed each time a file is scanned""" + if item: + _scan_success, _scanned_path = item + _progress_line = verbose and _scanned_path or fileutils.file_name(_scanned_path) + return style('Scanned: ') + style(_progress_line, fg=_scan_success and 'green' or 'red') + + scanning_errors = [] + files_count = 0 + with utils.progressmanager(scanned_files, item_show_func=scan_event, show_pos=True, verbose=verbose, quiet=quiet) as scanned: + while True: + try: + result = scanned.next() + scan_success, scanned_rel_path = result + if not scan_success: + scanning_errors.append(scanned_rel_path) + files_count += 1 + except StopIteration: + break + except KeyboardInterrupt: + print('\nAborted!') + pool.terminate() + break + # Compute stats ########################## @@ -409,12 +419,13 @@ def scan_event(item): scanning_time = total_time - indexing_time scan_summary['total_time'] = total_time scan_summary['scanning_time'] = scanning_time + files_scanned_per_second = round(float(files_count) / scanning_time , 2) scan_summary['files_scanned_per_second'] = files_scanned_per_second # Display stats ########################## - click.secho('Scanning done.' % locals(), fg=scanning_errors and 'red' or 'green', err=to_stdout) + click.secho('Scanning done.', fg=scanning_errors and 'red' or 'green', err=to_stdout) if scanning_errors: click.secho('Some files failed to scan properly. See scan for details:', fg='red', err=to_stdout) for errored_path in scanning_errors: @@ -422,13 +433,28 @@ def scan_event(item): click.secho('Scan statistics: %(files_count)d files scanned in %(total_time)ds.' % locals(), err=to_stdout) click.secho('Scan options: %(_scans)s with %(processes)d process(es).' % locals(), err=to_stdout) - click.secho('Scanning speed: {:.2} files per sec.'.format(files_scanned_per_second), err=to_stdout) + click.secho('Scanning speed: %(files_scanned_per_second)s files per sec.' % locals(), err=to_stdout) click.secho('Scanning time: %(scanning_time)ds.' % locals(), err=to_stdout, reset=True,) click.secho('Indexing time: %(indexing_time)ds.' % locals(), err=to_stdout) # finally return an iterator on cached results + scan_names = [] + if info: + scan_names.append('infos') + scan_names.extend(k for k, v in scanners.items() if v) cached_scan = scans_cache_class() - return files_count, cached_scan.iterate(with_infos=info) + return files_count, cached_scan.iterate(scan_names) + + +def _resource_logger(logfile_fd, resources): + """ + Log file path to the logfile_fd opened file descriptor for each resource and + yield back the resources. + """ + file_logger = ScanFileCache.log_file_path + for posix_path, rel_path in resources: + file_logger(logfile_fd, rel_path) + yield posix_path, rel_path def _scanit(paths, scanners, scans_cache_class, diag, timeout=DEFAULT_TIMEOUT, max_memory=DEFAULT_MAX_MEMORY): @@ -439,41 +465,35 @@ def _scanit(paths, scanners, scans_cache_class, diag, timeout=DEFAULT_TIMEOUT, m """ abs_path, rel_path = paths # always fetch infos and cache. - infos = scan_infos(abs_path) - scans_cache = None + infos = OrderedDict() + infos['path'] = rel_path + infos.update(scan_infos(abs_path, diag=diag)) + success = True - try: - # build a local instance of a cache - scans_cache = scans_cache_class() - is_cached = scans_cache.put_infos(rel_path, infos) + scans_cache = scans_cache_class() + is_cached = scans_cache.put_info(rel_path, infos) + has_scanners = any(scanners.values()) + if has_scanners: # Skip other scans if already cached - # ENSURE we only do tghis for files not directories + # FIXME: ENSURE we only do this for files not directories if not is_cached: # run the scan as an interruptiple task scans_runner = partial(scan_one, abs_path, scanners, diag) - - file_size = infos.get('size', 0) - # quota keyword args for interruptible kwargs = dict(timeout=timeout, max_memory=max_memory) - success, scan_result = interruptible(scans_runner, **kwargs) - if not success: # Use scan errors as the scan result for that file on failure this is # a top-level error not attachedd to a specific scanner, hence the # "scan" key is used for these errors - scan_result = {'scan_errors': [{'scan': [scan_result]}]} + scan_result = {'scan_errors': [scan_result]} scans_cache.put_scan(rel_path, infos, scan_result) # do not report success if some other errors happened if scan_result.get('scan_errors'): success = False - finally: - if scans_cache: - scans_cache.close() return success, rel_path @@ -497,7 +517,7 @@ def resource_paths(base_path): yield posix_path, rel_path -def scan_infos(input_file): +def scan_infos(input_file, diag=False): """ Scan one file or directory and return file_infos data. This always contains an extra 'errors' key with a list of error messages, @@ -509,23 +529,25 @@ def scan_infos(input_file): infos = get_file_infos(input_file, as_list=False) except Exception, e: # never fail but instead add an error message. - errors = dict(infos=(e.message, traceback.format_exc(),)) + messages = ['ERROR: infos: ' + e.message] + if diag: + messages.append('ERROR: infos: ' + traceback.format_exc()) # put errors last infos['scan_errors'] = errors return infos -def scan_one(input_file, scans, diag=False): +def scan_one(input_file, scanners, diag=False): """ Scan one file or directory and return a scanned data, calling every scan in the `scans` mapping of (scan name -> scan function). Scan data contain a - 'scan_errors' key with errors a dictionary keyed by "scan name" and a value as a - list of errors messages. If `diag` is True, 'scan_errors' error messages - also contain detailed diagnotics information, e.g. a traceback if available. + 'scan_errors' key with a list of error messages. + If `diag` is True, 'scan_errors' error messages also contain detailed diagnostic + information such as a traceback if available. """ scan_result = OrderedDict() scan_errors = [] - for scan_name, scan_func in scans.items(): + for scan_name, scan_func in scanners.items(): if not scan_func: continue try: @@ -537,10 +559,10 @@ def scan_one(input_file, scans, diag=False): except Exception, e: # never fail but instead add an error message and keep an empty scan: scan_result[scan_name] = [] - errs = [e.message] + messages = ['ERROR: ' + scan_name + ': ' + e.message] if diag: - errs.append(traceback.format_exc()) - scan_errors.append({scan_name: [e.message]}) + messages.append('ERROR: ' + scan_name + ': ' + traceback.format_exc()) + scan_errors.extend(messages) # put errors last, after scans proper scan_result['scan_errors'] = scan_errors return scan_result diff --git a/src/scancode/interrupt.py b/src/scancode/interrupt.py index 7ed90019d01..44faa54580f 100644 --- a/src/scancode/interrupt.py +++ b/src/scancode/interrupt.py @@ -104,18 +104,18 @@ def interruptible(func, *args, **kwargs): result = threads.next(timeout) if result == MEMORY_EXCEEDED: max_mb = megabytes(max_memory) - return False, 'Processing interrupted: excessive memory usage of more than %(max_mb)s.' % locals() + return False, 'ERROR: Processing interrupted: excessive memory usage of more than %(max_mb)s.' % locals() elif result == RUNTIME_EXCEEDED: - return False, 'Processing interrupted: timeout after %(timeout)d seconds.' % locals() + return False, 'ERROR: Processing interrupted: timeout after %(timeout)d seconds.' % locals() else: # we succeeded with quotas: return expected results return True, result except multiprocessing.TimeoutError: - return False, 'Processing interrupted: timeout after %(timeout)d seconds.' % locals() + return False, 'ERROR: Processing interrupted: timeout after %(timeout)d seconds.' % locals() except KeyboardInterrupt: - return False, 'Processing interrupted with Ctrl-C.' + return False, 'ERROR: Processing interrupted with Ctrl-C.' finally: # stop processing diff --git a/tests/licensedcode/data/index/test__add_rules.json b/tests/licensedcode/data/index/test__add_rules.json index 38f18afd14d..70699ae6c09 100644 --- a/tests/licensedcode/data/index/test__add_rules.json +++ b/tests/licensedcode/data/index/test__add_rules.json @@ -28,6 +28,9 @@ "conditioned": [ 74 ], + "minimum": [ + 61 + ], "including": [ 76, 139, @@ -151,9 +154,6 @@ "promote": [ 108 ], - "permitted": [ - 9 - ], "nor": [ 96 ], @@ -342,9 +342,6 @@ "promote": [ 86 ], - "permitted": [ - 13 - ], "nor": [ 74 ], diff --git a/tests/licensedcode/data/index/test__add_rules_with_templates.json b/tests/licensedcode/data/index/test__add_rules_with_templates.json index a45cc7ca528..80df55df84a 100644 --- a/tests/licensedcode/data/index/test__add_rules_with_templates.json +++ b/tests/licensedcode/data/index/test__add_rules_with_templates.json @@ -28,6 +28,9 @@ "conditioned": [ 72 ], + "minimum": [ + 59 + ], "including": [ 74, 137, @@ -150,9 +153,6 @@ "promote": [ 106 ], - "permitted": [ - 7 - ], "nor": [ 94 ], @@ -341,9 +341,6 @@ "promote": [ 84 ], - "permitted": [ - 13 - ], "nor": [ 72 ], diff --git a/tests/licensedcode/data/index/test_init_with_rules.json b/tests/licensedcode/data/index/test_init_with_rules.json index 38f18afd14d..70699ae6c09 100644 --- a/tests/licensedcode/data/index/test_init_with_rules.json +++ b/tests/licensedcode/data/index/test_init_with_rules.json @@ -28,6 +28,9 @@ "conditioned": [ 74 ], + "minimum": [ + 61 + ], "including": [ 76, 139, @@ -151,9 +154,6 @@ "promote": [ 108 ], - "permitted": [ - 9 - ], "nor": [ 96 ], @@ -342,9 +342,6 @@ "promote": [ 86 ], - "permitted": [ - 13 - ], "nor": [ 74 ], diff --git a/tests/licensedcode/data/licenses/artistic-2.0_and_gpl-1.0-plus_and_zlib.yml b/tests/licensedcode/data/licenses/artistic-2.0_and_gpl-1.0-plus_and_zlib.yml index 309405d0835..77439568f00 100644 --- a/tests/licensedcode/data/licenses/artistic-2.0_and_gpl-1.0-plus_and_zlib.yml +++ b/tests/licensedcode/data/licenses/artistic-2.0_and_gpl-1.0-plus_and_zlib.yml @@ -1,7 +1,7 @@ licenses: - - zlib - artistic-2.0 - gpl-1.0-plus + - zlib - artistic-2.0 - gpl-1.0-plus - artistic-2.0 diff --git a/tests/licensedcode/data/licenses/fraunhofer-fdk-aac-codec_1.yml b/tests/licensedcode/data/licenses/fraunhofer-fdk-aac-codec_1.yml index 0970123c370..3f46e4ff452 100644 --- a/tests/licensedcode/data/licenses/fraunhofer-fdk-aac-codec_1.yml +++ b/tests/licensedcode/data/licenses/fraunhofer-fdk-aac-codec_1.yml @@ -1,4 +1,4 @@ licenses: - fraunhofer-fdk-aac-codec -min_score: 95 + diff --git a/tests/licensedcode/data/licenses/ietf_4.yml b/tests/licensedcode/data/licenses/ietf_4.yml index 00a972782e6..fa1dfd4a1a0 100644 --- a/tests/licensedcode/data/licenses/ietf_4.yml +++ b/tests/licensedcode/data/licenses/ietf_4.yml @@ -1,3 +1,2 @@ licenses: - ietf -solid: yes diff --git a/tests/licensedcode/data/licenses/long-m32c-desc.c b/tests/licensedcode/data/licenses/long-m32c-desc.c new file mode 100644 index 00000000000..0cff7e0291b --- /dev/null +++ b/tests/licensedcode/data/licenses/long-m32c-desc.c @@ -0,0 +1,663 @@ +/* CPU data for m32c. + +THIS FILE IS MACHINE GENERATED WITH CGEN. + +Copyright 1996-2005 Free Software Foundation, Inc. + +This file is part of the GNU Binutils and/or GDB, the GNU debugger. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2, or (at your option) +any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street - Fifth Floor, Boston, MA 02110-1301, USA. + +*/ + +#include "sysdep.h" +#include +#include +#include "ansidecl.h" +#include "bfd.h" +#include "symcat.h" +#include "m32c-desc.h" +#include "m32c-opc.h" +#include "opintl.h" +#include "libiberty.h" +#include "xregex.h" + +/* Attributes. */ + +static const CGEN_ATTR_ENTRY bool_attr[] = +{ + { "#f", 0 }, + { "#t", 1 }, + { 0, 0 } +}; + +static const CGEN_ATTR_ENTRY MACH_attr[] ATTRIBUTE_UNUSED = +{ + { "base", MACH_BASE }, + { "m16c", MACH_M16C }, + { "m32c", MACH_M32C }, + { "max", MACH_MAX }, + { 0, 0 } +}; + +static const CGEN_ATTR_ENTRY ISA_attr[] ATTRIBUTE_UNUSED = +{ + { "m16c", ISA_M16C }, + { "m32c", ISA_M32C }, + { "max", ISA_MAX }, + { 0, 0 } +}; + +static const CGEN_ATTR_ENTRY RL_TYPE_attr[] ATTRIBUTE_UNUSED = +{ + { "NONE", RL_TYPE_NONE }, + { "JUMP", RL_TYPE_JUMP }, + { "1ADDR", RL_TYPE_1ADDR }, + { "2ADDR", RL_TYPE_2ADDR }, + { 0, 0 } +}; + +const CGEN_ATTR_TABLE m32c_cgen_ifield_attr_table[] = +{ + { "MACH", & MACH_attr[0], & MACH_attr[0] }, + { "ISA", & ISA_attr[0], & ISA_attr[0] }, + { "RL_TYPE", & RL_TYPE_attr[0], & RL_TYPE_attr[0] }, + { "VIRTUAL", &bool_attr[0], &bool_attr[0] }, + { "PCREL-ADDR", &bool_attr[0], &bool_attr[0] }, + { "ABS-ADDR", &bool_attr[0], &bool_attr[0] }, + { "RESERVED", &bool_attr[0], &bool_attr[0] }, + { "SIGN-OPT", &bool_attr[0], &bool_attr[0] }, + { "SIGNED", &bool_attr[0], &bool_attr[0] }, + { 0, 0, 0 } +}; + +const CGEN_ATTR_TABLE m32c_cgen_hardware_attr_table[] = +{ + { "MACH", & MACH_attr[0], & MACH_attr[0] }, + { "ISA", & ISA_attr[0], & ISA_attr[0] }, + { "RL_TYPE", & RL_TYPE_attr[0], & RL_TYPE_attr[0] }, + { "VIRTUAL", &bool_attr[0], &bool_attr[0] }, + { "CACHE-ADDR", &bool_attr[0], &bool_attr[0] }, + { "PC", &bool_attr[0], &bool_attr[0] }, + { "PROFILE", &bool_attr[0], &bool_attr[0] }, + { 0, 0, 0 } +}; + +const CGEN_ATTR_TABLE m32c_cgen_operand_attr_table[] = +{ + { "MACH", & MACH_attr[0], & MACH_attr[0] }, + { "ISA", & ISA_attr[0], & ISA_attr[0] }, + { "RL_TYPE", & RL_TYPE_attr[0], & RL_TYPE_attr[0] }, + { "VIRTUAL", &bool_attr[0], &bool_attr[0] }, + { "PCREL-ADDR", &bool_attr[0], &bool_attr[0] }, + { "ABS-ADDR", &bool_attr[0], &bool_attr[0] }, + { "SIGN-OPT", &bool_attr[0], &bool_attr[0] }, + { "SIGNED", &bool_attr[0], &bool_attr[0] }, + { "NEGATIVE", &bool_attr[0], &bool_attr[0] }, + { "RELAX", &bool_attr[0], &bool_attr[0] }, + { "SEM-ONLY", &bool_attr[0], &bool_attr[0] }, + { 0, 0, 0 } +}; + +const CGEN_ATTR_TABLE m32c_cgen_insn_attr_table[] = +{ + { "MACH", & MACH_attr[0], & MACH_attr[0] }, + { "ISA", & ISA_attr[0], & ISA_attr[0] }, + { "RL_TYPE", & RL_TYPE_attr[0], & RL_TYPE_attr[0] }, + { "ALIAS", &bool_attr[0], &bool_attr[0] }, + { "VIRTUAL", &bool_attr[0], &bool_attr[0] }, + { "UNCOND-CTI", &bool_attr[0], &bool_attr[0] }, + { "COND-CTI", &bool_attr[0], &bool_attr[0] }, + { "SKIP-CTI", &bool_attr[0], &bool_attr[0] }, + { "DELAY-SLOT", &bool_attr[0], &bool_attr[0] }, + { "RELAXABLE", &bool_attr[0], &bool_attr[0] }, + { "RELAXED", &bool_attr[0], &bool_attr[0] }, + { "NO-DIS", &bool_attr[0], &bool_attr[0] }, + { "PBB", &bool_attr[0], &bool_attr[0] }, + { 0, 0, 0 } +}; + +/* Instruction set variants. */ + +static const CGEN_ISA m32c_cgen_isa_table[] = { + { "m16c", 32, 32, 8, 56 }, + { "m32c", 32, 32, 8, 80 }, + { 0, 0, 0, 0, 0 } +}; + +/* Machine variants. */ + +static const CGEN_MACH m32c_cgen_mach_table[] = { + { "m16c", "m16c", MACH_M16C, 0 }, + { "m32c", "m32c", MACH_M32C, 0 }, + { 0, 0, 0, 0 } +}; + +static CGEN_KEYWORD_ENTRY m32c_cgen_opval_h_gr_entries[] = +{ + { "r0", 0, {0, {{{0, 0}}}}, 0, 0 }, + { "r1", 1, {0, {{{0, 0}}}}, 0, 0 }, + { "r2", 2, {0, {{{0, 0}}}}, 0, 0 }, + { "r3", 3, {0, {{{0, 0}}}}, 0, 0 } +}; + +CGEN_KEYWORD m32c_cgen_opval_h_gr = +{ + & m32c_cgen_opval_h_gr_entries[0], + 4, + 0, 0, 0, 0, "" +}; + +static CGEN_KEYWORD_ENTRY m32c_cgen_opval_h_gr_QI_entries[] = +{ + { "r0l", 0, {0, {{{0, 0}}}}, 0, 0 }, + { "r0h", 1, {0, {{{0, 0}}}}, 0, 0 }, + { "r1l", 2, {0, {{{0, 0}}}}, 0, 0 }, + { "r1h", 3, {0, {{{0, 0}}}}, 0, 0 } +}; + +CGEN_KEYWORD m32c_cgen_opval_h_gr_QI = +{ + & m32c_cgen_opval_h_gr_QI_entries[0], + 4, + 0, 0, 0, 0, "" +}; + +static CGEN_KEYWORD_ENTRY m32c_cgen_opval_h_gr_HI_entries[] = +{ + { "r0", 0, {0, {{{0, 0}}}}, 0, 0 }, + { "r1", 1, {0, {{{0, 0}}}}, 0, 0 }, + { "r2", 2, {0, {{{0, 0}}}}, 0, 0 }, + { "r3", 3, {0, {{{0, 0}}}}, 0, 0 } +}; + +CGEN_KEYWORD m32c_cgen_opval_h_gr_HI = +{ + & m32c_cgen_opval_h_gr_HI_entries[0], + 4, + 0, 0, 0, 0, "" +}; + +static CGEN_KEYWORD_ENTRY m32c_cgen_opval_h_gr_SI_entries[] = +{ + { "r2r0", 0, {0, {{{0, 0}}}}, 0, 0 }, + { "r3r1", 1, {0, {{{0, 0}}}}, 0, 0 } +}; + +CGEN_KEYWORD m32c_cgen_opval_h_gr_SI = +{ + & m32c_cgen_opval_h_gr_SI_entries[0], + 2, + 0, 0, 0, 0, "" +}; + +static CGEN_KEYWORD_ENTRY m32c_cgen_opval_h_gr_ext_QI_entries[] = +{ + { "r0l", 0, {0, {{{0, 0}}}}, 0, 0 }, + { "r1l", 1, {0, {{{0, 0}}}}, 0, 0 } +}; + +CGEN_KEYWORD m32c_cgen_opval_h_gr_ext_QI = +{ + & m32c_cgen_opval_h_gr_ext_QI_entries[0], + 2, + 0, 0, 0, 0, "" +}; + +static CGEN_KEYWORD_ENTRY m32c_cgen_opval_h_gr_ext_HI_entries[] = +{ + { "r0", 0, {0, {{{0, 0}}}}, 0, 0 }, + { "r1", 1, {0, {{{0, 0}}}}, 0, 0 } +}; + +CGEN_KEYWORD m32c_cgen_opval_h_gr_ext_HI = +{ + & m32c_cgen_opval_h_gr_ext_HI_entries[0], + 2, + 0, 0, 0, 0, "" +}; + +static CGEN_KEYWORD_ENTRY m32c_cgen_opval_h_r0l_entries[] = +{ + { "r0l", 0, {0, {{{0, 0}}}}, 0, 0 } +}; + +CGEN_KEYWORD m32c_cgen_opval_h_r0l = +{ + & m32c_cgen_opval_h_r0l_entries[0], + 1, + 0, 0, 0, 0, "" +}; + +static CGEN_KEYWORD_ENTRY m32c_cgen_opval_h_r0h_entries[] = +{ + { "r0h", 0, {0, {{{0, 0}}}}, 0, 0 } +}; + +CGEN_KEYWORD m32c_cgen_opval_h_r0h = +{ + & m32c_cgen_opval_h_r0h_entries[0], + 1, + 0, 0, 0, 0, "" +}; + +static CGEN_KEYWORD_ENTRY m32c_cgen_opval_h_r1l_entries[] = +{ + { "r1l", 0, {0, {{{0, 0}}}}, 0, 0 } +}; + +CGEN_KEYWORD m32c_cgen_opval_h_r1l = +{ + & m32c_cgen_opval_h_r1l_entries[0], + 1, + 0, 0, 0, 0, "" +}; + +static CGEN_KEYWORD_ENTRY m32c_cgen_opval_h_r1h_entries[] = +{ + { "r1h", 0, {0, {{{0, 0}}}}, 0, 0 } +}; + +CGEN_KEYWORD m32c_cgen_opval_h_r1h = +{ + & m32c_cgen_opval_h_r1h_entries[0], + 1, + 0, 0, 0, 0, "" +}; + +static CGEN_KEYWORD_ENTRY m32c_cgen_opval_h_r0_entries[] = +{ + { "r0", 0, {0, {{{0, 0}}}}, 0, 0 } +}; + +CGEN_KEYWORD m32c_cgen_opval_h_r0 = +{ + & m32c_cgen_opval_h_r0_entries[0], + 1, + 0, 0, 0, 0, "" +}; + +static CGEN_KEYWORD_ENTRY m32c_cgen_opval_h_r1_entries[] = +{ + { "r1", 0, {0, {{{0, 0}}}}, 0, 0 } +}; + +CGEN_KEYWORD m32c_cgen_opval_h_r1 = +{ + & m32c_cgen_opval_h_r1_entries[0], + 1, + 0, 0, 0, 0, "" +}; + +static CGEN_KEYWORD_ENTRY m32c_cgen_opval_h_r2_entries[] = +{ + { "r2", 0, {0, {{{0, 0}}}}, 0, 0 } +}; + +CGEN_KEYWORD m32c_cgen_opval_h_r2 = +{ + & m32c_cgen_opval_h_r2_entries[0], + 1, + 0, 0, 0, 0, "" +}; + +static CGEN_KEYWORD_ENTRY m32c_cgen_opval_h_r3_entries[] = +{ + { "r3", 0, {0, {{{0, 0}}}}, 0, 0 } +}; + +CGEN_KEYWORD m32c_cgen_opval_h_r3 = +{ + & m32c_cgen_opval_h_r3_entries[0], + 1, + 0, 0, 0, 0, "" +}; + +static CGEN_KEYWORD_ENTRY m32c_cgen_opval_h_r0l_r0h_entries[] = +{ + { "r0l", 0, {0, {{{0, 0}}}}, 0, 0 }, + { "r0h", 1, {0, {{{0, 0}}}}, 0, 0 } +}; + +CGEN_KEYWORD m32c_cgen_opval_h_r0l_r0h = +{ + & m32c_cgen_opval_h_r0l_r0h_entries[0], + 2, + 0, 0, 0, 0, "" +}; + +static CGEN_KEYWORD_ENTRY m32c_cgen_opval_h_r2r0_entries[] = +{ + { "r2r0", 0, {0, {{{0, 0}}}}, 0, 0 } +}; + +CGEN_KEYWORD m32c_cgen_opval_h_r2r0 = +{ + & m32c_cgen_opval_h_r2r0_entries[0], + 1, + 0, 0, 0, 0, "" +}; + +static CGEN_KEYWORD_ENTRY m32c_cgen_opval_h_r3r1_entries[] = +{ + { "r3r1", 0, {0, {{{0, 0}}}}, 0, 0 } +}; + +CGEN_KEYWORD m32c_cgen_opval_h_r3r1 = +{ + & m32c_cgen_opval_h_r3r1_entries[0], + 1, + 0, 0, 0, 0, "" +}; + +static CGEN_KEYWORD_ENTRY m32c_cgen_opval_h_r1r2r0_entries[] = +{ + { "r1r2r0", 0, {0, {{{0, 0}}}}, 0, 0 } +}; + +CGEN_KEYWORD m32c_cgen_opval_h_r1r2r0 = +{ + & m32c_cgen_opval_h_r1r2r0_entries[0], + 1, + 0, 0, 0, 0, "" +}; + +static CGEN_KEYWORD_ENTRY m32c_cgen_opval_h_ar_entries[] = +{ + { "a0", 0, {0, {{{0, 0}}}}, 0, 0 }, + { "a1", 1, {0, {{{0, 0}}}}, 0, 0 } +}; + +CGEN_KEYWORD m32c_cgen_opval_h_ar = +{ + & m32c_cgen_opval_h_ar_entries[0], + 2, + 0, 0, 0, 0, "" +}; + +static CGEN_KEYWORD_ENTRY m32c_cgen_opval_h_ar_QI_entries[] = +{ + { "a0", 0, {0, {{{0, 0}}}}, 0, 0 }, + { "a1", 1, {0, {{{0, 0}}}}, 0, 0 } +}; + +CGEN_KEYWORD m32c_cgen_opval_h_ar_QI = +{ + & m32c_cgen_opval_h_ar_QI_entries[0], + 2, + 0, 0, 0, 0, "" +}; + +static CGEN_KEYWORD_ENTRY m32c_cgen_opval_h_ar_HI_entries[] = +{ + { "a0", 0, {0, {{{0, 0}}}}, 0, 0 }, + { "a1", 1, {0, {{{0, 0}}}}, 0, 0 } +}; + +CGEN_KEYWORD m32c_cgen_opval_h_ar_HI = +{ + & m32c_cgen_opval_h_ar_HI_entries[0], + 2, + 0, 0, 0, 0, "" +}; + +static CGEN_KEYWORD_ENTRY m32c_cgen_opval_h_ar_SI_entries[] = +{ + { "a1a0", 0, {0, {{{0, 0}}}}, 0, 0 } +}; + +CGEN_KEYWORD m32c_cgen_opval_h_ar_SI = +{ + & m32c_cgen_opval_h_ar_SI_entries[0], + 1, + 0, 0, 0, 0, "" +}; + +static CGEN_KEYWORD_ENTRY m32c_cgen_opval_h_a0_entries[] = +{ + { "a0", 0, {0, {{{0, 0}}}}, 0, 0 } +}; + +CGEN_KEYWORD m32c_cgen_opval_h_a0 = +{ + & m32c_cgen_opval_h_a0_entries[0], + 1, + 0, 0, 0, 0, "" +}; + +static CGEN_KEYWORD_ENTRY m32c_cgen_opval_h_a1_entries[] = +{ + { "a1", 1, {0, {{{0, 0}}}}, 0, 0 } +}; + +CGEN_KEYWORD m32c_cgen_opval_h_a1 = +{ + & m32c_cgen_opval_h_a1_entries[0], + 1, + 0, 0, 0, 0, "" +}; + +static CGEN_KEYWORD_ENTRY m32c_cgen_opval_h_cond16_entries[] = +{ + { "geu", 0, {0, {{{0, 0}}}}, 0, 0 }, + { "c", 0, {0, {{{0, 0}}}}, 0, 0 }, + { "gtu", 1, {0, {{{0, 0}}}}, 0, 0 }, + { "eq", 2, {0, {{{0, 0}}}}, 0, 0 }, + { "z", 2, {0, {{{0, 0}}}}, 0, 0 }, + { "n", 3, {0, {{{0, 0}}}}, 0, 0 }, + { "le", 4, {0, {{{0, 0}}}}, 0, 0 }, + { "o", 5, {0, {{{0, 0}}}}, 0, 0 }, + { "ge", 6, {0, {{{0, 0}}}}, 0, 0 }, + { "ltu", 248, {0, {{{0, 0}}}}, 0, 0 }, + { "nc", 248, {0, {{{0, 0}}}}, 0, 0 }, + { "leu", 249, {0, {{{0, 0}}}}, 0, 0 }, + { "ne", 250, {0, {{{0, 0}}}}, 0, 0 }, + { "nz", 250, {0, {{{0, 0}}}}, 0, 0 }, + { "pz", 251, {0, {{{0, 0}}}}, 0, 0 }, + { "gt", 252, {0, {{{0, 0}}}}, 0, 0 }, + { "no", 253, {0, {{{0, 0}}}}, 0, 0 }, + { "lt", 254, {0, {{{0, 0}}}}, 0, 0 } +}; + +CGEN_KEYWORD m32c_cgen_opval_h_cond16 = +{ + & m32c_cgen_opval_h_cond16_entries[0], + 18, + 0, 0, 0, 0, "" +}; + +static CGEN_KEYWORD_ENTRY m32c_cgen_opval_h_cond16c_entries[] = +{ + { "geu", 0, {0, {{{0, 0}}}}, 0, 0 }, + { "c", 0, {0, {{{0, 0}}}}, 0, 0 }, + { "gtu", 1, {0, {{{0, 0}}}}, 0, 0 }, + { "eq", 2, {0, {{{0, 0}}}}, 0, 0 }, + { "z", 2, {0, {{{0, 0}}}}, 0, 0 }, + { "n", 3, {0, {{{0, 0}}}}, 0, 0 }, + { "ltu", 4, {0, {{{0, 0}}}}, 0, 0 }, + { "nc", 4, {0, {{{0, 0}}}}, 0, 0 }, + { "leu", 5, {0, {{{0, 0}}}}, 0, 0 }, + { "ne", 6, {0, {{{0, 0}}}}, 0, 0 }, + { "nz", 6, {0, {{{0, 0}}}}, 0, 0 }, + { "pz", 7, {0, {{{0, 0}}}}, 0, 0 }, + { "le", 8, {0, {{{0, 0}}}}, 0, 0 }, + { "o", 9, {0, {{{0, 0}}}}, 0, 0 }, + { "ge", 10, {0, {{{0, 0}}}}, 0, 0 }, + { "gt", 12, {0, {{{0, 0}}}}, 0, 0 }, + { "no", 13, {0, {{{0, 0}}}}, 0, 0 }, + { "lt", 14, {0, {{{0, 0}}}}, 0, 0 } +}; + +CGEN_KEYWORD m32c_cgen_opval_h_cond16c = +{ + & m32c_cgen_opval_h_cond16c_entries[0], + 18, + 0, 0, 0, 0, "" +}; + +static CGEN_KEYWORD_ENTRY m32c_cgen_opval_h_cond16j_entries[] = +{ + { "le", 8, {0, {{{0, 0}}}}, 0, 0 }, + { "o", 9, {0, {{{0, 0}}}}, 0, 0 }, + { "ge", 10, {0, {{{0, 0}}}}, 0, 0 }, + { "gt", 12, {0, {{{0, 0}}}}, 0, 0 }, + { "no", 13, {0, {{{0, 0}}}}, 0, 0 }, + { "lt", 14, {0, {{{0, 0}}}}, 0, 0 } +}; + +CGEN_KEYWORD m32c_cgen_opval_h_cond16j = +{ + & m32c_cgen_opval_h_cond16j_entries[0], + 6, + 0, 0, 0, 0, "" +}; + +static CGEN_KEYWORD_ENTRY m32c_cgen_opval_h_cond16j_5_entries[] = +{ + { "geu", 0, {0, {{{0, 0}}}}, 0, 0 }, + { "c", 0, {0, {{{0, 0}}}}, 0, 0 }, + { "gtu", 1, {0, {{{0, 0}}}}, 0, 0 }, + { "eq", 2, {0, {{{0, 0}}}}, 0, 0 }, + { "z", 2, {0, {{{0, 0}}}}, 0, 0 }, + { "n", 3, {0, {{{0, 0}}}}, 0, 0 }, + { "ltu", 4, {0, {{{0, 0}}}}, 0, 0 }, + { "nc", 4, {0, {{{0, 0}}}}, 0, 0 }, + { "leu", 5, {0, {{{0, 0}}}}, 0, 0 }, + { "ne", 6, {0, {{{0, 0}}}}, 0, 0 }, + { "nz", 6, {0, {{{0, 0}}}}, 0, 0 }, + { "pz", 7, {0, {{{0, 0}}}}, 0, 0 } +}; + +CGEN_KEYWORD m32c_cgen_opval_h_cond16j_5 = +{ + & m32c_cgen_opval_h_cond16j_5_entries[0], + 12, + 0, 0, 0, 0, "" +}; + +static CGEN_KEYWORD_ENTRY m32c_cgen_opval_h_cond32_entries[] = +{ + { "ltu", 0, {0, {{{0, 0}}}}, 0, 0 }, + { "nc", 0, {0, {{{0, 0}}}}, 0, 0 }, + { "leu", 1, {0, {{{0, 0}}}}, 0, 0 }, + { "ne", 2, {0, {{{0, 0}}}}, 0, 0 }, + { "nz", 2, {0, {{{0, 0}}}}, 0, 0 }, + { "pz", 3, {0, {{{0, 0}}}}, 0, 0 }, + { "no", 4, {0, {{{0, 0}}}}, 0, 0 }, + { "gt", 5, {0, {{{0, 0}}}}, 0, 0 }, + { "ge", 6, {0, {{{0, 0}}}}, 0, 0 }, + { "geu", 8, {0, {{{0, 0}}}}, 0, 0 }, + { "c", 8, {0, {{{0, 0}}}}, 0, 0 }, + { "gtu", 9, {0, {{{0, 0}}}}, 0, 0 }, + { "eq", 10, {0, {{{0, 0}}}}, 0, 0 }, + { "z", 10, {0, {{{0, 0}}}}, 0, 0 }, + { "n", 11, {0, {{{0, 0}}}}, 0, 0 }, + { "o", 12, {0, {{{0, 0}}}}, 0, 0 }, + { "le", 13, {0, {{{0, 0}}}}, 0, 0 }, + { "lt", 14, {0, {{{0, 0}}}}, 0, 0 } +}; + +CGEN_KEYWORD m32c_cgen_opval_h_cond32 = +{ + & m32c_cgen_opval_h_cond32_entries[0], + 18, + 0, 0, 0, 0, "" +}; + +static CGEN_KEYWORD_ENTRY m32c_cgen_opval_h_cr1_32_entries[] = +{ + { "dct0", 0, {0, {{{0, 0}}}}, 0, 0 }, + { "dct1", 1, {0, {{{0, 0}}}}, 0, 0 }, + { "flg", 2, {0, {{{0, 0}}}}, 0, 0 }, + { "svf", 3, {0, {{{0, 0}}}}, 0, 0 }, + { "drc0", 4, {0, {{{0, 0}}}}, 0, 0 }, + { "drc1", 5, {0, {{{0, 0}}}}, 0, 0 }, + { "dmd0", 6, {0, {{{0, 0}}}}, 0, 0 }, + { "dmd1", 7, {0, {{{0, 0}}}}, 0, 0 } +}; + +CGEN_KEYWORD m32c_cgen_opval_h_cr1_32 = +{ + & m32c_cgen_opval_h_cr1_32_entries[0], + 8, + 0, 0, 0, 0, "" +}; + +static CGEN_KEYWORD_ENTRY m32c_cgen_opval_h_cr2_32_entries[] = +{ + { "intb", 0, {0, {{{0, 0}}}}, 0, 0 }, + { "sp", 1, {0, {{{0, 0}}}}, 0, 0 }, + { "sb", 2, {0, {{{0, 0}}}}, 0, 0 }, + { "fb", 3, {0, {{{0, 0}}}}, 0, 0 }, + { "svp", 4, {0, {{{0, 0}}}}, 0, 0 }, + { "vct", 5, {0, {{{0, 0}}}}, 0, 0 }, + { "isp", 7, {0, {{{0, 0}}}}, 0, 0 } +}; + +CGEN_KEYWORD m32c_cgen_opval_h_cr2_32 = +{ + & m32c_cgen_opval_h_cr2_32_entries[0], + 7, + 0, 0, 0, 0, "" +}; + +static CGEN_KEYWORD_ENTRY m32c_cgen_opval_h_cr3_32_entries[] = +{ + { "dma0", 2, {0, {{{0, 0}}}}, 0, 0 }, + { "dma1", 3, {0, {{{0, 0}}}}, 0, 0 }, + { "dra0", 4, {0, {{{0, 0}}}}, 0, 0 }, + { "dra1", 5, {0, {{{0, 0}}}}, 0, 0 }, + { "dsa0", 6, {0, {{{0, 0}}}}, 0, 0 }, + { "dsa1", 7, {0, {{{0, 0}}}}, 0, 0 } +}; + +CGEN_KEYWORD m32c_cgen_opval_h_cr3_32 = +{ + & m32c_cgen_opval_h_cr3_32_entries[0], + 6, + 0, 0, 0, 0, "" +}; + +static CGEN_KEYWORD_ENTRY m32c_cgen_opval_h_cr_16_entries[] = +{ + { "intbl", 1, {0, {{{0, 0}}}}, 0, 0 }, + { "intbh", 2, {0, {{{0, 0}}}}, 0, 0 }, + { "flg", 3, {0, {{{0, 0}}}}, 0, 0 }, + { "isp", 4, {0, {{{0, 0}}}}, 0, 0 }, + { "sp", 5, {0, {{{0, 0}}}}, 0, 0 }, + { "sb", 6, {0, {{{0, 0}}}}, 0, 0 }, + { "fb", 7, {0, {{{0, 0}}}}, 0, 0 } +}; + +CGEN_KEYWORD m32c_cgen_opval_h_cr_16 = +{ + & m32c_cgen_opval_h_cr_16_entries[0], + 7, + 0, 0, 0, 0, "" +}; + +static CGEN_KEYWORD_ENTRY m32c_cgen_opval_h_flags_entries[] = +{ + { "c", 0, {0, {{{0, 0}}}}, 0, 0 }, + { "d", 1, {0, {{{0, 0}}}}, 0, 0 }, + { "z", 2, {0, {{{0, 0}}}}, 0, 0 }, + { "s", 3, {0, {{{0, 0}}}}, 0, 0 }, + { "b", 4, {0, {{{0, 0}}}}, 0, 0 }, + { "o", 5, {0, {{{0, 0}}}}, 0, 0 }, + { "i", 6, {0, {{{0, 0}}}}, 0, 0 }, + { "u", 7, {0, {{{0, 0}}}}, 0, 0 } +}; + + diff --git a/tests/licensedcode/data/licenses/long-m32c-desc.yml b/tests/licensedcode/data/licenses/long-m32c-desc.yml new file mode 100644 index 00000000000..c36c53c8b3b --- /dev/null +++ b/tests/licensedcode/data/licenses/long-m32c-desc.yml @@ -0,0 +1,3 @@ +licenses: + - gpl-2.0-plus +notes: from https://master.dl.sourceforge.net/project/mingw/OldFiles/Previous%20Release_%20binutils-2.17.50/binutils-2.17.50-20060824-1-src.tar.gz diff --git a/tests/licensedcode/data/licenses/long-s3cli-0.0.53-linux-amd64.go b/tests/licensedcode/data/licenses/long-s3cli-0.0.53-linux-amd64.go new file mode 100644 index 00000000000..4f8dd9755fc Binary files /dev/null and b/tests/licensedcode/data/licenses/long-s3cli-0.0.53-linux-amd64.go differ diff --git a/tests/licensedcode/data/licenses/long-s3cli-0.0.53-linux-amd64.yml b/tests/licensedcode/data/licenses/long-s3cli-0.0.53-linux-amd64.yml new file mode 100644 index 00000000000..d08fd03fdad --- /dev/null +++ b/tests/licensedcode/data/licenses/long-s3cli-0.0.53-linux-amd64.yml @@ -0,0 +1 @@ +notes: go exec \ No newline at end of file diff --git a/tests/licensedcode/test_detect.py b/tests/licensedcode/test_detect.py index 209d3fc2b0b..dace41ab427 100644 --- a/tests/licensedcode/test_detect.py +++ b/tests/licensedcode/test_detect.py @@ -881,14 +881,10 @@ def test_match_has_correct_positions_basic(self): matches = idx.match(query_string=querys) - line_by_pos = {x: 1 for x in range(0, 8)} - line_by_pos.update({x: 2 for x in range(8, 16)}) - line_by_pos.update({x: 3 for x in range(16, 24)}) - rule = [r for r in idx.rules_by_rid if r.identifier == 'gpl_69.RULE'][0] - m1 = LicenseMatch(rule=rule, qspan=Span(0, 7), ispan=Span(0, 7), line_by_pos=line_by_pos) - m2 = LicenseMatch(rule=rule, qspan=Span(8, 15), ispan=Span(0, 7), line_by_pos=line_by_pos) - m3 = LicenseMatch(rule=rule, qspan=Span(16, 23), ispan=Span(0, 7), line_by_pos=line_by_pos) + m1 = LicenseMatch(rule=rule, qspan=Span(0, 7), ispan=Span(0, 7), start_line=1, end_line=1) + m2 = LicenseMatch(rule=rule, qspan=Span(8, 15), ispan=Span(0, 7), start_line=2, end_line=2) + m3 = LicenseMatch(rule=rule, qspan=Span(16, 23), ispan=Span(0, 7), start_line=3, end_line=3) assert [m1, m2, m3] == matches def test_match_has_correct_line_positions_for_query_with_repeats(self): diff --git a/tests/licensedcode/test_detection_datadriven.py b/tests/licensedcode/test_detection_datadriven.py index 7ff0c8a592e..0a88b7f422f 100644 --- a/tests/licensedcode/test_detection_datadriven.py +++ b/tests/licensedcode/test_detection_datadriven.py @@ -172,7 +172,7 @@ def build_tests(license_tests, clazz): expected_failure=test.expected_failure, skip_test=test.skip and 'Skipping long test' or False, trace_text=TRACE_TEXTS - ) + ) # attach that method to our test class setattr(clazz, test_name, test_method) diff --git a/tests/licensedcode/test_index.py b/tests/licensedcode/test_index.py index 9d3201a61d6..36fbeb48fd7 100644 --- a/tests/licensedcode/test_index.py +++ b/tests/licensedcode/test_index.py @@ -174,82 +174,78 @@ def test_index_structures_with__add_rules(self): expected_index = { 'plain1_0': {u'redistribution': [0]}, - 'plain2_1': {u'redistribution': [0], u'yes': [2]}, - 'plain3_2': {u'allowed': [2], u'redistribution': [0], u'yes': [3]}, - 'plain4_3': {u'allowed': [2], u'redistribution': [0], u'yes': [4]}, - 'plain5_4': {u'allowed': [2], u'redistribution': [0]}, - 'tmpl10_5': {u'allowed': [2], u'any': [8], u'redistribution': [0], u'thing': [9]}, - 'tmpl2_6': {u'redistribution': [0]}, - 'tmpl3_7': {u'allowed': [2], u'redistribution': [0]}, - 'tmpl4_8': {u'allowed': [2], u'redistribution': [0]}, - 'tmpl5_2_10': {u'allowed': [2], u'redistribution': [0], u'yes': [5]}, - 'tmpl5_9': {u'allowed': [3], u'redistribution': [0]}, - 'tmpl6_11': {u'allowed': [2], u'redistribution': [0]}, - 'tmpl7_12': {u'allowed': [2], u'redistribution': [0]}, - 'tmpl8_13': {u'allowed': [2], u'redistribution': [0]}, - 'tmpl9_14': {u'allowed': [2], u'any': [8], u'redistribution': [0]} + 'plain2_1': {u'is': [1], u'redistribution': [0], u'yes': [2]}, + 'plain3_2': {u'is': [1], u'redistribution': [0], u'yes': [3]}, + 'plain4_3': {u'is': [1], u'redistribution': [0], u'yes': [4]}, + 'plain5_4': {u'is': [1], u'redistribution': [0]}, + 'tmpl10_5': {u'any': [8], u'is': [1], u'redistribution': [0], u'thing': [9]}, + 'tmpl2_6': {u'is': [1], u'redistribution': [0]}, + 'tmpl3_7': {u'is': [1], u'redistribution': [0]}, + 'tmpl4_8': {u'is': [1], u'redistribution': [0]}, + 'tmpl5_2_10': {u'is': [1], u'redistribution': [0], u'yes': [5]}, + 'tmpl5_9': {u'is': [1, 2], u'redistribution': [0]}, + 'tmpl6_11': {u'is': [1], u'redistribution': [0]}, + 'tmpl7_12': {u'is': [1], u'redistribution': [0]}, + 'tmpl8_13': {u'is': [1], u'redistribution': [0]}, + 'tmpl9_14': {u'any': [8], u'is': [1], u'redistribution': [0]} } - assert expected_index == idx._as_dict() expected_dict = { u'all': 1, - u'allowed': 5, + u'allowed': 0, u'and': 3, u'any': 7, u'for': 2, - u'is': 0, - u'redistribution': 4, + u'is': 4, + u'redistribution': 5, u'thing': 8, - u'yes': 6} - + u'yes': 6 + } assert expected_dict == idx.dictionary - expected_tids = [u'is', u'all', u'for', u'and', u'redistribution', u'allowed', u'yes', u'any', u'thing'] + expected_tids = [u'allowed', u'all', u'for', u'and', u'is', u'redistribution', u'yes', u'any', u'thing'] assert expected_tids == idx.tokens_by_tid expected_high_tids_msets_by_rid = [ {u'redistribution': 1}, - {u'redistribution': 1, u'yes': 1}, - {u'allowed': 1, u'redistribution': 1, u'yes': 1}, - {u'allowed': 1, u'redistribution': 1, u'yes': 1}, - {u'allowed': 1, u'redistribution': 1}, - {u'allowed': 1, u'any': 1, u'redistribution': 1, u'thing': 1}, - {u'redistribution': 1}, - {u'allowed': 1, u'redistribution': 1}, - {u'allowed': 1, u'redistribution': 1}, - {u'allowed': 1, u'redistribution': 1}, - {u'allowed': 1, u'redistribution': 1, u'yes': 1}, - {u'allowed': 1, u'redistribution': 1}, - {u'allowed': 1, u'redistribution': 1}, - {u'allowed': 1, u'redistribution': 1}, - {u'allowed': 1, u'any': 1, u'redistribution': 1} + {u'is': 1, u'redistribution': 1, u'yes': 1}, + {u'is': 1, u'redistribution': 1, u'yes': 1}, + {u'is': 1, u'redistribution': 1, u'yes': 1}, + {u'is': 1, u'redistribution': 1}, + {u'any': 1, u'is': 1, u'redistribution': 1, u'thing': 1}, + {u'is': 1, u'redistribution': 1}, + {u'is': 1, u'redistribution': 1}, + {u'is': 1, u'redistribution': 1}, + {u'is': 2, u'redistribution': 1}, + {u'is': 1, u'redistribution': 1, u'yes': 1}, + {u'is': 1, u'redistribution': 1}, + {u'is': 1, u'redistribution': 1}, + {u'is': 1, u'redistribution': 1}, + {u'any': 1, u'is': 1, u'redistribution': 1} ] - low_tids_msets_by_rid, high_tids_msets_by_rid = zip(*idx.tids_msets_by_rid) - htmset = [{idx.tokens_by_tid[tok]:freq for (tok, freq) in tids_mset.items()} for tids_mset in high_tids_msets_by_rid] assert expected_high_tids_msets_by_rid == htmset expected_low_tids_msets_by_rid = [ {}, - {u'is': 1}, - {u'is': 1}, - {u'for': 1, u'is': 1}, - {u'all': 1, u'for': 1, u'is': 1}, - {u'all': 2, u'and': 2, u'for': 1, u'is': 1}, - {u'is': 1}, - {u'is': 1}, - {u'for': 1, u'is': 1}, - {u'for': 1, u'is': 2}, - {u'all': 1, u'for': 1, u'is': 1}, - {u'all': 1, u'and': 1, u'for': 1, u'is': 1}, - {u'all': 2, u'and': 1, u'for': 1, u'is': 1}, - {u'all': 2, u'and': 2, u'for': 1, u'is': 1}, - {u'all': 2, u'and': 2, u'for': 1, u'is': 1} + {}, + {u'allowed': 1}, + {u'allowed': 1, u'for': 1}, + {u'all': 1, u'allowed': 1, u'for': 1}, + {u'all': 2, u'allowed': 1, u'and': 2, u'for': 1}, + {}, + {u'allowed': 1}, + {u'allowed': 1, u'for': 1}, + {u'allowed': 1, u'for': 1}, + {u'all': 1, u'allowed': 1, u'for': 1}, + {u'all': 1, u'allowed': 1, u'and': 1, u'for': 1}, + {u'all': 2, u'allowed': 1, u'and': 1, u'for': 1}, + {u'all': 2, u'allowed': 1, u'and': 2, u'for': 1}, + {u'all': 2, u'allowed': 1, u'and': 2, u'for': 1} ] - assert expected_low_tids_msets_by_rid == [{idx.tokens_by_tid[tok]: freq for tok, freq in tids_mset.items()} for tids_mset in low_tids_msets_by_rid] diff --git a/tests/licensedcode/test_match.py b/tests/licensedcode/test_match.py index b557918b2ff..cb4b8577095 100644 --- a/tests/licensedcode/test_match.py +++ b/tests/licensedcode/test_match.py @@ -86,20 +86,13 @@ def test_LicenseMatch_not_equal(self): assert not m1 == m4 - def test_LicenseMatch_lines(self): + def test_LicenseMatch_equals(self): rule = Rule(text_file='r1', licenses=['apache-2.0', 'gpl']) - line_by_pos = {x: 1 for x in range(0, 8)} - line_by_pos.update({x: 2 for x in range(8, 16)}) - line_by_pos.update({x: 3 for x in range(16, 24)}) - - m1 = LicenseMatch(rule=rule, matcher='chunk1', qspan=Span(0, 7), ispan=Span(0, 7), line_by_pos=line_by_pos) - assert (1, 1) == m1.lines() - m2 = LicenseMatch(rule=rule, matcher='chunk2', qspan=Span(0, 7), ispan=Span(0, 7), line_by_pos=line_by_pos) - assert (1, 1) == m2.lines() + m1 = LicenseMatch(rule=rule, matcher='chunk1', qspan=Span(0, 7), ispan=Span(0, 7), start_line=1, end_line=1) + m2 = LicenseMatch(rule=rule, matcher='chunk2', qspan=Span(0, 7), ispan=Span(0, 7), start_line=1, end_line=1) assert m1 == m2 - m3 = LicenseMatch(rule=rule, matcher='chunk3', qspan=Span(16, 23), ispan=Span(0, 7), line_by_pos=line_by_pos) - assert (3, 3) == m3.lines() + m3 = LicenseMatch(rule=rule, matcher='chunk3', qspan=Span(16, 23), ispan=Span(0, 7), start_line=3, end_line=3) assert m1 != m3 def test_LicenseMatch_comparisons(self): diff --git a/tests/licensedcode/test_match_seq.py b/tests/licensedcode/test_match_seq.py index 51353b0bd10..0b2a5565409 100644 --- a/tests/licensedcode/test_match_seq.py +++ b/tests/licensedcode/test_match_seq.py @@ -97,33 +97,30 @@ def test_match_seq_are_correct_on_apache(self): assert match_seq.MATCH_SEQ == match.matcher qtext, _itext = get_texts(match, location=query_loc, idx=idx) expected = u''' - The OpenSymphony Group All rights reserved - Redistribution and use in source and binary forms with or without - modification are permitted provided that the following conditions are met 1 - Redistributions of source code must retain the above copyright notice this - list of conditions and the following disclaimer 2 Redistributions in binary - form must reproduce the above copyright notice this list of conditions and - the following disclaimer in the documentation and or other materials provided - with the distribution 3 The end user documentation included with the - redistribution if any must include the following acknowledgment <4> This - product includes software developed by the OpenSymphony Group http www - opensymphony com <5> Alternately this acknowledgment may appear in the - software itself if and wherever such third party acknowledgments normally - appear The names OpenSymphony and The OpenSymphony Group must not be used to - endorse or promote products derived from this software without prior written - permission For written permission please contact license opensymphony com - Products derived from this software may not be called OpenSymphony or - [OsCore] nor may OpenSymphony or [OsCore] appear in their name without prior - written permission of the OpenSymphony Group THIS SOFTWARE IS PROVIDED AS IS - AND ANY EXPRESSED OR IMPLIED WARRANTIES INCLUDING BUT NOT LIMITED TO THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR ITS - CONTRIBUTORS BE LIABLE FOR ANY DIRECT INDIRECT INCIDENTAL SPECIAL EXEMPLARY - OR CONSEQUENTIAL DAMAGES INCLUDING BUT NOT LIMITED TO PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES LOSS OF USE DATA OR PROFITS OR BUSINESS - INTERRUPTION HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY WHETHER IN - CONTRACT STRICT LIABILITY OR TORT INCLUDING NEGLIGENCE OR OTHERWISE ARISING - IN ANY WAY OUT OF THE USE OF THIS SOFTWARE EVEN IF ADVISED OF THE POSSIBILITY - OF SUCH DAMAGE''' - + The OpenSymphony Group All rights reserved Redistribution and use in source and + binary forms with or without modification are permitted provided that the following + conditions are met 1 Redistributions of source code must retain the above copyright + notice this list of conditions and the following disclaimer 2 Redistributions in + binary form must reproduce the above copyright notice this list of conditions and the + following disclaimer in the documentation and or other materials provided with the + distribution 3 The end user documentation included with the redistribution if any + must include the following acknowledgment <4> + <5> + Alternately this acknowledgment may appear in the software itself if and wherever + such third party acknowledgments normally appear The names OpenSymphony and The + OpenSymphony Group must not be used to endorse or promote products derived from this + software without prior written permission For written permission please contact + license opensymphony com Products derived from this software may not be called + OpenSymphony or [OsCore] nor may OpenSymphony or [OsCore] appear in their name + without prior written permission of the OpenSymphony Group THIS SOFTWARE IS PROVIDED + AS IS AND ANY EXPRESSED OR IMPLIED WARRANTIES INCLUDING BUT NOT LIMITED TO THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR ITS CONTRIBUTORS BE + LIABLE FOR ANY DIRECT INDIRECT INCIDENTAL SPECIAL EXEMPLARY OR CONSEQUENTIAL DAMAGES + INCLUDING BUT NOT LIMITED TO PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES LOSS OF USE + DATA OR PROFITS OR BUSINESS INTERRUPTION HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY WHETHER IN CONTRACT STRICT LIABILITY OR TORT INCLUDING NEGLIGENCE OR + OTHERWISE ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE + ''' assert expected.split() == qtext.split() diff --git a/tests/licensedcode/test_match_set.py b/tests/licensedcode/test_match_set.py index 28b99430825..433966a40ee 100644 --- a/tests/licensedcode/test_match_set.py +++ b/tests/licensedcode/test_match_set.py @@ -35,7 +35,7 @@ class FilterTesting(TestCase): def test_compare_sets_tids_sets(self): - thresholds = Thresholds(high_len=3, low_len=0, length=3, min_high=2, small=False, min_len=2, max_gap_skip=0) + thresholds = Thresholds(high_len=3, low_len=0, length=3, min_high=2, small=False, min_len=2) qlow, qhigh = intbitset(), intbitset([3, 4, 6]) ilow, ihigh = intbitset(), intbitset([3, 4, 6]) @@ -43,7 +43,7 @@ def test_compare_sets_tids_sets(self): assert candidate def test_compare_sets_tids_sets_match_with_less_than_high_len(self): - thresholds = Thresholds(high_len=3, low_len=0, length=3, min_high=2, small=False, min_len=2, max_gap_skip=0) + thresholds = Thresholds(high_len=3, low_len=0, length=3, min_high=2, small=False, min_len=2) qlow, qhigh = intbitset(), intbitset([3, 4]) ilow, ihigh = intbitset(), intbitset([3, 4, 6]) @@ -51,7 +51,7 @@ def test_compare_sets_tids_sets_match_with_less_than_high_len(self): assert candidate def test_compare_sets_tids_sets_match_with_less_than_min_high(self): - thresholds = Thresholds(high_len=3, low_len=0, length=3, min_high=2, small=False, min_len=2, max_gap_skip=0) + thresholds = Thresholds(high_len=3, low_len=0, length=3, min_high=2, small=False, min_len=2) qlow, qhigh = intbitset(), intbitset([3]) ilow, ihigh = intbitset(), intbitset([3, 4, 6]) @@ -59,14 +59,14 @@ def test_compare_sets_tids_sets_match_with_less_than_min_high(self): assert not candidate def test_compare_sets_tids_sets_match_with_less_than_ilow_len(self): - thresholds = Thresholds(high_len=3, low_len=1, length=3, min_high=2, small=False, min_len=2, max_gap_skip=0) + thresholds = Thresholds(high_len=3, low_len=1, length=3, min_high=2, small=False, min_len=2) qlow, qhigh = intbitset(), intbitset([3, 4, 6]) ilow, ihigh = intbitset([1]), intbitset([3, 4, 6]) candidate = match_set.compare_sets(qhigh, qlow, ihigh, ilow, thresholds, match_set.tids_sets_intersector, match_set.tids_set_counter) assert candidate def test_compare_sets_tids_sets_match_with_more_than_min_and_low_len(self): - thresholds = Thresholds(high_len=3, low_len=1, length=4, min_high=2, small=False, min_len=2, max_gap_skip=0) + thresholds = Thresholds(high_len=3, low_len=1, length=4, min_high=2, small=False, min_len=2) qlow, qhigh = intbitset(), intbitset([3, 4, 6]) ilow, ihigh = intbitset([1]), intbitset([3, 4, 6]) @@ -74,17 +74,17 @@ def test_compare_sets_tids_sets_match_with_more_than_min_and_low_len(self): assert candidate def test_compare_sets_tids_sets_match_with_small_rule(self): - thresholds = Thresholds(high_len=3, low_len=1, length=4, min_high=2, small=True, min_len=2, max_gap_skip=0) + thresholds = Thresholds(high_len=3, low_len=1, length=4, min_high=2, small=True, min_len=2) qlow, qhigh = intbitset(), intbitset([3, 4, 6]) ilow, ihigh = intbitset([1]), intbitset([3, 4, 6]) candidate = match_set.compare_sets(qhigh, qlow, ihigh, ilow, thresholds, match_set.tids_sets_intersector, match_set.tids_set_counter) assert not candidate - thresholds = Thresholds(high_len=3, low_len=1, length=4, min_high=2, small=False, min_len=2, max_gap_skip=0) + thresholds = Thresholds(high_len=3, low_len=1, length=4, min_high=2, small=False, min_len=2) candidate = match_set.compare_sets(qhigh, qlow, ihigh, ilow, thresholds, match_set.tids_sets_intersector, match_set.tids_set_counter) assert candidate - thresholds = Thresholds(high_len=3, low_len=1, length=4, min_high=4, small=False, min_len=2, max_gap_skip=0) + thresholds = Thresholds(high_len=3, low_len=1, length=4, min_high=4, small=False, min_len=2) candidate = match_set.compare_sets(qhigh, qlow, ihigh, ilow, thresholds, match_set.tids_sets_intersector, match_set.tids_set_counter) assert not candidate diff --git a/tests/licensedcode/test_models.py b/tests/licensedcode/test_models.py index 1c0f6e059c5..5a18d3bc6b8 100644 --- a/tests/licensedcode/test_models.py +++ b/tests/licensedcode/test_models.py @@ -139,8 +139,8 @@ def test_LicenseMatch_small(self): r2_text = u'licensed under the GPL, licensed under the GPL' * 10 r2 = models.Rule(text_file='r1', licenses=['apache-1.1'], _text=r2_text) _idx = index.LicenseIndex([r1, r2]) - assert models.Thresholds(high_len=4, low_len=4, length=8, small=True, min_high=4, min_len=8, max_gap_skip=1) == r1.thresholds() - assert models.Thresholds(high_len=31, low_len=40, length=71, small=False, min_high=3, min_len=4, max_gap_skip=15) == r2.thresholds() + assert models.Thresholds(high_len=4, low_len=4, length=8, small=True, min_high=4, min_len=8) == r1.thresholds() + assert models.Thresholds(high_len=31, low_len=40, length=71, small=False, min_high=3, min_len=4) == r2.thresholds() r1_text = u'licensed under the GPL,{{}} licensed under the GPL' r1 = models.Rule(text_file='r1', licenses=['apache-1.1'], _text=r1_text) @@ -148,5 +148,5 @@ def test_LicenseMatch_small(self): r2 = models.Rule(text_file='r1', licenses=['apache-1.1'], _text=r2_text) _idx = index.LicenseIndex([r1, r2]) - assert models.Thresholds(high_len=4, low_len=4, length=8, small=True, min_high=4, min_len=8, max_gap_skip=1) == r1.thresholds() - assert models.Thresholds(high_len=31, low_len=40, length=71, small=False, min_high=3, min_len=4, max_gap_skip=15) == r2.thresholds() + assert models.Thresholds(high_len=4, low_len=4, length=8, small=True, min_high=4, min_len=8) == r1.thresholds() + assert models.Thresholds(high_len=31, low_len=40, length=71, small=False, min_high=3, min_len=4) == r2.thresholds() diff --git a/tests/licensedcode/test_query.py b/tests/licensedcode/test_query.py index ecaf48cc873..b4d269953ea 100644 --- a/tests/licensedcode/test_query.py +++ b/tests/licensedcode/test_query.py @@ -69,7 +69,7 @@ def test_Query_tokens_by_line_from_string(self): expected = [ [], [None], - [12, 0, 6, 3, 2, 0, 1, 10, 7], + [11, 0, 6, 4, 3, 0, 1, 9, 2], [], [None, None, None, None], [None, 0, None], @@ -94,7 +94,7 @@ def test_Query_tokens_by_line_from_string(self): assert expected_str == result_str - assert {0: 3, 1: 3, 2: 3, 3: 3, 4: 3, 5: 3, 6: 3, 7: 3, 8: 3, 9: 6} == qry.line_by_pos + assert [3, 3, 3, 3, 3, 3, 3, 3, 3, 6] == qry.line_by_pos idx = index.LicenseIndex([Rule(_text=rule_text, licenses=['bsd'])]) querys = 'and this is not a license' @@ -228,19 +228,12 @@ def test_Query_from_real_index_and_location(self): }, {'end': 36, 'start': 36, 'tokens': u'redistributions'}] assert expected == result - - runs = qry.query_runs - assert len(runs) == 2 - query_run = runs[0] - - expected_lbp = { - 0: 4, 1: 4, 2: 4, 3: 4, 4: 4, 5: 4, 6: 4, 7: 4, 8: 6, 9: 6, 10: 6, - 11: 6, 12: 6, 13: 7, 14: 7, 15: 7, 16: 7, 17: 7, 18: 8, 19: 9, - 20: 9, 21: 9, 22: 9, 23: 9, 24: 11, 25: 11, 26: 11, 27: 11, 28: 11, - 29: 11, 30: 11, 31: 11, 32: 11, 33: 11, 34: 11, 35: 11, 36: 15 - } - - assert expected_lbp == query_run.line_by_pos + + expected_lbp = [ + 4, 4, 4, 4, 4, 4, 4, 4, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 8, + 9, 9, 9, 9, 9, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 15 + ] + assert expected_lbp == qry.line_by_pos def test_query_and_index_tokens_are_identical_for_same_text(self): rule_dir = self.get_test_loc('query/rtos_exact/') @@ -267,57 +260,57 @@ def test_query_run_tokens_with_junk(self): # two junks q = Query(query_string='a the', idx=idx) + assert q.line_by_pos qrun = q.query_runs[0] - assert qrun.line_by_pos assert [0, 1] == qrun.tokens assert {} == qrun.unknowns_by_pos # one junk q = Query(query_string='a binary', idx=idx) qrun = q.query_runs[0] - assert qrun.line_by_pos + assert q.line_by_pos assert [0, 2] == qrun.tokens assert {} == qrun.unknowns_by_pos # one junk q = Query(query_string='binary the', idx=idx) qrun = q.query_runs[0] - assert qrun.line_by_pos + assert q.line_by_pos assert [2, 1] == qrun.tokens assert {} == qrun.unknowns_by_pos # one unknown at start q = Query(query_string='that binary', idx=idx) qrun = q.query_runs[0] - assert qrun.line_by_pos + assert q.line_by_pos assert [2] == qrun.tokens assert {-1: 1} == qrun.unknowns_by_pos # one unknown at end q = Query(query_string='binary that', idx=idx) qrun = q.query_runs[0] - assert qrun.line_by_pos + assert q.line_by_pos assert [2] == qrun.tokens assert {0: 1} == qrun.unknowns_by_pos # onw unknown in the middle q = Query(query_string='binary that a binary', idx=idx) qrun = q.query_runs[0] - assert qrun.line_by_pos + assert q.line_by_pos assert [2, 0, 2] == qrun.tokens assert {0: 1} == qrun.unknowns_by_pos # onw unknown in the middle q = Query(query_string='a binary that a binary', idx=idx) qrun = q.query_runs[0] - assert qrun.line_by_pos + assert q.line_by_pos assert [0, 2, 0, 2] == qrun.tokens assert {1: 1} == qrun.unknowns_by_pos # two unknowns in the middle q = Query(query_string='binary that was a binary', idx=idx) qrun = q.query_runs[0] - assert qrun.line_by_pos + assert q.line_by_pos assert [2, 0, 2] == qrun.tokens assert {0: 2} == qrun.unknowns_by_pos @@ -325,7 +318,7 @@ def test_query_run_tokens_with_junk(self): q = Query(query_string='hello dolly binary that was a binary end really', idx=idx) # u u u u u u qrun = q.query_runs[0] - assert qrun.line_by_pos + assert q.line_by_pos assert [2, 0, 2] == qrun.tokens assert {-1: 2, 0: 2, 2: 2} == qrun.unknowns_by_pos diff --git a/tests/scancode/data/failing/patchelf.expected.json b/tests/scancode/data/failing/patchelf.expected.json index c322c177f0c..fcabfb162dc 100644 --- a/tests/scancode/data/failing/patchelf.expected.json +++ b/tests/scancode/data/failing/patchelf.expected.json @@ -5,11 +5,7 @@ { "copyrights": [], "path": "patchelf.pdf", - "scan_errors": [ - { - "copyrights": ["unpack requires a string argument of length 8"] - } - ] + "scan_errors": ["ERROR: copyrights: unpack requires a string argument of length 8"] } ] } diff --git a/tests/scancode/test_cli.py b/tests/scancode/test_cli.py index af85eede4be..6ec875ca3ab 100644 --- a/tests/scancode/test_cli.py +++ b/tests/scancode/test_cli.py @@ -303,10 +303,16 @@ def test_scan_with_errors_and_diag_option_includes_full_traceback(monkeypatch): result = runner.invoke(cli.scancode, [ '--copyright', '--diag', test_file, result_file], catch_exceptions=True) assert result.exit_code == 0 assert 'Scanning done' in result.output - check_scan(test_env.get_test_loc('failing/patchelf.expected.json'), result_file, test_file) assert 'Some files failed to scan' in result.output assert 'patchelf.pdf' in result.output + result_json = json.loads(open(result_file).read()) + expected = 'ERROR: copyrights: unpack requires a string argument of length 8' + assert expected == result_json['files'][0]['scan_errors'][0] + + assert result_json['files'][0]['scan_errors'][1].startswith('ERROR: copyrights: Traceback (most recent call') + + def test_failing_scan_return_proper_exit_code(monkeypatch): monkeypatch.setattr(click._termui_impl, 'isatty', lambda _: True) test_file = test_env.get_test_loc('failing/patchelf.pdf') @@ -376,13 +382,13 @@ def test_scan_works_with_multiple_processes_and_timeouts(monkeypatch): assert result.exit_code == 0 assert 'Scanning done' in result.output expected = [ - {u'path': u'test1.txt', u'scan_errors': [{u'scan': [u'Processing interrupted: timeout after 1 seconds.']}]}, - {u'path': u'test2.txt', u'scan_errors': [{u'scan': [u'Processing interrupted: timeout after 1 seconds.']}]}, - {u'path': u'test3.txt', u'scan_errors': [{u'scan': [u'Processing interrupted: timeout after 1 seconds.']}]} + {u'path': u'test1.txt', u'scan_errors': [u'ERROR: Processing interrupted: timeout after 1 seconds.']}, + {u'path': u'test2.txt', u'scan_errors': [u'ERROR: Processing interrupted: timeout after 1 seconds.']}, + {u'path': u'test3.txt', u'scan_errors': [u'ERROR: Processing interrupted: timeout after 1 seconds.']} ] result_json = json.loads(open(result_file).read()) - assert any([scan_result in expected for scan_result in result_json['files']]) + assert sorted(expected) == sorted(result_json['files']) def test_scan_works_with_multiple_processes_and_memory_quota(monkeypatch): @@ -407,9 +413,9 @@ def test_scan_works_with_multiple_processes_and_memory_quota(monkeypatch): assert result.exit_code == 0 assert 'Scanning done' in result.output expected = [ - {u'path': u'apache-1.1.txt', u'scan_errors': [{u'scan': [u'Processing interrupted: excessive memory usage of more than 1MB.']}]}, - {u'path': u'apache-1.0.txt', u'scan_errors': [{u'scan': [u'Processing interrupted: excessive memory usage of more than 1MB.']}]}, - {u'path': u'patchelf.pdf', u'scan_errors': [{u'scan': [u'Processing interrupted: excessive memory usage of more than 1MB.']}]} + {u'path': u'apache-1.1.txt', u'scan_errors': [u'ERROR: Processing interrupted: excessive memory usage of more than 1MB.']}, + {u'path': u'apache-1.0.txt', u'scan_errors': [u'ERROR: Processing interrupted: excessive memory usage of more than 1MB.']}, + {u'path': u'patchelf.pdf', u'scan_errors': [u'ERROR: Processing interrupted: excessive memory usage of more than 1MB.']} ] result_json = json.loads(open(result_file).read()) - assert any([scan_result in expected for scan_result in result_json['files']]) + assert sorted(expected) == sorted(result_json['files']) diff --git a/tests/scancode/test_interrupt.py b/tests/scancode/test_interrupt.py index f0ce844464b..95767ce6783 100644 --- a/tests/scancode/test_interrupt.py +++ b/tests/scancode/test_interrupt.py @@ -63,7 +63,7 @@ def some_long_function(exec_time): return 'OK' result = interrupt.interruptible(some_long_function, 0.5, timeout=0.01) - assert (False, 'Processing interrupted: timeout after 0 seconds.') == result + assert (False, 'ERROR: Processing interrupted: timeout after 0 seconds.') == result def test_interruptible_stops_execution_on_memory(self): from time import sleep diff --git a/tests/scancode/test_scan_cache.py b/tests/scancode/test_scan_cache.py index 5f5b8422702..7a44dc98398 100644 --- a/tests/scancode/test_scan_cache.py +++ b/tests/scancode/test_scan_cache.py @@ -28,7 +28,7 @@ from commoncode.testcase import FileBasedTesting -from scancode.cache import ScanCache +from scancode.cache import ScanFileCache class TestCache(FileBasedTesting): @@ -38,8 +38,11 @@ def test_can_cache(self): test_file = self.get_test_loc('cache/package/package.json') from scancode import api package = api.get_package_infos(test_file) + file_info = dict(sha1='def') test_dir = self.get_temp_dir() - cache = ScanCache(test_dir) - cache.put_infos(path='abc', file_infos=dict(sha1='def')) - cache.put_scan(path='abc', file_infos=dict(sha1='def'), scan_result=package) + cache = ScanFileCache(test_dir) + cache.put_info(path='abc', file_info=file_info) + cache.put_scan(path='abc', file_info=file_info, scan_result=package) + assert file_info == cache.get_info(path='abc') + assert package == cache.get_scan(path='abc', file_info=file_info)