diff --git a/plugins/scancode-fingerprint/.gitignore b/plugins/scancode-fingerprint/.gitignore new file mode 100644 index 00000000000..e2cae228c89 --- /dev/null +++ b/plugins/scancode-fingerprint/.gitignore @@ -0,0 +1,2 @@ +/build/ +/dist/ diff --git a/plugins/scancode-fingerprint/MANIFEST.in b/plugins/scancode-fingerprint/MANIFEST.in new file mode 100644 index 00000000000..c2cc8541f4c --- /dev/null +++ b/plugins/scancode-fingerprint/MANIFEST.in @@ -0,0 +1,11 @@ +graft src + +include setup.py +include setup.cfg +include .gitignore +include README.md +include MANIFEST.in +include NOTICE +include apache-2.0.LICENSE + +global-exclude *.py[co] __pycache__ *.*~ diff --git a/plugins/scancode-fingerprint/NOTICE b/plugins/scancode-fingerprint/NOTICE new file mode 100644 index 00000000000..eefa408d354 --- /dev/null +++ b/plugins/scancode-fingerprint/NOTICE @@ -0,0 +1,25 @@ +Software license +================ + +Copyright (c) nexB Inc. and others. All rights reserved. +http://nexb.com and https://github.com/nexB/scancode-toolkit/ +The ScanCode software is licensed under the Apache License version 2.0. +Data generated with ScanCode require an acknowledgment. +ScanCode is a trademark of nexB Inc. + +You may not use this software except in compliance with the License. +You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +When you publish or redistribute any data created with ScanCode or any ScanCode +derivative work, you must accompany this data with the following acknowledgment: + + Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES + OR CONDITIONS OF ANY KIND, either express or implied. No content created from + ScanCode should be considered or used as legal advice. Consult an Attorney + for any legal advice. + ScanCode is a free software code scanning tool from nexB Inc. and others. + Visit https://github.com/nexB/scancode-toolkit/ for support and download. diff --git a/plugins/scancode-fingerprint/README.md b/plugins/scancode-fingerprint/README.md new file mode 100644 index 00000000000..61bac6385ba --- /dev/null +++ b/plugins/scancode-fingerprint/README.md @@ -0,0 +1,18 @@ +#### DESCRIPTION + +A ScanCode scan plugin to generate fingerprints using Simhash algorithm. The Simhash algorithm is a dimensionality +reduction technique. It maps high-dimensional vectors to small-sized fingerprints. + +Shingles are consecutive overlapping sequences of letters. By using these instead of the words directly, we can get an +idea about how the words are related to each other. Each shingle has its own fingerprint, which can be used to compare +the similarity between two shingles. The fingerprint is basically a hash with the property that relatively similar +values will be hashed to relatively similar hashes. After the hashes are computed we use hamming distance algorithm to +find the similarity between two files. + +#### USAGE +`scancode -f -- ` + +#### Example +* Scan the `samples/` directory for generating fingerprint. +* Save scan results to the `scancode_result.json` JSON file. +* command : `scancode -f --json-pp scancode_result.json samples` \ No newline at end of file diff --git a/plugins/scancode-fingerprint/apache-2.0.LICENSE b/plugins/scancode-fingerprint/apache-2.0.LICENSE new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/plugins/scancode-fingerprint/apache-2.0.LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/scancode-fingerprint/setup.cfg b/plugins/scancode-fingerprint/setup.cfg new file mode 100644 index 00000000000..917cf1e0f0d --- /dev/null +++ b/plugins/scancode-fingerprint/setup.cfg @@ -0,0 +1,8 @@ +[metadata] +license_file = NOTICE + +[bdist_wheel] +universal = 1 + +[aliases] +release = clean --all bdist_wheel diff --git a/plugins/scancode-fingerprint/setup.py b/plugins/scancode-fingerprint/setup.py new file mode 100644 index 00000000000..abcb5b00cfd --- /dev/null +++ b/plugins/scancode-fingerprint/setup.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +from __future__ import absolute_import +from __future__ import print_function + +from glob import glob +from os.path import basename +from os.path import join +from os.path import splitext + +from setuptools import find_packages +from setuptools import setup + + +desc = '''A ScanCode scan plugin to generate fingerprints using Simhash algorithm.''' + +setup( + name='scancode-fingerprint', + version='1.0.0', + license='Apache-2.0 with ScanCode acknowledgment', + description=desc, + long_description=desc, + author='nexB', + author_email='info@aboutcode.org', + url='https://github.com/nexB/scancode-toolkit/plugins/scancode-fingerprint', + packages=find_packages('src'), + package_dir={'': 'src'}, + py_modules=[splitext(basename(path))[0] for path in glob('src/*.py')], + include_package_data=True, + zip_safe=False, + classifiers=[ + # complete classifier list: http://pypi.python.org/pypi?%3Aaction=list_classifiers + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2.7', + 'Topic :: Utilities', + ], + keywords=[ + 'open source', 'scancode', 'fingerprint', 'simhash' + ], + install_requires=[ + 'scancode-toolkit', + ], + entry_points={ + 'scancode_scan': [ + 'fingerprint = plugin_fingerprint.plugin_fingerprint:FingerprintScanner', + ], + } + + +) diff --git a/plugins/scancode-fingerprint/src/plugin_fingerprint/__init__.py b/plugins/scancode-fingerprint/src/plugin_fingerprint/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/plugins/scancode-fingerprint/src/plugin_fingerprint/fingerprint.py b/plugins/scancode-fingerprint/src/plugin_fingerprint/fingerprint.py new file mode 100644 index 00000000000..acfcf3cf730 --- /dev/null +++ b/plugins/scancode-fingerprint/src/plugin_fingerprint/fingerprint.py @@ -0,0 +1,142 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/scancode-toolkit/ +# The ScanCode software is licensed under the Apache License version 2.0. +# Data generated with ScanCode require an acknowledgment. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with ScanCode or any ScanCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# ScanCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode-toolkit/ for support and download. + +import binascii +from bitarray import bitarray +from bitarray import bitdiff +from licensedcode.tokenize import ngrams +import hashlib + +HASH_LENGTH = 128 +SHINGLE_LENGTH = 3 + +class Simhash: + """ + Fingerprint class to generate fingerprints for files used for similarity matching + """ + def __init__(self): + self.tokens = [] + + def generate_fingerprint(self): + """ + Return fingerprint as a bitarray + """ + weighted_hash = self.get_weighted_hash() + fingerprint = self.process_weighted_hash(weighted_hash) + + return fingerprint + + def hex_digest(self): + """ + Return fingerprint as a hex string + """ + result = None + + if len(self.tokens): + fingerprint_binary = self.generate_fingerprint() + result = binascii.hexlify(fingerprint_binary) + return result + + def get_weighted_hash(self): + """ + Return a weighted array from the word token list. + """ + result = [0] * HASH_LENGTH + length = len(self.tokens) - SHINGLE_LENGTH + 1 + shingles = ngrams(self.tokens, SHINGLE_LENGTH) + + if length > 0: + for shingle in shingles: + shingle = ''.join(shingle) + self.process_shingles(shingle, result) + else: + self.process_shingles(''.join(self.tokens), result) + + return result + + def process_weighted_hash(self, weighted_hash): + """ + Return fingerprint from a weighted array as a bitarray. + """ + result = bitarray() + + for item in weighted_hash: + if item > 0: + result.append(1) + else: + result.append(0) + + return result + + @staticmethod + def bitarray_from_bytes(b): + """ + Return bitarray from a byte string, interpreted as machine values. + """ + a = bitarray() + a.frombytes(b) + + return a + + def process_shingles(self, shingle, weighted_list): + """ + Modify weighted list wrt to shingle + """ + hash = hashlib.md5(shingle.encode()).digest() + result = self.bitarray_from_bytes(hash) + + for idx, bit in enumerate(result): + if bit: + weighted_list[idx] += 1 + else: + weighted_list[idx] -= 1 + + return weighted_list + + def update(self, string): + """ + Update tokens by appending new tokens + """ + new_tokens = string.split() + self.tokens += new_tokens + + def bitarray_from_hex(self, fingerprint_hex): + """ + Return bitarray from a hex string. + """ + bytes = binascii.unhexlify(fingerprint_hex) + result = self.bitarray_from_bytes(bytes) + + return result + + def hamming_distance(self, fingerprint1, fingerprint2): + """ + Return hamming distance between two given fingerprints. + Hamming distance is the difference in the bits of two binary string. + Files with fingerprints whose hamming distance are less tends to be more similar. + """ + distance = bitdiff(fingerprint1, fingerprint2) + result = int(distance) + + return result diff --git a/plugins/scancode-fingerprint/src/plugin_fingerprint/plugin_fingerprint.py b/plugins/scancode-fingerprint/src/plugin_fingerprint/plugin_fingerprint.py new file mode 100644 index 00000000000..c1a651915fa --- /dev/null +++ b/plugins/scancode-fingerprint/src/plugin_fingerprint/plugin_fingerprint.py @@ -0,0 +1,73 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/scancode-toolkit/ +# The ScanCode software is licensed under the Apache License version 2.0. +# Data generated with ScanCode require an acknowledgment. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with ScanCode or any ScanCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# ScanCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode-toolkit/ for support and download. + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import attr + +from plugincode.scan import ScanPlugin +from plugincode.scan import scan_impl +from scancode import CommandLineOption +from scancode import OTHER_SCAN_GROUP +from scancode import SCAN_OPTIONS_GROUP +from plugin_fingerprint.fingerprint import Simhash + +@scan_impl +class FingerprintScanner(ScanPlugin): + """ + Scan a file Resource to generate fingerprint. + """ + resource_attributes = dict(fingerprint=attr.ib(default=None, repr=False)) + + sort_order = 1 + + options = [ + CommandLineOption(('-f', '--fingerprint'), + is_flag=True, default=False, + help='Scan to generate simhash fingerprints for similarity matching.', + help_group=OTHER_SCAN_GROUP) + ] + + def is_enabled(self, fingerprint, **kwargs): + return fingerprint + + def get_scanner(self, **kwargs): + return get_fingerprint + + +def get_fingerprint(location, **kwargs): + """ + Return a mapping of fingerprint generated for the file at `location`. + """ + with open(location, 'r') as f: + hashable = f.read() + + simhash = Simhash() + simhash.update(hashable) + result = simhash.hex_digest() + + return dict(fingerprint=result) diff --git a/plugins/scancode-fingerprint/tests/data/fingerprint/fingerprint-test1.java b/plugins/scancode-fingerprint/tests/data/fingerprint/fingerprint-test1.java new file mode 100644 index 00000000000..5223bb2b788 --- /dev/null +++ b/plugins/scancode-fingerprint/tests/data/fingerprint/fingerprint-test1.java @@ -0,0 +1,28 @@ +public class Test { + public static void main(String[] args) + { + String Branch = "CSE"; + int year = 2; + + switch (year) { + case 1: + System.out.println("elective courses : Advance english, Algebra"); + break; + case 2: + switch (Branch) + { + case "CSE": + case "CCE": + System.out.println("elective courses : Machine Learning, Big Data"); + break; + + case "ECE": + System.out.println("elective courses : Antenna Engineering"); + break; + + default: + System.out.println("Elective courses : Optimization"); + } + } + } +} \ No newline at end of file diff --git a/plugins/scancode-fingerprint/tests/data/fingerprint/fingerprint-test2.c b/plugins/scancode-fingerprint/tests/data/fingerprint/fingerprint-test2.c new file mode 100644 index 00000000000..b545146d943 --- /dev/null +++ b/plugins/scancode-fingerprint/tests/data/fingerprint/fingerprint-test2.c @@ -0,0 +1,16 @@ +#include +int main() +{ + int sum=0; + for(int i = 0; i<=10; i++){ + sum = sum+i; + if(i==5){ + goto addition; + } + } + + addition: + printf("%d", sum); + + return 0; +} \ No newline at end of file diff --git a/plugins/scancode-fingerprint/tests/data/fingerprint/fingerprint-test3.py b/plugins/scancode-fingerprint/tests/data/fingerprint/fingerprint-test3.py new file mode 100644 index 00000000000..6a7c7452b79 --- /dev/null +++ b/plugins/scancode-fingerprint/tests/data/fingerprint/fingerprint-test3.py @@ -0,0 +1,23 @@ +import array + +arr = array.array('i', [1, 2, 3]) + +print ("The new created array is : ", end=" ") +for i in range(0, 3): + print (arr[i], end=" ") + +print("\r") + +arr.append(4); + +print("The appended array is : ", end="") +for i in range(0, 4): + print (arr[i], end=" ") + +arr.insert(2, 5) + +print("\r") + +print ("The array after insertion is : ", end="") +for i in range(0, 5): + print (arr[i], end=" ") \ No newline at end of file diff --git a/plugins/scancode-fingerprint/tests/data/fingerprint/get_weighted_hash-test1.txt b/plugins/scancode-fingerprint/tests/data/fingerprint/get_weighted_hash-test1.txt new file mode 100644 index 00000000000..e83314b21d9 --- /dev/null +++ b/plugins/scancode-fingerprint/tests/data/fingerprint/get_weighted_hash-test1.txt @@ -0,0 +1,3 @@ +this +is +for testing \ No newline at end of file diff --git a/plugins/scancode-fingerprint/tests/data/fingerprint/get_weighted_hash-test2.txt b/plugins/scancode-fingerprint/tests/data/fingerprint/get_weighted_hash-test2.txt new file mode 100644 index 00000000000..2e65efe2a14 --- /dev/null +++ b/plugins/scancode-fingerprint/tests/data/fingerprint/get_weighted_hash-test2.txt @@ -0,0 +1 @@ +a \ No newline at end of file diff --git a/plugins/scancode-fingerprint/tests/data/fingerprint/similarity_matching1.py b/plugins/scancode-fingerprint/tests/data/fingerprint/similarity_matching1.py new file mode 100644 index 00000000000..a827ad3ee45 --- /dev/null +++ b/plugins/scancode-fingerprint/tests/data/fingerprint/similarity_matching1.py @@ -0,0 +1,588 @@ +import inspect +import cProfile +import pdb +import collections +import traceback +import logging +from functools import partial +from os import makedirs, getcwd +from os.path import join, abspath, exists, isdir +import requests +from appdirs import user_data_dir + +from pyprint.Printer import Printer + +from coala_utils.decorators import (enforce_signature, classproperty, + get_public_members) + +from coalib.bears.BEAR_KIND import BEAR_KIND +from coalib.output.printers.LogPrinter import LogPrinterMixin +from coalib.results.Result import Result +from coalib.results.TextPosition import ZeroOffsetError +from coalib.settings.FunctionMetadata import FunctionMetadata +from coalib.settings.Section import Section +from coalib.settings.Setting import Setting +from coalib.settings.ConfigurationGathering import get_config_directory + +from .meta import bearclass + + +def _setting_is_enabled(bear, key): + """ + Check setting key is in section. + + :param bear: Bear object. + :param key: Setting key. + :return: ``True`` if setting value is ``True``. Setting object if + setting key is in section else ``False``. + """ + if not isinstance(bear, Bear): + raise ValueError( + 'Positional argument bear is not an instance of Bear class.') + if key is None: + raise ValueError('No setting key passed.') + + if key not in bear.section: + return False + try: + return bool(bear.section[key]) + except ValueError: + pass + return bear.section[key] + + +def _is_debugged(bear): + """ + Check whether the bear is in debug mode according to its section-settings. + + :param bear: Bear object. + :return: True if ``debug_bears`` is ``True`` or if bear name specified + in ``debug_bears`` setting match with the bear parameter. + """ + setting = _setting_is_enabled(bear, key='debug_bears') + if isinstance(setting, bool): + return setting + return bear.name.lower() in map(str.lower, setting) + + +def _is_profiled(bear): + """ + Check whether the bear is in profile mode according to its section-settings. + + :param bear: Bear object. + :return: current working directory if ``profile`` is ``True``, False + if ``profile`` is ``False`` else return directory path + specified in ``profile``. + """ + setting = _setting_is_enabled(bear, key='profile') + if setting is True: + return getcwd() + if isinstance(setting, Setting): + return setting.value + return False + + +class Debugger(pdb.Pdb): + + def __init__(self, bear, *args, **kwargs): + if not isinstance(bear, Bear): + raise ValueError('Positional argument bear is not an instance of ' + 'Bear class.') + super(Debugger, self).__init__(*args, **kwargs) + self.bear = bear + + def do_quit(self, arg): + self.clear_all_breaks() + super().do_continue(arg) + return 1 + + do_q = do_quit + do_exit = do_quit + + def do_settings(self, arg): + md = self.bear.get_metadata() + section_params_dict = md.create_params_from_section( + self.bear.section) + for param in md.non_optional_params: + self.message('%s = %r' % (param, section_params_dict[param])) + for param in md.optional_params: + self.message('%s = %r' % (param, section_params_dict[param] if + param in section_params_dict else + md.optional_params[param][2])) + return 1 + + +class Bear(Printer, LogPrinterMixin, metaclass=bearclass): + """ + A bear contains the actual subroutine that is responsible for checking + source code for certain specifications. However it can actually do + whatever it wants with the files it gets. If you are missing some Result + type, feel free to contact us and/or help us extending the coalib. + + This is the base class for every bear. If you want to write a bear, you + will probably want to look at the GlobalBear and LocalBear classes that + inherit from this class. In any case you'll want to overwrite at least the + run method. You can send debug/warning/error messages through the + debug(), warn(), err() functions. These will send the + appropriate messages so that they are outputted. Be aware that if you use + err(), you are expected to also terminate the bear run-through + immediately. + + Settings are available at all times through self.section. + + To indicate which languages your bear supports, just give it the + ``LANGUAGES`` value which should be a set of string(s): + + >>> from dependency_management.requirements.PackageRequirement import ( + ... PackageRequirement) + >>> from dependency_management.requirements.PipRequirement import ( + ... PipRequirement) + >>> class SomeBear(Bear): + ... LANGUAGES = {'C', 'CPP','C#', 'D'} + + To indicate the requirements of the bear, assign ``REQUIREMENTS`` a set + with instances of ``PackageRequirements``. + + >>> class SomeBear(Bear): + ... REQUIREMENTS = { + ... PackageRequirement('pip', 'coala_decorators', '0.2.1')} + + If your bear uses requirements from a manager we have a subclass from, + you can use the subclass, such as ``PipRequirement``, without specifying + manager: + + >>> class SomeBear(Bear): + ... REQUIREMENTS = {PipRequirement('coala_decorators', '0.2.1')} + + To specify additional attributes to your bear, use the following: + + >>> class SomeBear(Bear): + ... AUTHORS = {'Jon Snow'} + ... AUTHORS_EMAILS = {'jon_snow@gmail.com'} + ... MAINTAINERS = {'Catelyn Stark'} + ... MAINTAINERS_EMAILS = {'catelyn_stark@gmail.com'} + ... LICENSE = 'AGPL-3.0' + ... ASCIINEMA_URL = 'https://asciinema.org/a/80761' + + If the maintainers are the same as the authors, they can be omitted: + + >>> class SomeBear(Bear): + ... AUTHORS = {'Jon Snow'} + ... AUTHORS_EMAILS = {'jon_snow@gmail.com'} + >>> SomeBear.maintainers + {'Jon Snow'} + >>> SomeBear.maintainers_emails + {'jon_snow@gmail.com'} + + If your bear needs to include local files, then specify it giving strings + containing relative file paths to the INCLUDE_LOCAL_FILES set: + + >>> class SomeBear(Bear): + ... INCLUDE_LOCAL_FILES = {'checkstyle.jar', 'google_checks.xml'} + + To keep track easier of what a bear can do, simply tell it to the CAN_FIX + and the CAN_DETECT sets. Possible values: + + >>> CAN_DETECT = {'Syntax', 'Formatting', 'Security', 'Complexity', 'Smell', + ... 'Unused Code', 'Redundancy', 'Variable Misuse', 'Spelling', + ... 'Memory Leak', 'Documentation', 'Duplication', 'Commented Code', + ... 'Grammar', 'Missing Import', 'Unreachable Code', 'Undefined Element', + ... 'Code Simplification', 'Statistics'} + >>> CAN_FIX = {'Syntax', ...} + + Specifying something to CAN_FIX makes it obvious that it can be detected + too, so it may be omitted: + + >>> class SomeBear(Bear): + ... CAN_DETECT = {'Syntax', 'Security'} + ... CAN_FIX = {'Redundancy'} + >>> list(sorted(SomeBear.can_detect)) + ['Redundancy', 'Security', 'Syntax'] + + Every bear has a data directory which is unique to that particular bear: + + >>> class SomeBear(Bear): pass + >>> class SomeOtherBear(Bear): pass + >>> SomeBear.data_dir == SomeOtherBear.data_dir + False + + BEAR_DEPS contains bear classes that are to be executed before this bear + gets executed. The results of these bears will then be passed to the + run method as a dict via the dependency_results argument. The dict + will have the name of the Bear as key and the list of its results as + results: + + >>> class SomeBear(Bear): pass + >>> class SomeOtherBear(Bear): + ... BEAR_DEPS = {SomeBear} + >>> SomeOtherBear.BEAR_DEPS + {} + + Every bear resides in some directory which is specified by the + source_location attribute: + + >>> class SomeBear(Bear): pass + >>> SomeBear.source_location + '...Bear.py' + + Every linter bear makes use of an executable tool for its operations. + The SEE_MORE attribute provides a link to the main page of the linter + tool: + + >>> class PyLintBear(Bear): + ... SEE_MORE = 'https://www.pylint.org/' + >>> PyLintBear.SEE_MORE + 'https://www.pylint.org/' + + In the future, bears will not survive without aspects. aspects are defined + as part of the ``class`` statement's parameter list. According to the + classic ``CAN_DETECT`` and ``CAN_FIX`` attributes, aspects can either be + only ``'detect'``-able or also ``'fix'``-able: + + >>> from coalib.bearlib.aspects.Metadata import CommitMessage + + >>> class aspectsCommitBear(Bear, aspects={ + ... 'detect': [CommitMessage.Shortlog.ColonExistence], + ... 'fix': [CommitMessage.Shortlog.TrailingPeriod], + ... }, languages=['Python']): + ... pass + + >>> aspectsCommitBear.aspects['detect'] + [] + >>> aspectsCommitBear.aspects['fix'] + [] + + To indicate the bear uses raw files, set ``USE_RAW_FILES`` to True: + + >>> class RawFileBear(Bear): + ... USE_RAW_FILES = True + >>> RawFileBear.USE_RAW_FILES + True + + However if ``USE_RAW_FILES`` is enabled the Bear is in charge of managing + the file (opening the file, closing the file, reading the file, etc). + """ + + LANGUAGES = set() + REQUIREMENTS = set() + AUTHORS = set() + AUTHORS_EMAILS = set() + MAINTAINERS = set() + MAINTAINERS_EMAILS = set() + PLATFORMS = {'any'} + LICENSE = '' + INCLUDE_LOCAL_FILES = set() + CAN_DETECT = set() + CAN_FIX = set() + ASCIINEMA_URL = '' + SEE_MORE = '' + BEAR_DEPS = set() + USE_RAW_FILES = False + + @classproperty + def name(cls): + """ + :return: The name of the bear + """ + return cls.__name__ + + @classproperty + def can_detect(cls): + """ + :return: A set that contains everything a bear can detect, gathering + information from what it can fix too. + """ + return cls.CAN_DETECT | cls.CAN_FIX + + @classproperty + def source_location(cls): + """ + :return: The file path where the bear was fetched from. + """ + return inspect.getfile(cls) + + @classproperty + def maintainers(cls): + """ + :return: A set containing ``MAINTAINERS`` if specified, else takes + ``AUTHORS`` by default. + """ + return cls.AUTHORS if cls.MAINTAINERS == set() else cls.MAINTAINERS + + @classproperty + def maintainers_emails(cls): + """ + :return: A set containing ``MAINTAINERS_EMAILS`` if specified, else + takes ``AUTHORS_EMAILS`` by default. + """ + return (cls.AUTHORS_EMAILS if cls.MAINTAINERS_EMAILS == set() + else cls.MAINTAINERS_EMAILS) + + @enforce_signature + def __init__(self, + section: Section, + message_queue, + timeout=0): + """ + Constructs a new bear. + + :param section: The section object where bear settings are + contained. + :param message_queue: The queue object for messages. Can be ``None``. + :param timeout: The time the bear is allowed to run. To set no + time limit, use 0. + :raises TypeError: Raised when ``message_queue`` is no queue. + :raises RuntimeError: Raised when bear requirements are not fulfilled. + """ + Printer.__init__(self) + + if message_queue is not None and not hasattr(message_queue, 'put'): + raise TypeError('message_queue has to be a Queue or None.') + + self.section = section + self.message_queue = message_queue + self.timeout = timeout + self.debugger = _is_debugged(bear=self) + self.profile = _is_profiled(bear=self) + + if self.profile and self.debugger: + raise ValueError( + 'Cannot run debugger and profiler at the same time.') + + self.setup_dependencies() + cp = type(self).check_prerequisites() + if cp is not True: + error_string = ('The bear ' + self.name + + ' does not fulfill all requirements.') + if cp is not False: + error_string += ' ' + cp + + self.err(error_string) + raise RuntimeError(error_string) + + def _print(self, output, **kwargs): + self.debug(output) + + def log_message(self, log_message, timestamp=None, **kwargs): + if self.message_queue is not None: + self.message_queue.put(log_message) + + def run(self, *args, dependency_results=None, **kwargs): + raise NotImplementedError + + def _dump_bear_profile_data(self, profiler): + filename = '{}_{}.prof'.format(self.section.name, self.name) + path = join(self.profile, filename) + if not isdir(self.profile): + try: + makedirs(self.profile) + except FileExistsError: + logging.error('File exists :'.format(self.profile)) + raise SystemExit(2) + + profiler.dump_stats(path) + + def profile_run(self, *args, profiler=None, **kwargs): + profiler = cProfile.Profile() if profiler is None else profiler + bear_results = profiler.runcall(self.run, *args, **kwargs) + if isinstance(bear_results, collections.Iterable): + results = [] + iterator = iter(bear_results) + while True: + try: + result = profiler.runcall(next, iterator) + results.append(result) + except StopIteration: + break + else: + results = bear_results + self._dump_bear_profile_data(profiler) + return results + + def run_bear_from_section(self, args, kwargs): + try: + # Don't get `language` setting from `section.contents` + if self.section.language and ( + 'language' in self.get_metadata()._optional_params or + 'language' in self.get_metadata()._non_optional_params): + kwargs['language'] = self.section.language + kwargs.update( + self.get_metadata().create_params_from_section(self.section)) + except ValueError as err: + self.warn('The bear {} cannot be executed.'.format( + self.name), str(err)) + return + if self.debugger: + return debug_run(self.run, Debugger(bear=self), *args, **kwargs) + elif self.profile: + return self.profile_run(*args, **kwargs) + else: + return self.run(*args, **kwargs) + + def execute(self, *args, debug=False, **kwargs): + name = self.name + try: + self.debug('Running bear {}...'.format(name)) + + # If `dependency_results` kwargs is defined but there are no + # dependency results (usually in Bear that has no dependency) + # delete the `dependency_results` kwargs, since most Bears don't + # define `dependency_results` kwargs in its `run()` function. + if ('dependency_results' in kwargs and + kwargs['dependency_results'] is None and + not self.BEAR_DEPS): + del kwargs['dependency_results'] + + # If it's already a list it won't change it + result = self.run_bear_from_section(args, kwargs) + return [] if result is None else list(result) + except (Exception, SystemExit) as exc: + if debug and not isinstance(exc, SystemExit): + raise + + if isinstance(exc, ZeroOffsetError): + self.err('Bear {} violated one-based offset convention.' + .format(name), str(exc)) + + if (self.kind() == BEAR_KIND.LOCAL + and ('log_level' not in self.section + or self.section['log_level'].value != 'DEBUG')): + self.err('Bear {} failed to run on file {}. Take a look ' + 'at debug messages (`-V`) for further ' + 'information.'.format(name, args[0])) + elif ('log_level' not in self.section + or self.section['log_level'].value != 'DEBUG'): + self.err('Bear {} failed to run. Take a look ' + 'at debug messages (`-V`) for further ' + 'information.'.format(name)) + + self.debug( + 'The bear {bear} raised an exception. If you are the author ' + 'of this bear, please make sure to catch all exceptions. If ' + 'not and this error annoys you, you might want to get in ' + 'contact with the author of this bear.\n\nTraceback ' + 'information is provided below:\n\n{traceback}' + '\n'.format(bear=name, traceback=traceback.format_exc())) + + @staticmethod + def kind(): + """ + :return: The kind of the bear + """ + raise NotImplementedError + + @classmethod + def get_metadata(cls): + """ + :return: Metadata for the run function. However parameters like + ``self`` or parameters implicitly used by coala (e.g. + filename for local bears) are already removed. + """ + return FunctionMetadata.from_function( + cls.run, + omit={'self', 'dependency_results', 'language'}) + + @classmethod + def __json__(cls): + """ + Override JSON export of ``Bear`` object. + """ + # json cannot serialize properties, so drop them + _dict = {key: value for key, value in get_public_members(cls).items() + if not isinstance(value, property)} + metadata = cls.get_metadata() + non_optional_params = metadata.non_optional_params + optional_params = metadata.optional_params + _dict['metadata'] = { + 'desc': metadata.desc, + 'non_optional_params': ({param: non_optional_params[param][0]} + for param in non_optional_params), + 'optional_params': ({param: optional_params[param][0]} + for param in optional_params)} + if hasattr(cls, 'languages'): + _dict['languages'] = (str(language) for language in cls.languages) + return _dict + + @classmethod + def missing_dependencies(cls, lst): + """ + Checks if the given list contains all dependencies. + + :param lst: A list of all already resolved bear classes (not + instances). + :return: A set of missing dependencies. + """ + return set(cls.BEAR_DEPS) - set(lst) + + @classmethod + def get_non_optional_settings(cls, recurse=True): + """ + This method has to determine which settings are needed by this bear. + The user will be prompted for needed settings that are not available + in the settings file so don't include settings where a default value + would do. + + Note: This function also queries settings from bear dependencies in + recursive manner. Though circular dependency chains are a challenge to + achieve, this function would never return on them! + + :param recurse: Get the settings recursively from its dependencies. + :return: A dictionary of needed settings as keys and a tuple of + help text and annotation as values. + """ + non_optional_settings = {} + + if recurse: + for dependency in cls.BEAR_DEPS: + non_optional_settings.update( + dependency.get_non_optional_settings()) + + non_optional_settings.update(cls.get_metadata().non_optional_params) + + return non_optional_settings + + @staticmethod + def setup_dependencies(): + """ + This is a user defined function that can download and set up + dependencies (via download_cached_file or arbitrary other means) in an + OS independent way. + """ + + @classmethod + def check_prerequisites(cls): + """ + Checks whether needed runtime prerequisites of the bear are satisfied. + + This function gets executed at construction. + + Section value requirements shall be checked inside the ``run`` method. + >>> from dependency_management.requirements.PipRequirement import ( + ... PipRequirement) + >>> class SomeBear(Bear): + ... REQUIREMENTS = {PipRequirement('pip')} + + >>> SomeBear.check_prerequisites() + True + + >>> class SomeOtherBear(Bear): + ... REQUIREMENTS = {PipRequirement('really_bad_package')} + + >>> SomeOtherBear.check_prerequisites() + 'really_bad_package is not installed. You can install it using ...' + + >>> class anotherBear(Bear): + ... REQUIREMENTS = {PipRequirement('bad_package', '0.0.1')} + + >>> anotherBear.check_prerequisites() + 'bad_package 0.0.1 is not installed. You can install it using ...' + + :return: True if prerequisites are satisfied, else False or a string + that serves a more detailed description of what's missing. + """ + for requirement in cls.REQUIREMENTS: + if not requirement.is_installed(): + return str(requirement) + ' is not installed. You can ' + ( + 'install it using ') + ( + ' '.join(requirement.install_command())) + return True diff --git a/plugins/scancode-fingerprint/tests/data/fingerprint/similarity_matching2.py b/plugins/scancode-fingerprint/tests/data/fingerprint/similarity_matching2.py new file mode 100644 index 00000000000..6563ae0e452 --- /dev/null +++ b/plugins/scancode-fingerprint/tests/data/fingerprint/similarity_matching2.py @@ -0,0 +1,672 @@ +import inspect +import cProfile +import pdb +import collections +import traceback +import logging +from functools import partial +from os import makedirs, getcwd +from os.path import join, abspath, exists, isdir +import requests +from appdirs import user_data_dir + +from pyprint.Printer import Printer + +from coala_utils.decorators import (enforce_signature, classproperty, + get_public_members) + +from coalib.bears.BEAR_KIND import BEAR_KIND +from coalib.output.printers.LogPrinter import LogPrinterMixin +from coalib.results.Result import Result +from coalib.results.TextPosition import ZeroOffsetError +from coalib.settings.FunctionMetadata import FunctionMetadata +from coalib.settings.Section import Section +from coalib.settings.Setting import Setting +from coalib.settings.ConfigurationGathering import get_config_directory + +from .meta import bearclass + + +def _setting_is_enabled(bear, key): + """ + Check setting key is in section. + + :param bear: Bear object. + :param key: Setting key. + :return: ``True`` if setting value is ``True``. Setting object if + setting key is in section else ``False``. + """ + if not isinstance(bear, Bear): + raise ValueError( + 'Positional argument bear is not an instance of Bear class.') + if key is None: + raise ValueError('No setting key passed.') + + if key not in bear.section: + return False + try: + return bool(bear.section[key]) + except ValueError: + pass + return bear.section[key] + + +def _is_debugged(bear): + """ + Check whether the bear is in debug mode according to its section-settings. + + :param bear: Bear object. + :return: True if ``debug_bears`` is ``True`` or if bear name specified + in ``debug_bears`` setting match with the bear parameter. + """ + setting = _setting_is_enabled(bear, key='debug_bears') + if isinstance(setting, bool): + return setting + return bear.name.lower() in map(str.lower, setting) + + +def _is_profiled(bear): + """ + Check whether the bear is in profile mode according to its section-settings. + + :param bear: Bear object. + :return: current working directory if ``profile`` is ``True``, False + if ``profile`` is ``False`` else return directory path + specified in ``profile``. + """ + setting = _setting_is_enabled(bear, key='profile') + if setting is True: + return getcwd() + if isinstance(setting, Setting): + return setting.value + return False + + +class Debugger(pdb.Pdb): + + def __init__(self, bear, *args, **kwargs): + if not isinstance(bear, Bear): + raise ValueError('Positional argument bear is not an instance of ' + 'Bear class.') + super(Debugger, self).__init__(*args, **kwargs) + self.bear = bear + + def do_quit(self, arg): + self.clear_all_breaks() + super().do_continue(arg) + return 1 + + do_q = do_quit + do_exit = do_quit + + def do_settings(self, arg): + md = self.bear.get_metadata() + section_params_dict = md.create_params_from_section( + self.bear.section) + for param in md.non_optional_params: + self.message('%s = %r' % (param, section_params_dict[param])) + for param in md.optional_params: + self.message('%s = %r' % (param, section_params_dict[param] if + param in section_params_dict else + md.optional_params[param][2])) + return 1 + + +def debug_run(func, dbg=None, *args, **kwargs): + dbg = Debugger() if dbg is None else dbg + bear_results = dbg.runcall(func, *args, **kwargs) + if isinstance(bear_results, collections.Iterable): + results = [] + iterator = iter(bear_results) + try: + while True: + result = dbg.runcall(next, iterator) + results.append(result) + except StopIteration: + return results + else: + return bear_results + + +class Bear(Printer, LogPrinterMixin, metaclass=bearclass): + """ + A bear contains the actual subroutine that is responsible for checking + source code for certain specifications. However it can actually do + whatever it wants with the files it gets. If you are missing some Result + type, feel free to contact us and/or help us extending the coalib. + + This is the base class for every bear. If you want to write a bear, you + will probably want to look at the GlobalBear and LocalBear classes that + inherit from this class. In any case you'll want to overwrite at least the + run method. You can send debug/warning/error messages through the + debug(), warn(), err() functions. These will send the + appropriate messages so that they are outputted. Be aware that if you use + err(), you are expected to also terminate the bear run-through + immediately. + + Settings are available at all times through self.section. + + To indicate which languages your bear supports, just give it the + ``LANGUAGES`` value which should be a set of string(s): + + >>> from dependency_management.requirements.PackageRequirement import ( + ... PackageRequirement) + >>> from dependency_management.requirements.PipRequirement import ( + ... PipRequirement) + >>> class SomeBear(Bear): + ... LANGUAGES = {'C', 'CPP','C#', 'D'} + + To indicate the requirements of the bear, assign ``REQUIREMENTS`` a set + with instances of ``PackageRequirements``. + + >>> class SomeBear(Bear): + ... REQUIREMENTS = { + ... PackageRequirement('pip', 'coala_decorators', '0.2.1')} + + If your bear uses requirements from a manager we have a subclass from, + you can use the subclass, such as ``PipRequirement``, without specifying + manager: + + >>> class SomeBear(Bear): + ... REQUIREMENTS = {PipRequirement('coala_decorators', '0.2.1')} + + To specify additional attributes to your bear, use the following: + + >>> class SomeBear(Bear): + ... AUTHORS = {'Jon Snow'} + ... AUTHORS_EMAILS = {'jon_snow@gmail.com'} + ... MAINTAINERS = {'Catelyn Stark'} + ... MAINTAINERS_EMAILS = {'catelyn_stark@gmail.com'} + ... LICENSE = 'AGPL-3.0' + ... ASCIINEMA_URL = 'https://asciinema.org/a/80761' + + If the maintainers are the same as the authors, they can be omitted: + + >>> class SomeBear(Bear): + ... AUTHORS = {'Jon Snow'} + ... AUTHORS_EMAILS = {'jon_snow@gmail.com'} + >>> SomeBear.maintainers + {'Jon Snow'} + >>> SomeBear.maintainers_emails + {'jon_snow@gmail.com'} + + If your bear needs to include local files, then specify it giving strings + containing relative file paths to the INCLUDE_LOCAL_FILES set: + + >>> class SomeBear(Bear): + ... INCLUDE_LOCAL_FILES = {'checkstyle.jar', 'google_checks.xml'} + + To keep track easier of what a bear can do, simply tell it to the CAN_FIX + and the CAN_DETECT sets. Possible values: + + >>> CAN_DETECT = {'Syntax', 'Formatting', 'Security', 'Complexity', 'Smell', + ... 'Unused Code', 'Redundancy', 'Variable Misuse', 'Spelling', + ... 'Memory Leak', 'Documentation', 'Duplication', 'Commented Code', + ... 'Grammar', 'Missing Import', 'Unreachable Code', 'Undefined Element', + ... 'Code Simplification', 'Statistics'} + >>> CAN_FIX = {'Syntax', ...} + + Specifying something to CAN_FIX makes it obvious that it can be detected + too, so it may be omitted: + + >>> class SomeBear(Bear): + ... CAN_DETECT = {'Syntax', 'Security'} + ... CAN_FIX = {'Redundancy'} + >>> list(sorted(SomeBear.can_detect)) + ['Redundancy', 'Security', 'Syntax'] + + Every bear has a data directory which is unique to that particular bear: + + >>> class SomeBear(Bear): pass + >>> class SomeOtherBear(Bear): pass + >>> SomeBear.data_dir == SomeOtherBear.data_dir + False + + BEAR_DEPS contains bear classes that are to be executed before this bear + gets executed. The results of these bears will then be passed to the + run method as a dict via the dependency_results argument. The dict + will have the name of the Bear as key and the list of its results as + results: + + >>> class SomeBear(Bear): pass + >>> class SomeOtherBear(Bear): + ... BEAR_DEPS = {SomeBear} + >>> SomeOtherBear.BEAR_DEPS + {} + + Every bear resides in some directory which is specified by the + source_location attribute: + + >>> class SomeBear(Bear): pass + >>> SomeBear.source_location + '...Bear.py' + + Every linter bear makes use of an executable tool for its operations. + The SEE_MORE attribute provides a link to the main page of the linter + tool: + + >>> class PyLintBear(Bear): + ... SEE_MORE = 'https://www.pylint.org/' + >>> PyLintBear.SEE_MORE + 'https://www.pylint.org/' + + In the future, bears will not survive without aspects. aspects are defined + as part of the ``class`` statement's parameter list. According to the + classic ``CAN_DETECT`` and ``CAN_FIX`` attributes, aspects can either be + only ``'detect'``-able or also ``'fix'``-able: + + >>> from coalib.bearlib.aspects.Metadata import CommitMessage + + >>> class aspectsCommitBear(Bear, aspects={ + ... 'detect': [CommitMessage.Shortlog.ColonExistence], + ... 'fix': [CommitMessage.Shortlog.TrailingPeriod], + ... }, languages=['Python']): + ... pass + + >>> aspectsCommitBear.aspects['detect'] + [] + >>> aspectsCommitBear.aspects['fix'] + [] + + To indicate the bear uses raw files, set ``USE_RAW_FILES`` to True: + + >>> class RawFileBear(Bear): + ... USE_RAW_FILES = True + >>> RawFileBear.USE_RAW_FILES + True + + However if ``USE_RAW_FILES`` is enabled the Bear is in charge of managing + the file (opening the file, closing the file, reading the file, etc). + """ + + LANGUAGES = set() + REQUIREMENTS = set() + AUTHORS = set() + AUTHORS_EMAILS = set() + MAINTAINERS = set() + MAINTAINERS_EMAILS = set() + PLATFORMS = {'any'} + LICENSE = '' + INCLUDE_LOCAL_FILES = set() + CAN_DETECT = set() + CAN_FIX = set() + ASCIINEMA_URL = '' + SEE_MORE = '' + BEAR_DEPS = set() + USE_RAW_FILES = False + + @classproperty + def name(cls): + """ + :return: The name of the bear + """ + return cls.__name__ + + @classproperty + def can_detect(cls): + """ + :return: A set that contains everything a bear can detect, gathering + information from what it can fix too. + """ + return cls.CAN_DETECT | cls.CAN_FIX + + @classproperty + def source_location(cls): + """ + :return: The file path where the bear was fetched from. + """ + return inspect.getfile(cls) + + @classproperty + def maintainers(cls): + """ + :return: A set containing ``MAINTAINERS`` if specified, else takes + ``AUTHORS`` by default. + """ + return cls.AUTHORS if cls.MAINTAINERS == set() else cls.MAINTAINERS + + @classproperty + def maintainers_emails(cls): + """ + :return: A set containing ``MAINTAINERS_EMAILS`` if specified, else + takes ``AUTHORS_EMAILS`` by default. + """ + return (cls.AUTHORS_EMAILS if cls.MAINTAINERS_EMAILS == set() + else cls.MAINTAINERS_EMAILS) + + @enforce_signature + def __init__(self, + section: Section, + message_queue, + timeout=0): + """ + Constructs a new bear. + + :param section: The section object where bear settings are + contained. + :param message_queue: The queue object for messages. Can be ``None``. + :param timeout: The time the bear is allowed to run. To set no + time limit, use 0. + :raises TypeError: Raised when ``message_queue`` is no queue. + :raises RuntimeError: Raised when bear requirements are not fulfilled. + """ + Printer.__init__(self) + + if message_queue is not None and not hasattr(message_queue, 'put'): + raise TypeError('message_queue has to be a Queue or None.') + + self.section = section + self.message_queue = message_queue + self.timeout = timeout + self.debugger = _is_debugged(bear=self) + self.profile = _is_profiled(bear=self) + + if self.profile and self.debugger: + raise ValueError( + 'Cannot run debugger and profiler at the same time.') + + self.setup_dependencies() + cp = type(self).check_prerequisites() + if cp is not True: + error_string = ('The bear ' + self.name + + ' does not fulfill all requirements.') + if cp is not False: + error_string += ' ' + cp + + self.err(error_string) + raise RuntimeError(error_string) + + def _print(self, output, **kwargs): + self.debug(output) + + def log_message(self, log_message, timestamp=None, **kwargs): + if self.message_queue is not None: + self.message_queue.put(log_message) + + def run(self, *args, dependency_results=None, **kwargs): + raise NotImplementedError + + def _dump_bear_profile_data(self, profiler): + filename = '{}_{}.prof'.format(self.section.name, self.name) + path = join(self.profile, filename) + if not isdir(self.profile): + try: + makedirs(self.profile) + except FileExistsError: + logging.error('File exists :'.format(self.profile)) + raise SystemExit(2) + + profiler.dump_stats(path) + + def profile_run(self, *args, profiler=None, **kwargs): + profiler = cProfile.Profile() if profiler is None else profiler + bear_results = profiler.runcall(self.run, *args, **kwargs) + if isinstance(bear_results, collections.Iterable): + results = [] + iterator = iter(bear_results) + while True: + try: + result = profiler.runcall(next, iterator) + results.append(result) + except StopIteration: + break + else: + results = bear_results + self._dump_bear_profile_data(profiler) + return results + + def run_bear_from_section(self, args, kwargs): + try: + # Don't get `language` setting from `section.contents` + if self.section.language and ( + 'language' in self.get_metadata()._optional_params or + 'language' in self.get_metadata()._non_optional_params): + kwargs['language'] = self.section.language + kwargs.update( + self.get_metadata().create_params_from_section(self.section)) + except ValueError as err: + self.warn('The bear {} cannot be executed.'.format( + self.name), str(err)) + return + if self.debugger: + return debug_run(self.run, Debugger(bear=self), *args, **kwargs) + elif self.profile: + return self.profile_run(*args, **kwargs) + else: + return self.run(*args, **kwargs) + + def execute(self, *args, debug=False, **kwargs): + name = self.name + try: + self.debug('Running bear {}...'.format(name)) + + # If `dependency_results` kwargs is defined but there are no + # dependency results (usually in Bear that has no dependency) + # delete the `dependency_results` kwargs, since most Bears don't + # define `dependency_results` kwargs in its `run()` function. + if ('dependency_results' in kwargs and + kwargs['dependency_results'] is None and + not self.BEAR_DEPS): + del kwargs['dependency_results'] + + # If it's already a list it won't change it + result = self.run_bear_from_section(args, kwargs) + return [] if result is None else list(result) + except (Exception, SystemExit) as exc: + if debug and not isinstance(exc, SystemExit): + raise + + if isinstance(exc, ZeroOffsetError): + self.err('Bear {} violated one-based offset convention.' + .format(name), str(exc)) + + if (self.kind() == BEAR_KIND.LOCAL + and ('log_level' not in self.section + or self.section['log_level'].value != 'DEBUG')): + self.err('Bear {} failed to run on file {}. Take a look ' + 'at debug messages (`-V`) for further ' + 'information.'.format(name, args[0])) + elif ('log_level' not in self.section + or self.section['log_level'].value != 'DEBUG'): + self.err('Bear {} failed to run. Take a look ' + 'at debug messages (`-V`) for further ' + 'information.'.format(name)) + + self.debug( + 'The bear {bear} raised an exception. If you are the author ' + 'of this bear, please make sure to catch all exceptions. If ' + 'not and this error annoys you, you might want to get in ' + 'contact with the author of this bear.\n\nTraceback ' + 'information is provided below:\n\n{traceback}' + '\n'.format(bear=name, traceback=traceback.format_exc())) + + @staticmethod + def kind(): + """ + :return: The kind of the bear + """ + raise NotImplementedError + + @classmethod + def get_metadata(cls): + """ + :return: Metadata for the run function. However parameters like + ``self`` or parameters implicitly used by coala (e.g. + filename for local bears) are already removed. + """ + return FunctionMetadata.from_function( + cls.run, + omit={'self', 'dependency_results', 'language'}) + + @classmethod + def __json__(cls): + """ + Override JSON export of ``Bear`` object. + """ + # json cannot serialize properties, so drop them + _dict = {key: value for key, value in get_public_members(cls).items() + if not isinstance(value, property)} + metadata = cls.get_metadata() + non_optional_params = metadata.non_optional_params + optional_params = metadata.optional_params + _dict['metadata'] = { + 'desc': metadata.desc, + 'non_optional_params': ({param: non_optional_params[param][0]} + for param in non_optional_params), + 'optional_params': ({param: optional_params[param][0]} + for param in optional_params)} + if hasattr(cls, 'languages'): + _dict['languages'] = (str(language) for language in cls.languages) + return _dict + + @classmethod + def missing_dependencies(cls, lst): + """ + Checks if the given list contains all dependencies. + + :param lst: A list of all already resolved bear classes (not + instances). + :return: A set of missing dependencies. + """ + return set(cls.BEAR_DEPS) - set(lst) + + @classmethod + def get_non_optional_settings(cls, recurse=True): + """ + This method has to determine which settings are needed by this bear. + The user will be prompted for needed settings that are not available + in the settings file so don't include settings where a default value + would do. + + Note: This function also queries settings from bear dependencies in + recursive manner. Though circular dependency chains are a challenge to + achieve, this function would never return on them! + + :param recurse: Get the settings recursively from its dependencies. + :return: A dictionary of needed settings as keys and a tuple of + help text and annotation as values. + """ + non_optional_settings = {} + + if recurse: + for dependency in cls.BEAR_DEPS: + non_optional_settings.update( + dependency.get_non_optional_settings()) + + non_optional_settings.update(cls.get_metadata().non_optional_params) + + return non_optional_settings + + @staticmethod + def setup_dependencies(): + """ + This is a user defined function that can download and set up + dependencies (via download_cached_file or arbitrary other means) in an + OS independent way. + """ + + @classmethod + def check_prerequisites(cls): + """ + Checks whether needed runtime prerequisites of the bear are satisfied. + + This function gets executed at construction. + + Section value requirements shall be checked inside the ``run`` method. + >>> from dependency_management.requirements.PipRequirement import ( + ... PipRequirement) + >>> class SomeBear(Bear): + ... REQUIREMENTS = {PipRequirement('pip')} + + >>> SomeBear.check_prerequisites() + True + + >>> class SomeOtherBear(Bear): + ... REQUIREMENTS = {PipRequirement('really_bad_package')} + + >>> SomeOtherBear.check_prerequisites() + 'really_bad_package is not installed. You can install it using ...' + + >>> class anotherBear(Bear): + ... REQUIREMENTS = {PipRequirement('bad_package', '0.0.1')} + + >>> anotherBear.check_prerequisites() + 'bad_package 0.0.1 is not installed. You can install it using ...' + + :return: True if prerequisites are satisfied, else False or a string + that serves a more detailed description of what's missing. + """ + for requirement in cls.REQUIREMENTS: + if not requirement.is_installed(): + return str(requirement) + ' is not installed. You can ' + ( + 'install it using ') + ( + ' '.join(requirement.install_command())) + return True + + def get_config_dir(self): + """ + Gives the directory where the configuration file is. + + :return: Directory of the config file. + """ + return get_config_directory(self.section) + + def download_cached_file(self, url, filename): + """ + Downloads the file if needed and caches it for the next time. If a + download happens, the user will be informed. + + Take a sane simple bear: + + >>> from queue import Queue + >>> bear = Bear(Section("a section"), Queue()) + + We can now carelessly query for a neat file that doesn't exist yet: + + >>> from os import remove + >>> if exists(join(bear.data_dir, "a_file")): + ... remove(join(bear.data_dir, "a_file")) + >>> file = bear.download_cached_file("https://github.com/", "a_file") + + If we download it again, it'll be much faster as no download occurs: + + >>> newfile = bear.download_cached_file("https://github.com/", "a_file") + >>> newfile == file + True + + :param url: The URL to download the file from. + :param filename: The filename it should get, e.g. "test.txt". + :return: A full path to the file ready for you to use! + """ + filename = join(self.data_dir, filename) + if exists(filename): + return filename + + self.info('Downloading {filename!r} for bear {bearname} from {url}.' + .format(filename=filename, bearname=self.name, url=url)) + + response = requests.get(url, stream=True, timeout=20) + response.raise_for_status() + + with open(filename, 'wb') as file: + for chunk in response.iter_content(chunk_size=16 * 1024): + file.write(chunk) + return filename + + @classproperty + def data_dir(cls): + """ + Returns a directory that may be used by the bear to store stuff. Every + bear has an own directory dependent on their name. + """ + data_dir = abspath(join(user_data_dir('coala-bears'), cls.name)) + + makedirs(data_dir, exist_ok=True) + return data_dir + + @property + def new_result(self): + """ + Returns a partial for creating a result with this bear already bound. + """ + return partial(Result.from_values, self) \ No newline at end of file diff --git a/plugins/scancode-fingerprint/tests/data/fingerprint/similarity_matching3.py b/plugins/scancode-fingerprint/tests/data/fingerprint/similarity_matching3.py new file mode 100644 index 00000000000..9862c3d5195 --- /dev/null +++ b/plugins/scancode-fingerprint/tests/data/fingerprint/similarity_matching3.py @@ -0,0 +1,174 @@ +from itertools import chain +import os +from types import MappingProxyType + +from pyprint.ClosableObject import ClosableObject + +from coala_utils.string_processing import escape +from coalib.settings.Section import Section + + +class ConfWriter(ClosableObject): + + def __init__(self, + file_name, + key_value_delimiters=('=',), + comment_separators=('#',), + key_delimiters=(',', ' '), + section_name_surroundings=MappingProxyType({'[': ']'}), + section_override_delimiters=('.',), + unsavable_keys=('save',), + key_value_append_delimiters=('+=',)): + ClosableObject.__init__(self) + + self.__file_name = file_name + if (os.path.isdir(self.__file_name)): + self.__file_name = os.path.join(self.__file_name, '.coafile') + + self.__file = open(self.__file_name, 'w') + self.__key_value_delimiters = key_value_delimiters + self.__key_value_append_delimiters = key_value_append_delimiters + self.__comment_separators = comment_separators + self.__key_delimiters = key_delimiters + self.__section_name_surroundings = section_name_surroundings + self.__section_override_delimiters = section_override_delimiters + self.__unsavable_keys = unsavable_keys + self.__closed = False + + self.__key_delimiter = self.__key_delimiters[0] + self.__key_value_delimiter = key_value_delimiters[0] + self.__key_value_append_delimiter = key_value_append_delimiters[0] + (self.__section_name_surrounding_beg, + self.__section_name_surrounding_end) = ( + tuple(self.__section_name_surroundings.items())[0]) + + def _close(self): + self.__file.close() + + def write_sections(self, sections): + assert not self.__closed + + for section in sections: + self.write_section(sections[section]) + + def write_section(self, section): + assert not self.__closed + + if not isinstance(section, Section): + raise TypeError + + self.__write_section_name(section.name) + + keys = [] + val = None + section_iter = section.__iter__(ignore_defaults=True) + try: + while True: + setting = section[next(section_iter)] + if (str(setting) == val and + not self.is_comment(setting.key) and + ( + (setting.key not in self.__unsavable_keys) or + (not setting.from_cli))): + keys.append(setting.key) + elif ((setting.key not in self.__unsavable_keys) or + (not setting.from_cli)): + self.__write_key_val(keys, val, section.defaults) + keys = [setting.key] + val = str(setting) + except StopIteration: + self.__write_key_val(keys, val, section.defaults) + + def __write_section_name(self, name): + assert not self.__closed + + self.__file.write(self.__section_name_surrounding_beg + name + + self.__section_name_surrounding_end + '\n') + + def __write_key_val(self, keys, val, defaults): + assert not self.__closed + + if keys == []: + return + + if all(self.is_comment(key) for key in keys): + self.__file.write(val + '\n') + return + + # Add escape characters as appropriate + keys = [escape(key, chain(['\\'], + self.__key_value_delimiters, + self.__comment_separators, + self.__key_delimiters, + self.__section_override_delimiters)) + for key in keys] + val = escape(val, chain(['\\'], self.__comment_separators)) + + append_keys = [] + other_keys = [] + for key in keys: + if (defaults and + (key in defaults and + val.startswith(str(defaults[key])+','))): + append_keys.append(key) + else: + other_keys.append(key) + self.__write_keys_val_to_file(append_keys, other_keys, val, defaults) + + def __write_keys_val_to_file(self, append_keys, other_keys, val, defaults): + """ + This method helps in grouping keys with common appendable values and + writing all keys to the file. + :param append_keys: The keys that have to be written with an append + delimiter. + :param other_keys: The keys that have to be written with a normal + delimiter. + :param val: The value to be written for the keys. + :param defaults: The defaults of the section the keys and value are + to be written to. + """ + if append_keys: + append_keys = sorted(append_keys, + key=lambda key: len(defaults[str(key)]), + reverse=True) + write_keys = [] + def_val = None + for key in append_keys: + if str(defaults[key]) == def_val: + write_keys.append(key) + continue + else: + if write_keys: + append_val = self.__get_append_val(val, def_val) + self.__write_value( + write_keys, + append_val, + self.__key_value_append_delimiter) + write_keys = [key] + def_val = str(defaults[key]) + append_val = self.__get_append_val(val, def_val) + self.__write_value(write_keys, + append_val, + self.__key_value_append_delimiter) + + if other_keys: + self.__write_value(other_keys, + val, + self.__key_value_delimiter) + + def __get_append_val(self, val, def_val): + def_val_list = def_val.split(self.__key_delimiter) + append_val = (self.__key_delimiter + ' ').join( + [v.strip() for v in val.split( + self.__key_delimiter) + if v not in def_val_list]) + return append_val + + def __write_value(self, keys, val, delimiter): + self.__file.write((self.__key_delimiter + ' ').join(keys) + + ' ' + delimiter + ' ' + + val + '\n') + + @staticmethod + def is_comment(key): + return key.lower().startswith('comment') \ No newline at end of file diff --git a/plugins/scancode-fingerprint/tests/data/fingerprint/similarity_matching4.py b/plugins/scancode-fingerprint/tests/data/fingerprint/similarity_matching4.py new file mode 100644 index 00000000000..b847ea2be89 --- /dev/null +++ b/plugins/scancode-fingerprint/tests/data/fingerprint/similarity_matching4.py @@ -0,0 +1,127 @@ +from datetime import datetime +from collections import Counter +import json +import io +import logging +import logging.config + + +class CounterHandler(logging.Handler): + """ + A logging handler which counts the number of calls + for each logging level. + """ + _call_counter = Counter() + + @classmethod + def reset(cls): + """ + Reset the counter to 0 for all levels + """ + cls._call_counter.clear() + + @classmethod + def emit(cls, record): + cls._call_counter[record.levelname] += 1 + + @classmethod + def get_num_calls_for_level(cls, level): + """ + Returns the number of calls registered for a given log level. + """ + return cls._call_counter[level] + + +def configure_logging(color=True): + """ + Configures the logging with hard coded dictionary. + """ + import sys + + # reset counter handler + CounterHandler.reset() + + logging.config.dictConfig({ + 'version': 1, + 'handlers': { + 'colored': { + 'class': 'logging.StreamHandler', + 'formatter': 'colored' if color else 'plain', + 'stream': sys.stderr + }, + 'counter': { + 'class': 'coalib.output.Logging.CounterHandler' + } + }, + 'root': { + 'level': 'DEBUG', + 'handlers': ['colored', 'counter'] + }, + 'formatters': { + 'colored': { + '()': 'colorlog.ColoredFormatter', + 'format': '%(log_color)s[%(levelname)s]%(reset)s[%(asctime)s]' + ' %(message)s', + 'datefmt': '%X', + 'log_colors': { + 'ERROR': 'red', + 'WARNING': 'yellow', + 'INFO': 'blue', + 'DEBUG': 'green' + } + }, + 'plain': { + 'format': '[%(levelname)s][%(asctime)s] %(message)s', + 'datefmt': '%X', + } + } + }) + + +def configure_json_logging(): + """ + Configures logging for JSON. + :return: Returns a ``StringIO`` that captures the logs as JSON. + """ + stream = io.StringIO() + + # reset counter handler + CounterHandler.reset() + + logging.config.dictConfig({ + 'version': 1, + 'handlers': { + 'json': { + 'class': 'logging.StreamHandler', + 'formatter': 'json', + 'stream': stream + }, + 'counter': { + 'class': 'coalib.output.Logging.CounterHandler' + } + }, + 'root': { + 'level': 'DEBUG', + 'handlers': ['json', 'counter'] + }, + 'formatters': { + 'json': { + '()': 'coalib.output.Logging.JSONFormatter', + } + } + }) + return stream + + +class JSONFormatter(logging.Formatter): + """ + JSON formatter for python logging. + """ + @staticmethod + def format(record): + message = { + 'timestamp': datetime.utcfromtimestamp(record.created).isoformat(), + 'message': record.getMessage(), + 'level': record.levelname, + } + return json.dumps(message) \ No newline at end of file diff --git a/plugins/scancode-fingerprint/tests/data/fingerprint/similarity_matching5.py b/plugins/scancode-fingerprint/tests/data/fingerprint/similarity_matching5.py new file mode 100644 index 00000000000..737fd45e791 --- /dev/null +++ b/plugins/scancode-fingerprint/tests/data/fingerprint/similarity_matching5.py @@ -0,0 +1,1017 @@ +import copy +import logging +import platform +import os +from collections import OrderedDict + +from termcolor import colored + +try: + # This import has side effects and is needed to make input() behave nicely + import readline # pylint: disable=unused-import +except ImportError: # pragma: no cover + pass + +from coalib.misc.DictUtilities import inverse_dicts +from coalib.misc.Exceptions import log_exception +from coalib.misc.DeprecationUtilities import check_deprecation +from coalib.bearlib.spacing.SpacingHelper import SpacingHelper +from coalib.results.Result import Result +from coalib.results.result_actions.ApplyPatchAction import ApplyPatchAction +from coalib.results.result_actions.OpenEditorAction import OpenEditorAction +from coalib.results.result_actions.IgnoreResultAction import IgnoreResultAction +from coalib.results.result_actions.DoNothingAction import DoNothingAction +from coalib.results.result_actions.GeneratePatchesAction import ( + GeneratePatchesAction) +from coalib.results.result_actions.ShowAppliedPatchesAction import ( + ShowAppliedPatchesAction) +from coalib.results.result_actions.PrintDebugMessageAction import ( + PrintDebugMessageAction) +from coalib.results.result_actions.PrintMoreInfoAction import ( + PrintMoreInfoAction) +from coalib.results.result_actions.ShowPatchAction import ShowPatchAction +from coalib.results.RESULT_SEVERITY import ( + RESULT_SEVERITY, RESULT_SEVERITY_COLORS) +from coalib.settings.Setting import Setting +from coala_utils.string_processing.Core import join_names + +from pygments import highlight +from pygments.formatters import TerminalTrueColorFormatter +from pygments.filters import VisibleWhitespaceFilter +from pygments.lexers import TextLexer, get_lexer_for_filename +from pygments.style import Style +from pygments.token import Token +from pygments.util import ClassNotFound + + +class BackgroundSourceRangeStyle(Style): + styles = { + Token: 'bold bg:#BB4D3E #111' + } + + +class BackgroundMessageStyle(Style): + styles = { + Token: 'bold bg:#eee #111' + } + + +class NoColorStyle(Style): + styles = { + Token: 'noinherit' + } + + +def highlight_text(no_color, text, style, lexer=TextLexer()): + formatter = TerminalTrueColorFormatter(style=style) + if no_color: + formatter = TerminalTrueColorFormatter(style=NoColorStyle) + return highlight(text, lexer, formatter)[:-1] + + +STR_GET_VAL_FOR_SETTING = ('Please enter a value for the setting \"{}\" ({}) ' + 'needed by {} for section \"{}\": ') +STR_LINE_DOESNT_EXIST = ('The line belonging to the following result ' + 'cannot be printed because it refers to a line ' + "that doesn't seem to exist in the given file.") +STR_PROJECT_WIDE = 'Project wide:' +STR_ENTER_NUMBER = 'Enter number (Ctrl-{} to exit): '.format( + 'Z' if platform.system() == 'Windows' else 'D') +STR_INVALID_OPTION = '*** Invalid Option: ({}) ***\n' +WARNING_COLOR = 'red' +FILE_NAME_COLOR = 'blue' +FILE_LINES_COLOR = 'blue' +CAPABILITY_COLOR = 'green' +HIGHLIGHTED_CODE_COLOR = 'red' +SUCCESS_COLOR = 'green' +REQUIRED_SETTINGS_COLOR = 'green' +CLI_ACTIONS = (OpenEditorAction(), + ApplyPatchAction(), + PrintDebugMessageAction(), + PrintMoreInfoAction(), + ShowPatchAction(), + IgnoreResultAction(), + ShowAppliedPatchesAction(), + GeneratePatchesAction()) +DIFF_EXCERPT_MAX_SIZE = 4 + + +def color_letter(console_printer, line): + x = -1 + y = -1 + letter = '' + for i, l in enumerate(line, 0): + if line[i] == '(': + x = i + if line[i] == ')': + y = i + if l.isupper() and x != -1: + letter = l + first_part = line[:x+1] + second_part = line[y:] + + console_printer.print(first_part, end='') + console_printer.print(letter, color='blue', end='') + console_printer.print(second_part) + + +def format_lines(lines, symbol='', line_nr=''): + def sym(x): return ']' if x is '[' else x + return '\n'.join('{}{:>4}{} {}'.format(symbol, line_nr, sym(symbol), line) + for line in lines.rstrip('\n').split('\n')) + + +def print_section_beginning(console_printer, section): + """ + Will be called after initialization current_section in + begin_section() + :param console_printer: Object to print messages on the console. + :param section: The section that will get executed now. + """ + console_printer.print('Executing section {name}...'.format( + name=section.name)) + + +def nothing_done(log_printer=None): + """ + Will be called after processing a coafile when nothing had to be done, + i.e. no section was enabled/targeted. + :param log_printer: A LogPrinter object. + """ + logging.warning('No existent section was targeted or enabled. Nothing to ' + 'do.') + + +def acquire_actions_and_apply(console_printer, + section, + file_diff_dict, + result, + file_dict, + cli_actions=None, + apply_single=False): + """ + Acquires applicable actions and applies them. + :param console_printer: Object to print messages on the console. + :param section: Name of section to which the result belongs. + :param file_diff_dict: Dictionary containing filenames as keys and Diff + objects as values. + :param result: A derivative of Result. + :param file_dict: A dictionary containing all files with filename as + key. + :param apply_single: The action that should be applied for all results. + If it's not selected, has a value of False. + :param cli_actions: The list of cli actions available. + """ + cli_actions = CLI_ACTIONS if cli_actions is None else cli_actions + failed_actions = set() + applied_actions = {} + + while True: + action_dict = {} + metadata_list = [] + + for action in cli_actions: + if action.is_applicable(result, + file_dict, + file_diff_dict, + tuple(applied_actions.keys())) is True: + metadata = action.get_metadata() + action_dict[metadata.name] = action + metadata_list.append(metadata) + + if not metadata_list: + return + + # User can always choose no action which is guaranteed to succeed + continue_interaction = ask_for_action_and_apply( + console_printer, + section, + metadata_list, + action_dict, + failed_actions, + result, + file_diff_dict, + file_dict, + applied_actions, + apply_single=apply_single + ) + if not continue_interaction: + break + + +def print_lines(console_printer, + file_dict, + sourcerange): + """ + Prints the lines between the current and the result line. If needed + they will be shortened. + :param console_printer: Object to print messages on the console. + :param file_dict: A dictionary containing all files as values with + filenames as key. + :param sourcerange: The SourceRange object referring to the related + lines to print. + """ + no_color = not console_printer.print_colored + for i in range(sourcerange.start.line, sourcerange.end.line + 1): + # Print affected file's line number in the sidebar. + console_printer.print(format_lines(lines='', line_nr=i, symbol='['), + color=FILE_LINES_COLOR, + end='') + + line = file_dict[sourcerange.file][i - 1].rstrip('\n') + try: + lexer = get_lexer_for_filename(sourcerange.file) + except ClassNotFound: + lexer = TextLexer() + lexer.add_filter(VisibleWhitespaceFilter( + spaces=True, tabs=True, + tabsize=SpacingHelper.DEFAULT_TAB_WIDTH)) + # highlight() combines lexer and formatter to output a ``str`` + # object. + printed_chars = 0 + if i == sourcerange.start.line and sourcerange.start.column: + console_printer.print(highlight_text( + no_color, line[:sourcerange.start.column - 1], + BackgroundMessageStyle, lexer), end='') + + printed_chars = sourcerange.start.column - 1 + + if i == sourcerange.end.line and sourcerange.end.column: + console_printer.print(highlight_text( + no_color, line[printed_chars:sourcerange.end.column - 1], + BackgroundSourceRangeStyle, lexer), end='') + + console_printer.print(highlight_text( + no_color, line[sourcerange.end.column - 1:], + BackgroundSourceRangeStyle, lexer), end='') + console_printer.print('') + else: + console_printer.print(highlight_text( + no_color, line[printed_chars:], BackgroundMessageStyle, lexer), + end='') + console_printer.print('') + + +def print_result(console_printer, + section, + file_diff_dict, + result, + file_dict, + interactive=True, + apply_single=False): + """ + Prints the result to console. + :param console_printer: Object to print messages on the console. + :param section: Name of section to which the result belongs. + :param file_diff_dict: Dictionary containing filenames as keys and Diff + objects as values. + :param result: A derivative of Result. + :param file_dict: A dictionary containing all files with filename as + key. + :param apply_single: The action that should be applied for all results. + If it's not selected, has a value of False. + :param interactive: Variable to check whether or not to + offer the user actions interactively. + """ + no_color = not console_printer.print_colored + if not isinstance(result, Result): + logging.warning('One of the results can not be printed since it is ' + 'not a valid derivative of the coala result ' + 'class.') + return + + if hasattr(section, 'name'): + console_printer.print('**** {bear} [Section: {section} | Severity: ' + '{severity}] ****' + .format(bear=result.origin, + section=section.name, + severity=RESULT_SEVERITY.__str__( + result.severity)), + color=RESULT_SEVERITY_COLORS[result.severity]) + else: + console_printer.print('**** {bear} [Section {section} | Severity ' + '{severity}] ****' + .format(bear=result.origin, section='', + severity=RESULT_SEVERITY.__str__( + result.severity)), + color=RESULT_SEVERITY_COLORS[result.severity]) + lexer = TextLexer() + result.message = highlight_text(no_color, result.message, + BackgroundMessageStyle, lexer) + console_printer.print(format_lines(result.message, symbol='!')) + + if interactive: + cli_actions = CLI_ACTIONS + show_patch_action = ShowPatchAction() + if show_patch_action.is_applicable( + result, file_dict, file_diff_dict) is True: + diff_size = sum(len(diff) for diff in result.diffs.values()) + if diff_size <= DIFF_EXCERPT_MAX_SIZE: + show_patch_action.apply_from_section(result, + file_dict, + file_diff_dict, + section) + cli_actions = tuple(action for action in cli_actions + if not isinstance(action, ShowPatchAction)) + else: + print_diffs_info(result.diffs, console_printer) + acquire_actions_and_apply(console_printer, + section, + file_diff_dict, + result, + file_dict, + cli_actions, + apply_single=apply_single) + + +def print_diffs_info(diffs, printer): + """ + Prints diffs information (number of additions and deletions) to the console. + :param diffs: List of Diff objects containing corresponding diff info. + :param printer: Object responsible for printing diffs on console. + """ + for filename, diff in sorted(diffs.items()): + additions, deletions = diff.stats() + printer.print( + format_lines('+{additions} -{deletions} in {file}'.format( + file=filename, + additions=additions, + deletions=deletions), '!'), + color='green') + + +def print_results_formatted(log_printer, + section, + result_list, + file_dict, + *args): + """ + Prints results through the format string from the format setting done by + user. + :param log_printer: Printer responsible for logging the messages. + :param section: The section to which the results belong. + :param result_list: List of Result objects containing the corresponding + results. + """ + default_format = ('id:{id}:origin:{origin}:file:{file}:line:{line}:' + 'column:{column}:end_line:{end_line}:end_column:' + '{end_column}:severity:{severity}:severity_str:' + '{severity_str}:message:{message}') + format_str = str(section.get('format', default_format)) + + if format_str == 'True': + format_str = default_format + + for result in result_list: + severity_str = RESULT_SEVERITY.__str__(result.severity) + format_args = vars(result) + try: + if len(result.affected_code) == 0: + format_args['affected_code'] = None + print(format_str.format(file=None, + line=None, + end_line=None, + column=None, + end_column=None, + severity_str=severity_str, + message=result.message, + **format_args)) + continue + + for range in result.affected_code: + format_args['affected_code'] = range + format_args['source_lines'] = range.affected_source(file_dict) + print(format_str.format(file=range.start.file, + line=range.start.line, + end_line=range.end.line, + column=range.start.column, + end_column=range.end.column, + severity_str=severity_str, + message=result.message, + **format_args)) + except KeyError as exception: + log_exception( + 'Unable to print the result with the given format string.', + exception) + + +def print_bears_formatted(bears, format=None): + format_str = format or ('name:{name}:can_detect:{can_detect}:' + 'can_fix:{can_fix}:description:{description}') + print('\n\n'.join(format_str.format(name=bear.name, + can_detect=bear.CAN_DETECT, + can_fix=bear.CAN_FIX, + description=bear.get_metadata().desc) + for bear in bears)) + + +def print_affected_files(console_printer, + log_printer, + result, + file_dict): + """ + Prints all the affected files and affected lines within them. + :param console_printer: Object to print messages on the console. + :param log_printer: Printer responsible for logging the messages. + :param result: The result to print the context for. + :param file_dict: A dictionary containing all files with filename as + key. + """ + if len(result.affected_code) == 0: + console_printer.print('\n' + STR_PROJECT_WIDE, + color=FILE_NAME_COLOR) + else: + for sourcerange in result.affected_code: + if ( + sourcerange.file is not None and + sourcerange.file not in file_dict): + logging.warning('The context for the result ({}) cannot ' + 'be printed because it refers to a file ' + "that doesn't seem to exist ({})" + '.'.format(result, sourcerange.file)) + else: + print_affected_lines(console_printer, + file_dict, + sourcerange) + + +def print_results_no_input(log_printer, + section, + result_list, + file_dict, + file_diff_dict, + console_printer, + apply_single=False): + """ + Prints all non interactive results in a section + :param log_printer: Printer responsible for logging the messages. + :param section: The section to which the results belong to. + :param result_list: List containing the results + :param file_dict: A dictionary containing all files with filename as + key. + :param file_diff_dict: A dictionary that contains filenames as keys and + diff objects as values. + :param apply_single: The action that should be applied for all results. + If it's not selected, has a value of False. + :param console_printer: Object to print messages on the console. + """ + for result in result_list: + + print_affected_files(console_printer, + None, + result, + file_dict) + + print_result(console_printer, + section, + file_diff_dict, + result, + file_dict, + interactive=False, + apply_single=apply_single) + + +def print_results(log_printer, + section, + result_list, + file_dict, + file_diff_dict, + console_printer, + apply_single=False): + """ + Prints all the results in a section. + :param log_printer: Printer responsible for logging the messages. + :param section: The section to which the results belong to. + :param result_list: List containing the results + :param file_dict: A dictionary containing all files with filename as + key. + :param file_diff_dict: A dictionary that contains filenames as keys and + diff objects as values. + :param apply_single: The action that should be applied for all results. + If it's not selected, has a value of False. + :param console_printer: Object to print messages on the console. + """ + for result in sorted(result_list): + + print_affected_files(console_printer, + None, + result, + file_dict) + + print_result(console_printer, + section, + file_diff_dict, + result, + file_dict, + apply_single=apply_single) + + +def print_affected_lines(console_printer, file_dict, sourcerange): + """ + Prints the lines affected by the bears. + :param console_printer: Object to print messages on the console. + :param file_dict: A dictionary containing all files with filename + as key. + :param sourcerange: The SourceRange object referring to the related + lines to print. + """ + console_printer.print('\n' + os.path.relpath(sourcerange.file), + color=FILE_NAME_COLOR) + + if sourcerange.start.line is not None: + if len(file_dict[sourcerange.file]) < sourcerange.end.line: + console_printer.print(format_lines(lines=STR_LINE_DOESNT_EXIST, + line_nr=sourcerange.end.line, + symbol='!')) + else: + print_lines(console_printer, + file_dict, + sourcerange) + + +def require_setting(setting_name, arr, section): + """ + This method is responsible for prompting a user about a missing setting and + taking its value as input from the user. + :param setting_name: Name of the setting missing + :param arr: A list containing a description in [0] and the name + of the bears who need this setting in [1]. + :param section: The section the action corresponds to. + :param return: Returns the setting value that was requested from the + user. + """ + needed = join_names(arr[1:]) + + # Don't use input, it can't deal with escapes! + + print(colored(STR_GET_VAL_FOR_SETTING.format(setting_name, arr[0], needed, + section.name), + REQUIRED_SETTINGS_COLOR)) + return input() + + +def acquire_settings(log_printer, settings_names_dict, section): + """ + This method prompts the user for the given settings. + :param log_printer: + Printer responsible for logging the messages. This is needed to comply + with the interface. + :param settings_names_dict: + A dictionary with the settings name as key and a list containing a + description in [0] and the name of the bears who need this setting in + [1] and following. + Example: + :: + {"UseTabs": ["describes whether tabs should be used instead of spaces", + "SpaceConsistencyBear", + "SomeOtherBear"]} + :param section: + The section the action corresponds to. + :return: + A dictionary with the settings name as key and the given value as + value. + """ + if not isinstance(settings_names_dict, dict): + raise TypeError('The settings_names_dict parameter has to be a ' + 'dictionary.') + + result = {} + for setting_name, arr in sorted(settings_names_dict.items(), + key=lambda x: (join_names(x[1][1:]), x[0])): + value = require_setting(setting_name, arr, section) + result.update({setting_name: value} if value is not None else {}) + + return result + + +def get_action_info(section, action, failed_actions): + """ + Gets all the required Settings for an action. It updates the section with + the Settings. + :param section: The section the action corresponds to. + :param action: The action to get the info for. + :param failed_actions: A set of all actions that have failed. A failed + action remains in the list until it is successfully + executed. + :return: Action name and the updated section. + """ + params = action.non_optional_params + + for param_name in params: + if param_name not in section or action.name in failed_actions: + question = format_lines( + "Please enter a value for the parameter '{}' ({}): " + .format(param_name, params[param_name][0]), symbol='!') + section.append(Setting(param_name, input(question))) + + return action.name, section + + +def choose_action(console_printer, actions, apply_single=False): + """ + Presents the actions available to the user and takes as input the action + the user wants to choose. + :param console_printer: Object to print messages on the console. + :param actions: Actions available to the user. + :param apply_single: The action that should be applied for all results. + If it's not selected, has a value of False. + :return: Return a tuple of lists, a list with the names of + actions that needs to be applied and a list with + with the description of the actions. + """ + actions_desc = [] + actions_name = [] + if apply_single: + for i, action in enumerate(actions, 0): + if apply_single == action.desc: + return ([action.desc], [action.name]) + return (['Do (N)othing'], ['Do (N)othing']) + else: + while True: + for i, action in enumerate(actions, 0): + output = '{:>2}. {}' if i != 0 else '*{}. {}' + color_letter(console_printer, format_lines(output.format( + i, action.desc), symbol='[')) + + line = format_lines(STR_ENTER_NUMBER, symbol='[') + + choice = input(line) + choice = str(choice) + + for c in choice: + c = str(c) + str_c = c + actions_desc_len = len(actions_desc) + if c.isnumeric(): + for i, action in enumerate(actions, 0): + c = int(c) + if i == c: + actions_desc.append(action.desc) + actions_name.append(action.name) + break + elif c.isalpha(): + c = c.upper() + c = '(' + c + ')' + for i, action in enumerate(actions, 1): + if c in action.desc: + actions_desc.append(action.desc) + actions_name.append(action.name) + break + if actions_desc_len == len(actions_desc): + console_printer.print(STR_INVALID_OPTION.format(str_c), + color=WARNING_COLOR) + + if not choice: + actions_desc.append(DoNothingAction().get_metadata().desc) + actions_name.append(DoNothingAction().get_metadata().name) + return (actions_desc, actions_name) + + +def try_to_apply_action(action_name, + chosen_action, + console_printer, + section, + metadata_list, + action_dict, + failed_actions, + result, + file_diff_dict, + file_dict, + applied_actions): + """ + Try to apply the given action. + :param action_name: The name of the action. + :param chosen_action: The action object that will be applied. + :param console_printer: Object to print messages on the console. + :param section: Currently active section. + :param metadata_list: Contains metadata for all the actions. + :param action_dict: Contains the action names as keys and their + references as values. + :param failed_actions: A set of all actions that have failed. A failed + action remains in the list until it is successfully + executed. + :param result: Result corresponding to the actions. + :param file_diff_dict: If it is an action which applies a patch, this + contains the diff of the patch to be applied to + the file with filename as keys. + :param applied_actions: A dictionary that contains the result, file_dict, + file_diff_dict and the section for an action. + :param file_dict: Dictionary with filename as keys and its contents + as values. + """ + try: + chosen_action.apply_from_section(result, + file_dict, + file_diff_dict, + section) + if not isinstance(chosen_action, DoNothingAction): + console_printer.print( + format_lines(chosen_action.SUCCESS_MESSAGE, symbol='['), + color=SUCCESS_COLOR) + applied_actions[action_name] = [copy.copy(result), copy.copy( + file_dict), + copy.copy(file_diff_dict), + copy.copy(section)] + result.set_applied_actions(applied_actions) + failed_actions.discard(action_name) + except Exception as exception: # pylint: disable=broad-except + logging.error('Failed to execute the action {} with error: {}.' + .format(action_name, exception)) + failed_actions.add(action_name) + + +def ask_for_action_and_apply(console_printer, + section, + metadata_list, + action_dict, + failed_actions, + result, + file_diff_dict, + file_dict, + applied_actions, + apply_single=False): + """ + Asks the user for an action and applies it. + :param console_printer: Object to print messages on the console. + :param section: Currently active section. + :param metadata_list: Contains metadata for all the actions. + :param action_dict: Contains the action names as keys and their + references as values. + :param failed_actions: A set of all actions that have failed. A failed + action remains in the list until it is successfully + executed. + :param result: Result corresponding to the actions. + :param file_diff_dict: If it is an action which applies a patch, this + contains the diff of the patch to be applied to + the file with filename as keys. + :param file_dict: Dictionary with filename as keys and its contents + as values. + :param apply_single: The action that should be applied for all results. + If it's not selected, has a value of False. + :param applied_actions: A dictionary that contains the result, file_dict, + file_diff_dict and the section for an action. + :return: Returns a boolean value. True will be returned, if + it makes sense that the user may choose to execute + another action, False otherwise. + If apply_single isn't set, always return False. + """ + do_nothing_action = DoNothingAction() + metadata_list.insert(0, do_nothing_action.get_metadata()) + action_dict[do_nothing_action.get_metadata().name] = DoNothingAction() + + actions_desc, actions_name = choose_action(console_printer, metadata_list, + apply_single) + + if apply_single: + for index, action_details in enumerate(metadata_list, 1): + if apply_single == action_details.desc: + action_name, section = get_action_info( + section, metadata_list[index - 1], failed_actions) + chosen_action = action_dict[action_details.name] + try_to_apply_action(action_name, + chosen_action, + console_printer, + section, + metadata_list, + action_dict, + failed_actions, + result, + file_diff_dict, + file_dict, + applied_actions) + return False + else: + for action_choice, action_choice_name in zip(actions_desc, + actions_name): + chosen_action = action_dict[action_choice_name] + action_choice_made = action_choice + for index, action_details in enumerate(metadata_list, 1): + if action_choice_made in action_details.desc: + action_name, section = get_action_info( + section, metadata_list[index-1], failed_actions) + try_to_apply_action(action_name, + chosen_action, + console_printer, + section, + metadata_list, + action_dict, + failed_actions, + result, + file_diff_dict, + file_dict, + applied_actions) + + if action_choice == 'Do (N)othing': + return False + return True + + +def show_enumeration(console_printer, + title, + items, + indentation, + no_items_text): + """ + This function takes as input an iterable object (preferably a list or + a dict) and prints it in a stylized format. If the iterable object is + empty, it prints a specific statement given by the user. An e.g : + Title: + * Item 1 + * Item 2 + :param console_printer: Object to print messages on the console. + :param title: Title of the text to be printed + :param items: The iterable object. + :param indentation: Number of spaces to indent every line by. + :param no_items_text: Text printed when iterable object is empty. + """ + if not items: + console_printer.print(indentation + no_items_text) + else: + console_printer.print(indentation + title) + if isinstance(items, dict): + for key, value in items.items(): + console_printer.print(indentation + ' * ' + key + ': ' + + value[0]) + else: + for item in items: + console_printer.print(indentation + ' * ' + item) + console_printer.print() + + +def show_bear(bear, + show_description, + show_params, + console_printer, + args=None): + """ + Displays all information about a bear. + :param bear: The bear to be displayed. + :param show_description: This parameter is deprecated. + :param show_params: This parameter is deprecated. + :param console_printer: Object to print messages on the console. + :param args: Args passed to coala command. + """ + console_printer.print(bear.name, color='blue') + + metadata = bear.get_metadata() + + check_deprecation(OrderedDict([ + ('show_description', show_description), + ('show_params', show_params)])) + + if show_description: + console_printer.print( + ' ' + metadata.desc.replace('\n', '\n ')) + console_printer.print() # Add a newline + + if show_params: + show_enumeration( + console_printer, 'Supported languages:', + bear.LANGUAGES, + ' ', + 'The bear does not provide information about which languages ' + 'it can analyze.') + + if (args and args.show_settings) or show_params: + show_enumeration(console_printer, + 'Needed Settings:', + metadata.non_optional_params, + ' ', + 'No needed settings.') + show_enumeration(console_printer, + 'Optional Settings:', + metadata.optional_params, + ' ', + 'No optional settings.') + + if show_params: + show_enumeration(console_printer, + 'Can detect:', + bear.can_detect, + ' ', + 'This bear does not provide information about what ' + 'categories it can detect.') + show_enumeration(console_printer, + 'Can fix:', + bear.CAN_FIX, + ' ', + 'This bear cannot fix issues or does not provide ' + 'information about what categories it can fix.') + console_printer.print( + ' Path:\n' + ' ' + repr(bear.source_location) + '\n') + + +def print_bears(bears, + show_description, + show_params, + console_printer, + args=None): + """ + Presents all bears being used in a stylized manner. + :param bears: It's a dictionary with bears as keys and list of + sections containing those bears as values. + :param show_description: This parameter is deprecated. + :param show_params: This parameter is deprecated. + :param console_printer: Object to print messages on the console. + :param args: Args passed to coala command. + """ + check_deprecation(OrderedDict([ + ('show_description', show_description), + ('show_params', show_params)])) + + if not bears and not (args and args.json): + console_printer.print('No bears to show. Did you forget to install ' + 'the `coala-bears` package? Try `pip3 install ' + 'coala-bears`.') + return + + results = [bear for bear, _ in sorted(bears.items(), + key=lambda bear_tuple: + bear_tuple[0].name.lower())] + if args and args.json: + from coalib.output.JSONEncoder import create_json_encoder + JSONEncoder = create_json_encoder(use_relpath=args.relpath) + json_output = {} + if args.show_details: + json_output['bears'] = results + else: + json_output['bears'] = [bear.name for bear in results] + + import json + json_formatted_output = json.dumps(json_output, + cls=JSONEncoder, + sort_keys=True, + indent=2, + separators=(',', ': ')) + if args.output: + filename = args.output[0] + with open(filename, 'w') as fp: + fp.write(json_formatted_output) + else: + print(json_formatted_output) + elif args and args.format: + print_bears_formatted(results) + else: + for bear in results: + show_bear(bear, + show_description, + show_params, + console_printer, + args) + + +def show_bears(local_bears, + global_bears, + show_description, + show_params, + console_printer, + args=None): + """ + Extracts all the bears from each enabled section or the sections in the + targets and passes a dictionary to the show_bears_callback method. + :param local_bears: Dictionary of local bears with section names + as keys and bear list as values. + :param global_bears: Dictionary of global bears with section + names as keys and bear list as values. + :param show_description: This parameter is deprecated. + :param show_params: This parameter is deprecated. + :param console_printer: Object to print messages on the console. + :param args: Args passed to coala command. + """ + bears = inverse_dicts(local_bears, global_bears) + + print_bears(bears, show_description, show_params, console_printer, args) + + +def show_language_bears_capabilities(language_bears_capabilities, + console_printer): + """ + Displays what the bears can detect and fix. + :param language_bears_capabilities: + Dictionary with languages as keys and their bears' capabilities as + values. The capabilities are stored in a tuple of two elements where the + first one represents what the bears can detect, and the second one what + they can fix. + :param console_printer: + Object to print messages on the console. + """ + if not language_bears_capabilities: + console_printer.print('There is no bear available for this language') + else: + for language, capabilities in language_bears_capabilities.items(): + if capabilities[0]: + console_printer.print( + 'coala can do the following for ', end='') + console_printer.print(language.upper(), color='blue') + console_printer.print(' Can detect only: ', end='') + console_printer.print( + ', '.join(sorted(capabilities[0])), color=CAPABILITY_COLOR) + if capabilities[1]: + console_printer.print(' Can fix : ', end='') + console_printer.print( + ', '.join(sorted(capabilities[1])), + color=CAPABILITY_COLOR) + else: + console_printer.print('coala does not support ', color='red', + end='') + console_printer.print(language, color='blue') diff --git a/plugins/scancode-fingerprint/tests/data/fingerprint/similarity_matching6.py b/plugins/scancode-fingerprint/tests/data/fingerprint/similarity_matching6.py new file mode 100644 index 00000000000..565a5226b4d --- /dev/null +++ b/plugins/scancode-fingerprint/tests/data/fingerprint/similarity_matching6.py @@ -0,0 +1,925 @@ +import copy +import logging +import platform +import os +from collections import OrderedDict + +from termcolor import colored + +try: + # This import has side effects and is needed to make input() behave nicely + import readline # pylint: disable=unused-import +except ImportError: # pragma: no cover + pass + +from coalib.misc.DictUtilities import inverse_dicts +from coalib.misc.Exceptions import log_exception +from coalib.misc.DeprecationUtilities import check_deprecation +from coalib.bearlib.spacing.SpacingHelper import SpacingHelper +from coalib.results.Result import Result +from coalib.results.result_actions.ApplyPatchAction import ApplyPatchAction +from coalib.results.result_actions.OpenEditorAction import OpenEditorAction +from coalib.results.result_actions.IgnoreResultAction import IgnoreResultAction +from coalib.results.result_actions.DoNothingAction import DoNothingAction +from pygments import highlight +from pygments.formatters import TerminalTrueColorFormatter +from pygments.filters import VisibleWhitespaceFilter +from pygments.lexers import TextLexer, get_lexer_for_filename +from pygments.style import Style +from pygments.token import Token +from pygments.util import ClassNotFound + + +class BackgroundSourceRangeStyle(Style): + styles = { + Token: 'bold bg:#BB4D3E #111' + } + + +class BackgroundMessageStyle(Style): + styles = { + Token: 'bold bg:#eee #111' + } + + +class NoColorStyle(Style): + styles = { + Token: 'noinherit' + } + + +def highlight_text(no_color, text, style, lexer=TextLexer()): + formatter = TerminalTrueColorFormatter(style=style) + if no_color: + formatter = TerminalTrueColorFormatter(style=NoColorStyle) + return highlight(text, lexer, formatter)[:-1] + + +STR_GET_VAL_FOR_SETTING = ('Please enter a value for the setting \"{}\" ({}) ' + 'needed by {} for section \"{}\": ') +STR_LINE_DOESNT_EXIST = ('The line belonging to the following result ' + 'cannot be printed because it refers to a line ' + "that doesn't seem to exist in the given file.") +STR_PROJECT_WIDE = 'Project wide:' +STR_ENTER_NUMBER = 'Enter number (Ctrl-{} to exit): '.format( + 'Z' if platform.system() == 'Windows' else 'D') +STR_INVALID_OPTION = '*** Invalid Option: ({}) ***\n' +WARNING_COLOR = 'red' +FILE_NAME_COLOR = 'blue' +FILE_LINES_COLOR = 'blue' +CAPABILITY_COLOR = 'green' +HIGHLIGHTED_CODE_COLOR = 'red' +SUCCESS_COLOR = 'green' +REQUIRED_SETTINGS_COLOR = 'green' +CLI_ACTIONS = (OpenEditorAction(), + ApplyPatchAction(), + PrintDebugMessageAction(), + PrintMoreInfoAction(), + ShowPatchAction(), + IgnoreResultAction(), + ShowAppliedPatchesAction(), + GeneratePatchesAction()) +DIFF_EXCERPT_MAX_SIZE = 4 + + +def nothing_done(log_printer=None): + """ + Will be called after processing a coafile when nothing had to be done, + i.e. no section was enabled/targeted. + :param log_printer: A LogPrinter object. + """ + logging.warning('No existent section was targeted or enabled. Nothing to ' + 'do.') + + +def acquire_actions_and_apply(console_printer, + section, + file_diff_dict, + result, + file_dict, + cli_actions=None, + apply_single=False): + """ + Acquires applicable actions and applies them. + :param console_printer: Object to print messages on the console. + :param section: Name of section to which the result belongs. + :param file_diff_dict: Dictionary containing filenames as keys and Diff + objects as values. + :param result: A derivative of Result. + :param file_dict: A dictionary containing all files with filename as + key. + :param apply_single: The action that should be applied for all results. + If it's not selected, has a value of False. + :param cli_actions: The list of cli actions available. + """ + cli_actions = CLI_ACTIONS if cli_actions is None else cli_actions + failed_actions = set() + applied_actions = {} + + while True: + action_dict = {} + metadata_list = [] + + for action in cli_actions: + if action.is_applicable(result, + file_dict, + file_diff_dict, + tuple(applied_actions.keys())) is True: + metadata = action.get_metadata() + action_dict[metadata.name] = action + metadata_list.append(metadata) + + if not metadata_list: + return + + # User can always choose no action which is guaranteed to succeed + continue_interaction = ask_for_action_and_apply( + console_printer, + section, + metadata_list, + action_dict, + failed_actions, + result, + file_diff_dict, + file_dict, + applied_actions, + apply_single=apply_single + ) + if not continue_interaction: + break + + + +def color_letter(console_printer, line): + x = -1 + y = -1 + letter = '' + for i, l in enumerate(line, 0): + if line[i] == '(': + x = i + if line[i] == ')': + y = i + if l.isupper() and x != -1: + letter = l + first_part = line[:x+1] + second_part = line[y:] + + console_printer.print(first_part, end='') + console_printer.print(letter, color='blue', end='') + console_printer.print(second_part) + + +def format_lines(lines, symbol='', line_nr=''): + def sym(x): return ']' if x is '[' else x + return '\n'.join('{}{:>4}{} {}'.format(symbol, line_nr, sym(symbol), line) + for line in lines.rstrip('\n').split('\n')) + + +def print_section_beginning(console_printer, section): + """ + Will be called after initialization current_section in + begin_section() + :param console_printer: Object to print messages on the console. + :param section: The section that will get executed now. + """ + console_printer.print('Executing section {name}...'.format( + name=section.name)) + + + +def acquire_actions_and_apply(console_printer, + section, + file_diff_dict, + result, + file_dict, + cli_actions=None, + apply_single=False): + """ + Acquires applicable actions and applies them. + :param console_printer: Object to print messages on the console. + :param section: Name of section to which the result belongs. + :param file_diff_dict: Dictionary containing filenames as keys and Diff + objects as values. + :param result: A derivative of Result. + :param file_dict: A dictionary containing all files with filename as + key. + :param apply_single: The action that should be applied for all results. + If it's not selected, has a value of False. + :param cli_actions: The list of cli actions available. + """ + cli_actions = CLI_ACTIONS if cli_actions is None else cli_actions + failed_actions = set() + applied_actions = {} + + while True: + action_dict = {} + metadata_list = [] + + for action in cli_actions: + if action.is_applicable(result, + file_dict, + file_diff_dict, + tuple(applied_actions.keys())) is True: + metadata = action.get_metadata() + action_dict[metadata.name] = action + metadata_list.append(metadata) + + if not metadata_list: + return + + # User can always choose no action which is guaranteed to succeed + continue_interaction = ask_for_action_and_apply( + console_printer, + section, + metadata_list, + action_dict, + failed_actions, + result, + file_diff_dict, + file_dict, + applied_actions, + apply_single=apply_single + ) + if not continue_interaction: + break + + +def print_lines(console_printer, + file_dict, + sourcerange): + """ + Prints the lines between the current and the result line. If needed + they will be shortened. + :param console_printer: Object to print messages on the console. + :param file_dict: A dictionary containing all files as values with + filenames as key. + :param sourcerange: The SourceRange object referring to the related + lines to print. + """ + no_color = not console_printer.print_colored + for i in range(sourcerange.start.line, sourcerange.end.line + 1): + # Print affected file's line number in the sidebar. + console_printer.print(format_lines(lines='', line_nr=i, symbol='['), + color=FILE_LINES_COLOR, + end='') + + line = file_dict[sourcerange.file][i - 1].rstrip('\n') + try: + lexer = get_lexer_for_filename(sourcerange.file) + except ClassNotFound: + lexer = TextLexer() + lexer.add_filter(VisibleWhitespaceFilter( + spaces=True, tabs=True, + tabsize=SpacingHelper.DEFAULT_TAB_WIDTH)) + # highlight() combines lexer and formatter to output a ``str`` + # object. + printed_chars = 0 + if i == sourcerange.start.line and sourcerange.start.column: + console_printer.print(highlight_text( + no_color, line[:sourcerange.start.column - 1], + BackgroundMessageStyle, lexer), end='') + + printed_chars = sourcerange.start.column - 1 + + if i == sourcerange.end.line and sourcerange.end.column: + console_printer.print(highlight_text( + no_color, line[printed_chars:sourcerange.end.column - 1], + BackgroundSourceRangeStyle, lexer), end='') + + console_printer.print(highlight_text( + no_color, line[sourcerange.end.column - 1:], + BackgroundSourceRangeStyle, lexer), end='') + console_printer.print('') + else: + console_printer.print(highlight_text( + no_color, line[printed_chars:], BackgroundMessageStyle, lexer), + end='') + console_printer.print('') + + +def print_diffs_info(diffs, printer): + """ + Prints diffs information (number of additions and deletions) to the console. + :param diffs: List of Diff objects containing corresponding diff info. + :param printer: Object responsible for printing diffs on console. + """ + for filename, diff in sorted(diffs.items()): + additions, deletions = diff.stats() + printer.print( + format_lines('+{additions} -{deletions} in {file}'.format( + file=filename, + additions=additions, + deletions=deletions), '!'), + color='green') + + +def print_results_formatted(log_printer, + section, + result_list, + file_dict, + *args): + """ + Prints results through the format string from the format setting done by + user. + :param log_printer: Printer responsible for logging the messages. + :param section: The section to which the results belong. + :param result_list: List of Result objects containing the corresponding + results. + """ + default_format = ('id:{id}:origin:{origin}:file:{file}:line:{line}:' + 'column:{column}:end_line:{end_line}:end_column:' + '{end_column}:severity:{severity}:severity_str:' + '{severity_str}:message:{message}') + format_str = str(section.get('format', default_format)) + + if format_str == 'True': + format_str = default_format + + for result in result_list: + severity_str = RESULT_SEVERITY.__str__(result.severity) + format_args = vars(result) + try: + if len(result.affected_code) == 0: + format_args['affected_code'] = None + print(format_str.format(file=None, + line=None, + end_line=None, + column=None, + end_column=None, + severity_str=severity_str, + message=result.message, + **format_args)) + continue + + for range in result.affected_code: + format_args['affected_code'] = range + format_args['source_lines'] = range.affected_source(file_dict) + print(format_str.format(file=range.start.file, + line=range.start.line, + end_line=range.end.line, + column=range.start.column, + end_column=range.end.column, + severity_str=severity_str, + message=result.message, + **format_args)) + except KeyError as exception: + log_exception( + 'Unable to print the result with the given format string.', + exception) + + +def print_bears_formatted(bears, format=None): + format_str = format or ('name:{name}:can_detect:{can_detect}:' + 'can_fix:{can_fix}:description:{description}') + print('\n\n'.join(format_str.format(name=bear.name, + can_detect=bear.CAN_DETECT, + can_fix=bear.CAN_FIX, + description=bear.get_metadata().desc) + for bear in bears)) + + +def print_affected_files(console_printer, + log_printer, + result, + file_dict): + """ + Prints all the affected files and affected lines within them. + :param console_printer: Object to print messages on the console. + :param log_printer: Printer responsible for logging the messages. + :param result: The result to print the context for. + :param file_dict: A dictionary containing all files with filename as + key. + """ + if len(result.affected_code) == 0: + console_printer.print('\n' + STR_PROJECT_WIDE, + color=FILE_NAME_COLOR) + else: + for sourcerange in result.affected_code: + if ( + sourcerange.file is not None and + sourcerange.file not in file_dict): + logging.warning('The context for the result ({}) cannot ' + 'be printed because it refers to a file ' + "that doesn't seem to exist ({})" + '.'.format(result, sourcerange.file)) + else: + print_affected_lines(console_printer, + file_dict, + sourcerange) + + +def print_results_no_input(log_printer, + section, + result_list, + file_dict, + file_diff_dict, + console_printer, + apply_single=False): + """ + Prints all non interactive results in a section + :param log_printer: Printer responsible for logging the messages. + :param section: The section to which the results belong to. + :param result_list: List containing the results + :param file_dict: A dictionary containing all files with filename as + key. + :param file_diff_dict: A dictionary that contains filenames as keys and + diff objects as values. + :param apply_single: The action that should be applied for all results. + If it's not selected, has a value of False. + :param console_printer: Object to print messages on the console. + """ + for result in result_list: + + print_affected_files(console_printer, + None, + result, + file_dict) + + print_result(console_printer, + section, + file_diff_dict, + result, + file_dict, + interactive=False, + apply_single=apply_single) + + +def print_results(log_printer, + section, + result_list, + file_dict, + file_diff_dict, + console_printer, + apply_single=False): + """ + Prints all the results in a section. + :param log_printer: Printer responsible for logging the messages. + :param section: The section to which the results belong to. + :param result_list: List containing the results + :param file_dict: A dictionary containing all files with filename as + key. + :param file_diff_dict: A dictionary that contains filenames as keys and + diff objects as values. + :param apply_single: The action that should be applied for all results. + If it's not selected, has a value of False. + :param console_printer: Object to print messages on the console. + """ + for result in sorted(result_list): + + print_affected_files(console_printer, + None, + result, + file_dict) + + print_result(console_printer, + section, + file_diff_dict, + result, + file_dict, + apply_single=apply_single) + + +def print_affected_lines(console_printer, file_dict, sourcerange): + """ + Prints the lines affected by the bears. + :param console_printer: Object to print messages on the console. + :param file_dict: A dictionary containing all files with filename + as key. + :param sourcerange: The SourceRange object referring to the related + lines to print. + """ + console_printer.print('\n' + os.path.relpath(sourcerange.file), + color=FILE_NAME_COLOR) + + if sourcerange.start.line is not None: + if len(file_dict[sourcerange.file]) < sourcerange.end.line: + console_printer.print(format_lines(lines=STR_LINE_DOESNT_EXIST, + line_nr=sourcerange.end.line, + symbol='!')) + else: + print_lines(console_printer, + file_dict, + sourcerange) + + +def require_setting(setting_name, arr, section): + """ + This method is responsible for prompting a user about a missing setting and + taking its value as input from the user. + :param setting_name: Name of the setting missing + :param arr: A list containing a description in [0] and the name + of the bears who need this setting in [1]. + :param section: The section the action corresponds to. + :param return: Returns the setting value that was requested from the + user. + """ + needed = join_names(arr[1:]) + + # Don't use input, it can't deal with escapes! + + print(colored(STR_GET_VAL_FOR_SETTING.format(setting_name, arr[0], needed, + section.name), + REQUIRED_SETTINGS_COLOR)) + return input() + + +def acquire_settings(log_printer, settings_names_dict, section): + """ + This method prompts the user for the given settings. + :param log_printer: + Printer responsible for logging the messages. This is needed to comply + with the interface. + :param settings_names_dict: + A dictionary with the settings name as key and a list containing a + description in [0] and the name of the bears who need this setting in + [1] and following. + Example: + :: + {"UseTabs": ["describes whether tabs should be used instead of spaces", + "SpaceConsistencyBear", + "SomeOtherBear"]} + :param section: + The section the action corresponds to. + :return: + A dictionary with the settings name as key and the given value as + value. + """ + if not isinstance(settings_names_dict, dict): + raise TypeError('The settings_names_dict parameter has to be a ' + 'dictionary.') + + result = {} + for setting_name, arr in sorted(settings_names_dict.items(), + key=lambda x: (join_names(x[1][1:]), x[0])): + value = require_setting(setting_name, arr, section) + result.update({setting_name: value} if value is not None else {}) + + return result + +def choose_action(console_printer, actions, apply_single=False): + """ + Presents the actions available to the user and takes as input the action + the user wants to choose. + :param console_printer: Object to print messages on the console. + :param actions: Actions available to the user. + :param apply_single: The action that should be applied for all results. + If it's not selected, has a value of False. + :return: Return a tuple of lists, a list with the names of + actions that needs to be applied and a list with + with the description of the actions. + """ + actions_desc = [] + actions_name = [] + if apply_single: + for i, action in enumerate(actions, 0): + if apply_single == action.desc: + return ([action.desc], [action.name]) + return (['Do (N)othing'], ['Do (N)othing']) + else: + while True: + for i, action in enumerate(actions, 0): + output = '{:>2}. {}' if i != 0 else '*{}. {}' + color_letter(console_printer, format_lines(output.format( + i, action.desc), symbol='[')) + + line = format_lines(STR_ENTER_NUMBER, symbol='[') + + choice = input(line) + choice = str(choice) + + for c in choice: + c = str(c) + str_c = c + actions_desc_len = len(actions_desc) + if c.isnumeric(): + for i, action in enumerate(actions, 0): + c = int(c) + if i == c: + actions_desc.append(action.desc) + actions_name.append(action.name) + break + elif c.isalpha(): + c = c.upper() + c = '(' + c + ')' + for i, action in enumerate(actions, 1): + if c in action.desc: + actions_desc.append(action.desc) + actions_name.append(action.name) + break + if actions_desc_len == len(actions_desc): + console_printer.print(STR_INVALID_OPTION.format(str_c), + color=WARNING_COLOR) + + if not choice: + actions_desc.append(DoNothingAction().get_metadata().desc) + actions_name.append(DoNothingAction().get_metadata().name) + return (actions_desc, actions_name) + + +def try_to_apply_action(action_name, + chosen_action, + console_printer, + section, + metadata_list, + action_dict, + failed_actions, + result, + file_diff_dict, + file_dict, + applied_actions): + """ + Try to apply the given action. + :param action_name: The name of the action. + :param chosen_action: The action object that will be applied. + :param console_printer: Object to print messages on the console. + :param section: Currently active section. + :param metadata_list: Contains metadata for all the actions. + :param action_dict: Contains the action names as keys and their + references as values. + :param failed_actions: A set of all actions that have failed. A failed + action remains in the list until it is successfully + executed. + :param result: Result corresponding to the actions. + :param file_diff_dict: If it is an action which applies a patch, this + contains the diff of the patch to be applied to + the file with filename as keys. + :param applied_actions: A dictionary that contains the result, file_dict, + file_diff_dict and the section for an action. + :param file_dict: Dictionary with filename as keys and its contents + as values. + """ + try: + chosen_action.apply_from_section(result, + file_dict, + file_diff_dict, + section) + if not isinstance(chosen_action, DoNothingAction): + console_printer.print( + format_lines(chosen_action.SUCCESS_MESSAGE, symbol='['), + color=SUCCESS_COLOR) + applied_actions[action_name] = [copy.copy(result), copy.copy( + file_dict), + copy.copy(file_diff_dict), + copy.copy(section)] + result.set_applied_actions(applied_actions) + failed_actions.discard(action_name) + except Exception as exception: # pylint: disable=broad-except + logging.error('Failed to execute the action {} with error: {}.' + .format(action_name, exception)) + failed_actions.add(action_name) + +def print_lines(console_printer, + file_dict, + sourcerange): + """ + Prints the lines between the current and the result line. If needed + they will be shortened. + :param console_printer: Object to print messages on the console. + :param file_dict: A dictionary containing all files as values with + filenames as key. + :param sourcerange: The SourceRange object referring to the related + lines to print. + """ + no_color = not console_printer.print_colored + for i in range(sourcerange.start.line, sourcerange.end.line + 1): + # Print affected file's line number in the sidebar. + console_printer.print(format_lines(lines='', line_nr=i, symbol='['), + color=FILE_LINES_COLOR, + end='') + + line = file_dict[sourcerange.file][i - 1].rstrip('\n') + try: + lexer = get_lexer_for_filename(sourcerange.file) + except ClassNotFound: + lexer = TextLexer() + lexer.add_filter(VisibleWhitespaceFilter( + spaces=True, tabs=True, + tabsize=SpacingHelper.DEFAULT_TAB_WIDTH)) + # highlight() combines lexer and formatter to output a ``str`` + # object. + printed_chars = 0 + if i == sourcerange.start.line and sourcerange.start.column: + console_printer.print(highlight_text( + no_color, line[:sourcerange.start.column - 1], + BackgroundMessageStyle, lexer), end='') + + printed_chars = sourcerange.start.column - 1 + + if i == sourcerange.end.line and sourcerange.end.column: + console_printer.print(highlight_text( + no_color, line[printed_chars:sourcerange.end.column - 1], + BackgroundSourceRangeStyle, lexer), end='') + + console_printer.print(highlight_text( + no_color, line[sourcerange.end.column - 1:], + BackgroundSourceRangeStyle, lexer), end='') + console_printer.print('') + else: + console_printer.print(highlight_text( + no_color, line[printed_chars:], BackgroundMessageStyle, lexer), + end='') + console_printer.print('') + + + +def ask_for_action_and_apply(console_printer, + section, + metadata_list, + action_dict, + failed_actions, + result, + file_diff_dict, + file_dict, + applied_actions, + apply_single=False): + """ + Asks the user for an action and applies it. + :param console_printer: Object to print messages on the console. + :param section: Currently active section. + :param metadata_list: Contains metadata for all the actions. + :param action_dict: Contains the action names as keys and their + references as values. + :param failed_actions: A set of all actions that have failed. A failed + action remains in the list until it is successfully + executed. + :param result: Result corresponding to the actions. + :param file_diff_dict: If it is an action which applies a patch, this + contains the diff of the patch to be applied to + the file with filename as keys. + :param file_dict: Dictionary with filename as keys and its contents + as values. + :param apply_single: The action that should be applied for all results. + If it's not selected, has a value of False. + :param applied_actions: A dictionary that contains the result, file_dict, + file_diff_dict and the section for an action. + :return: Returns a boolean value. True will be returned, if + it makes sense that the user may choose to execute + another action, False otherwise. + If apply_single isn't set, always return False. + """ + do_nothing_action = DoNothingAction() + metadata_list.insert(0, do_nothing_action.get_metadata()) + action_dict[do_nothing_action.get_metadata().name] = DoNothingAction() + + actions_desc, actions_name = choose_action(console_printer, metadata_list, + apply_single) + + if apply_single: + for index, action_details in enumerate(metadata_list, 1): + if apply_single == action_details.desc: + action_name, section = get_action_info( + section, metadata_list[index - 1], failed_actions) + chosen_action = action_dict[action_details.name] + try_to_apply_action(action_name, + chosen_action, + console_printer, + section, + metadata_list, + action_dict, + failed_actions, + result, + file_diff_dict, + file_dict, + applied_actions) + return False + else: + for action_choice, action_choice_name in zip(actions_desc, + actions_name): + chosen_action = action_dict[action_choice_name] + action_choice_made = action_choice + for index, action_details in enumerate(metadata_list, 1): + if action_choice_made in action_details.desc: + action_name, section = get_action_info( + section, metadata_list[index-1], failed_actions) + try_to_apply_action(action_name, + chosen_action, + console_printer, + section, + metadata_list, + action_dict, + failed_actions, + result, + file_diff_dict, + file_dict, + applied_actions) + + if action_choice == 'Do (N)othing': + return False + return True + + +def show_enumeration(console_printer, + title, + items, + indentation, + no_items_text): + """ + This function takes as input an iterable object (preferably a list or + a dict) and prints it in a stylized format. If the iterable object is + empty, it prints a specific statement given by the user. An e.g : + Title: + * Item 1 + * Item 2 + :param console_printer: Object to print messages on the console. + :param title: Title of the text to be printed + :param items: The iterable object. + :param indentation: Number of spaces to indent every line by. + :param no_items_text: Text printed when iterable object is empty. + """ + if not items: + console_printer.print(indentation + no_items_text) + else: + console_printer.print(indentation + title) + if isinstance(items, dict): + for key, value in items.items(): + console_printer.print(indentation + ' * ' + key + ': ' + + value[0]) + else: + for item in items: + console_printer.print(indentation + ' * ' + item) + console_printer.print() + + +def show_bear(bear, + show_description, + show_params, + console_printer, + args=None): + """ + Displays all information about a bear. + :param bear: The bear to be displayed. + :param show_description: This parameter is deprecated. + :param show_params: This parameter is deprecated. + :param console_printer: Object to print messages on the console. + :param args: Args passed to coala command. + """ + console_printer.print(bear.name, color='blue') + + metadata = bear.get_metadata() + + check_deprecation(OrderedDict([ + ('show_description', show_description), + ('show_params', show_params)])) + + if show_description: + console_printer.print( + ' ' + metadata.desc.replace('\n', '\n ')) + console_printer.print() # Add a newline + + if show_params: + show_enumeration( + console_printer, 'Supported languages:', + bear.LANGUAGES, + ' ', + 'The bear does not provide information about which languages ' + 'it can analyze.') + + if (args and args.show_settings) or show_params: + show_enumeration(console_printer, + 'Needed Settings:', + metadata.non_optional_params, + ' ', + 'No needed settings.') + show_enumeration(console_printer, + 'Optional Settings:', + metadata.optional_params, + ' ', + 'No optional settings.') + + if show_params: + show_enumeration(console_printer, + 'Can detect:', + bear.can_detect, + ' ', + 'This bear does not provide information about what ' + 'categories it can detect.') + show_enumeration(console_printer, + 'Can fix:', + bear.CAN_FIX, + ' ', + 'This bear cannot fix issues or does not provide ' + 'information about what categories it can fix.') + console_printer.print( + ' Path:\n' + ' ' + repr(bear.source_location) + '\n') + +def show_bears(local_bears, + global_bears, + show_description, + show_params, + console_printer, + args=None): + """ + Extracts all the bears from each enabled section or the sections in the + targets and passes a dictionary to the show_bears_callback method. + :param local_bears: Dictionary of local bears with section names + as keys and bear list as values. + :param global_bears: Dictionary of global bears with section + names as keys and bear list as values. + :param show_description: This parameter is deprecated. + :param show_params: This parameter is deprecated. + :param console_printer: Object to print messages on the console. + :param args: Args passed to coala command. + """ + bears = inverse_dicts(local_bears, global_bears) + + print_bears(bears, show_description, show_params, console_printer, args) diff --git a/plugins/scancode-fingerprint/tests/data/fingerprint/token-test.txt b/plugins/scancode-fingerprint/tests/data/fingerprint/token-test.txt new file mode 100644 index 00000000000..63db5edad6c --- /dev/null +++ b/plugins/scancode-fingerprint/tests/data/fingerprint/token-test.txt @@ -0,0 +1,5 @@ +This is for testing purpose +It should +work +fine +. \ No newline at end of file diff --git a/plugins/scancode-fingerprint/tests/data/plugin_fingerprint/files/factorial-test.java b/plugins/scancode-fingerprint/tests/data/plugin_fingerprint/files/factorial-test.java new file mode 100644 index 00000000000..19432f37cf1 --- /dev/null +++ b/plugins/scancode-fingerprint/tests/data/plugin_fingerprint/files/factorial-test.java @@ -0,0 +1,15 @@ +public class Factorial +{ + public static void main(String[] args) + { final int NUM_FACTS = 100; + for(int i = 0; i < NUM_FACTS; i++) + System.out.println( i + "! is " + factorial(i)); + } + + public static int factorial(int n) + { int result = 1; + for(int i = 2; i <= n; i++) + result *= i; + return result; + } +} \ No newline at end of file diff --git a/plugins/scancode-fingerprint/tests/data/plugin_fingerprint/files/hello-test.c b/plugins/scancode-fingerprint/tests/data/plugin_fingerprint/files/hello-test.c new file mode 100644 index 00000000000..32b0c06a9df --- /dev/null +++ b/plugins/scancode-fingerprint/tests/data/plugin_fingerprint/files/hello-test.c @@ -0,0 +1,6 @@ +#include +int main() +{ + printf("Hello, World! This is for testing purpose"); + return 0; +} \ No newline at end of file diff --git a/plugins/scancode-fingerprint/tests/data/plugin_fingerprint/files/lorem-test.txt b/plugins/scancode-fingerprint/tests/data/plugin_fingerprint/files/lorem-test.txt new file mode 100644 index 00000000000..b00c9ef3b0a --- /dev/null +++ b/plugins/scancode-fingerprint/tests/data/plugin_fingerprint/files/lorem-test.txt @@ -0,0 +1,19 @@ +Lorem ipsum dolor sit amet, sonet consul nec at. Ne utinam utroque salutandi vix. Nam eius ridens feugiat in, ne nulla maiorum quo. Modus disputando at pri, utinam vituperatoribus te usu, omnis sensibus est in. Usu ad utinam dolorum, justo euripidis ei usu. Et mei scriptorem omittantur. + +Paulo consulatu deterruisset vix an, mea bonorum verterem no. Sit mollis regione apeirian te. Mutat liberavisse vel no. Ut noster atomorum iudicabit his, mel feugait voluptaria suscipiantur at, nec at appetere splendide adversarium. + +An sit porro putent eligendi, no facilis perpetua vituperatoribus sea, ex ius dolor denique. Quas corpora mei et, at possit copiosae consulatu vis. Nulla contentiones te qui. Ne elit timeam mei, omnesque maluisset an has. Id quis alterum saperet eum. Id vim modo postulant, his no debet legere. + +Tation maiorum accusata eu mea, ius convenire petentium ne. Eum sale antiopam et. Ignota torquatos per ei, at sea molestie adipisci, ius eu mutat atqui aliquando. Et paulo denique eleifend nec, utinam oblique id vix. Graeci habemus epicurei et per, nusquam repudiare temporibus cu sit, in has conceptam reprimique. + +Natum everti ex qui, id officiis tractatos repudiandae est, atomorum tincidunt reformidans mel eu. Sea dico inermis ne. Ex duo illum essent, porro iriure eam in. Mei hinc alienum principes cu. Ex cetero integre quo, sit at elit vulputate cotidieque, id sea nisl percipitur. Luptatum quaestio duo an. + +Unum nemore maiorum ad usu, mei graeci putent convenire at. Habeo possit id vis. Ei quando tation recusabo duo. Ut vocibus antiopam mel, meis doming oblique per eu, putent facilisi forensibus sea at. Harum semper maiestatis quo cu, ei cetero probatus vix. Sea quem persius ex, est animal verterem dissentiet te, at aeque solet dignissim cum. + +Ad mel molestie percipit argumentum, munere doming noluisse pro id, audire contentiones in pro. An saepe albucius splendide cum. Eu enim saperet sit, eu veri error alienum quo. Ius ne idque numquam. Ex eam dico duis, cu eam imperdiet assueverit, vim simul regione dolores ne. + +Omnes postea urbanitas et his, quo cu fuisset efficiantur conclusionemque, quo et prompta accusata. Omnes salutandi sea te, tincidunt percipitur in vim. Ex tollit tantas deseruisse nec. Et duo rebum disputando, rebum assum expetendis has cu, sea ex quidam option impedit. + +Summo evertitur persecuti vel ne. Et pri paulo reformidans, at qui facilisi quaerendum. Ne everti eirmod usu, nam movet aliquip dissentiet te, no eum idque summo erroribus. Ea ubique omnesque expetenda cum. Has ea inani laudem quidam, dolore moderatius ius no. Sed eleifend consectetuer ad. + +Usu quas inciderint an, eu porro accusamus sit. Per error aeque postea eu, sit ad possit virtute, eum causae volutpat no. Agam quaestio usu an, sint deleniti pri an. Consul patrioque no sea, te ius error congue adversarium. \ No newline at end of file diff --git a/plugins/scancode-fingerprint/tests/data/plugin_fingerprint/fingerprints.expected.json b/plugins/scancode-fingerprint/tests/data/plugin_fingerprint/fingerprints.expected.json new file mode 100644 index 00000000000..6f0534340a2 --- /dev/null +++ b/plugins/scancode-fingerprint/tests/data/plugin_fingerprint/fingerprints.expected.json @@ -0,0 +1,39 @@ +{ + "headers": [ + { + "tool_name": "scancode-toolkit", + "options": { + "input": "", + "--fingerprint": true, + "--json": "", + "--strip-root": true + }, + "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "message": null, + "errors": [], + "extra_data": { + "files_count": 3 + } + } + ], + "files": [ + { + "path": "factorial-test.java", + "type": "file", + "fingerprint": "01cfb437aced3f7d8e4184f4d8673127", + "scan_errors": [] + }, + { + "path": "hello-test.c", + "type": "file", + "fingerprint": "bb274e73955fbebd97c9c37e7f1e09d2", + "scan_errors": [] + }, + { + "path": "lorem-test.txt", + "type": "file", + "fingerprint": "72c54ab1f6b56cb97df6152182eb95b3", + "scan_errors": [] + } + ] +} diff --git a/plugins/scancode-fingerprint/tests/test_fingerprint.py b/plugins/scancode-fingerprint/tests/test_fingerprint.py new file mode 100644 index 00000000000..f5fffa0f78e --- /dev/null +++ b/plugins/scancode-fingerprint/tests/test_fingerprint.py @@ -0,0 +1,211 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/scancode-toolkit/ +# The ScanCode software is licensed under the Apache License version 2.0. +# Data generated with ScanCode require an acknowledgment. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with ScanCode or any ScanCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# ScanCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode-toolkit/ for support and download. + +from __future__ import absolute_import, print_function + +import os + +from bitarray import bitarray +from commoncode.testcase import FileBasedTesting +from plugin_fingerprint.fingerprint import Simhash +from plugin_fingerprint.fingerprint import HASH_LENGTH + +class TestFingerprint(FileBasedTesting): + + test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + + def test_get_weighted_hash1(self): + simhash = Simhash() + test_file = self.get_test_loc('fingerprint/get_weighted_hash-test1.txt') + + with open(test_file, 'r') as f: + hashable = f.read() + + simhash.update(hashable) + result = simhash.get_weighted_hash() + expected = [0, -2, 2, 0, 0, -2, 2, 2, -2, 0, -2, 0, 0, 0, -2, 0, 2, 0, -2, -2, 0, 0, 2, 0, 2, 2, 0, 2, 0, 0, 0, -2, 2, 0, 0, 0, -2, 2, 0, 0, 0, 0, 0, 2, -2, 0, 0, -2, 2, 0, -2, 2, 2, -2, 0, -2, 2, 0, -2, 2, 0, 2, 2, 0, 0, 0, 2, 0, 0, -2, 0, 0, 0, 2, -2, 0, 0, 0, -2, 0, 0, 2, 0, 2, 0, 0, 0, 0, -2, -2, 2, 2, 0, 0, -2, -2, -2, -2, -2, 0, 0, 2, 0, 2, 2, -2, 0, 2, 2, -2, 0, -2, -2, 2, 0, 2, 0, 0, 0, 0, 2, -2, 0, 2, -2, -2, 0, -2] + assert result == expected + + def test_get_weighted_hash2(self): + simhash = Simhash() + test_file = self.get_test_loc('fingerprint/get_weighted_hash-test2.txt') + + with open(test_file, 'r') as f: + hashable = f.read() + + simhash.update(hashable) + result = simhash.get_weighted_hash() + expected = [-1, -1, -1, -1, 1, 1, -1, -1, 1, 1, -1, -1, -1, -1, -1, 1, -1, 1, 1, 1, -1, 1, -1, 1, 1, -1, 1, 1, 1, -1, -1, 1, 1, 1, -1, -1, -1, -1, -1, -1, 1, 1, 1, 1, -1, -1, -1, 1, 1, -1, 1, 1, -1, 1, 1, -1, 1, -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, 1, -1, -1, -1, 1, 1, 1, -1, -1, -1, -1, 1, 1, 1, -1, -1, 1, 1, -1, -1, 1, 1, 1, 1, -1, -1, -1, 1, -1, -1, 1, 1, -1, 1, -1, -1, 1, -1, 1, 1, 1, -1, 1, 1, 1, -1, -1, 1, -1, -1, 1, 1, -1, -1, 1, 1, -1, -1, -1, -1, 1] + assert result == expected + + def test_process_weighted_hash1(self): + simhash = Simhash() + result = simhash.process_weighted_hash([1, 2, 3, 4]) + expected = bitarray('1111') + assert result == expected + + def test_process_weighted_hash2(self): + simhash = Simhash() + result = simhash.process_weighted_hash([0, 0, 0, 0]) + expected = bitarray('0000') + assert result == expected + + def test_process_weighted_hash3(self): + simhash = Simhash() + result = simhash.process_weighted_hash([1, -2, -3, 4, 5, -6, 7, 0, 0]) + expected = bitarray('100110100') + assert result == expected + + def test_process_shingles1(self): + simhash = Simhash() + weighted_hash = [1] * HASH_LENGTH + expected = [0, 0, 2, 2, 0, 0, 0, 2, 2, 2, 2, 2, 0, 2, 0, 0, 0, 2, 2, 2, 0, 0, 2, 2, 0, 2, 2, 2, 2, 0, 2, 2, 2, 2, 2, 0, 2, 2, 0, 0, 0, 0, 2, 2, 0, 0, 2, 0, 2, 0, 0, 0, 2, 2, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 2, 2, 2, 2, 2, 0, 2, 2, 0, 2, 2, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 2, 2, 2, 2, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 2, 2, 2, 0, 2, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0] + result = simhash.process_shingles('Thisisfortesting', weighted_hash) + assert result == expected + + def test_process_shingles2(self): + simhash = Simhash() + weighted_hash = [0, 0, 2, 2, 0, 0, 0, 2, 2, 2, 2, 2, 0, 2, 0, 0, 0, 2, 2, 2, 0, 0, 2, 2, 0, 2, 2, 2, 2, 0, 2, 2, 2, 2, 2, 0, 2, 2, 0, 0, 0, 0, 2, 2, 0, 0, 2, 0, 2, 0, 0, 0, 2, 2, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 2, 2, 2, 2, 2, 0, 2, 2, 0, 2, 2, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 2, 2, 2, 2, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 2, 2, 2, 0, 2, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0] + expected = [1, 1, 3, 1, -1, 1, 1, 3, 1, 1, 1, 3, 1, 1, 1, -1, -1, 1, 3, 3, 1, 1, 3, 1, 1, 3, 3, 3, 1, 1, 3, 1, 1, 3, 3, -1, 1, 1, -1, -1, 1, -1, 3, 1, 1, -1, 3, 1, 3, 1, 1, -1, 3, 3, 1, 1, 1, 1, 3, 1, -1, 1, -1, 1, 1, -1, 3, 1, 1, 3, 1, 1, 1, 3, 1, 1, 3, -1, 3, 1, 1, 1, 3, -1, 1, -1, 1, 1, 1, -1, 1, 3, 1, 1, 1, 1, 3, 1, 1, -1, 1, 1, -1, 1, -1, 1, -1, 1, 3, 3, -1, 1, -1, 1, -1, -1, 3, -1, 1, 3, 1, 1, -1, 1, 1, 3, 1, -1] + result = simhash.process_shingles('tryforanotherone', weighted_hash) + assert result == expected + + def test_process_shingles3(self): + simhash = Simhash() + weighted_hash = [0, 0, 2, 2, 0, 0, 0, 2, 2, 2, 2, 2, 0, 2, 0, 0, 0, 2, 2, 2, 0, 0, 2, 2, 0, 2, 2, 2, 2, 0, 2, 2, 2, 2, 2, 0, 2, 2, 0, 0, 0, 0, 2, 2, 0, 0, 2, 0, 2, 0, 0, 0, 2, 2, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 2, 2, 2, 2, 2, 0, 2, 2, 0, 2, 2, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 2, 2, 2, 2, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 2, 2, 2, 0, 2, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0] + expected = [1, -1, 1, 1, -1, -1, 1, 3, 1, 1, 1, 1, -1, 1, -1, -1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 3, 1, 1, 1, 3, 1, 3, 3, 1, 1, 3, -1, 1, 1, 1, 1, 3, 1, 1, 3, 1, 1, -1, 1, -1, 3, 1, 1, -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, 1, 1, 1, -1, 1, 3, 1, 1, 1, 1, 1, 3, 1, 1, 3, -1, 1, 3, -1, -1, -1, -1, 1, 1, 1, 1, 1, -1, 3, 3, 3, 3, 1, 1, -1, 1, 1, -1, 3, 1, -1, -1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, 3, 1, -1, 1, -1, 1, 1, -1, 1] + result = simhash.process_shingles('for(inti=0;i