|
| 1 | +# |
| 2 | +# Copyright (c) 2018 nexB Inc. and others. All rights reserved. |
| 3 | +# http://nexb.com and https://github.com/nexB/scancode-toolkit/ |
| 4 | +# The ScanCode software is licensed under the Apache License version 2.0. |
| 5 | +# Data generated with ScanCode require an acknowledgment. |
| 6 | +# ScanCode is a trademark of nexB Inc. |
| 7 | +# |
| 8 | +# You may not use this software except in compliance with the License. |
| 9 | +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 |
| 10 | +# Unless required by applicable law or agreed to in writing, software distributed |
| 11 | +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR |
| 12 | +# CONDITIONS OF ANY KIND, either express or implied. See the License for the |
| 13 | +# specific language governing permissions and limitations under the License. |
| 14 | +# |
| 15 | +# When you publish or redistribute any data created with ScanCode or any ScanCode |
| 16 | +# derivative work, you must accompany this data with the following acknowledgment: |
| 17 | +# |
| 18 | +# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES |
| 19 | +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from |
| 20 | +# ScanCode should be considered or used as legal advice. Consult an Attorney |
| 21 | +# for any legal advice. |
| 22 | +# ScanCode is a free software code scanning tool from nexB Inc. and others. |
| 23 | +# Visit https://github.com/nexB/scancode-toolkit/ for support and download. |
| 24 | + |
| 25 | +from __future__ import absolute_import |
| 26 | +from __future__ import print_function |
| 27 | +from __future__ import unicode_literals |
| 28 | + |
| 29 | +from collections import OrderedDict |
| 30 | +import io |
| 31 | +from os import path |
| 32 | + |
| 33 | +import attr |
| 34 | +import pytest |
| 35 | + |
| 36 | +from commoncode import compat |
| 37 | +from commoncode import saneyaml |
| 38 | +from commoncode.system import py2 |
| 39 | +from commoncode.testcase import FileDrivenTesting |
| 40 | +from commoncode.testcase import get_test_file_pairs |
| 41 | +from commoncode.text import python_safe_name |
| 42 | +from typecode.contenttype import get_type |
| 43 | + |
| 44 | + |
| 45 | +""" |
| 46 | +Data-driven file type test utilities. |
| 47 | +""" |
| 48 | + |
| 49 | +test_env = FileDrivenTesting() |
| 50 | +test_env.test_data_dir = path.join(path.dirname(__file__), 'data') |
| 51 | + |
| 52 | + |
| 53 | +@attr.s(slots=True) |
| 54 | +class FileTypeTest(object): |
| 55 | + data_file = attr.ib(default=None) |
| 56 | + test_file = attr.ib(default=None) |
| 57 | + # one of holders, copyrights, authors |
| 58 | + what = attr.ib(default=attr.Factory(list)) |
| 59 | + mime_type = attr.ib(default=attr.Factory(list)) |
| 60 | + file_type = attr.ib(default=attr.Factory(list)) |
| 61 | + programming_language = attr.ib(default=attr.Factory(list)) |
| 62 | + is_binary = attr.ib(default=attr.Factory(list)) |
| 63 | + is_text = attr.ib(default=attr.Factory(list)) |
| 64 | + is_archive = attr.ib(default=attr.Factory(list)) |
| 65 | + is_media = attr.ib(default=attr.Factory(list)) |
| 66 | + is_source = attr.ib(default=attr.Factory(list)) |
| 67 | + is_script = attr.ib(default=attr.Factory(list)) |
| 68 | + |
| 69 | + expected_failures = attr.ib(default=attr.Factory(list)) |
| 70 | + notes = attr.ib(default=None) |
| 71 | + |
| 72 | + def __attrs_post_init__(self, *args, **kwargs): |
| 73 | + if self.data_file: |
| 74 | + try: |
| 75 | + with io.open(self.data_file, encoding='utf-8') as df: |
| 76 | + for key, value in saneyaml.load(df.read()).items(): |
| 77 | + if value: |
| 78 | + setattr(self, key, value) |
| 79 | + except: |
| 80 | + import traceback |
| 81 | + msg = 'file://' + self.data_file + '\n' + repr(self) + '\n' + traceback.format_exc() |
| 82 | + raise Exception(msg) |
| 83 | + |
| 84 | + def to_dict(self): |
| 85 | + """ |
| 86 | + Serialize self to an ordered mapping. |
| 87 | + """ |
| 88 | + filtered = [field for field in attr.fields(FileTypeTest) |
| 89 | + if '_file' in field.name] |
| 90 | + fields_filter = attr.filters.exclude(*filtered) |
| 91 | + data = attr.asdict(self, filter=fields_filter, dict_factory=OrderedDict) |
| 92 | + return OrderedDict([ |
| 93 | + (key, value) for key, value in data.items() |
| 94 | + # do not dump false and empties |
| 95 | + if value]) |
| 96 | + |
| 97 | + def dumps(self): |
| 98 | + """ |
| 99 | + Return a string representation of self in YAML block format. |
| 100 | + """ |
| 101 | + return saneyaml.dump(self.to_dict()) |
| 102 | + |
| 103 | + def dump(self, check_exists=False): |
| 104 | + """ |
| 105 | + Dump a representation of self to a .yml data_file in YAML block format. |
| 106 | + """ |
| 107 | + if check_exists and path.exists(self.data_file): |
| 108 | + raise Exception(self.data_file) |
| 109 | + with io.open(self.data_file, 'w', encoding='utf-8') as df: |
| 110 | + df.write(self.dumps()) |
| 111 | + |
| 112 | + |
| 113 | +def load_filetype_tests(test_dir=test_env.test_data_dir): |
| 114 | + """ |
| 115 | + Yield an iterable of FileTypeTest loaded from test data files in `test_dir`. |
| 116 | + """ |
| 117 | + test_dir = path.join(test_dir, 'filetest') |
| 118 | + |
| 119 | + all_test_files = get_test_file_pairs(test_dir) |
| 120 | + |
| 121 | + for data_file, test_file in all_test_files: |
| 122 | + yield FileTypeTest(data_file, test_file) |
| 123 | + |
| 124 | + |
| 125 | +def filetype_detector(location): |
| 126 | + """ |
| 127 | + Return detected filetype info |
| 128 | + """ |
| 129 | + collector = get_type(location) |
| 130 | + return { |
| 131 | + 'mime_type': collector.mimetype_file or None, |
| 132 | + 'file_type': collector.filetype_file or None, |
| 133 | + 'programming_language': collector.programming_language or None, |
| 134 | + 'is_binary': bool(collector.is_binary), |
| 135 | + 'is_text': bool(collector.is_text), |
| 136 | + 'is_archive': bool(collector.is_archive), |
| 137 | + 'is_media': bool(collector.is_media), |
| 138 | + 'is_source': bool(collector.is_source), |
| 139 | + 'is_script': bool(collector.is_script), |
| 140 | + } |
| 141 | + |
| 142 | + |
| 143 | +def make_filetype_test_functions(test, index, test_data_dir=test_env.test_data_dir, regen=False): |
| 144 | + """ |
| 145 | + Build and return a test function closing on tests arguments and the function name. |
| 146 | + """ |
| 147 | + |
| 148 | + def closure_test_function(*args, **kwargs): |
| 149 | + results = filetype_detector(test_file) |
| 150 | + |
| 151 | + expected_yaml = test.dumps() |
| 152 | + |
| 153 | + for wht in test.what: |
| 154 | + setattr(test, wht, results.get(wht)) |
| 155 | + results_yaml = test.dumps() |
| 156 | + |
| 157 | + if regen: |
| 158 | + test.dump() |
| 159 | + if expected_yaml != results_yaml: |
| 160 | + expected_yaml = ( |
| 161 | + 'data file: file://' + data_file + |
| 162 | + '\ntest file: file://' + test_file + '\n' |
| 163 | + ) + expected_yaml |
| 164 | + |
| 165 | + assert expected_yaml == results_yaml |
| 166 | + |
| 167 | + data_file = test.data_file |
| 168 | + test_file = test.test_file |
| 169 | + what = test.what |
| 170 | + |
| 171 | + tfn = test_file.replace(test_data_dir, '').strip('\\/\\') |
| 172 | + whats = '_'.join(what) |
| 173 | + test_name = 'test_%(tfn)s_%(index)s' % locals() |
| 174 | + test_name = python_safe_name(test_name) |
| 175 | + |
| 176 | + # onPython2 we need a plain non-unicode string |
| 177 | + if py2 and isinstance(test_name, compat.unicode): |
| 178 | + test_name = test_name.encode('utf-8') |
| 179 | + |
| 180 | + closure_test_function.__name__ = test_name |
| 181 | + |
| 182 | + if test.expected_failures: |
| 183 | + closure_test_function = pytest.mark.xfail(closure_test_function) |
| 184 | + |
| 185 | + return closure_test_function, test_name |
| 186 | + |
| 187 | + |
| 188 | +def build_tests(filetype_tests, clazz, test_data_dir=test_env.test_data_dir, regen=False): |
| 189 | + """ |
| 190 | + Dynamically build test methods from a sequence of FileTypeTests and attach |
| 191 | + these method to the clazz test class. |
| 192 | + """ |
| 193 | + for i, test in enumerate(sorted(filetype_tests, key=lambda x:x.test_file)): |
| 194 | + # closure on the test params |
| 195 | + if test.expected_failures: |
| 196 | + actual_regen = False |
| 197 | + else: |
| 198 | + actual_regen = regen |
| 199 | + method, name = make_filetype_test_functions(test, i, test_data_dir, actual_regen) |
| 200 | + # attach that method to our test class |
| 201 | + setattr(clazz, name, method) |
0 commit comments