diff --git a/setup.py b/setup.py index 24d719ca254..6ee9cfa1f69 100644 --- a/setup.py +++ b/setup.py @@ -78,7 +78,7 @@ def read(*names, **kwargs): # caching 'zc.lockfile >= 1.0.0, < 2.0.0', 'yg.lockfile >= 2.0.0, < 3.0.0', - 'diskcache >= 1.7.0, < 1.8.0', + 'diskcache >= 2.0.0, < 3.0.0', # textcode 'Beautifulsoup >= 3.2.0, <4.0.0', @@ -99,6 +99,7 @@ def read(*names, **kwargs): 'jinja2 >= 2.7.0, < 3.0.0', 'MarkupSafe >= 0.23', 'colorama', + 'simplejson', # packagedcode 'requests >= 2.7.0, < 3.0.0', diff --git a/src/scancode/__init__.py b/src/scancode/__init__.py index 358c65207f8..6364695a7fd 100644 --- a/src/scancode/__init__.py +++ b/src/scancode/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2015 nexB Inc. and others. All rights reserved. +# Copyright (c) 2016 nexB Inc. and others. All rights reserved. # http://nexb.com and https://github.com/nexB/scancode-toolkit/ # The ScanCode software is licensed under the Apache License version 2.0. # Data generated with ScanCode require an acknowledgment. @@ -23,3 +23,21 @@ # Visit https://github.com/nexB/scancode-toolkit/ for support and download. __version__ = '2.0.0rc2' + +from os.path import dirname +from os.path import abspath +from os.path import getsize +from os.path import getmtime +from os.path import join +from os.path import exists + +from commoncode import fileutils + +scan_src_dir = abspath(dirname(__file__)) +src_dir = dirname(scan_src_dir) +root_dir = dirname(src_dir) +cache_dir = join(root_dir, '.cache') +scans_cache_dir = join(cache_dir, 'scan_results_caches') + +if not exists(scans_cache_dir): + fileutils.create_dir(scans_cache_dir) diff --git a/src/scancode/api.py b/src/scancode/api.py index 4e191452079..1010a25d3af 100644 --- a/src/scancode/api.py +++ b/src/scancode/api.py @@ -142,11 +142,11 @@ def get_licenses(location, min_score=0): result['matched_rule']['licenses'] = match.rule.licenses # TODO: add debug details such as matcher # result['matched_rule']['matcher'] = match.matcher - + yield result -def get_file_infos(location): +def get_file_infos(location, as_list=True): """ Return a list of dictionaries of informations collected from the file or directory at location. @@ -177,7 +177,10 @@ def get_file_infos(location): infos['is_media'] = is_file and T.is_media or None infos['is_source'] = is_file and T.is_source or None infos['is_script'] = is_file and T.is_script or None - return [infos] + if as_list: + return [infos] + else: + return infos def get_package_infos(location): diff --git a/src/scancode/cache.py b/src/scancode/cache.py new file mode 100644 index 00000000000..44661d96925 --- /dev/null +++ b/src/scancode/cache.py @@ -0,0 +1,162 @@ +# +# Copyright (c) 2016 nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/scancode-toolkit/ +# The ScanCode software is licensed under the Apache License version 2.0. +# Data generated with ScanCode require an acknowledgment. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with ScanCode or any ScanCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# ScanCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode-toolkit/ for support and download. + +from __future__ import absolute_import, print_function + +from collections import OrderedDict +import os +import sys + +from commoncode import fileutils +from commoncode import timeutils + +from scancode import scans_cache_dir + +""" +Caching scans on disk: A cache of all the scan results. + +Each scan results for a file or directory is cached on disk. + +The approach is to use to cache: + - the results of a scan, excluding file infos keyed by the hash of a scanned file + - the file infos, keyed by the path of a scanned file + +Once a scan is completed, we iterate the caches to output the scan results using this +procedure: iterate the cached file infos and for each lookup the scan details in the +cached scan results. This iteration is driving the final streaming of results to the +output format (e.g. JSON). + +Finally once a scan is completed the cache is destroyed to free up disk space. +""" + +# Tracing flags +TRACE = False + +def logger_debug(*args): + pass + +if TRACE: + import logging + + logger = logging.getLogger(__name__) + # logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) + logging.basicConfig(stream=sys.stdout) + logger.setLevel(logging.DEBUG) + + def logger_debug(*args): + return logger.debug(' '.join(isinstance(a, basestring) and a or repr(a) for a in args)) + + +class ScanCache(object): + """ + A file-based cache for scan results. + This is NOT thread-safe, but is multi-process safe. + """ + def __init__(self, cache_dir): + fileutils.create_dir(cache_dir) + + # create a unique temp directory in cache_dir + self.cache_base_dir = fileutils.get_temp_dir(cache_dir, prefix=timeutils.time2tstamp() + '-') + + # and subdirs for infos and scans caches + self.cache_infos_dir = os.path.join(self.cache_base_dir, 'infos') + fileutils.create_dir(self.cache_infos_dir) + self.cache_scans_dir = os.path.join(self.cache_base_dir, 'scans') + fileutils.create_dir(self.cache_scans_dir) + + # workaround for https://github.com/grantjenks/python-diskcache/issues/32 + from diskcache import Disk + class DiskWithNoHighPickleProtocol(Disk): + "Subclass of diskcache.Disk that always use the lowest pickle protocol." + def __init__(self, directory, size_threshold, pickle_protocol): + super(DiskWithNoHighPickleProtocol, self).__init__(directory, size_threshold, pickle_protocol) + self._protocol = 0 + + # and finially cache instances + from diskcache import Cache + self.infos = Cache(self.cache_infos_dir, disk=DiskWithNoHighPickleProtocol) + self.scans = Cache(self.cache_scans_dir, disk=DiskWithNoHighPickleProtocol) + + def scan_key(self, path, file_infos): + """ + Return a scan cache key for a path and file_infos. + """ + sha1 = file_infos['sha1'] + # we may eventually store directories, in which case we use the path as a key + return sha1 or path + + def put_infos(self, path, file_infos): + """ + Put file_infos for path in the cache and return True if the file referenced + in file_infos has already been scanned or False otherwise. + """ + self.infos.set(path, file_infos) + has_cached_details = self.scan_key(path, file_infos) in self.scans + if TRACE: + logger_debug('put_infos:', 'path:', path, 'has_cached_details:', has_cached_details, 'file_infos:', file_infos, '\n') + logger_debug('put_infos:', 'cached_infos:', self.infos[path], '\n') + + return has_cached_details + + def put_scan(self, path, file_infos, scan_result): + """ + Put scan_result in the cache. Also put file_infos in the cache if needed. + """ + scan_key = self.scan_key(path, file_infos) + self.scans.add(scan_key, scan_result) + if TRACE: + logger_debug('put_scan:', 'scan_key:', scan_key, 'file_infos:', file_infos, 'scan_result:', scan_result, '\n') + logger_debug('put_scan:', 'cached_infos:', self.infos[path], '\n') + logger_debug('put_scan:', 'scan_key:', scan_key, 'cached_scan:', self.scans[scan_key], '\n') + + def iterate(self, with_infos=True): + """ + Return an iterator of scan data for all cached scans e.g. the whole cache. + """ + for path in self.infos: + file_infos = self.infos[path] + scan_result = OrderedDict(path=path) + if with_infos: + # infos is always collected but only returnedd if asked: + # we flatten these as direct attributes of a file object + scan_result.update(file_infos.items()) + + scan_key = self.scan_key(path, file_infos) + scan_details = self.scans[scan_key] + scan_result.update(scan_details) + if TRACE: + logger_debug('iterate:', 'scan_details:', scan_details, 'for path:', path, 'scan_key:', scan_key, '\n') + yield scan_result + + def clear(self, *args): + """ + Purge the cache by deleting the corresponding cached data files. + """ + self.infos.close() + self.scans.close() + fileutils.delete(self.cache_base_dir) + + +def get_scans_cache(): + return ScanCache(cache_dir=scans_cache_dir) diff --git a/src/scancode/cli.py b/src/scancode/cli.py index 8dfb45516b6..501e89c43b6 100644 --- a/src/scancode/cli.py +++ b/src/scancode/cli.py @@ -26,13 +26,13 @@ from collections import OrderedDict from functools import partial -import json import os import sys from types import GeneratorType import click from click.termui import style +import simplejson as json from commoncode import ignore from commoncode import fileutils @@ -43,6 +43,8 @@ from scancode import __version__ as version from scancode import utils +from scancode.cache import get_scans_cache + from scancode.format import as_template from scancode.format import as_html_app from scancode.format import create_html_app_assets @@ -260,15 +262,27 @@ def scancode(ctx, input, output_file, copyright, license, package, license = True package = True - results = scan(input, copyright, license, package, email, url, info, license_score, verbose, quiet) - save_results(results, format, input, output_file) + scans_cache = get_scans_cache() + try: + files_count, results = scan(input, copyright, license, package, email, url, info, license_score, verbose, quiet, scans_cache) + save_results(files_count, results, format, input, output_file) + finally: + # cleanup + scans_cache.clear() def scan(input_path, copyright=True, license=True, package=True, - email=False, url=False, info=True, license_score=0, verbose=False, quiet=False): + email=False, url=False, info=True, license_score=0, + verbose=False, quiet=False, + scans_cache=None): """ - Do the scans proper, return a list of file_results. + Return a tuple of (file_count, scan_results) where scan_results is an iterable. + Run each requested scan proper: each individual file scan is cached on disk to + free memory. Then the whole set of scans is loaded from the cache and streamed at + the end. """ + assert scans_cache + # save paths to report paths relative to the original input original_input = fileutils.as_posixpath(input_path) abs_input = fileutils.as_posixpath(os.path.abspath(os.path.expanduser(input_path))) @@ -278,7 +292,7 @@ def scan(input_path, copyright=True, license=True, package=True, # note: "flag and function" expressions return the function if flag is True # note: the order of the scans matters to show things in logical order scanners = OrderedDict([ - ('infos' , info and get_file_infos), + # ('infos' , info and get_file_infos), ('licenses' , license and get_licenses_with_score), ('copyrights' , copyright and get_copyrights), ('packages' , package and get_package_infos), @@ -286,8 +300,6 @@ def scan(input_path, copyright=True, license=True, package=True, ('urls' , url and get_urls), ]) - file_results = [] - # note: we inline progress display functions to close on some args def scan_start(): @@ -323,20 +335,37 @@ def scan_end(): quiet=quiet ) as progressive_resources: - for resource in progressive_resources: + for files_count, resource in enumerate(progressive_resources): + # actual path of the file being scanned res = fileutils.as_posixpath(resource) - # fix paths: keep the path as relative to the original input relative_path = utils.get_relative_path(original_input, abs_input, res) - scan_result = OrderedDict(path=relative_path) - # Should we yield instead? - scan_result.update(scan_one(res, scanners)) - file_results.append(scan_result) - # TODO: eventually merge scans for the same files path... - # TODO: fix absolute paths as relative to original input argument... + # always fetch infos and cache. + infos = scan_infos(res) + is_cached = scans_cache.put_infos(relative_path, infos) - return file_results + # Skip other scans if already cached + if is_cached: + continue + scan_result = scan_one(res, scanners) + scans_cache.put_scan(relative_path, infos, scan_result) + files_count += 1 + return files_count, scans_cache.iterate(with_infos=info) + + +def scan_infos(input_file): + """ + Scan one file or directory and return file_infos data. + """ + infos = OrderedDict() + try: + infos = get_file_infos(input_file, as_list=False) + except Exception, e: + # never fail but instead add an error message. + # FIXME: this should not be stored at the individual scan level + return dict(errors=e.message) + return infos def scan_one(input_file, scans): @@ -344,27 +373,24 @@ def scan_one(input_file, scans): Scan one file or directory and return a scanned data, calling every scan in the `scans` mapping of (scan name -> scan function). """ - scanned_file = OrderedDict() + scan_result = OrderedDict() for scan_name, scan_func in scans.items(): if not scan_func: continue try: - scan = scan_func(input_file) - if isinstance(scan, GeneratorType): - scan = list(scan) - # this is special and we flatten these as direct attributes of a file object - if scan_name == 'infos': - for file_infos in scan: - scanned_file.update(file_infos.items()) - else: - scanned_file[scan_name] = scan + scan_details = scan_func(input_file) + # consume generators + if isinstance(scan_details, GeneratorType): + scan_details = list(scan_details) + scan_result[scan_name] = scan_details except Exception, e: # never fail but instead add an error message. - scanned_file[scan_name] = {'errors': e.message} - return scanned_file + # FIXME: this should not be stored at the individual scan level + scan_result[scan_name] = {'errors': e.message} + return scan_result -def save_results(scanned_files, format, input, output_file): +def save_results(files_count, scanned_files, format, input, output_file): """ Save results to file or screen. """ @@ -397,9 +423,10 @@ def save_results(scanned_files, format, input, output_file): meta = OrderedDict() meta['scancode_notice'] = acknowledgment_text_json meta['scancode_version'] = version - meta['files_count'] = len(scanned_files) + meta['files_count'] = files_count # TODO: add scanning options to meta meta['files'] = scanned_files - output_file.write(json.dumps(meta, indent=2)) + # json.dump(meta, output_file, indent=2) + json.dump(meta, output_file, indent=2 * ' ', iterable_as_array=True) else: raise Exception('unknown format') diff --git a/src/scancode/format.py b/src/scancode/format.py index df6e64e93e9..9fb327a317a 100644 --- a/src/scancode/format.py +++ b/src/scancode/format.py @@ -34,6 +34,8 @@ from os.path import isfile from os.path import join +import simplejson as json + from commoncode import fileutils @@ -130,10 +132,10 @@ def create_html_app_assets(results, output_file): fileutils.copytree(assets_dir, target_dir) # write json data - import json root_path, assets_dir = get_html_app_files_dirs(output_file) with open(join(root_path, assets_dir, 'data.json'), 'w') as f: - f.write('data=' + json.dumps(results)) + f.write('data=') + json.dump(results, f, iterable_as_array=True) # create help file with open(join(root_path, assets_dir, 'help.html'), 'w') as f: diff --git a/tests/scancode/data/api/package/package.json b/tests/scancode/data/api/package/package.json new file mode 100644 index 00000000000..247e1784187 --- /dev/null +++ b/tests/scancode/data/api/package/package.json @@ -0,0 +1,96 @@ +{ + "name": "async", + "description": "Higher-order functions and common patterns for asynchronous code", + "main": "lib/async.js", + "author": { + "name": "Caolan McMahon" + }, + "version": "1.2.1", + "keywords": [ + "async", + "callback", + "utility", + "module" + ], + "repository": { + "type": "git", + "url": "git+https://github.com/caolan/async.git" + }, + "bugs": { + "url": "https://github.com/caolan/async/issues" + }, + "license": "MIT", + "devDependencies": { + "benchmark": "github:bestiejs/benchmark.js", + "coveralls": "^2.11.2", + "jshint": "~2.7.0", + "lodash": ">=2.4.1", + "mkdirp": "~0.5.1", + "nodeunit": ">0.0.0", + "nyc": "^2.1.0", + "uglify-js": "1.2.x", + "yargs": "~3.9.1" + }, + "jam": { + "main": "lib/async.js", + "include": [ + "lib/async.js", + "README.md", + "LICENSE" + ], + "categories": [ + "Utilities" + ] + }, + "scripts": { + "test": "npm run-script lint && nodeunit test/test-async.js", + "lint": "jshint lib/*.js test/*.js perf/*.js", + "coverage": "nyc npm test && nyc report", + "coveralls": "nyc npm test && nyc report --reporter=text-lcov | coveralls" + }, + "spm": { + "main": "lib/async.js" + }, + "volo": { + "main": "lib/async.js", + "ignore": [ + "**/.*", + "node_modules", + "bower_components", + "test", + "tests" + ] + }, + "gitHead": "b66e85d1cca8c8056313253f22d18f571e7001d2", + "homepage": "https://github.com/caolan/async#readme", + "_id": "async@1.2.1", + "_shasum": "a4816a17cd5ff516dfa2c7698a453369b9790de0", + "_from": "async@*", + "_npmVersion": "2.9.0", + "_nodeVersion": "2.0.2", + "_npmUser": { + "name": "aearly", + "email": "alexander.early@gmail.com" + }, + "maintainers": [ + { + "name": "caolan", + "email": "caolan.mcmahon@gmail.com" + }, + { + "name": "beaugunderson", + "email": "beau@beaugunderson.com" + }, + { + "name": "aearly", + "email": "alexander.early@gmail.com" + } + ], + "dist": { + "shasum": "a4816a17cd5ff516dfa2c7698a453369b9790de0", + "tarball": "http://registry.npmjs.org/async/-/async-1.2.1.tgz" + }, + "directories": {}, + "_resolved": "https://registry.npmjs.org/async/-/async-1.2.1.tgz", + "readme": "ERROR: No README data found!" +} diff --git a/tests/scancode/data/cache/package/package.json b/tests/scancode/data/cache/package/package.json new file mode 100644 index 00000000000..247e1784187 --- /dev/null +++ b/tests/scancode/data/cache/package/package.json @@ -0,0 +1,96 @@ +{ + "name": "async", + "description": "Higher-order functions and common patterns for asynchronous code", + "main": "lib/async.js", + "author": { + "name": "Caolan McMahon" + }, + "version": "1.2.1", + "keywords": [ + "async", + "callback", + "utility", + "module" + ], + "repository": { + "type": "git", + "url": "git+https://github.com/caolan/async.git" + }, + "bugs": { + "url": "https://github.com/caolan/async/issues" + }, + "license": "MIT", + "devDependencies": { + "benchmark": "github:bestiejs/benchmark.js", + "coveralls": "^2.11.2", + "jshint": "~2.7.0", + "lodash": ">=2.4.1", + "mkdirp": "~0.5.1", + "nodeunit": ">0.0.0", + "nyc": "^2.1.0", + "uglify-js": "1.2.x", + "yargs": "~3.9.1" + }, + "jam": { + "main": "lib/async.js", + "include": [ + "lib/async.js", + "README.md", + "LICENSE" + ], + "categories": [ + "Utilities" + ] + }, + "scripts": { + "test": "npm run-script lint && nodeunit test/test-async.js", + "lint": "jshint lib/*.js test/*.js perf/*.js", + "coverage": "nyc npm test && nyc report", + "coveralls": "nyc npm test && nyc report --reporter=text-lcov | coveralls" + }, + "spm": { + "main": "lib/async.js" + }, + "volo": { + "main": "lib/async.js", + "ignore": [ + "**/.*", + "node_modules", + "bower_components", + "test", + "tests" + ] + }, + "gitHead": "b66e85d1cca8c8056313253f22d18f571e7001d2", + "homepage": "https://github.com/caolan/async#readme", + "_id": "async@1.2.1", + "_shasum": "a4816a17cd5ff516dfa2c7698a453369b9790de0", + "_from": "async@*", + "_npmVersion": "2.9.0", + "_nodeVersion": "2.0.2", + "_npmUser": { + "name": "aearly", + "email": "alexander.early@gmail.com" + }, + "maintainers": [ + { + "name": "caolan", + "email": "caolan.mcmahon@gmail.com" + }, + { + "name": "beaugunderson", + "email": "beau@beaugunderson.com" + }, + { + "name": "aearly", + "email": "alexander.early@gmail.com" + } + ], + "dist": { + "shasum": "a4816a17cd5ff516dfa2c7698a453369b9790de0", + "tarball": "http://registry.npmjs.org/async/-/async-1.2.1.tgz" + }, + "directories": {}, + "_resolved": "https://registry.npmjs.org/async/-/async-1.2.1.tgz", + "readme": "ERROR: No README data found!" +} diff --git a/tests/scancode/test_api.py b/tests/scancode/test_api.py new file mode 100644 index 00000000000..683cf00982b --- /dev/null +++ b/tests/scancode/test_api.py @@ -0,0 +1,49 @@ +# +# Copyright (c) 2016 nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/scancode-toolkit/ +# The ScanCode software is licensed under the Apache License version 2.0. +# Data generated with ScanCode require an acknowledgment. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with ScanCode or any ScanCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# ScanCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode-toolkit/ for support and download. + +from __future__ import absolute_import, print_function + +import os + +from commoncode.testcase import FileBasedTesting + +from scancode import api + + +class TestAPI(FileBasedTesting): + test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + + def test_get_package_infos_can_pickle(self): + test_file = self.get_test_loc('api/package/package.json') + package = api.get_package_infos(test_file) + + import pickle + import cPickle + try: + _pickled = pickle.dumps(package, pickle.HIGHEST_PROTOCOL) + _cpickled = cPickle.dumps(package, pickle.HIGHEST_PROTOCOL) + self.fail('pickle.HIGHEST_PROTOCOL used to fail to pickle this data') + except: + _pickled = pickle.dumps(package) + _cpickled = cPickle.dumps(package) diff --git a/tests/scancode/test_cli.py b/tests/scancode/test_cli.py index d32b40b52d2..8853be74c33 100644 --- a/tests/scancode/test_cli.py +++ b/tests/scancode/test_cli.py @@ -92,7 +92,7 @@ def test_package_option_detects_packages(monkeypatch): test_dir = test_env.get_test_loc('package', copy=True) runner = CliRunner() result_file = test_env.get_temp_file('json') - result = runner.invoke(cli.scancode, ['--package', test_dir, result_file]) + result = runner.invoke(cli.scancode, ['--package', test_dir, result_file], catch_exceptions=True) assert result.exit_code == 0 assert 'Scanning done' in result.output assert 'package.json' in result.output @@ -105,7 +105,7 @@ def test_verbose_option_with_packages(monkeypatch): test_dir = test_env.get_test_loc('package', copy=True) runner = CliRunner() result_file = test_env.get_temp_file('json') - result = runner.invoke(cli.scancode, ['--package', '--verbose', test_dir, result_file]) + result = runner.invoke(cli.scancode, ['--package', '--verbose', test_dir, result_file], catch_exceptions=True) assert result.exit_code == 0 assert 'Scanning done' in result.output assert 'package.json' in result.output @@ -118,7 +118,7 @@ def test_copyright_option_detects_copyrights(monkeypatch): test_dir = test_env.get_test_loc('copyright', copy=True) runner = CliRunner() result_file = test_env.get_temp_file('json') - result = runner.invoke(cli.scancode, ['--copyright', test_dir, result_file]) + result = runner.invoke(cli.scancode, ['--copyright', test_dir, result_file], catch_exceptions=True) assert result.exit_code == 0 assert 'Scanning done' in result.output assert os.path.exists(result_file) @@ -130,7 +130,7 @@ def test_verbose_option_with_copyrights(monkeypatch): test_dir = test_env.get_test_loc('copyright', copy=True) runner = CliRunner() result_file = test_env.get_temp_file('json') - result = runner.invoke(cli.scancode, ['--copyright', '--verbose', test_dir, result_file]) + result = runner.invoke(cli.scancode, ['--copyright', '--verbose', test_dir, result_file], catch_exceptions=True) assert result.exit_code == 0 assert 'Scanning done' in result.output assert 'copyright_acme_c-c.c' in result.output @@ -143,7 +143,7 @@ def test_license_option_detects_licenses(monkeypatch): test_dir = test_env.get_test_loc('license', copy=True) runner = CliRunner() result_file = test_env.get_temp_file('json') - result = runner.invoke(cli.scancode, ['--license', test_dir, result_file]) + result = runner.invoke(cli.scancode, ['--license', test_dir, result_file], catch_exceptions=True) assert result.exit_code == 0 assert 'Scanning done' in result.output assert os.path.exists(result_file) @@ -155,7 +155,7 @@ def test_scancode_skip_vcs_files_and_dirs_by_default(monkeypatch): test_dir = test_env.extract_test_tar('ignore/vcs.tgz') runner = CliRunner() result_file = test_env.get_temp_file('json') - result = runner.invoke(cli.scancode, ['--copyright', test_dir, result_file]) + result = runner.invoke(cli.scancode, ['--copyright', test_dir, result_file], catch_exceptions=True) assert result.exit_code == 0 scan_result = _load_json_result(result_file, test_dir) # a single test.tst file and its directory that is not a VCS file should be listed @@ -172,12 +172,12 @@ def test_usage_and_help_return_a_correct_script_name_on_all_platforms(monkeypatc # this was showing up on Windows assert 'scancode-script.py' not in result.output - result = runner.invoke(cli.scancode, []) + result = runner.invoke(cli.scancode, [], catch_exceptions=True) assert 'Usage: scancode [OPTIONS]' in result.output # this was showing up on Windows assert 'scancode-script.py' not in result.output - result = runner.invoke(cli.scancode, ['-xyz']) + result = runner.invoke(cli.scancode, ['-xyz'], catch_exceptions=True) # this was showing up on Windows assert 'scancode-script.py' not in result.output @@ -187,7 +187,7 @@ def test_scan_info_does_collect_infos(monkeypatch): test_dir = test_env.extract_test_tar('info/basic.tgz') runner = CliRunner() result_file = test_env.get_temp_file('json') - result = runner.invoke(cli.scancode, ['--info', test_dir, result_file]) + result = runner.invoke(cli.scancode, ['--info', test_dir, result_file], catch_exceptions=True) assert result.exit_code == 0 assert 'Scanning done' in result.output check_scan(test_env.get_test_loc('info/basic.expected.json'), result_file, test_dir) @@ -198,7 +198,7 @@ def test_scan_info_license_copyrights(monkeypatch): test_dir = test_env.extract_test_tar('info/basic.tgz') runner = CliRunner() result_file = test_env.get_temp_file('json') - result = runner.invoke(cli.scancode, ['--info', '--license', '--copyright', test_dir, result_file]) + result = runner.invoke(cli.scancode, ['--info', '--license', '--copyright', test_dir, result_file], catch_exceptions=True) assert result.exit_code == 0 assert 'Scanning done' in result.output check_scan(test_env.get_test_loc('info/all.expected.json'), result_file, test_dir) @@ -209,7 +209,7 @@ def test_scan_email_url_info(monkeypatch): test_dir = test_env.extract_test_tar('info/basic.tgz') runner = CliRunner() result_file = test_env.get_temp_file('json') - result = runner.invoke(cli.scancode, ['--email', '--url', '--info', test_dir, result_file]) + result = runner.invoke(cli.scancode, ['--email', '--url', '--info', test_dir, result_file], catch_exceptions=True) assert result.exit_code == 0 assert 'Scanning done' in result.output check_scan(test_env.get_test_loc('info/email_url_info.expected.json'), result_file, test_dir) @@ -220,7 +220,7 @@ def test_paths_are_posix_paths_in_html_app_format_output(monkeypatch): test_dir = test_env.get_test_loc('posix_path', copy=True) runner = CliRunner() result_file = test_env.get_temp_file(extension='html', file_name='test_html') - result = runner.invoke(cli.scancode, [ '--copyright', '--format', 'html-app', test_dir, result_file]) + result = runner.invoke(cli.scancode, [ '--copyright', '--format', 'html-app', test_dir, result_file], catch_exceptions=True) assert result.exit_code == 0 assert 'Scanning done' in result.output # the data we want to test is in the data.json file @@ -233,7 +233,7 @@ def test_paths_are_posix_in_html_format_output(monkeypatch): test_dir = test_env.get_test_loc('posix_path', copy=True) runner = CliRunner() result_file = test_env.get_temp_file('html') - result = runner.invoke(cli.scancode, [ '--copyright', '--format', 'html', test_dir, result_file]) + result = runner.invoke(cli.scancode, [ '--copyright', '--format', 'html', test_dir, result_file], catch_exceptions=True) assert result.exit_code == 0 assert 'Scanning done' in result.output assert '/posix_path/copyright_acme_c-c.c' in open(result_file).read() @@ -244,7 +244,7 @@ def test_paths_are_posix_in_json_format_output(monkeypatch): test_dir = test_env.get_test_loc('posix_path', copy=True) runner = CliRunner() result_file = test_env.get_temp_file('json') - result = runner.invoke(cli.scancode, [ '--copyright', '--format', 'json', test_dir, result_file]) + result = runner.invoke(cli.scancode, [ '--copyright', '--format', 'json', test_dir, result_file], catch_exceptions=True) assert result.exit_code == 0 assert 'Scanning done' in result.output assert '/posix_path/copyright_acme_c-c.c' in open(result_file).read() @@ -255,7 +255,7 @@ def test_format_with_custom_filename_fails_for_directory(monkeypatch): test_dir = test_env.get_test_loc('posix_path', copy=True) runner = CliRunner() result_file = test_env.get_temp_file('html') - result = runner.invoke(cli.scancode, [ '--format', test_dir, test_dir, result_file]) + result = runner.invoke(cli.scancode, [ '--format', test_dir, test_dir, result_file], catch_exceptions=True) assert result.exit_code != 0 assert 'Invalid template file' in result.output @@ -266,7 +266,7 @@ def test_format_with_custom_filename(monkeypatch): runner = CliRunner() template = test_env.get_test_loc('template/sample-template.html') result_file = test_env.get_temp_file('html') - result = runner.invoke(cli.scancode, [ '--format', template, test_dir, result_file]) + result = runner.invoke(cli.scancode, [ '--format', template, test_dir, result_file], catch_exceptions=True) assert result.exit_code == 0 assert 'Custom Template' in open(result_file).read() @@ -276,7 +276,7 @@ def test_scanned_path_is_present_in_html_app_output(monkeypatch): test_dir = test_env.get_test_loc('html_app') runner = CliRunner() result_file = test_env.get_temp_file('test.html') - result = runner.invoke(cli.scancode, [ '--copyright', '--format', 'html-app', test_dir, result_file]) + result = runner.invoke(cli.scancode, [ '--copyright', '--format', 'html-app', test_dir, result_file], catch_exceptions=True) assert result.exit_code == 0 assert 'Scanning done' in result.output html_file = open(result_file).read() @@ -288,8 +288,7 @@ def test_scan_should_not_fail_on_faulty_pdf_or_pdfminer_bug_but_instead_report_e test_file = test_env.get_test_loc('failing/patchelf.pdf') runner = CliRunner() result_file = test_env.get_temp_file('test.json') - print('====>', cli.scancode, [ '--copyright', test_file, result_file]) - result = runner.invoke(cli.scancode, [ '--copyright', test_file, result_file]) + result = runner.invoke(cli.scancode, [ '--copyright', test_file, result_file], catch_exceptions=True) assert result.exit_code == 0 assert 'Scanning done' in result.output check_scan(test_env.get_test_loc('failing/patchelf.expected.json'), result_file, test_file) diff --git a/tests/scancode/test_scan_cache.py b/tests/scancode/test_scan_cache.py new file mode 100644 index 00000000000..5f5b8422702 --- /dev/null +++ b/tests/scancode/test_scan_cache.py @@ -0,0 +1,45 @@ +# +# Copyright (c) 2016 nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/scancode-toolkit/ +# The ScanCode software is licensed under the Apache License version 2.0. +# Data generated with ScanCode require an acknowledgment. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with ScanCode or any ScanCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# ScanCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode-toolkit/ for support and download. + +from __future__ import absolute_import, print_function + +import os + +from commoncode.testcase import FileBasedTesting + +from scancode.cache import ScanCache + + +class TestCache(FileBasedTesting): + test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + + def test_can_cache(self): + test_file = self.get_test_loc('cache/package/package.json') + from scancode import api + package = api.get_package_infos(test_file) + + test_dir = self.get_temp_dir() + cache = ScanCache(test_dir) + cache.put_infos(path='abc', file_infos=dict(sha1='def')) + cache.put_scan(path='abc', file_infos=dict(sha1='def'), scan_result=package) diff --git a/thirdparty/prod/diskcache-1.7.0-py2.py3-none-any.whl b/thirdparty/prod/diskcache-1.7.0-py2.py3-none-any.whl deleted file mode 100644 index d4227475687..00000000000 Binary files a/thirdparty/prod/diskcache-1.7.0-py2.py3-none-any.whl and /dev/null differ diff --git a/thirdparty/prod/diskcache-2.0.2-py2.py3-none-any.whl b/thirdparty/prod/diskcache-2.0.2-py2.py3-none-any.whl new file mode 100644 index 00000000000..fc93d3ca8af Binary files /dev/null and b/thirdparty/prod/diskcache-2.0.2-py2.py3-none-any.whl differ diff --git a/thirdparty/prod/diskcache.ABOUT b/thirdparty/prod/diskcache.ABOUT index 3b68e8e149d..ca004087253 100644 --- a/thirdparty/prod/diskcache.ABOUT +++ b/thirdparty/prod/diskcache.ABOUT @@ -1,13 +1,13 @@ -about_resource: diskcache-1.7.0-py2.py3-none-any.whl +about_resource: diskcache-2.0.2-py2.py3-none-any.whl name: python-diskcache -version: 1.7.0 +version: 2.0.2 owner: Grant Jenks home_url: https://github.com/grantjenks/python-diskcache/ -download_url: https://pypi.python.org/packages/0f/fc/b24e3d66525e09c621817ae1d3804e7ec605e5d91cf1c2da3d755c4b9e52/diskcache-1.7.0.tar.gz#md5=8fcce03a1b25b37b376a0982c50d70eb +download_url: https://pypi.python.org/packages/9f/71/36193e75dfa37bebad9a46d80bb003c52602de69ad2b945d0987a80c2d59/diskcache-2.0.2.tar.gz#md5=c3d75a2a99e87da6844bd882e7be82fe vcs_tool: git vcs_repository: https://github.com/grantjenks/python-diskcache.git diff --git a/thirdparty/prod/simplejson-3.10.0-cp27-cp27m-linux_i686.whl b/thirdparty/prod/simplejson-3.10.0-cp27-cp27m-linux_i686.whl new file mode 100644 index 00000000000..7fc12591cc0 Binary files /dev/null and b/thirdparty/prod/simplejson-3.10.0-cp27-cp27m-linux_i686.whl differ diff --git a/thirdparty/prod/simplejson-3.10.0-cp27-cp27m-linux_x86_64.whl b/thirdparty/prod/simplejson-3.10.0-cp27-cp27m-linux_x86_64.whl new file mode 100644 index 00000000000..e81390dcc59 Binary files /dev/null and b/thirdparty/prod/simplejson-3.10.0-cp27-cp27m-linux_x86_64.whl differ diff --git a/thirdparty/prod/simplejson-3.10.0-cp27-cp27m-macosx_10_11_x86_64.whl b/thirdparty/prod/simplejson-3.10.0-cp27-cp27m-macosx_10_11_x86_64.whl new file mode 100644 index 00000000000..baa8ac76187 Binary files /dev/null and b/thirdparty/prod/simplejson-3.10.0-cp27-cp27m-macosx_10_11_x86_64.whl differ diff --git a/thirdparty/prod/simplejson-3.10.0-cp27-cp27m-win32.whl b/thirdparty/prod/simplejson-3.10.0-cp27-cp27m-win32.whl new file mode 100644 index 00000000000..383a9a05c41 Binary files /dev/null and b/thirdparty/prod/simplejson-3.10.0-cp27-cp27m-win32.whl differ diff --git a/thirdparty/prod/simplejson-3.10.0-cp27-cp27m-win_amd64.whl b/thirdparty/prod/simplejson-3.10.0-cp27-cp27m-win_amd64.whl new file mode 100644 index 00000000000..4d046a12550 Binary files /dev/null and b/thirdparty/prod/simplejson-3.10.0-cp27-cp27m-win_amd64.whl differ diff --git a/thirdparty/prod/simplejson-3.10.0-cp27-cp27mu-linux_i686.whl b/thirdparty/prod/simplejson-3.10.0-cp27-cp27mu-linux_i686.whl new file mode 100644 index 00000000000..5c4e6081030 Binary files /dev/null and b/thirdparty/prod/simplejson-3.10.0-cp27-cp27mu-linux_i686.whl differ diff --git a/thirdparty/prod/simplejson-3.10.0-cp27-cp27mu-linux_x86_64.whl b/thirdparty/prod/simplejson-3.10.0-cp27-cp27mu-linux_x86_64.whl new file mode 100644 index 00000000000..c23d949acc4 Binary files /dev/null and b/thirdparty/prod/simplejson-3.10.0-cp27-cp27mu-linux_x86_64.whl differ diff --git a/thirdparty/prod/simplejson-3.10.0.tar.gz b/thirdparty/prod/simplejson-3.10.0.tar.gz new file mode 100644 index 00000000000..6a8512cfc43 Binary files /dev/null and b/thirdparty/prod/simplejson-3.10.0.tar.gz differ diff --git a/thirdparty/prod/simplejson.ABOUT b/thirdparty/prod/simplejson.ABOUT new file mode 100644 index 00000000000..e7164febb7b --- /dev/null +++ b/thirdparty/prod/simplejson.ABOUT @@ -0,0 +1,24 @@ +component: + name: simplejson + version: 3.10.0 + + home_url: https://github.com/simplejson/simplejson + owner: Bob Ippolito + copyright: Copyright (c) 2006 Bob Ippolito + license_expression: mit or afl-2.1 + license_text_file: simplejson.LICENSE + + vcs_tool: git + vcs_repository: https://github.com/simplejson/simplejson.git + + files: + - simplejson-3.10.0-cp27-cp27m-linux_i686.whl + - simplejson-3.10.0-cp27-cp27m-linux_x86_64.whl + - simplejson-3.10.0-cp27-cp27m-macosx_10_11_x86_64.whl + - simplejson-3.10.0-cp27-cp27m-win_amd64.whl + - simplejson-3.10.0-cp27-cp27m-win32.whl + - simplejson-3.10.0-cp27-cp27mu-linux_i686.whl + - simplejson-3.10.0-cp27-cp27mu-linux_x86_64.whl + - simplejson-3.10.0.tar.gz + + \ No newline at end of file diff --git a/thirdparty/prod/simplejson.LICENSE b/thirdparty/prod/simplejson.LICENSE new file mode 100644 index 00000000000..e05f49c3fd0 --- /dev/null +++ b/thirdparty/prod/simplejson.LICENSE @@ -0,0 +1,79 @@ +simplejson is dual-licensed software. It is available under the terms +of the MIT license, or the Academic Free License version 2.1. The full +text of each license agreement is included below. This code is also +licensed to the Python Software Foundation (PSF) under a Contributor +Agreement. + +MIT License +=========== + +Copyright (c) 2006 Bob Ippolito + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +Academic Free License v. 2.1 +============================ + +Copyright (c) 2006 Bob Ippolito. All rights reserved. + +This Academic Free License (the "License") applies to any original work of authorship (the "Original Work") whose owner (the "Licensor") has placed the following notice immediately following the copyright notice for the Original Work: + +Licensed under the Academic Free License version 2.1 + +1) Grant of Copyright License. Licensor hereby grants You a world-wide, royalty-free, non-exclusive, perpetual, sublicenseable license to do the following: + +a) to reproduce the Original Work in copies; + +b) to prepare derivative works ("Derivative Works") based upon the Original Work; + +c) to distribute copies of the Original Work and Derivative Works to the public; + +d) to perform the Original Work publicly; and + +e) to display the Original Work publicly. + +2) Grant of Patent License. Licensor hereby grants You a world-wide, royalty-free, non-exclusive, perpetual, sublicenseable license, under patent claims owned or controlled by the Licensor that are embodied in the Original Work as furnished by the Licensor, to make, use, sell and offer for sale the Original Work and Derivative Works. + +3) Grant of Source Code License. The term "Source Code" means the preferred form of the Original Work for making modifications to it and all available documentation describing how to modify the Original Work. Licensor hereby agrees to provide a machine-readable copy of the Source Code of the Original Work along with each copy of the Original Work that Licensor distributes. Licensor reserves the right to satisfy this obligation by placing a machine-readable copy of the Source Code in an information repository reasonably calculated to permit inexpensive and convenient access by You for as long as Licensor continues to distribute the Original Work, and by publishing the address of that information repository in a notice immediately following the copyright notice that applies to the Original Work. + +4) Exclusions From License Grant. Neither the names of Licensor, nor the names of any contributors to the Original Work, nor any of their trademarks or service marks, may be used to endorse or promote products derived from this Original Work without express prior written permission of the Licensor. Nothing in this License shall be deemed to grant any rights to trademarks, copyrights, patents, trade secrets or any other intellectual property of Licensor except as expressly stated herein. No patent license is granted to make, use, sell or offer to sell embodiments of any patent claims other than the licensed claims defined in Section 2. No right is granted to the trademarks of Licensor even if such marks are included in the Original Work. Nothing in this License shall be interpreted to prohibit Licensor from licensing under different terms from this License any Original Work that Licensor otherwise would have a right to license. + +5) This section intentionally omitted. + +6) Attribution Rights. You must retain, in the Source Code of any Derivative Works that You create, all copyright, patent or trademark notices from the Source Code of the Original Work, as well as any notices of licensing and any descriptive text identified therein as an "Attribution Notice." You must cause the Source Code for any Derivative Works that You create to carry a prominent Attribution Notice reasonably calculated to inform recipients that You have modified the Original Work. + +7) Warranty of Provenance and Disclaimer of Warranty. Licensor warrants that the copyright in and to the Original Work and the patent rights granted herein by Licensor are owned by the Licensor or are sublicensed to You under the terms of this License with the permission of the contributor(s) of those copyrights and patent rights. Except as expressly stated in the immediately proceeding sentence, the Original Work is provided under this License on an "AS IS" BASIS and WITHOUT WARRANTY, either express or implied, including, without limitation, the warranties of NON-INFRINGEMENT, MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY OF THE ORIGINAL WORK IS WITH YOU. This DISCLAIMER OF WARRANTY constitutes an essential part of this License. No license to Original Work is granted hereunder except under this disclaimer. + +8) Limitation of Liability. Under no circumstances and under no legal theory, whether in tort (including negligence), contract, or otherwise, shall the Licensor be liable to any person for any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or the use of the Original Work including, without limitation, damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses. This limitation of liability shall not apply to liability for death or personal injury resulting from Licensor's negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. + +9) Acceptance and Termination. If You distribute copies of the Original Work or a Derivative Work, You must make a reasonable effort under the circumstances to obtain the express assent of recipients to the terms of this License. Nothing else but this License (or another written agreement between Licensor and You) grants You permission to create Derivative Works based upon the Original Work or to exercise any of the rights granted in Section 1 herein, and any attempt to do so except under the terms of this License (or another written agreement between Licensor and You) is expressly prohibited by U.S. copyright law, the equivalent laws of other countries, and by international treaty. Therefore, by exercising any of the rights granted to You in Section 1 herein, You indicate Your acceptance of this License and all of its terms and conditions. + +10) Termination for Patent Action. This License shall terminate automatically and You may no longer exercise any of the rights granted to You by this License as of the date You commence an action, including a cross-claim or counterclaim, against Licensor or any licensee alleging that the Original Work infringes a patent. This termination provision shall not apply for an action alleging patent infringement by combinations of the Original Work with other software or hardware. + +11) Jurisdiction, Venue and Governing Law. Any action or suit relating to this License may be brought only in the courts of a jurisdiction wherein the Licensor resides or in which Licensor conducts its primary business, and under the laws of that jurisdiction excluding its conflict-of-law provisions. The application of the United Nations Convention on Contracts for the International Sale of Goods is expressly excluded. Any use of the Original Work outside the scope of this License or after its termination shall be subject to the requirements and penalties of the U.S. Copyright Act, 17 U.S.C. ยง 101 et seq., the equivalent laws of other countries, and international treaty. This section shall survive the termination of this License. + +12) Attorneys Fees. In any action to enforce the terms of this License or seeking damages relating thereto, the prevailing party shall be entitled to recover its costs and expenses, including, without limitation, reasonable attorneys' fees and costs incurred in connection with such action, including any appeal of such action. This section shall survive the termination of this License. + +13) Miscellaneous. This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. + +14) Definition of "You" in This License. "You" throughout this License, whether in upper or lower case, means an individual or a legal entity exercising rights under, and complying with all of the terms of, this License. For legal entities, "You" includes any entity that controls, is controlled by, or is under common control with you. For purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + +15) Right to Use. You may use the Original Work in all ways not otherwise restricted or conditioned by this License or by law, and Licensor promises not to interfere with or be responsible for such uses by You. + +This license is Copyright (C) 2003-2004 Lawrence E. Rosen. All rights reserved. Permission is hereby granted to copy and distribute this license without modification. This license may not be modified without the express written permission of its copyright owner.