diff --git a/requirements.txt b/requirements.txt index ab2e6ecb4e5..d822ff3d871 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,7 @@ commoncode==31.0.0b4 construct==2.10.68 container-inspector==31.0.0 cryptography==36.0.2 -debian-inspector==30.0.0 +debian-inspector==31.0.0b1 dockerfile-parse==1.2.0 dparse2==0.6.1 extractcode==31.0.0 diff --git a/setup-mini.cfg b/setup-mini.cfg index 17eb423d02c..5afd9a586e7 100644 --- a/setup-mini.cfg +++ b/setup-mini.cfg @@ -71,7 +71,7 @@ install_requires = colorama >= 0.3.9 commoncode==31.0.0b4 container-inspector >= 31.0.0 - debian-inspector >= 30.0.0 + debian-inspector >= 31.0.0b1 dparse2 >= 0.6.1 fasteners fingerprints >= 0.6.0 @@ -112,6 +112,8 @@ install_requires = xmltodict >= 0.11.0 zipp >= 3.0.0; python_version < "3.9" typecode >= 30.0.0 +# typecode[full] >= 30.0.0 +# extractcode[full] >= 31.0.0 [options.packages.find] @@ -128,6 +130,8 @@ testing = pytest-xdist >= 2 aboutcode-toolkit >= 7.0.2 twine + black + isort docs = Sphinx >= 3.3.1 diff --git a/setup.cfg b/setup.cfg index 68e55e75c6e..c2e333f516e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -71,7 +71,7 @@ install_requires = colorama >= 0.3.9 commoncode==31.0.0b4 container-inspector >= 31.0.0 - debian-inspector >= 30.0.0 + debian-inspector >= 31.0.0b1 dparse2 >= 0.6.1 fasteners fingerprints >= 0.6.0 diff --git a/src/packagedcode/alpine.py b/src/packagedcode/alpine.py index 8caa4a226b5..fa6287598d4 100644 --- a/src/packagedcode/alpine.py +++ b/src/packagedcode/alpine.py @@ -1772,4 +1772,3 @@ def detect_licenses(locin, locout): # get_apkindex_licenses(loc, 'alpine-licenses.csv') get_apkbuild_licenses(loc, 'alpine-licenses.csv') detect_licenses('alpine-licenses.csv', 'alpine-licenses-detection.csv') - diff --git a/src/packagedcode/models.py b/src/packagedcode/models.py index 4f140ad9b01..1c6a2c498f9 100644 --- a/src/packagedcode/models.py +++ b/src/packagedcode/models.py @@ -25,6 +25,7 @@ from commoncode.datautils import Mapping from commoncode.datautils import String from commoncode.fileutils import as_posixpath +from commoncode.resource import Resource from typecode import contenttype """ @@ -1201,7 +1202,7 @@ def __attrs_post_init__(self, *args, **kwargs): self.package_uid = build_package_uid(self.purl) def to_dict(self): - return super().to_dict(with_details=False) + return super().to_dict(with_details=False) @classmethod def from_package_data(cls, package_data, datafile_path): @@ -1340,6 +1341,24 @@ def get_packages_files(self, codebase): yield resource +@attr.attributes(slots=True) +class PackageWithResources(Package): + """ + A Package with Resources. + """ + + resources = List( + item_type=Resource, + label='List of Resources', + help='List of Resources for this package.', + ) + + def to_dict(self): + package_data = super().to_dict() + package_data['resources'] = [resource.to_dict() for resource in self.resources] + return package_data + + def get_files_for_packages(codebase): """ Yield tuple of (Resource, package_uid) for all resources in codebase that are diff --git a/src/packagedcode/plugin_package.py b/src/packagedcode/plugin_package.py index de43ead4578..4bf9b7a7f51 100644 --- a/src/packagedcode/plugin_package.py +++ b/src/packagedcode/plugin_package.py @@ -25,6 +25,7 @@ from packagedcode.models import Dependency from packagedcode.models import Package from packagedcode.models import PackageData +from packagedcode.models import PackageWithResources TRACE = os.environ.get('SCANCODE_DEBUG_PACKAGE', False) @@ -153,6 +154,36 @@ def process_codebase(self, codebase, strip_root=False, **kwargs): create_package_and_deps(codebase, strip_root=strip_root, **kwargs) +def get_installed_packages(root_dir, processes=2, **kwargs): + """ + Yield Package and their Resources as they are found in `root_dir` + """ + from scancode import cli + + _, codebase = cli.run_scan( + input=root_dir, + processes=processes, + quiet=True, + verbose=False, + max_in_memory=0, + return_results=False, + return_codebase=True, + system_package=True, + ) + + packages_by_uid = {} + for package in codebase.attributes.packages: + p = PackageWithResources.from_dict(package) + packages_by_uid[p.package_uid] = p + + for resource in codebase.walk(): + for package_uid in resource.for_packages: + p = packages_by_uid[package_uid] + p.resources.append(resource) + + yield from packages_by_uid.values() + + def create_package_and_deps(codebase, strip_root=False, **kwargs): """ Create and save top-level Package and Dependency from the parsed diff --git a/src/scancode/cli.py b/src/scancode/cli.py index 4649c62a471..9b4cb0a78b2 100644 --- a/src/scancode/cli.py +++ b/src/scancode/cli.py @@ -504,7 +504,9 @@ def run_scan( echo_func=None, timing=False, keep_temp_files=False, + # TODO: Review return_results as it does not return Packages and Dependencies return_results=True, + return_codebase=False, test_mode=False, test_slow_mode=False, test_error_mode=False, @@ -522,6 +524,12 @@ def run_scan( error. See scancode() for arguments details. """ + assert not (return_results is True and return_codebase is True), 'Only one of return_results and return_codebase can be True' + + if return_codebase and max_in_memory > 0: + # We're keeping temp files otherwise the codebase is gone + keep_temp_files = True + plugins_option_defaults = {clio.name: clio.default for clio in plugin_options} requested_options = dict(plugins_option_defaults) requested_options.update(kwargs) @@ -667,7 +675,7 @@ def echo_func(*_args, **_kwargs): 'selected when using only scan data.') raise ScancodeCliUsageError(msg) - if not output_plugins and not return_results: + if not output_plugins and not (return_results or return_codebase): msg = ('ERROR: Missing output option(s): at least one output ' 'option is required to save scan results.') raise ScancodeCliUsageError(msg) @@ -1002,6 +1010,8 @@ def echo_func(*_args, **_kwargs): # the structure is exactly the same as the JSON output from formattedcode.output_json import get_results results = get_results(codebase, as_list=True, **requested_options) + elif return_codebase: + results = codebase finally: # remove temporary files @@ -1657,3 +1667,9 @@ def get_pretty_params(ctx, generic_paths=False): options.append((cli_opt, value)) return dict(sorted(args) + sorted(options)) + + +if __name__ == '__main__': + # We have this __main__ block so that we can run scancode as a script. + # This is needed so we can use the Python debugging features in VSCode. + scancode() diff --git a/tests/packagedcode/data/plugin/get_installed_packages-expected.json b/tests/packagedcode/data/plugin/get_installed_packages-expected.json new file mode 100644 index 00000000000..384cf8c2a72 --- /dev/null +++ b/tests/packagedcode/data/plugin/get_installed_packages-expected.json @@ -0,0 +1,485 @@ +[ + { + "type": "deb", + "namespace": null, + "name": "libncurses5", + "version": "6.1-1ubuntu1.18.04", + "qualifiers": { + "architecture": "amd64" + }, + "subpath": null, + "primary_language": null, + "description": "shared libraries for terminal handling\n The ncurses library routines are a terminal-independent method of\n updating character screens with reasonable optimization.\n .\n This package contains the shared libraries necessary to run programs\n compiled with ncurses.", + "release_date": null, + "parties": [ + { + "type": null, + "role": "maintainer", + "name": "Ubuntu Developers ", + "email": null, + "url": null + } + ], + "keywords": [ + "libs" + ], + "homepage_url": "https://invisible-island.net/ncurses/", + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "source_packages": [ + "pkg:deb/ncurses" + ], + "extra_data": { + "multi_arch": "same", + "missing_file_references": [ + { + "path": "lib/x86_64-linux-gnu/libncurses.so.5.9", + "size": 0, + "sha1": null, + "md5": "23c8a935fa4fc7290d55cc5df3ef56b1", + "sha256": null, + "sha512": null, + "extra_data": {} + }, + { + "path": "usr/lib/x86_64-linux-gnu/libform.so.5.9", + "size": 0, + "sha1": null, + "md5": "98b70f283324e89db5787a018a54adf4", + "sha256": null, + "sha512": null, + "extra_data": {} + }, + { + "path": "usr/lib/x86_64-linux-gnu/libmenu.so.5.9", + "size": 0, + "sha1": null, + "md5": "e3a0f5154928da2da234920343ac14b2", + "sha256": null, + "sha512": null, + "extra_data": {} + }, + { + "path": "usr/lib/x86_64-linux-gnu/libpanel.so.5.9", + "size": 0, + "sha1": null, + "md5": "a927e7d76753bb85f5a784b653d337d2", + "sha256": null, + "sha512": null, + "extra_data": {} + } + ] + }, + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, + "package_uid": "pkg:deb/libncurses5@6.1-1ubuntu1.18.04?architecture=amd64&uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_paths": [ + "basic-rootfs.tar.gz/var/lib/dpkg/status" + ], + "datasource_ids": [ + "debian_installed_status_db" + ], + "resources": [ + { + "path": "basic-rootfs.tar.gz/usr/share/doc/libncurses5/copyright", + "type": "file", + "package_data": [ + { + "type": "deb", + "namespace": null, + "name": "libncurses5", + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": "Copyright (c) 1998-2016 Free Software Foundation, Inc.\nCopyright (c) 2001 by Pradeep Padala\nCopyright (c) 1994 X Consortium\nCopyright (c) 1980, 1991, 1992, 1993 The Regents of the University of California\nCopyright 1996-2007 by Thomas E. Dickey", + "license_expression": "x11-fsf AND x11-xconsortium AND bsd-new AND x11-fsf", + "declared_license": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "extra_data": {}, + "dependencies": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, + "datasource_id": "debian_copyright_in_package", + "purl": "pkg:deb/libncurses5" + } + ], + "for_packages": [ + "pkg:deb/libncurses5@6.1-1ubuntu1.18.04?architecture=amd64&uuid=fixed-uid-done-for-testing-5642512d1758" + ], + "scan_errors": [] + }, + { + "path": "basic-rootfs.tar.gz/var/lib/dpkg/info/libncurses5:amd64.md5sums", + "type": "file", + "package_data": [ + { + "type": "deb", + "namespace": null, + "name": "libncurses5", + "version": null, + "qualifiers": { + "arch": "amd64" + }, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "source_packages": [], + "file_references": [ + { + "path": "lib/x86_64-linux-gnu/libncurses.so.5.9", + "size": 0, + "sha1": null, + "md5": "23c8a935fa4fc7290d55cc5df3ef56b1", + "sha256": null, + "sha512": null, + "extra_data": {} + }, + { + "path": "usr/lib/x86_64-linux-gnu/libform.so.5.9", + "size": 0, + "sha1": null, + "md5": "98b70f283324e89db5787a018a54adf4", + "sha256": null, + "sha512": null, + "extra_data": {} + }, + { + "path": "usr/lib/x86_64-linux-gnu/libmenu.so.5.9", + "size": 0, + "sha1": null, + "md5": "e3a0f5154928da2da234920343ac14b2", + "sha256": null, + "sha512": null, + "extra_data": {} + }, + { + "path": "usr/lib/x86_64-linux-gnu/libpanel.so.5.9", + "size": 0, + "sha1": null, + "md5": "a927e7d76753bb85f5a784b653d337d2", + "sha256": null, + "sha512": null, + "extra_data": {} + } + ], + "extra_data": {}, + "dependencies": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, + "datasource_id": "debian_installed_md5sums", + "purl": "pkg:deb/libncurses5?arch=amd64" + } + ], + "for_packages": [ + "pkg:deb/libncurses5@6.1-1ubuntu1.18.04?architecture=amd64&uuid=fixed-uid-done-for-testing-5642512d1758" + ], + "scan_errors": [] + } + ], + "purl": "pkg:deb/libncurses5@6.1-1ubuntu1.18.04?architecture=amd64" + }, + { + "type": "deb", + "namespace": null, + "name": "libndp0", + "version": "1.4-2ubuntu0.16.04.1", + "qualifiers": { + "architecture": "amd64" + }, + "subpath": null, + "primary_language": null, + "description": "Library for Neighbor Discovery Protocol\n libndp is a library for the IPv6 Neighbor Discovery Protocol (NDP). It\n contains functions for building and parsing NDP messages, and provides\n a high-level interface for sending and receiving NDP messages on a\n network interface.", + "release_date": null, + "parties": [ + { + "type": null, + "role": "maintainer", + "name": "Ubuntu Developers ", + "email": null, + "url": null + } + ], + "keywords": [ + "libs" + ], + "homepage_url": "http://libndp.org", + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "source_packages": [ + "pkg:deb/libndp" + ], + "extra_data": { + "multi_arch": "same", + "missing_file_references": [ + { + "path": "usr/lib/x86_64-linux-gnu/libndp.so.0.0.2", + "size": 0, + "sha1": null, + "md5": "5d26434efecc08048ab72357af804ef7", + "sha256": null, + "sha512": null, + "extra_data": {} + }, + { + "path": "usr/share/doc/libndp0/changelog.Debian.gz", + "size": 0, + "sha1": null, + "md5": "60d977e0c9a9fb07c1f8ae3090ea6f48", + "sha256": null, + "sha512": null, + "extra_data": {} + } + ] + }, + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, + "package_uid": "pkg:deb/libndp0@1.4-2ubuntu0.16.04.1?architecture=amd64&uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_paths": [ + "basic-rootfs.tar.gz/var/lib/dpkg/status" + ], + "datasource_ids": [ + "debian_installed_status_db" + ], + "resources": [ + { + "path": "basic-rootfs.tar.gz/usr/share/doc/libndp0/copyright", + "type": "file", + "package_data": [ + { + "type": "deb", + "namespace": null, + "name": "libndp0", + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": "Copyright 2013 Jiri Pirko \nCopyright 2014 Andrew Ayer ", + "license_expression": "(lgpl-2.1-plus AND lgpl-2.1-plus AND lgpl-2.1) AND (lgpl-2.1-plus AND lgpl-2.1-plus AND lgpl-2.1)", + "declared_license": [ + "LGPL-2.1+", + "LGPL-2.1+", + "LGPL-2.1+" + ], + "notice_text": null, + "source_packages": [], + "file_references": [], + "extra_data": {}, + "dependencies": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, + "datasource_id": "debian_copyright_in_package", + "purl": "pkg:deb/libndp0" + } + ], + "for_packages": [ + "pkg:deb/libndp0@1.4-2ubuntu0.16.04.1?architecture=amd64&uuid=fixed-uid-done-for-testing-5642512d1758", + "pkg:deb/libndp0@1.4-2ubuntu0.16.04.1?architecture=amd64&uuid=fixed-uid-done-for-testing-5642512d1758" + ], + "scan_errors": [] + }, + { + "path": "basic-rootfs.tar.gz/usr/share/doc/libndp0/copyright", + "type": "file", + "package_data": [ + { + "type": "deb", + "namespace": null, + "name": "libndp0", + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": "Copyright 2013 Jiri Pirko \nCopyright 2014 Andrew Ayer ", + "license_expression": "(lgpl-2.1-plus AND lgpl-2.1-plus AND lgpl-2.1) AND (lgpl-2.1-plus AND lgpl-2.1-plus AND lgpl-2.1)", + "declared_license": [ + "LGPL-2.1+", + "LGPL-2.1+", + "LGPL-2.1+" + ], + "notice_text": null, + "source_packages": [], + "file_references": [], + "extra_data": {}, + "dependencies": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, + "datasource_id": "debian_copyright_in_package", + "purl": "pkg:deb/libndp0" + } + ], + "for_packages": [ + "pkg:deb/libndp0@1.4-2ubuntu0.16.04.1?architecture=amd64&uuid=fixed-uid-done-for-testing-5642512d1758", + "pkg:deb/libndp0@1.4-2ubuntu0.16.04.1?architecture=amd64&uuid=fixed-uid-done-for-testing-5642512d1758" + ], + "scan_errors": [] + }, + { + "path": "basic-rootfs.tar.gz/var/lib/dpkg/info/libndp0:amd64.md5sums", + "type": "file", + "package_data": [ + { + "type": "deb", + "namespace": null, + "name": "libndp0", + "version": null, + "qualifiers": { + "arch": "amd64" + }, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "source_packages": [], + "file_references": [ + { + "path": "usr/lib/x86_64-linux-gnu/libndp.so.0.0.2", + "size": 0, + "sha1": null, + "md5": "5d26434efecc08048ab72357af804ef7", + "sha256": null, + "sha512": null, + "extra_data": {} + }, + { + "path": "usr/share/doc/libndp0/changelog.Debian.gz", + "size": 0, + "sha1": null, + "md5": "60d977e0c9a9fb07c1f8ae3090ea6f48", + "sha256": null, + "sha512": null, + "extra_data": {} + }, + { + "path": "usr/share/doc/libndp0/copyright", + "size": 0, + "sha1": null, + "md5": "3f4ecdd67d5b9427cdc66847bdd11cf4", + "sha256": null, + "sha512": null, + "extra_data": {} + } + ], + "extra_data": {}, + "dependencies": [], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, + "datasource_id": "debian_installed_md5sums", + "purl": "pkg:deb/libndp0?arch=amd64" + } + ], + "for_packages": [ + "pkg:deb/libndp0@1.4-2ubuntu0.16.04.1?architecture=amd64&uuid=fixed-uid-done-for-testing-5642512d1758" + ], + "scan_errors": [] + } + ], + "purl": "pkg:deb/libndp0@1.4-2ubuntu0.16.04.1?architecture=amd64" + } +] \ No newline at end of file diff --git a/tests/packagedcode/packages_test_utils.py b/tests/packagedcode/packages_test_utils.py index bc6787dc771..6d4d7356486 100644 --- a/tests/packagedcode/packages_test_utils.py +++ b/tests/packagedcode/packages_test_utils.py @@ -14,6 +14,7 @@ from commoncode import testcase from commoncode import text +from scancode.cli_test_utils import purl_with_fake_uuid from scancode_config import REGEN_TEST_FIXTURES @@ -36,7 +37,14 @@ def check_package_data(self, package_data, expected_loc, regen=REGEN_TEST_FIXTUR regen=regen, ) - def check_packages_data(self, packages_data, expected_loc, must_exist=True, regen=REGEN_TEST_FIXTURES): + def check_packages_data( + self, + packages_data, + expected_loc, + must_exist=True, + remove_uuid=True, + regen=REGEN_TEST_FIXTURES + ): """ Helper to test a list of package_data objects against an expected JSON file. """ @@ -44,6 +52,15 @@ def check_packages_data(self, packages_data, expected_loc, must_exist=True, rege results = [] for package_data in packages_data: + if remove_uuid and hasattr(package_data, 'package_uid'): + package_data.package_uid = purl_with_fake_uuid(package_data.package_uid) + if remove_uuid and hasattr(package_data, 'resources'): + for resource in package_data.resources: + normalized_package_uids = [ + purl_with_fake_uuid(package_uid) + for package_uid in resource.for_packages + ] + resource.for_packages = normalized_package_uids compute_and_set_license_expression(package_data) results.append(package_data.to_dict()) diff --git a/tests/packagedcode/test_plugin_package.py b/tests/packagedcode/test_plugin_package.py index 4bbb4886a22..abff7824976 100644 --- a/tests/packagedcode/test_plugin_package.py +++ b/tests/packagedcode/test_plugin_package.py @@ -12,6 +12,7 @@ from commoncode.system import on_windows +from packagedcode.plugin_package import get_installed_packages from packages_test_utils import PackageTester from scancode.cli_test_utils import check_json_scan from scancode.cli_test_utils import run_scan_click @@ -211,3 +212,10 @@ def test_package_list_command(self, regen=REGEN_TEST_FIXTURES): with open(expected_file, 'w') as ef: ef.write(result.output) assert result.output == open(expected_file).read() + + def test_package_get_installed_packages(self): + test_dir = self.extract_test_tar('debian/basic-rootfs.tar.gz') + expected_file = self.get_test_loc('plugin/get_installed_packages-expected.json') + result_file = self.get_temp_file('results.json') + results = list(get_installed_packages(test_dir)) + self.check_packages_data(results, expected_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES)