diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e20f8ad9..4a6828f0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,14 @@ vNext ----- +Version 21.6.11 +--------------- + +- Do not fail if a Codebase does not have a common shared root #23 +- Consider all Resource attributes when building a VirtualCodebase #23 +- Do not ignore by default sccs and rcs dirs https://github.com/nexB/scancode-toolkit/issues/1422 + + Version 21.6.10 --------------- diff --git a/src/commoncode/ignore.py b/src/commoncode/ignore.py index 477c0225..881b93f5 100644 --- a/src/commoncode/ignore.py +++ b/src/commoncode/ignore.py @@ -184,7 +184,6 @@ def get_ignores(location, include_defaults=True): 'CVS': 'Default ignore: CVS artifact', '.cvsignore': 'Default ignore: CVS config artifact', - '*/SCCS': 'Default ignore: CVS artifact', '*/_MTN': 'Default ignore: Monotone artifact', '*/_darcs': 'Default ignore: Darcs artifact', diff --git a/src/commoncode/resource.py b/src/commoncode/resource.py index af4098a4..ebf3421f 100644 --- a/src/commoncode/resource.py +++ b/src/commoncode/resource.py @@ -12,6 +12,7 @@ import posixpath import traceback import sys +from pathlib import Path from collections import deque from functools import partial @@ -1530,7 +1531,7 @@ def _get_or_create_parent(self, path, parent_by_path): Note: the root path and root Resource must already be in `parent_by_path` or else this function does not work. """ - parent_path = parent_directory(path).rstrip('/').rstrip('\\').lstrip("/") + parent_path = parent_directory(path).rstrip('/').rstrip('\\').lstrip('/') existing_parent = parent_by_path.get(parent_path) if existing_parent: return existing_parent @@ -1548,6 +1549,13 @@ def _get_or_create_parent(self, path, parent_by_path): parent_by_path[parent_path] = parent_resource return parent_resource + def _set_new_root_directory(self, resources_data, new_root_directory_path): + for resource_data in resources_data: + resource_path = Path(resource_data['path']) + new_resource_path = Path(new_root_directory_path) + new_resource_path = new_resource_path.joinpath(resource_path) + resource_data['path'] = str(new_resource_path) + def _populate(self, scan_data): """ Populate this codebase with Resource objects. @@ -1645,6 +1653,21 @@ def _populate(self, scan_data): sample_resource_path = sample_resource_data['path'] sample_resource_path = sample_resource_path.strip('/') root_path = sample_resource_path.split('/')[0] + + # Check to see if the Resources from the scan we received has a common root directory. + for resource_data in resources_data: + resource_path = resource_data.get('path') + resource_path = resource_path.strip('/') + resource_root_path = resource_path.split('/')[0] + # If not, set a common root directory for all Resources. + if resource_root_path != root_path: + self._set_new_root_directory( + resources_data=resources_data, + new_root_directory_path='virtual_root' + ) + root_path = 'virtual_root' + break + root_name = root_path root_is_file = False root_data = self._create_empty_resource_data() diff --git a/tests/data/resource/virtual_codebase/docker-hello-world.json b/tests/data/resource/virtual_codebase/docker-hello-world.json new file mode 100644 index 00000000..86eafc18 --- /dev/null +++ b/tests/data/resource/virtual_codebase/docker-hello-world.json @@ -0,0 +1,100 @@ +{ +"headers": [ +{ + "tool_name": "scanpipe", + "tool_version": "21.5.12", + "notice": "Generated with ScanCode.io and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied.\nNo content created from ScanCode.io should be considered or used as legal advice.\nConsult an Attorney for any legal advice.\nScanCode.io is a free software code scanning tool from nexB Inc. and others\nlicensed under the Apache License version 2.0.\nScanCode is a trademark of nexB Inc.\nVisit https://github.com/nexB/scancode.io for support and download.\n", + "uuid": "2f8e9a35-048e-4d9f-ae56-feea8863d6eb", + "created_date": "2021-06-10T13:56:44.418Z", + "input_files": [ + "hello_world.tar" + ], + "runs": [ + { + "pipeline_name": "docker", + "description": "A pipeline to analyze a Docker image.", + "uuid": "106296b1-6334-4393-9f1a-209459954812", + "created_date": "2021-06-10T13:56:44.421302Z", + "task_id": "c851f2e7-7cc1-4952-8d93-1cdee4a147bd", + "task_start_date": "2021-06-10T13:56:44.434313Z", + "task_end_date": "2021-06-10T13:56:44.819773Z", + "task_exitcode": 0, + "task_output": "", + "log": "2021-06-10 13:56:44.43 Pipeline [docker] starting\n2021-06-10 13:56:44.43 Step [extract_images] starting\n2021-06-10 13:56:44.44 Step [extract_images] completed in 0.00 seconds\n2021-06-10 13:56:44.44 Step [extract_layers] starting\n2021-06-10 13:56:44.44 Step [extract_layers] completed in 0.00 seconds\n2021-06-10 13:56:44.45 Step [find_images_linux_distro] starting\n2021-06-10 13:56:44.45 Step [find_images_linux_distro] completed in 0.00 seconds\n2021-06-10 13:56:44.45 Step [collect_images_information] starting\n2021-06-10 13:56:44.45 Step [collect_images_information] completed in 0.00 seconds\n2021-06-10 13:56:44.46 Step [collect_and_create_codebase_resources] starting\n2021-06-10 13:56:44.46 Step [collect_and_create_codebase_resources] completed in 0.01 seconds\n2021-06-10 13:56:44.47 Step [collect_and_create_system_packages] starting\n2021-06-10 13:56:44.47 Step [collect_and_create_system_packages] completed in 0.00 seconds\n2021-06-10 13:56:44.47 Step [tag_uninteresting_codebase_resources] starting\n2021-06-10 13:56:44.48 Step [tag_uninteresting_codebase_resources] completed in 0.00 seconds\n2021-06-10 13:56:44.48 Step [tag_empty_files] starting\n2021-06-10 13:56:44.48 Step [tag_empty_files] completed in 0.00 seconds\n2021-06-10 13:56:44.48 Step [scan_for_application_packages] starting\n2021-06-10 13:56:44.53 Step [scan_for_application_packages] completed in 0.05 seconds\n2021-06-10 13:56:44.54 Step [scan_for_files] starting\n2021-06-10 13:56:44.80 Step [scan_for_files] completed in 0.26 seconds\n2021-06-10 13:56:44.80 Step [analyze_scanned_files] starting\n2021-06-10 13:56:44.81 Step [analyze_scanned_files] completed in 0.00 seconds\n2021-06-10 13:56:44.81 Step [tag_not_analyzed_codebase_resources] starting\n2021-06-10 13:56:44.81 Step [tag_not_analyzed_codebase_resources] completed in 0.00 seconds\n2021-06-10 13:56:44.81 Pipeline completed\n", + "execution_time": 0 + } + ], + "extra_data": { + "images": [ + { + "os": "linux", + "tags": [], + "author": null, + "distro": null, + "labels": {}, + "sha256": null, + "comment": null, + "created": "2021-03-05T23:25:25.230064203Z", + "history": [ + { + "created": "2021-03-05T23:25:25.052495292Z", + "created_by": "/bin/sh -c #(nop) COPY file:7bf12aab75c3867a023fe3b8bd6d113d43a4fcc415f3cc27cbcf0fff37b65a02 in / " + }, + { + "created": "2021-03-05T23:25:25.230064203Z", + "created_by": "/bin/sh -c #(nop) CMD [\"/hello\"]", + "empty_layer": true + } + ], + "variant": null, + "image_id": "d1165f2212346b2bab48cb01c1e39ee8ad1be46b87873d9ca7a4e434980a7726", + "os_version": null, + "architecture": "amd64", + "image_format": "docker", + "config_digest": "sha256:d1165f2212346b2bab48cb01c1e39ee8ad1be46b87873d9ca7a4e434980a7726", + "docker_version": "19.03.12" + } + ] + } +}], +"packages": [ +], +"files": [ +{ + "for_packages": [], + "compliance_alert": "", + "path": "/hello_world.tar-extract/f22b99068db93900abe17f7f5e09ec775c2826ecfe9db961fea68293744144bd/hello", + "size": 13336, + "sha1": "ab2ceef6fe98ef7acd088f5d79034dc859ff0d27", + "md5": "773e022bbbd0b4cc88edb6c7d12daf5d", + "copyrights": [], + "holders": [], + "authors": [], + "licenses": [], + "license_expressions": [], + "emails": [], + "urls": [ + { + "url": "https://hub.docker.com/", + "end_line": 19, + "start_line": 19 + }, + { + "url": "https://docs.docker.com/get-started/", + "end_line": 21, + "start_line": 21 + } + ], + "status": "no-licenses", + "type": "file", + "extra_data": {}, + "name": "hello", + "extension": "", + "programming_language": "", + "mime_type": "application/x-executable", + "file_type": "ELF 64-bit LSB executable, x86-64, version 1 (SYSV), statically linked, stripped", + "is_binary": true, + "is_text": false, + "is_archive": false +}] +} \ No newline at end of file diff --git a/tests/data/resource/virtual_codebase/zephyr-binary.json b/tests/data/resource/virtual_codebase/zephyr-binary.json new file mode 100644 index 00000000..2f1c1a3f --- /dev/null +++ b/tests/data/resource/virtual_codebase/zephyr-binary.json @@ -0,0 +1,220 @@ +{ +"headers": [ +{ + "tool_name": "scanpipe", + "tool_version": "21.5.12", + "notice": "Generated with ScanCode.io and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied.\nNo content created from ScanCode.io should be considered or used as legal advice.\nConsult an Attorney for any legal advice.\nScanCode.io is a free software code scanning tool from nexB Inc. and others\nlicensed under the Apache License version 2.0.\nScanCode is a trademark of nexB Inc.\nVisit https://github.com/nexB/scancode.io for support and download.\n", + "uuid": "d0d9bf3d-980a-4822-ac5b-e4949470499d", + "created_date": "2021-06-10T23:17:30.522Z", + "input_files": [ + "SBOM-Plugfest.zip" + ], + "runs": [ + { + "pipeline_name": "scan_codebase", + "description": "A pipeline to scan a codebase with ScanCode-toolkit.\n\nThe input files are copied to the project codebase/ directory and extracted\nin place before running the scan.\nAlternatively, the code can be manually copied to the project codebase/\ndirectory.", + "uuid": "13efbd78-3921-4895-b4bd-e0b393b92ec3", + "created_date": "2021-06-10T23:17:30.526728Z", + "task_id": "f68d383a-4910-4e05-ad64-82ef1f0b5888", + "task_start_date": "2021-06-10T23:17:30.546024Z", + "task_end_date": "2021-06-10T23:17:36.396574Z", + "task_exitcode": 0, + "task_output": "", + "log": "2021-06-10 23:17:30.54 Pipeline [scan_codebase] starting\n2021-06-10 23:17:30.54 Step [copy_inputs_to_codebase_directory] starting\n2021-06-10 23:17:30.55 Step [copy_inputs_to_codebase_directory] completed in 0.00 seconds\n2021-06-10 23:17:30.55 Step [run_extractcode] starting\n2021-06-10 23:17:30.83 Step [run_extractcode] completed in 0.28 seconds\n2021-06-10 23:17:30.84 Step [run_scancode] starting\n2021-06-10 23:17:36.33 Step [run_scancode] completed in 5.49 seconds\n2021-06-10 23:17:36.34 Step [build_inventory_from_scan] starting\n2021-06-10 23:17:36.37 Step [build_inventory_from_scan] completed in 0.03 seconds\n2021-06-10 23:17:36.37 Step [csv_output] starting\n2021-06-10 23:17:36.39 Step [csv_output] completed in 0.02 seconds\n2021-06-10 23:17:36.39 Pipeline completed\n", + "execution_time": 5 + } + ], + "extra_data": {} +}], +"packages": [ +], +"files": [ +{ + "for_packages": [], + "compliance_alert": "", + "path": "SBOM-Plugfest.zip", + "size": 1762970, + "sha1": "db3fa3422912685484860aec79960f3ef3de95a8", + "md5": "8a9423ff930d0afd9399600f351b0507", + "copyrights": [], + "holders": [], + "authors": [], + "licenses": [], + "license_expressions": [], + "emails": [], + "urls": [], + "status": "", + "type": "file", + "extra_data": {}, + "name": "SBOM-Plugfest.zip", + "extension": ".zip", + "programming_language": "", + "mime_type": "application/zip", + "file_type": "Zip archive data, at least v2.0 to extract", + "is_binary": true, + "is_text": false, + "is_archive": true +}, +{ + "for_packages": [], + "compliance_alert": "", + "path": "SBOM-Plugfest.zip-extract", + "size": 0, + "sha1": "", + "md5": "", + "copyrights": [], + "holders": [], + "authors": [], + "licenses": [], + "license_expressions": [], + "emails": [], + "urls": [], + "status": "", + "type": "directory", + "extra_data": {}, + "name": "SBOM-Plugfest.zip-extract", + "extension": "", + "programming_language": "", + "mime_type": "", + "file_type": "", + "is_binary": false, + "is_text": false, + "is_archive": false +}, +{ + "for_packages": [], + "compliance_alert": "", + "path": "SBOM-Plugfest.zip-extract/SBOM-Plugfest", + "size": 0, + "sha1": "", + "md5": "", + "copyrights": [], + "holders": [], + "authors": [], + "licenses": [], + "license_expressions": [], + "emails": [], + "urls": [], + "status": "", + "type": "directory", + "extra_data": {}, + "name": "SBOM-Plugfest", + "extension": "", + "programming_language": "", + "mime_type": "", + "file_type": "", + "is_binary": false, + "is_text": false, + "is_archive": false +}, +{ + "for_packages": [], + "compliance_alert": "", + "path": "SBOM-Plugfest.zip-extract/SBOM-Plugfest/Readme.md", + "size": 238, + "sha1": "b3edec38166cb4747ce67e7e229cbe9279e011c3", + "md5": "2c2fdcabb9e3a46bbc60f9e235ec0389", + "copyrights": [], + "holders": [], + "authors": [], + "licenses": [], + "license_expressions": [], + "emails": [], + "urls": [ + { + "url": "https://github.com/LibreSolar/charge-controller-firmware/tree/sbom-plugfest", + "end_line": 7, + "start_line": 7 + } + ], + "status": "", + "type": "file", + "extra_data": {}, + "name": "Readme.md", + "extension": ".md", + "programming_language": "", + "mime_type": "text/plain", + "file_type": "ASCII text", + "is_binary": false, + "is_text": true, + "is_archive": false +}, +{ + "for_packages": [], + "compliance_alert": "", + "path": "SBOM-Plugfest.zip-extract/SBOM-Plugfest/zephyr.bin", + "size": 95600, + "sha1": "3e5510346a723c4ecdd698330436d1d89f2e6428", + "md5": "fdc8b20a54f7960b204abce20f8dae98", + "copyrights": [], + "holders": [], + "authors": [], + "licenses": [], + "license_expressions": [], + "emails": [], + "urls": [], + "status": "", + "type": "file", + "extra_data": {}, + "name": "zephyr.bin", + "extension": ".bin", + "programming_language": "", + "mime_type": "application/octet-stream", + "file_type": "data", + "is_binary": true, + "is_text": false, + "is_archive": false +}, +{ + "for_packages": [], + "compliance_alert": "", + "path": "SBOM-Plugfest.zip-extract/SBOM-Plugfest/zephyr.elf", + "size": 1397388, + "sha1": "a0c9fca9a551a935026c86fb56c4af563f0a926f", + "md5": "b7f937d8a1fbd71a0a48f16c845ed36e", + "copyrights": [], + "holders": [], + "authors": [], + "licenses": [], + "license_expressions": [], + "emails": [], + "urls": [], + "status": "", + "type": "file", + "extra_data": {}, + "name": "zephyr.elf", + "extension": ".elf", + "programming_language": "", + "mime_type": "application/x-executable", + "file_type": "ELF 32-bit LSB executable, ARM, EABI5 version 1 (SYSV), statically linked, with debug_info, not stripped", + "is_binary": true, + "is_text": false, + "is_archive": false +}, +{ + "for_packages": [], + "compliance_alert": "", + "path": "SBOM-Plugfest.zip-extract/SBOM-Plugfest/zephyr.hex", + "size": 269060, + "sha1": "c70fc409d4003e437ea2e84d9546de8083333ab2", + "md5": "2505d99bb160d8b05e8892026209923d", + "copyrights": [], + "holders": [], + "authors": [], + "licenses": [], + "license_expressions": [], + "emails": [], + "urls": [], + "status": "", + "type": "file", + "extra_data": {}, + "name": "zephyr.hex", + "extension": ".hex", + "programming_language": "", + "mime_type": "text/plain", + "file_type": "ASCII text, with CRLF line terminators", + "is_binary": false, + "is_text": true, + "is_archive": false +}] +} \ No newline at end of file diff --git a/tests/test_resource.py b/tests/test_resource.py index 1506ab19..03feb068 100644 --- a/tests/test_resource.py +++ b/tests/test_resource.py @@ -13,8 +13,6 @@ from os.path import exists from os.path import join -import pytest - from commoncode.fileutils import parent_directory from commoncode.testcase import FileBasedTesting from commoncode.resource import Codebase @@ -42,13 +40,16 @@ def test_walk_defaults(self): ] assert [(r.name, r.is_file) for r in results] == expected - @pytest.mark.xfail(reason='FIXME: a fix for ticket #1422 is needed') - def test_Codebase_with_only_ignores_should_not_fail_to_create(self): + def test_Codebase_do_not_ignore_by_default_older_sccs_and_rcs_dirs(self): + # See https://github.com/nexB/scancode-toolkit/issues/1422 from commoncode.fileutils import create_dir test_codebase = self.get_temp_dir() create_dir(join(test_codebase, 'sccs', 'a')) create_dir(join(test_codebase, 'rcs', 'b')) - Codebase(test_codebase) + codebase = Codebase(test_codebase) + results = list(codebase.walk(topdown=True, skip_root=True)) + expected = ['rcs', 'b', 'sccs', 'a'] + assert [r.name for r in results] == expected def test_walk_topdown(self): test_codebase = self.get_test_loc('resource/codebase') @@ -1055,7 +1056,7 @@ def test_virtual_codebase_can_process_minimal_resources_with_only_path(self): ]) ] assert [r.to_dict() for r in codebase.walk()] == expected - + def test_VirtualCodebase_account_fingerprint_attribute(self): test_file = self.get_test_loc("resource/virtual_codebase/fingerprint_attribute.json") codebase = VirtualCodebase(test_file) @@ -1334,14 +1335,24 @@ def test_VirtualCodebase_scanning_full_root(self): assert "/Users/sesser/code/nexb/scancode-toolkit/samples/README" == resource.path assert 1 == codebase.compute_counts()[0] - def test_VirtualCodebase_can_compute_counts_witrh_null(self): - # was failing with + def test_VirtualCodebase_can_compute_counts_with_null(self): + # was failing with # size_count += child.size # TypeError: unsupported operand type(s) for +=: 'int' and 'NoneType' test_file = self.get_test_loc("resource/virtual_codebase/node-16-slim.json") codebase = VirtualCodebase(test_file) codebase.compute_counts() + def test_VirtualCodebase_can_be_created_with_single_path(self): + test_file = self.get_test_loc("resource/virtual_codebase/docker-hello-world.json") + VirtualCodebase(test_file) + + def test_VirtualCodebase_can_be_created_without_RecursionError(self): + # was failing with RecursionError: maximum recursion depth exceeded + test_file = self.get_test_loc("resource/virtual_codebase/zephyr-binary.json") + VirtualCodebase(test_file) + + class TestResource(FileBasedTesting): test_data_dir = join(dirname(__file__), 'data')