From 3b794f1a3165abc8ce526a603ab12878a1f769d8 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 11 Jun 2021 02:48:08 +0200 Subject: [PATCH 1/5] Add failing test for #23 We have a recursion-error Signed-off-by: Philippe Ombredanne --- .../virtual_codebase/docker-hello-world.json | 100 ++++++++ .../virtual_codebase/zephyr-binary.json | 220 ++++++++++++++++++ tests/test_resource.py | 15 +- 3 files changed, 333 insertions(+), 2 deletions(-) create mode 100644 tests/data/resource/virtual_codebase/docker-hello-world.json create mode 100644 tests/data/resource/virtual_codebase/zephyr-binary.json diff --git a/tests/data/resource/virtual_codebase/docker-hello-world.json b/tests/data/resource/virtual_codebase/docker-hello-world.json new file mode 100644 index 00000000..86eafc18 --- /dev/null +++ b/tests/data/resource/virtual_codebase/docker-hello-world.json @@ -0,0 +1,100 @@ +{ +"headers": [ +{ + "tool_name": "scanpipe", + "tool_version": "21.5.12", + "notice": "Generated with ScanCode.io and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied.\nNo content created from ScanCode.io should be considered or used as legal advice.\nConsult an Attorney for any legal advice.\nScanCode.io is a free software code scanning tool from nexB Inc. and others\nlicensed under the Apache License version 2.0.\nScanCode is a trademark of nexB Inc.\nVisit https://github.com/nexB/scancode.io for support and download.\n", + "uuid": "2f8e9a35-048e-4d9f-ae56-feea8863d6eb", + "created_date": "2021-06-10T13:56:44.418Z", + "input_files": [ + "hello_world.tar" + ], + "runs": [ + { + "pipeline_name": "docker", + "description": "A pipeline to analyze a Docker image.", + "uuid": "106296b1-6334-4393-9f1a-209459954812", + "created_date": "2021-06-10T13:56:44.421302Z", + "task_id": "c851f2e7-7cc1-4952-8d93-1cdee4a147bd", + "task_start_date": "2021-06-10T13:56:44.434313Z", + "task_end_date": "2021-06-10T13:56:44.819773Z", + "task_exitcode": 0, + "task_output": "", + "log": "2021-06-10 13:56:44.43 Pipeline [docker] starting\n2021-06-10 13:56:44.43 Step [extract_images] starting\n2021-06-10 13:56:44.44 Step [extract_images] completed in 0.00 seconds\n2021-06-10 13:56:44.44 Step [extract_layers] starting\n2021-06-10 13:56:44.44 Step [extract_layers] completed in 0.00 seconds\n2021-06-10 13:56:44.45 Step [find_images_linux_distro] starting\n2021-06-10 13:56:44.45 Step [find_images_linux_distro] completed in 0.00 seconds\n2021-06-10 13:56:44.45 Step [collect_images_information] starting\n2021-06-10 13:56:44.45 Step [collect_images_information] completed in 0.00 seconds\n2021-06-10 13:56:44.46 Step [collect_and_create_codebase_resources] starting\n2021-06-10 13:56:44.46 Step [collect_and_create_codebase_resources] completed in 0.01 seconds\n2021-06-10 13:56:44.47 Step [collect_and_create_system_packages] starting\n2021-06-10 13:56:44.47 Step [collect_and_create_system_packages] completed in 0.00 seconds\n2021-06-10 13:56:44.47 Step [tag_uninteresting_codebase_resources] starting\n2021-06-10 13:56:44.48 Step [tag_uninteresting_codebase_resources] completed in 0.00 seconds\n2021-06-10 13:56:44.48 Step [tag_empty_files] starting\n2021-06-10 13:56:44.48 Step [tag_empty_files] completed in 0.00 seconds\n2021-06-10 13:56:44.48 Step [scan_for_application_packages] starting\n2021-06-10 13:56:44.53 Step [scan_for_application_packages] completed in 0.05 seconds\n2021-06-10 13:56:44.54 Step [scan_for_files] starting\n2021-06-10 13:56:44.80 Step [scan_for_files] completed in 0.26 seconds\n2021-06-10 13:56:44.80 Step [analyze_scanned_files] starting\n2021-06-10 13:56:44.81 Step [analyze_scanned_files] completed in 0.00 seconds\n2021-06-10 13:56:44.81 Step [tag_not_analyzed_codebase_resources] starting\n2021-06-10 13:56:44.81 Step [tag_not_analyzed_codebase_resources] completed in 0.00 seconds\n2021-06-10 13:56:44.81 Pipeline completed\n", + "execution_time": 0 + } + ], + "extra_data": { + "images": [ + { + "os": "linux", + "tags": [], + "author": null, + "distro": null, + "labels": {}, + "sha256": null, + "comment": null, + "created": "2021-03-05T23:25:25.230064203Z", + "history": [ + { + "created": "2021-03-05T23:25:25.052495292Z", + "created_by": "/bin/sh -c #(nop) COPY file:7bf12aab75c3867a023fe3b8bd6d113d43a4fcc415f3cc27cbcf0fff37b65a02 in / " + }, + { + "created": "2021-03-05T23:25:25.230064203Z", + "created_by": "/bin/sh -c #(nop) CMD [\"/hello\"]", + "empty_layer": true + } + ], + "variant": null, + "image_id": "d1165f2212346b2bab48cb01c1e39ee8ad1be46b87873d9ca7a4e434980a7726", + "os_version": null, + "architecture": "amd64", + "image_format": "docker", + "config_digest": "sha256:d1165f2212346b2bab48cb01c1e39ee8ad1be46b87873d9ca7a4e434980a7726", + "docker_version": "19.03.12" + } + ] + } +}], +"packages": [ +], +"files": [ +{ + "for_packages": [], + "compliance_alert": "", + "path": "/hello_world.tar-extract/f22b99068db93900abe17f7f5e09ec775c2826ecfe9db961fea68293744144bd/hello", + "size": 13336, + "sha1": "ab2ceef6fe98ef7acd088f5d79034dc859ff0d27", + "md5": "773e022bbbd0b4cc88edb6c7d12daf5d", + "copyrights": [], + "holders": [], + "authors": [], + "licenses": [], + "license_expressions": [], + "emails": [], + "urls": [ + { + "url": "https://hub.docker.com/", + "end_line": 19, + "start_line": 19 + }, + { + "url": "https://docs.docker.com/get-started/", + "end_line": 21, + "start_line": 21 + } + ], + "status": "no-licenses", + "type": "file", + "extra_data": {}, + "name": "hello", + "extension": "", + "programming_language": "", + "mime_type": "application/x-executable", + "file_type": "ELF 64-bit LSB executable, x86-64, version 1 (SYSV), statically linked, stripped", + "is_binary": true, + "is_text": false, + "is_archive": false +}] +} \ No newline at end of file diff --git a/tests/data/resource/virtual_codebase/zephyr-binary.json b/tests/data/resource/virtual_codebase/zephyr-binary.json new file mode 100644 index 00000000..2f1c1a3f --- /dev/null +++ b/tests/data/resource/virtual_codebase/zephyr-binary.json @@ -0,0 +1,220 @@ +{ +"headers": [ +{ + "tool_name": "scanpipe", + "tool_version": "21.5.12", + "notice": "Generated with ScanCode.io and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied.\nNo content created from ScanCode.io should be considered or used as legal advice.\nConsult an Attorney for any legal advice.\nScanCode.io is a free software code scanning tool from nexB Inc. and others\nlicensed under the Apache License version 2.0.\nScanCode is a trademark of nexB Inc.\nVisit https://github.com/nexB/scancode.io for support and download.\n", + "uuid": "d0d9bf3d-980a-4822-ac5b-e4949470499d", + "created_date": "2021-06-10T23:17:30.522Z", + "input_files": [ + "SBOM-Plugfest.zip" + ], + "runs": [ + { + "pipeline_name": "scan_codebase", + "description": "A pipeline to scan a codebase with ScanCode-toolkit.\n\nThe input files are copied to the project codebase/ directory and extracted\nin place before running the scan.\nAlternatively, the code can be manually copied to the project codebase/\ndirectory.", + "uuid": "13efbd78-3921-4895-b4bd-e0b393b92ec3", + "created_date": "2021-06-10T23:17:30.526728Z", + "task_id": "f68d383a-4910-4e05-ad64-82ef1f0b5888", + "task_start_date": "2021-06-10T23:17:30.546024Z", + "task_end_date": "2021-06-10T23:17:36.396574Z", + "task_exitcode": 0, + "task_output": "", + "log": "2021-06-10 23:17:30.54 Pipeline [scan_codebase] starting\n2021-06-10 23:17:30.54 Step [copy_inputs_to_codebase_directory] starting\n2021-06-10 23:17:30.55 Step [copy_inputs_to_codebase_directory] completed in 0.00 seconds\n2021-06-10 23:17:30.55 Step [run_extractcode] starting\n2021-06-10 23:17:30.83 Step [run_extractcode] completed in 0.28 seconds\n2021-06-10 23:17:30.84 Step [run_scancode] starting\n2021-06-10 23:17:36.33 Step [run_scancode] completed in 5.49 seconds\n2021-06-10 23:17:36.34 Step [build_inventory_from_scan] starting\n2021-06-10 23:17:36.37 Step [build_inventory_from_scan] completed in 0.03 seconds\n2021-06-10 23:17:36.37 Step [csv_output] starting\n2021-06-10 23:17:36.39 Step [csv_output] completed in 0.02 seconds\n2021-06-10 23:17:36.39 Pipeline completed\n", + "execution_time": 5 + } + ], + "extra_data": {} +}], +"packages": [ +], +"files": [ +{ + "for_packages": [], + "compliance_alert": "", + "path": "SBOM-Plugfest.zip", + "size": 1762970, + "sha1": "db3fa3422912685484860aec79960f3ef3de95a8", + "md5": "8a9423ff930d0afd9399600f351b0507", + "copyrights": [], + "holders": [], + "authors": [], + "licenses": [], + "license_expressions": [], + "emails": [], + "urls": [], + "status": "", + "type": "file", + "extra_data": {}, + "name": "SBOM-Plugfest.zip", + "extension": ".zip", + "programming_language": "", + "mime_type": "application/zip", + "file_type": "Zip archive data, at least v2.0 to extract", + "is_binary": true, + "is_text": false, + "is_archive": true +}, +{ + "for_packages": [], + "compliance_alert": "", + "path": "SBOM-Plugfest.zip-extract", + "size": 0, + "sha1": "", + "md5": "", + "copyrights": [], + "holders": [], + "authors": [], + "licenses": [], + "license_expressions": [], + "emails": [], + "urls": [], + "status": "", + "type": "directory", + "extra_data": {}, + "name": "SBOM-Plugfest.zip-extract", + "extension": "", + "programming_language": "", + "mime_type": "", + "file_type": "", + "is_binary": false, + "is_text": false, + "is_archive": false +}, +{ + "for_packages": [], + "compliance_alert": "", + "path": "SBOM-Plugfest.zip-extract/SBOM-Plugfest", + "size": 0, + "sha1": "", + "md5": "", + "copyrights": [], + "holders": [], + "authors": [], + "licenses": [], + "license_expressions": [], + "emails": [], + "urls": [], + "status": "", + "type": "directory", + "extra_data": {}, + "name": "SBOM-Plugfest", + "extension": "", + "programming_language": "", + "mime_type": "", + "file_type": "", + "is_binary": false, + "is_text": false, + "is_archive": false +}, +{ + "for_packages": [], + "compliance_alert": "", + "path": "SBOM-Plugfest.zip-extract/SBOM-Plugfest/Readme.md", + "size": 238, + "sha1": "b3edec38166cb4747ce67e7e229cbe9279e011c3", + "md5": "2c2fdcabb9e3a46bbc60f9e235ec0389", + "copyrights": [], + "holders": [], + "authors": [], + "licenses": [], + "license_expressions": [], + "emails": [], + "urls": [ + { + "url": "https://github.com/LibreSolar/charge-controller-firmware/tree/sbom-plugfest", + "end_line": 7, + "start_line": 7 + } + ], + "status": "", + "type": "file", + "extra_data": {}, + "name": "Readme.md", + "extension": ".md", + "programming_language": "", + "mime_type": "text/plain", + "file_type": "ASCII text", + "is_binary": false, + "is_text": true, + "is_archive": false +}, +{ + "for_packages": [], + "compliance_alert": "", + "path": "SBOM-Plugfest.zip-extract/SBOM-Plugfest/zephyr.bin", + "size": 95600, + "sha1": "3e5510346a723c4ecdd698330436d1d89f2e6428", + "md5": "fdc8b20a54f7960b204abce20f8dae98", + "copyrights": [], + "holders": [], + "authors": [], + "licenses": [], + "license_expressions": [], + "emails": [], + "urls": [], + "status": "", + "type": "file", + "extra_data": {}, + "name": "zephyr.bin", + "extension": ".bin", + "programming_language": "", + "mime_type": "application/octet-stream", + "file_type": "data", + "is_binary": true, + "is_text": false, + "is_archive": false +}, +{ + "for_packages": [], + "compliance_alert": "", + "path": "SBOM-Plugfest.zip-extract/SBOM-Plugfest/zephyr.elf", + "size": 1397388, + "sha1": "a0c9fca9a551a935026c86fb56c4af563f0a926f", + "md5": "b7f937d8a1fbd71a0a48f16c845ed36e", + "copyrights": [], + "holders": [], + "authors": [], + "licenses": [], + "license_expressions": [], + "emails": [], + "urls": [], + "status": "", + "type": "file", + "extra_data": {}, + "name": "zephyr.elf", + "extension": ".elf", + "programming_language": "", + "mime_type": "application/x-executable", + "file_type": "ELF 32-bit LSB executable, ARM, EABI5 version 1 (SYSV), statically linked, with debug_info, not stripped", + "is_binary": true, + "is_text": false, + "is_archive": false +}, +{ + "for_packages": [], + "compliance_alert": "", + "path": "SBOM-Plugfest.zip-extract/SBOM-Plugfest/zephyr.hex", + "size": 269060, + "sha1": "c70fc409d4003e437ea2e84d9546de8083333ab2", + "md5": "2505d99bb160d8b05e8892026209923d", + "copyrights": [], + "holders": [], + "authors": [], + "licenses": [], + "license_expressions": [], + "emails": [], + "urls": [], + "status": "", + "type": "file", + "extra_data": {}, + "name": "zephyr.hex", + "extension": ".hex", + "programming_language": "", + "mime_type": "text/plain", + "file_type": "ASCII text, with CRLF line terminators", + "is_binary": false, + "is_text": true, + "is_archive": false +}] +} \ No newline at end of file diff --git a/tests/test_resource.py b/tests/test_resource.py index d7140661..2aa56ab2 100644 --- a/tests/test_resource.py +++ b/tests/test_resource.py @@ -1325,14 +1325,25 @@ def test_VirtualCodebase_scanning_full_root(self): assert "/Users/sesser/code/nexb/scancode-toolkit/samples/README" == resource.path assert 1 == codebase.compute_counts()[0] - def test_VirtualCodebase_can_compute_counts_witrh_null(self): - # was failing with + def test_VirtualCodebase_can_compute_counts_with_null(self): + # was failing with # size_count += child.size # TypeError: unsupported operand type(s) for +=: 'int' and 'NoneType' test_file = self.get_test_loc("resource/virtual_codebase/node-16-slim.json") codebase = VirtualCodebase(test_file) codebase.compute_counts() + def test_VirtualCodebase_can_be_created_with_single_path(self): + test_file = self.get_test_loc("resource/virtual_codebase/docker-hello-world.json") + VirtualCodebase(test_file) + + def test_VirtualCodebase_can_be_created_without_RecursionError(self): + # was failing with RecursionError: maximum recursion depth exceeded + test_file = self.get_test_loc("resource/virtual_codebase/zephyr-binary.json") + VirtualCodebase(test_file) + + + class TestResource(FileBasedTesting): test_data_dir = join(dirname(__file__), 'data') From dc061bff14ed17928761a524fdc9c6d24609004e Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Thu, 10 Jun 2021 19:13:53 -0700 Subject: [PATCH 2/5] Set a root path if one is not present #23 Signed-off-by: Jono Yang --- src/commoncode/resource.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/commoncode/resource.py b/src/commoncode/resource.py index 0af6f368..25b7e567 100644 --- a/src/commoncode/resource.py +++ b/src/commoncode/resource.py @@ -12,6 +12,7 @@ import posixpath import traceback import sys +from pathlib import Path from collections import deque from functools import partial @@ -1530,7 +1531,7 @@ def _get_or_create_parent(self, path, parent_by_path): Note: the root path and root Resource must already be in `parent_by_path` or else this function does not work. """ - parent_path = parent_directory(path).rstrip('/').rstrip('\\').lstrip("/") + parent_path = parent_directory(path).rstrip('/').rstrip('\\').lstrip('/') existing_parent = parent_by_path.get(parent_path) if existing_parent: return existing_parent @@ -1548,6 +1549,13 @@ def _get_or_create_parent(self, path, parent_by_path): parent_by_path[parent_path] = parent_resource return parent_resource + def _set_new_root_directory(self, resources_data, new_root_directory_path): + for resource_data in resources_data: + resource_path = Path(resource_data['path']) + new_resource_path = Path(new_root_directory_path) + new_resource_path = new_resource_path.joinpath(resource_path) + resource_data['path'] = str(new_resource_path) + def _populate(self, scan_data): """ Populate this codebase with Resource objects. @@ -1642,6 +1650,21 @@ def _populate(self, scan_data): sample_resource_path = sample_resource_data['path'] sample_resource_path = sample_resource_path.strip('/') root_path = sample_resource_path.split('/')[0] + + # Check to see if the Resources from the scan we received has a common root directory. + for resource_data in resources_data: + resource_path = resource_data.get('path') + resource_path = resource_path.strip('/') + resource_root_path = resource_path.split('/')[0] + # If not, set a common root directory for all Resources. + if resource_root_path != root_path: + self._set_new_root_directory( + resources_data=resources_data, + new_root_directory_path='virtual_root' + ) + root_path = 'virtual_root' + break + root_name = root_path root_is_file = False root_data = self._create_empty_resource_data() From e1419777aeec15abad6f3125b5a64781f2efe54c Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 11 Jun 2021 13:35:58 +0200 Subject: [PATCH 3/5] Add CHANGELOG entry Signed-off-by: Philippe Ombredanne --- CHANGELOG.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e20f8ad9..9b846e76 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,12 @@ vNext ----- +Version 21.6.11 +--------------- + +- Do not fail if a Codebase does not have a common shared root + + Version 21.6.10 --------------- From d4c7a2e291c199ca4bd7a1e0eaa92c7e1b2bbdc1 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 11 Jun 2021 13:41:45 +0200 Subject: [PATCH 4/5] Update changelog Signed-off-by: Philippe Ombredanne --- CHANGELOG.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9b846e76..b4c253cb 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,7 +8,8 @@ vNext Version 21.6.11 --------------- -- Do not fail if a Codebase does not have a common shared root +- Do not fail if a Codebase does not have a common shared root #23 +- Consider all Resource attributes when building a VirtualCodebase #23 Version 21.6.10 From 02ec56d1dc9d79c96def6717f09ac85e1bbd3f97 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 11 Jun 2021 13:57:22 +0200 Subject: [PATCH 5/5] Do not ignore by default sccs and rcs dirs Fix for https://github.com/nexB/scancode-toolkit/issues/1422 Signed-off-by: Philippe Ombredanne --- CHANGELOG.rst | 1 + src/commoncode/ignore.py | 1 - tests/test_resource.py | 14 +++++++------- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b4c253cb..4a6828f0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,7 @@ Version 21.6.11 - Do not fail if a Codebase does not have a common shared root #23 - Consider all Resource attributes when building a VirtualCodebase #23 +- Do not ignore by default sccs and rcs dirs https://github.com/nexB/scancode-toolkit/issues/1422 Version 21.6.10 diff --git a/src/commoncode/ignore.py b/src/commoncode/ignore.py index 477c0225..881b93f5 100644 --- a/src/commoncode/ignore.py +++ b/src/commoncode/ignore.py @@ -184,7 +184,6 @@ def get_ignores(location, include_defaults=True): 'CVS': 'Default ignore: CVS artifact', '.cvsignore': 'Default ignore: CVS config artifact', - '*/SCCS': 'Default ignore: CVS artifact', '*/_MTN': 'Default ignore: Monotone artifact', '*/_darcs': 'Default ignore: Darcs artifact', diff --git a/tests/test_resource.py b/tests/test_resource.py index 8dcf0764..03feb068 100644 --- a/tests/test_resource.py +++ b/tests/test_resource.py @@ -13,8 +13,6 @@ from os.path import exists from os.path import join -import pytest - from commoncode.fileutils import parent_directory from commoncode.testcase import FileBasedTesting from commoncode.resource import Codebase @@ -42,13 +40,16 @@ def test_walk_defaults(self): ] assert [(r.name, r.is_file) for r in results] == expected - @pytest.mark.xfail(reason='FIXME: a fix for ticket #1422 is needed') - def test_Codebase_with_only_ignores_should_not_fail_to_create(self): + def test_Codebase_do_not_ignore_by_default_older_sccs_and_rcs_dirs(self): + # See https://github.com/nexB/scancode-toolkit/issues/1422 from commoncode.fileutils import create_dir test_codebase = self.get_temp_dir() create_dir(join(test_codebase, 'sccs', 'a')) create_dir(join(test_codebase, 'rcs', 'b')) - Codebase(test_codebase) + codebase = Codebase(test_codebase) + results = list(codebase.walk(topdown=True, skip_root=True)) + expected = ['rcs', 'b', 'sccs', 'a'] + assert [r.name for r in results] == expected def test_walk_topdown(self): test_codebase = self.get_test_loc('resource/codebase') @@ -1055,7 +1056,7 @@ def test_virtual_codebase_can_process_minimal_resources_with_only_path(self): ]) ] assert [r.to_dict() for r in codebase.walk()] == expected - + def test_VirtualCodebase_account_fingerprint_attribute(self): test_file = self.get_test_loc("resource/virtual_codebase/fingerprint_attribute.json") codebase = VirtualCodebase(test_file) @@ -1352,7 +1353,6 @@ def test_VirtualCodebase_can_be_created_without_RecursionError(self): VirtualCodebase(test_file) - class TestResource(FileBasedTesting): test_data_dir = join(dirname(__file__), 'data')