diff --git a/.gitattributes b/.gitattributes index c446d380..b79df5cd 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,2 @@ # Ignore all Git auto CR/LF line endings conversions -* binary +* -text diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 12171865..b396dc7f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,14 @@ Release notes vNext ----- + +Version 21.4.28 +--------------- + +*2020-04-28* +- Add new function to get a Resource path stripped from its root path segment + + Version 21.1.21 --------------- diff --git a/src/commoncode/resource.py b/src/commoncode/resource.py index cbecaed6..a92fcb8e 100644 --- a/src/commoncode/resource.py +++ b/src/commoncode/resource.py @@ -60,6 +60,7 @@ from commoncode.fileutils import splitext_name from commoncode import ignore +from commoncode import paths """ This module provides Codebase and Resource objects as an abstraction for files @@ -1054,6 +1055,12 @@ def type(self, value): else: self.is_file = False + def get_path(self, strip_root=False): + if strip_root: + return strip_first_path_segment(self.path) + else: + return self.path + @property def is_dir(self): # note: we only store is_file @@ -1354,6 +1361,29 @@ def get_path(root_location, location, full_root=False, strip_root=False): return posix_loc.replace(posix_root_loc, '', 1) +def strip_first_path_segment(path): + """ + Return a POSIX path stripped from its first path segment. + + For example:: + >>> strip_first_path_segment('') + '' + >>> strip_first_path_segment('foo') + 'foo' + >>> strip_first_path_segment('foo/bar/baz') + 'bar/baz' + >>> strip_first_path_segment('/foo/bar/baz/') + 'bar/baz' + >>> strip_first_path_segment('foo/') + 'foo/' + """ + segments = paths.split(path) + if not segments or len(segments) == 1: + return path + stripped = segments[1:] + return '/'.join(stripped) + + def get_codebase_cache_dir(temp_dir): """ Return a new, created and unique per-run cache storage directory path rooted @@ -1506,12 +1536,13 @@ def _get_or_create_parent(self, path, parent_by_path): """ Return a parent resource for a given `path` from `parent_by_path`. - If a parent resource for a `path` does not exist in `parent_by_path`, it is created recursively. + If a parent resource for a `path` does not exist in `parent_by_path`, it + is created recursively. - Note: the root path and root Resource must already be in `parent_by_path` or else this - function does not work. + Note: the root path and root Resource must already be in + `parent_by_path` or else this function does not work. """ - parent_path = parent_directory(path).rstrip('/').rstrip('\\') + parent_path = parent_directory(path).rstrip('/').rstrip('\\').lstrip("/") existing_parent = parent_by_path.get(parent_path) if existing_parent: return existing_parent @@ -1519,7 +1550,13 @@ def _get_or_create_parent(self, path, parent_by_path): parent_name = file_base_name(parent_path) parent_is_file = False parent_resource_data = self._create_empty_resource_data() - parent_resource = self._create_resource(parent_name, parent_parent, parent_is_file, parent_path, parent_resource_data) + parent_resource = self._create_resource( + parent_name, + parent_parent, + parent_is_file, + parent_path, + parent_resource_data, + ) parent_by_path[parent_path] = parent_resource return parent_resource @@ -1620,7 +1657,12 @@ def _populate(self, scan_data): root_name = root_path root_is_file = False root_data = self._create_empty_resource_data() - root_resource = self._create_root_resource(root_name, root_path, root_is_file, root_data) + root_resource = self._create_root_resource( + name=root_name, + path=root_path, + is_file=root_is_file, + root_data=root_data, + ) # To help recreate the resource tree we keep a mapping by path of any # parent resource @@ -1630,24 +1672,32 @@ def _populate(self, scan_data): path = resource_data.get('path') # Append virtual_root path to imported Resource path if we are merging multiple scans if multiple_input: - path = os.path.join(root_path, path) + path = posixpath.join(root_path, path) + name = resource_data.get('name', None) if not name: name = file_name(path) + is_file = resource_data.get('type', 'file') == 'file' existing_parent = parent_by_path.get(path) if existing_parent: - # We update the empty parent Resouorce we in _get_or_create_parent() with the data - # from the scan + # We update the empty parent Resouorce we in + # _get_or_create_parent() with the data from the scan for k, v in resource_data.items(): setattr(existing_parent, k, v) self.save_resource(existing_parent) else: - # Note: `root_path`: `root_resource` must be in `parent_by_path` in order for - # `_get_or_create_parent` to work + # Note: `root_path`: `root_resource` must be in `parent_by_path` + # in order for `_get_or_create_parent` to work parent = self._get_or_create_parent(path, parent_by_path) - resource = self._create_resource(name, parent, is_file, path, resource_data) + resource = self._create_resource( + name=name, + parent=parent, + is_file=is_file, + path=path, + resource_data=resource_data, + ) # Files are not parents (for now), so we do not need to add this # to the parent_by_path mapping @@ -1665,7 +1715,14 @@ def _create_root_resource(self, name, path, is_file, root_data): if root_data: root_data = remove_properties_and_basics(root_data) root = self.resource_class( - name=name, location=None, path=path, rid=0, pid=None, is_file=is_file, **root_data) + name=name, + location=None, + path=path, + rid=0, + pid=None, + is_file=is_file, + **root_data, + ) self.resource_ids.add(0) self.resources[0] = root diff --git a/src/commoncode/system.py b/src/commoncode/system.py index 3496bc8d..48b777b9 100644 --- a/src/commoncode/system.py +++ b/src/commoncode/system.py @@ -20,7 +20,6 @@ import getpass import os -import subprocess import sys diff --git a/tests/data/resource/virtual_codebase/path_full_root.json b/tests/data/resource/virtual_codebase/path_full_root.json new file mode 100644 index 00000000..dc768ec5 --- /dev/null +++ b/tests/data/resource/virtual_codebase/path_full_root.json @@ -0,0 +1,36 @@ +{ + "scancode_notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "scancode_version": "2.2.1.post12.6d07756e", + "scancode_options": { + "--info": true, + "--license-score": 0, + "--full-root": true, + "--format": "json-pp" + }, + "files_count": 1, + "files": [ + { + "path": "/Users/sesser/code/nexb/scancode-toolkit/samples/README", + "type": "file", + "name": "README", + "base_name": "README", + "extension": "", + "date": "2017-09-22", + "size": 236, + "sha1": "2e07e32c52d607204fad196052d70e3d18fb8636", + "md5": "effc6856ef85a9250fb1a470792b3f38", + "files_count": null, + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "scan_errors": [] + } + ] + } + \ No newline at end of file diff --git a/tests/data/resource/virtual_codebase/stripped-and-skipped-root.json b/tests/data/resource/virtual_codebase/stripped-and-skipped-root.json new file mode 100644 index 00000000..9d6acdd4 --- /dev/null +++ b/tests/data/resource/virtual_codebase/stripped-and-skipped-root.json @@ -0,0 +1,19 @@ +{ + "files": [ + { + "path": "samples", + "type": "directory", + "scan_errors": [] + }, + { + "path": "samples/README", + "type": "file", + "scan_errors": [] + }, + { + "path": "samples/screenshot.png", + "type": "file", + "scan_errors": [] + } + ] +} \ No newline at end of file diff --git a/tests/test_resource.py b/tests/test_resource.py index 697646c6..61cd28ed 100644 --- a/tests/test_resource.py +++ b/tests/test_resource.py @@ -802,6 +802,13 @@ def test_virtual_codebase_walk_skip_root_basic(self): ] assert expected == [(r.name, r.is_file) for r in results] + def test_virtual_codebase_get_path_with_strip_root_and_walk_with_skip_root(self): + scan_data = self.get_test_loc('resource/virtual_codebase/stripped-and-skipped-root.json') + virtual_codebase = VirtualCodebase(location=scan_data) + results = [r.get_path(strip_root=True) for r in virtual_codebase.walk(skip_root=True)] + expected = ['README', 'screenshot.png'] + assert results == expected + def test_virtual_codebase_walk_filtered_with_filtered_root(self): scan_data = self.get_test_loc('resource/virtual_codebase/virtual_codebase.json') virtual_codebase = VirtualCodebase(location=scan_data) @@ -1323,6 +1330,13 @@ def test_VirtualCodebase_create_from_multiple_scans(self): ] assert expected == results + def test_VirtualCodebase_scanning_full_root(self): + test_file = self.get_test_loc("resource/virtual_codebase/path_full_root.json") + codebase = VirtualCodebase(test_file) + resource = sorted(r for r in codebase.walk())[0] + assert resource.path == "/Users/sesser/code/nexb/scancode-toolkit/samples/README" + assert codebase.compute_counts()[0] == 1 + class TestResource(FileBasedTesting): test_data_dir = join(dirname(__file__), 'data')