From e55bcb7801fd84c6b4015f71098e62b4ff594d61 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Fri, 17 Nov 2017 16:55:08 -0800 Subject: [PATCH 1/4] Simplify DeltaCode output #16 * Modify Delta object to_dict function to return only minimal info (category and path). * Modify CSV ouput function to handle new data structure. * Refactor 9 failing tests and related test files. Signed-off-by: John M. Horan --- src/deltacode/__init__.py | 27 ++++--- src/deltacode/cli.py | 49 +++++++------ tests/data/cli/added1.csv | 20 ++--- tests/data/cli/modified1.csv | 18 ++--- tests/data/cli/removed1.csv | 18 ++--- tests/data/cli/renamed1.csv | 20 ++--- tests/test_deltacode.py | 138 +++++++++++------------------------ 7 files changed, 121 insertions(+), 169 deletions(-) diff --git a/src/deltacode/__init__.py b/src/deltacode/__init__.py index 8750a4dc..0a68fcd9 100644 --- a/src/deltacode/__init__.py +++ b/src/deltacode/__init__.py @@ -90,7 +90,7 @@ def determine_delta(self): for path, old_files in old_index.items(): for old_file in old_files: old_files_to_visit -= 1 - + if old_file.type != 'file': continue @@ -100,7 +100,7 @@ def determine_delta(self): except KeyError: deltas['removed'].append(Delta(None, old_file, 'removed')) continue - + # make sure everything is accounted for assert new_files_to_visit == 0 assert old_files_to_visit == 0 @@ -130,7 +130,7 @@ def to_dict(self): """ if self.deltas == None: return - + return OrderedDict([ ('added', [d.to_dict() for d in self.deltas.get('added')]), ('removed', [d.to_dict() for d in self.deltas.get('removed')]), @@ -155,18 +155,23 @@ def to_dict(self): if self.new_file == None and self.old_file == None: return - if self.new_file == None: + if self.category == 'added': + return OrderedDict([ + ('category', 'added'), + ('path', self.new_file.path) + ]) + elif self.category == 'removed': return OrderedDict([ - ('new', None), - ('old', self.old_file.to_dict()), + ('category', 'removed'), + ('path', self.old_file.path) ]) - elif self.old_file == None: + elif self.category == 'modified': return OrderedDict([ - ('new', self.new_file.to_dict()), - ('old', None), + ('category', 'modified'), + ('path', self.old_file.path) ]) else: return OrderedDict([ - ('new', self.new_file.to_dict()), - ('old', self.old_file.to_dict()), + ('category', 'unmodified'), + ('path', self.old_file.path) ]) diff --git a/src/deltacode/cli.py b/src/deltacode/cli.py index 16787815..5ca41ce8 100644 --- a/src/deltacode/cli.py +++ b/src/deltacode/cli.py @@ -16,9 +16,9 @@ def generate_csv(data, result_file): Using the OrderedDict generated by DeltaCode.to_dict(), create a .csv file containing the primary information from the Delta objects. """ - category, new, new_filename, new_sha1, new_size, new_type, new_orig, old,\ - old_filename, old_sha1, old_size, old_type, old_orig = '', '', '', '',\ - '', '', '', '', '', '', '', '', '' + category = '' + path = '' + tuple = () tuple_list = [] deltas = data @@ -26,29 +26,14 @@ def generate_csv(data, result_file): for delta in deltas: category = delta for f in deltas[delta]: - new = '' if delta == 'removed' else f['new']['path'] - new_filename = '' if delta == 'removed' else f['new']['name'] - new_sha1 = '' if delta == 'removed' else f['new']['sha1'] - new_size = '' if delta == 'removed' else f['new']['size'] - new_type = '' if delta == 'removed' else f['new']['type'] - new_orig = '' if delta == 'removed' else f['new']['original_path'] - old = '' if delta == 'added' else f['old']['path'] - old_filename = '' if delta == 'added' else f['old']['name'] - old_sha1 = '' if delta == 'added' else f['old']['sha1'] - old_size = '' if delta == 'added' else f['old']['size'] - old_type = '' if delta == 'added' else f['old']['type'] - old_orig = '' if delta == 'added' else f['old']['original_path'] - - tuple = (category, new, old, new_filename, old_filename, new_sha1, - old_sha1, new_size, old_size, new_type, old_type, new_orig, old_orig) + category = f['category'] + path = f['path'] + tuple = (category, path) tuple_list.append(tuple) with open(result_file, 'wb') as out: csv_out = csv.writer(out) - csv_out.writerow(['Type of delta', 'New scan path', 'Old scan path', - 'new_filename', 'old_filename', 'new_sha1', 'old_sha1', 'new_size', - 'old_size', 'new_type', 'old_type', 'new_original_path', 'old_original_path']) - + csv_out.writerow(['Type of delta', 'Path']) for row in tuple_list: csv_out.writerow(row) @@ -81,7 +66,7 @@ def cli(new, old, csv_file, json_file): # do the delta delta = DeltaCode(new, old) data = delta.to_dict() - + # output to csv if csv_file: @@ -92,3 +77,21 @@ def cli(new, old, csv_file, json_file): # print to stdout else: print(json.dumps(data, indent=4)) + + # for key in delta.deltas: + # print('\nkey in delta.deltas = {}\n'. format(key)) + # print('\ndelta.deltas[key] = {}\n'. format(delta.deltas[key])) + # for value in delta.deltas[key]: + # print('\n+++++\n') + # print('\nvalue = \n\n{}\n'.format(value)) + # print('\nvalue.new_file = \n\n{}\n'.format(value.new_file)) + # print('\nvalue.old_file = \n\n{}\n'.format(value.old_file)) + # print('\nvalue.category = \n\n{}\n'.format(value.category)) + # print('\nvalue.to_dict() = \n\n{}\n'.format(value.to_dict())) + # print('\n+++++\n') + + # print(delta.deltas) + # for key in delta.deltas: + # for value in delta.deltas[key]: + # print('=====================================') + # print(value.to_dict()) diff --git a/tests/data/cli/added1.csv b/tests/data/cli/added1.csv index 8a693df0..6b584e0b 100644 --- a/tests/data/cli/added1.csv +++ b/tests/data/cli/added1.csv @@ -1,10 +1,10 @@ -Type of delta,New scan path,Old scan path,new_filename,old_filename,new_sha1,old_sha1,new_size,old_size,new_type,old_type,new_original_path,old_original_path -added,a/a5.py,,a5.py,,a0c7dab0c0f07cc8ef8fac7729359b6997ed50cd,,200,,file,,codebase_01_1_file_added/a/a5.py, -unmodified,a/a3.py,a/a3.py,a3.py,a3.py,fd5d3589c825f448546d7dcec36da3e567d35fe9,fd5d3589c825f448546d7dcec36da3e567d35fe9,200,200,file,file,codebase_01_1_file_added/a/a3.py,codebase_01/a/a3.py -unmodified,b/b4.py,b/b4.py,b4.py,b4.py,98c9e6bed78b1513c28e666016cb35a50708c36e,98c9e6bed78b1513c28e666016cb35a50708c36e,200,200,file,file,codebase_01_1_file_added/b/b4.py,codebase_01/b/b4.py -unmodified,a/a2.py,a/a2.py,a2.py,a2.py,310797523e47db8481aeb06f1634317285115091,310797523e47db8481aeb06f1634317285115091,200,200,file,file,codebase_01_1_file_added/a/a2.py,codebase_01/a/a2.py -unmodified,b/b2.py,b/b2.py,b2.py,b2.py,3340d86b1da9323067db8022f86dc97cfccee1d0,3340d86b1da9323067db8022f86dc97cfccee1d0,200,200,file,file,codebase_01_1_file_added/b/b2.py,codebase_01/b/b2.py -unmodified,b/b1.py,b/b1.py,b1.py,b1.py,70f6ce80985578b5104db0abc578cf5a05e78f4b,70f6ce80985578b5104db0abc578cf5a05e78f4b,200,200,file,file,codebase_01_1_file_added/b/b1.py,codebase_01/b/b1.py -unmodified,b/b3.py,b/b3.py,b3.py,b3.py,e49d4463662414bee5ad2d2e5c1fbd704f33b84e,e49d4463662414bee5ad2d2e5c1fbd704f33b84e,200,200,file,file,codebase_01_1_file_added/b/b3.py,codebase_01/b/b3.py -unmodified,a/a4.py,a/a4.py,a4.py,a4.py,6f71666c46446c29d3f45feef5419ae76fb86a5b,6f71666c46446c29d3f45feef5419ae76fb86a5b,200,200,file,file,codebase_01_1_file_added/a/a4.py,codebase_01/a/a4.py -unmodified,a/a1.py,a/a1.py,a1.py,a1.py,84b647771481d39dd3a53f6dc210c26abac37748,84b647771481d39dd3a53f6dc210c26abac37748,200,200,file,file,codebase_01_1_file_added/a/a1.py,codebase_01/a/a1.py +Type of delta,Path +added,a/a5.py +unmodified,a/a3.py +unmodified,b/b4.py +unmodified,a/a2.py +unmodified,b/b2.py +unmodified,b/b1.py +unmodified,b/b3.py +unmodified,a/a4.py +unmodified,a/a1.py diff --git a/tests/data/cli/modified1.csv b/tests/data/cli/modified1.csv index d1e50ef7..214f0815 100644 --- a/tests/data/cli/modified1.csv +++ b/tests/data/cli/modified1.csv @@ -1,9 +1,9 @@ -Type of delta,New scan path,Old scan path,new_filename,old_filename,new_sha1,old_sha1,new_size,old_size,new_type,old_type,new_original_path,old_original_path -modified,a/a4.py,a/a4.py,a4.py,a4.py,56aacf4e8ebcd7d8365e68cf11df933a237d33eb,6f71666c46446c29d3f45feef5419ae76fb86a5b,246,200,file,file,codebase_01_1_file_modified_not_renamed/a/a4.py,codebase_01/a/a4.py -unmodified,a/a3.py,a/a3.py,a3.py,a3.py,fd5d3589c825f448546d7dcec36da3e567d35fe9,fd5d3589c825f448546d7dcec36da3e567d35fe9,200,200,file,file,codebase_01_1_file_modified_not_renamed/a/a3.py,codebase_01/a/a3.py -unmodified,b/b4.py,b/b4.py,b4.py,b4.py,98c9e6bed78b1513c28e666016cb35a50708c36e,98c9e6bed78b1513c28e666016cb35a50708c36e,200,200,file,file,codebase_01_1_file_modified_not_renamed/b/b4.py,codebase_01/b/b4.py -unmodified,b/b1.py,b/b1.py,b1.py,b1.py,70f6ce80985578b5104db0abc578cf5a05e78f4b,70f6ce80985578b5104db0abc578cf5a05e78f4b,200,200,file,file,codebase_01_1_file_modified_not_renamed/b/b1.py,codebase_01/b/b1.py -unmodified,a/a2.py,a/a2.py,a2.py,a2.py,310797523e47db8481aeb06f1634317285115091,310797523e47db8481aeb06f1634317285115091,200,200,file,file,codebase_01_1_file_modified_not_renamed/a/a2.py,codebase_01/a/a2.py -unmodified,b/b2.py,b/b2.py,b2.py,b2.py,3340d86b1da9323067db8022f86dc97cfccee1d0,3340d86b1da9323067db8022f86dc97cfccee1d0,200,200,file,file,codebase_01_1_file_modified_not_renamed/b/b2.py,codebase_01/b/b2.py -unmodified,b/b3.py,b/b3.py,b3.py,b3.py,e49d4463662414bee5ad2d2e5c1fbd704f33b84e,e49d4463662414bee5ad2d2e5c1fbd704f33b84e,200,200,file,file,codebase_01_1_file_modified_not_renamed/b/b3.py,codebase_01/b/b3.py -unmodified,a/a1.py,a/a1.py,a1.py,a1.py,84b647771481d39dd3a53f6dc210c26abac37748,84b647771481d39dd3a53f6dc210c26abac37748,200,200,file,file,codebase_01_1_file_modified_not_renamed/a/a1.py,codebase_01/a/a1.py +Type of delta,Path +modified,a/a4.py +unmodified,a/a3.py +unmodified,b/b4.py +unmodified,b/b1.py +unmodified,a/a2.py +unmodified,b/b2.py +unmodified,b/b3.py +unmodified,a/a1.py diff --git a/tests/data/cli/removed1.csv b/tests/data/cli/removed1.csv index 0570ad1c..26e4c983 100644 --- a/tests/data/cli/removed1.csv +++ b/tests/data/cli/removed1.csv @@ -1,9 +1,9 @@ -Type of delta,New scan path,Old scan path,new_filename,old_filename,new_sha1,old_sha1,new_size,old_size,new_type,old_type,new_original_path,old_original_path -removed,,a/a4.py,,a4.py,,6f71666c46446c29d3f45feef5419ae76fb86a5b,,200,,file,,codebase_01/a/a4.py -unmodified,a/a3.py,a/a3.py,a3.py,a3.py,fd5d3589c825f448546d7dcec36da3e567d35fe9,fd5d3589c825f448546d7dcec36da3e567d35fe9,200,200,file,file,codebase_01_1_file_removed/a/a3.py,codebase_01/a/a3.py -unmodified,b/b4.py,b/b4.py,b4.py,b4.py,98c9e6bed78b1513c28e666016cb35a50708c36e,98c9e6bed78b1513c28e666016cb35a50708c36e,200,200,file,file,codebase_01_1_file_removed/b/b4.py,codebase_01/b/b4.py -unmodified,b/b1.py,b/b1.py,b1.py,b1.py,70f6ce80985578b5104db0abc578cf5a05e78f4b,70f6ce80985578b5104db0abc578cf5a05e78f4b,200,200,file,file,codebase_01_1_file_removed/b/b1.py,codebase_01/b/b1.py -unmodified,a/a2.py,a/a2.py,a2.py,a2.py,310797523e47db8481aeb06f1634317285115091,310797523e47db8481aeb06f1634317285115091,200,200,file,file,codebase_01_1_file_removed/a/a2.py,codebase_01/a/a2.py -unmodified,b/b2.py,b/b2.py,b2.py,b2.py,3340d86b1da9323067db8022f86dc97cfccee1d0,3340d86b1da9323067db8022f86dc97cfccee1d0,200,200,file,file,codebase_01_1_file_removed/b/b2.py,codebase_01/b/b2.py -unmodified,b/b3.py,b/b3.py,b3.py,b3.py,e49d4463662414bee5ad2d2e5c1fbd704f33b84e,e49d4463662414bee5ad2d2e5c1fbd704f33b84e,200,200,file,file,codebase_01_1_file_removed/b/b3.py,codebase_01/b/b3.py -unmodified,a/a1.py,a/a1.py,a1.py,a1.py,84b647771481d39dd3a53f6dc210c26abac37748,84b647771481d39dd3a53f6dc210c26abac37748,200,200,file,file,codebase_01_1_file_removed/a/a1.py,codebase_01/a/a1.py +Type of delta,Path +removed,a/a4.py +unmodified,a/a3.py +unmodified,b/b4.py +unmodified,b/b1.py +unmodified,a/a2.py +unmodified,b/b2.py +unmodified,b/b3.py +unmodified,a/a1.py diff --git a/tests/data/cli/renamed1.csv b/tests/data/cli/renamed1.csv index c7cd4162..91d18c7a 100644 --- a/tests/data/cli/renamed1.csv +++ b/tests/data/cli/renamed1.csv @@ -1,10 +1,10 @@ -Type of delta,New scan path,Old scan path,new_filename,old_filename,new_sha1,old_sha1,new_size,old_size,new_type,old_type,new_original_path,old_original_path -added,a/a4_renamed_not_modified.py,,a4_renamed_not_modified.py,,6f71666c46446c29d3f45feef5419ae76fb86a5b,,200,,file,,codebase_01_1_file_renamed_not_modified/a/a4_renamed_not_modified.py, -removed,,a/a4.py,,a4.py,,6f71666c46446c29d3f45feef5419ae76fb86a5b,,200,,file,,codebase_01/a/a4.py -unmodified,a/a3.py,a/a3.py,a3.py,a3.py,fd5d3589c825f448546d7dcec36da3e567d35fe9,fd5d3589c825f448546d7dcec36da3e567d35fe9,200,200,file,file,codebase_01_1_file_renamed_not_modified/a/a3.py,codebase_01/a/a3.py -unmodified,b/b4.py,b/b4.py,b4.py,b4.py,98c9e6bed78b1513c28e666016cb35a50708c36e,98c9e6bed78b1513c28e666016cb35a50708c36e,200,200,file,file,codebase_01_1_file_renamed_not_modified/b/b4.py,codebase_01/b/b4.py -unmodified,b/b1.py,b/b1.py,b1.py,b1.py,70f6ce80985578b5104db0abc578cf5a05e78f4b,70f6ce80985578b5104db0abc578cf5a05e78f4b,200,200,file,file,codebase_01_1_file_renamed_not_modified/b/b1.py,codebase_01/b/b1.py -unmodified,b/b2.py,b/b2.py,b2.py,b2.py,3340d86b1da9323067db8022f86dc97cfccee1d0,3340d86b1da9323067db8022f86dc97cfccee1d0,200,200,file,file,codebase_01_1_file_renamed_not_modified/b/b2.py,codebase_01/b/b2.py -unmodified,b/b3.py,b/b3.py,b3.py,b3.py,e49d4463662414bee5ad2d2e5c1fbd704f33b84e,e49d4463662414bee5ad2d2e5c1fbd704f33b84e,200,200,file,file,codebase_01_1_file_renamed_not_modified/b/b3.py,codebase_01/b/b3.py -unmodified,a/a1.py,a/a1.py,a1.py,a1.py,84b647771481d39dd3a53f6dc210c26abac37748,84b647771481d39dd3a53f6dc210c26abac37748,200,200,file,file,codebase_01_1_file_renamed_not_modified/a/a1.py,codebase_01/a/a1.py -unmodified,a/a2.py,a/a2.py,a2.py,a2.py,310797523e47db8481aeb06f1634317285115091,310797523e47db8481aeb06f1634317285115091,200,200,file,file,codebase_01_1_file_renamed_not_modified/a/a2.py,codebase_01/a/a2.py +Type of delta,Path +added,a/a4_renamed_not_modified.py +removed,a/a4.py +unmodified,a/a3.py +unmodified,b/b4.py +unmodified,b/b1.py +unmodified,b/b2.py +unmodified,b/b3.py +unmodified,a/a1.py +unmodified,a/a2.py diff --git a/tests/test_deltacode.py b/tests/test_deltacode.py index 881ffcf5..da4330a8 100644 --- a/tests/test_deltacode.py +++ b/tests/test_deltacode.py @@ -46,16 +46,16 @@ def test_align_and_index_scans(self): assert delta.new.files_count == new_index_length assert delta.old.files_count == old_index_length - + def test_DeltaCode_ecos_failed_counts_assertion(self): new_scan = self.get_test_loc('deltacode/ecos-failed-counts-assertion-new.json') old_scan = self.get_test_loc('deltacode/ecos-failed-counts-assertion-old.json') result = DeltaCode(new_scan, old_scan) - + assert result.new.files_count == 11408 assert result.old.files_count == 8631 - + def test_DeltaCode_abcm_aligned(self): new_scan = self.get_test_loc('deltacode/abcm-aligned-new.json') old_scan = self.get_test_loc('deltacode/abcm-aligned-old.json') @@ -184,15 +184,15 @@ def test_DeltaCode_delta_len_error(self): with pytest.raises(AssertionError): result.determine_delta() - + def test_DeltaCode_to_dict_original_path_openssl(self): test_scan_new = self.get_test_loc('deltacode/to-dict-openssl-new.json') test_scan_old = self.get_test_loc('deltacode/to-dict-openssl-old.json') deltacode = DeltaCode(test_scan_new, test_scan_old) result = deltacode.to_dict() - - assert (len(result.get('added')) + len(result.get('removed')) + + assert (len(result.get('added')) + len(result.get('removed')) + len(result.get('modified')) + len(result.get('unmodified'))) == 2459 assert len(result.get('added')) == 76 @@ -209,14 +209,14 @@ def test_DeltaCode_to_dict_original_path_dropbear(self): deltacode = DeltaCode(test_scan_new, test_scan_old) result = deltacode.to_dict() - - assert (len(result.get('added')) + len(result.get('removed')) + + assert (len(result.get('added')) + len(result.get('removed')) + len(result.get('modified')) + len(result.get('unmodified'))) == 733 assert len(result.get('added')) == 0 assert len(result.get('removed')) == 0 assert len(result.get('modified')) == 17 - assert len(result.get('unmodified')) == 716 + assert len(result.get('unmodified')) == 716 def test_DeltaCode_to_dict_original_path_zlib(self): test_scan_new = self.get_test_loc('deltacode/to-dict-zlib-new.json') @@ -227,30 +227,30 @@ def test_DeltaCode_to_dict_original_path_zlib(self): deltacode = DeltaCode(test_scan_new, test_scan_old) result = deltacode.to_dict() - - assert (len(result.get('added')) + len(result.get('removed')) + + assert (len(result.get('added')) + len(result.get('removed')) + len(result.get('modified')) + len(result.get('unmodified'))) == 259 assert len(result.get('added')) == 0 assert len(result.get('removed')) == 6 assert len(result.get('modified')) == 34 assert len(result.get('unmodified')) == 219 - + def test_DeltaCode_to_dict_original_path_added1(self): test_scan_new = self.get_test_loc('deltacode/to-dict-new-added1.json') test_scan_old = self.get_test_loc('deltacode/to-dict-old-added1.json') deltacode = DeltaCode(test_scan_new, test_scan_old) result = deltacode.to_dict() - - assert (len(result.get('added')) + len(result.get('removed')) + + assert (len(result.get('added')) + len(result.get('removed')) + len(result.get('modified')) + len(result.get('unmodified'))) == 9 assert len(result.get('added')) == 1 assert len(result.get('removed')) == 0 assert len(result.get('modified')) == 0 assert len(result.get('unmodified')) == 8 - + def test_DeltaCode_to_dict_original_path_full_root(self): test_scan_new = self.get_test_loc('deltacode/to-dict-align-trees-simple-new.json') # Our old scan uses --full-root option in scancode @@ -259,14 +259,14 @@ def test_DeltaCode_to_dict_original_path_full_root(self): deltacode = DeltaCode(test_scan_new, test_scan_old) result = deltacode.to_dict() - assert (len(result.get('added')) + len(result.get('removed')) + assert (len(result.get('added')) + len(result.get('removed')) + len(result.get('modified')) + len(result.get('unmodified'))) == 33 assert len(result.get('added')) == 0 assert len(result.get('removed')) == 0 assert len(result.get('modified')) == 0 assert len(result.get('unmodified')) == 33 - + def test_DeltaCode_to_dict_simple_file_added(self): new_scan = self.get_test_loc('deltacode/new_added1.json') old_scan = self.get_test_loc('deltacode/old_added1.json') @@ -274,7 +274,7 @@ def test_DeltaCode_to_dict_simple_file_added(self): deltacode = DeltaCode(new_scan, old_scan) result = deltacode.to_dict() - assert (len(result.get('added')) + len(result.get('removed')) + assert (len(result.get('added')) + len(result.get('removed')) + len(result.get('modified')) + len(result.get('unmodified'))) == 9 assert len(result.get('added')) == 1 @@ -289,14 +289,14 @@ def test_DeltaCode_to_dict_simple_file_modified(self): deltacode = DeltaCode(new_scan, old_scan) result = deltacode.to_dict() - assert (len(result.get('added')) + len(result.get('removed')) + assert (len(result.get('added')) + len(result.get('removed')) + len(result.get('modified')) + len(result.get('unmodified'))) == 8 assert len(result.get('added')) == 0 assert len(result.get('removed')) == 0 assert len(result.get('modified')) == 1 assert len(result.get('unmodified')) == 7 - + def test_DeltaCode_to_dict_simple_unmodified(self): test_file = self.get_test_loc('deltacode/to-dict-unmodified.json') @@ -306,30 +306,16 @@ def test_DeltaCode_to_dict_simple_unmodified(self): ('added', []), ('removed', []), ('modified', []), - ('unmodified',[ + ('unmodified', [ OrderedDict([ - ('new', OrderedDict([ - ('path', 'test/unmodified.txt'), - ('type', 'file',), - ('name', 'unmodified.txt'), - ('size', 11), - ('sha1', '4f499c82f79e5372c293010f931ad2798ddf3e8e'), - ('original_path', 'test/unmodified.txt'), - ])), - ('old', OrderedDict([ - ('path', 'test/unmodified.txt'), - ('type', 'file',), - ('name', 'unmodified.txt'), - ('size', 11), - ('sha1', '4f499c82f79e5372c293010f931ad2798ddf3e8e'), - ('original_path', 'test/unmodified.txt'), - ])) + ('category', 'unmodified'), + ('path', u'test/unmodified.txt') ]) - ]), + ]) ]) result = deltacode.to_dict() - + assert result == expected def test_DeltaCode_to_dict_empty(self): @@ -380,7 +366,7 @@ def test_DeltaCode_None_paths(self): assert result.old.files == None assert result.deltas == None - + def test_Delta_to_dict_removed(self): old = models.File({ 'path': 'path/removed.txt', @@ -391,21 +377,14 @@ def test_Delta_to_dict_removed(self): 'original_path': '' }) expected = { - 'new': None, - 'old': { - 'path': 'path/removed.txt', - 'type': 'file', - 'name': 'removed.txt', - 'size': 20, - 'sha1': 'a', - 'original_path': '' - }, + 'category': 'removed', + 'path': 'path/removed.txt' } - + delta = deltacode.Delta(None, old, 'removed') assert delta.to_dict() == expected - + def test_Delta_to_dict_added(self): new = models.File({ 'path': 'path/added.txt', @@ -416,21 +395,14 @@ def test_Delta_to_dict_added(self): 'original_path': '' }) expected = { - 'new': { - 'path': 'path/added.txt', - 'type': 'file', - 'name': 'added.txt', - 'size': 20, - 'sha1': 'a', - 'original_path': '' - }, - 'old': None + 'category': 'added', + 'path': 'path/added.txt' } - + delta = deltacode.Delta(new, None, 'added') assert delta.to_dict() == expected - + def test_Delta_to_dict_modified(self): new = models.File({ 'path': 'path/modified.txt', @@ -450,22 +422,8 @@ def test_Delta_to_dict_modified(self): }) expected = { - 'new': { - 'path': 'path/modified.txt', - 'type': 'file', - 'name': 'modified.txt', - 'size': 20, - 'sha1': 'a', - 'original_path': '' - }, - 'old': { - 'path': 'path/modified.txt', - 'type': 'file', - 'name': 'modified.txt', - 'size': 21, - 'sha1': 'b', - 'original_path': '' - } + 'category': 'modified', + 'path': 'path/modified.txt' } delta = deltacode.Delta(new, old, 'modified') @@ -491,22 +449,8 @@ def test_Delta_to_dict_unmodified(self): }) expected = { - 'new': { - 'path': 'path/unmodified.txt', - 'type': 'file', - 'name': 'unmodified.txt', - 'size': 20, - 'sha1': 'a', - 'original_path': '' - }, - 'old': { - 'path': 'path/unmodified.txt', - 'type': 'file', - 'name': 'unmodified.txt', - 'size': 20, - 'sha1': 'a', - 'original_path': '' - } + 'category': 'unmodified', + 'path': 'path/unmodified.txt' } delta = deltacode.Delta(new, old, 'unmodified') @@ -517,7 +461,7 @@ def test_Delta_to_dict_empty(self): delta = deltacode.Delta() assert delta.to_dict() == None - + def test_Delta_create_object_removed(self): new = None old = models.File({'path': 'path/removed.txt'}) @@ -537,7 +481,7 @@ def test_Delta_create_object_added(self): assert delta.new_file.path == 'path/added.txt' assert delta.old_file == None assert delta.category == 'added' - + def test_Delta_create_object_modified(self): new = models.File({'path': 'path/modified.txt', 'sha1': 'a'}) old = models.File({'path': 'path/modified.txt', 'sha1': 'b'}) @@ -549,7 +493,7 @@ def test_Delta_create_object_modified(self): assert delta.old_file.path == 'path/modified.txt' assert delta.old_file.sha1 == 'b' assert delta.category == 'modified' - + def test_Delta_create_object_unmodified(self): new = models.File({'path': 'path/unmodified.txt', 'sha1': 'a'}) old = models.File({'path': 'path/unmodified.txt', 'sha1': 'a'}) From 515c4fda88c204b2b7cd111dab67bda7f388da64 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Sat, 18 Nov 2017 10:21:10 -0800 Subject: [PATCH 2/4] Refactor generate_csv() and generate_json() # 16 Signed-off-by: John M. Horan --- src/deltacode/cli.py | 52 ++++++++++++-------------------------------- 1 file changed, 14 insertions(+), 38 deletions(-) diff --git a/src/deltacode/cli.py b/src/deltacode/cli.py index 5ca41ce8..fac5130f 100644 --- a/src/deltacode/cli.py +++ b/src/deltacode/cli.py @@ -3,12 +3,15 @@ # from __future__ import absolute_import +from collections import OrderedDict + import csv import json import click from deltacode import DeltaCode +from deltacode import __version__ def generate_csv(data, result_file): @@ -16,20 +19,8 @@ def generate_csv(data, result_file): Using the OrderedDict generated by DeltaCode.to_dict(), create a .csv file containing the primary information from the Delta objects. """ - category = '' - path = '' - - tuple = () - tuple_list = [] deltas = data - - for delta in deltas: - category = delta - for f in deltas[delta]: - category = f['category'] - path = f['path'] - tuple = (category, path) - tuple_list.append(tuple) + tuple_list = [(f['category'], f['path']) for delta in deltas for f in deltas[delta]] with open(result_file, 'wb') as out: csv_out = csv.writer(out) @@ -38,14 +29,19 @@ def generate_csv(data, result_file): csv_out.writerow(row) -def generate_json(data, result_file): +def generate_json(delta, result_file): """ - Using the OrderedDict generated by DeltaCode.to_dict(), create a .json file + Using the DeltaCode object, create a .json file containing the primary information from the Delta objects. """ - # TODO: Add json file headers here + json_dict = OrderedDict([ + ('deltacode_version', __version__), + ('deltacode_stats', delta.get_stats()), + ('deltas', delta.to_dict()) + ]) + with open(result_file, 'w') as outfile: - json.dump(data, outfile, indent=4) + json.dump(json_dict, outfile, indent=4) @click.command() @@ -62,36 +58,16 @@ def cli(new, old, csv_file, json_file): .json file (-j or -json-file) at a user-designated location. If no file option is selected, the JSON results are printed to the console. """ - # do the delta delta = DeltaCode(new, old) data = delta.to_dict() - # output to csv if csv_file: generate_csv(data, csv_file) # generate JSON output elif json_file: - generate_json(data, json_file) + generate_json(delta, json_file) # print to stdout else: print(json.dumps(data, indent=4)) - - # for key in delta.deltas: - # print('\nkey in delta.deltas = {}\n'. format(key)) - # print('\ndelta.deltas[key] = {}\n'. format(delta.deltas[key])) - # for value in delta.deltas[key]: - # print('\n+++++\n') - # print('\nvalue = \n\n{}\n'.format(value)) - # print('\nvalue.new_file = \n\n{}\n'.format(value.new_file)) - # print('\nvalue.old_file = \n\n{}\n'.format(value.old_file)) - # print('\nvalue.category = \n\n{}\n'.format(value.category)) - # print('\nvalue.to_dict() = \n\n{}\n'.format(value.to_dict())) - # print('\n+++++\n') - - # print(delta.deltas) - # for key in delta.deltas: - # for value in delta.deltas[key]: - # print('=====================================') - # print(value.to_dict()) From 054888e05fe937d5e10986ed1e8c901a5761e722 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Mon, 20 Nov 2017 13:14:46 -0800 Subject: [PATCH 3/4] Refactor JSON- and CSV-output functions #16 Signed-off-by: John M. Horan --- src/deltacode/cli.py | 18 ++++++++---------- tests/test_cli.py | 14 ++++---------- 2 files changed, 12 insertions(+), 20 deletions(-) diff --git a/src/deltacode/cli.py b/src/deltacode/cli.py index fac5130f..cfd19628 100644 --- a/src/deltacode/cli.py +++ b/src/deltacode/cli.py @@ -14,18 +14,17 @@ from deltacode import __version__ -def generate_csv(data, result_file): +def generate_csv(delta, result_file): """ - Using the OrderedDict generated by DeltaCode.to_dict(), create a .csv file + Using the DeltaCode object and its to_dict() method, create a .csv file containing the primary information from the Delta objects. """ - deltas = data - tuple_list = [(f['category'], f['path']) for delta in deltas for f in deltas[delta]] + output = delta.to_dict() with open(result_file, 'wb') as out: csv_out = csv.writer(out) csv_out.writerow(['Type of delta', 'Path']) - for row in tuple_list: + for row in [(f.get('category'), f.get('path')) for d in output for f in output.get(d)]: csv_out.writerow(row) @@ -34,14 +33,14 @@ def generate_json(delta, result_file): Using the DeltaCode object, create a .json file containing the primary information from the Delta objects. """ - json_dict = OrderedDict([ + output = OrderedDict([ ('deltacode_version', __version__), ('deltacode_stats', delta.get_stats()), ('deltas', delta.to_dict()) ]) with open(result_file, 'w') as outfile: - json.dump(json_dict, outfile, indent=4) + json.dump(output, outfile, indent=4) @click.command() @@ -60,14 +59,13 @@ def cli(new, old, csv_file, json_file): """ # do the delta delta = DeltaCode(new, old) - data = delta.to_dict() # output to csv if csv_file: - generate_csv(data, csv_file) + generate_csv(delta, csv_file) # generate JSON output elif json_file: generate_json(delta, json_file) # print to stdout else: - print(json.dumps(data, indent=4)) + print(json.dumps(delta.to_dict())) diff --git a/tests/test_cli.py b/tests/test_cli.py index ed595fe3..f3b57313 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -58,11 +58,8 @@ def test_generate_csv_added(self): old_scan = self.get_test_loc('cli/old_added1.json') delta = DeltaCode(new_scan, old_scan) - data = delta.to_dict() - result_file = self.get_temp_file('.csv') - cli.generate_csv(data, result_file) - + cli.generate_csv(delta, result_file) expected_file = self.get_test_loc('cli/added1.csv') check_csvs(result_file, expected_file) @@ -71,9 +68,8 @@ def test_generate_csv_modified(self): old_scan = self.get_test_loc('cli/old_modified1.json') delta = DeltaCode(new_scan, old_scan) - data = delta.to_dict() result_file = self.get_temp_file('.csv') - cli.generate_csv(data, result_file) + cli.generate_csv(delta, result_file) expected_file = self.get_test_loc('cli/modified1.csv') check_csvs(result_file, expected_file) @@ -82,9 +78,8 @@ def test_generate_csv_removed(self): old_scan = self.get_test_loc('cli/old_removed1.json') delta = DeltaCode(new_scan, old_scan) - data = delta.to_dict() result_file = self.get_temp_file('.csv') - cli.generate_csv(data, result_file) + cli.generate_csv(delta, result_file) expected_file = self.get_test_loc('cli/removed1.csv') check_csvs(result_file, expected_file) @@ -93,8 +88,7 @@ def test_generate_csv_renamed(self): old_scan = self.get_test_loc('cli/old_renamed1.json') delta = DeltaCode(new_scan, old_scan) - data = delta.to_dict() result_file = self.get_temp_file('.csv') - cli.generate_csv(data, result_file) + cli.generate_csv(delta, result_file) expected_file = self.get_test_loc('cli/renamed1.csv') check_csvs(result_file, expected_file) From 5a426280020e756caf60816240c1f021b7834058 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Mon, 20 Nov 2017 16:21:20 -0800 Subject: [PATCH 4/4] Refactor generate_csv() #16 * Remove call to Delta.to_dict(). Signed-off-by: John M. Horan --- src/deltacode/cli.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/deltacode/cli.py b/src/deltacode/cli.py index cfd19628..189f0f5e 100644 --- a/src/deltacode/cli.py +++ b/src/deltacode/cli.py @@ -16,15 +16,13 @@ def generate_csv(delta, result_file): """ - Using the DeltaCode object and its to_dict() method, create a .csv file + Using the DeltaCode object, create a .csv file containing the primary information from the Delta objects. """ - output = delta.to_dict() - with open(result_file, 'wb') as out: csv_out = csv.writer(out) csv_out.writerow(['Type of delta', 'Path']) - for row in [(f.get('category'), f.get('path')) for d in output for f in output.get(d)]: + for row in [(f.category, f.old_file.path if f.category == 'removed' else f.new_file.path) for d in delta.deltas for f in delta.deltas.get(d)]: csv_out.writerow(row)