From 7309796779e5a194a3644c491361095ce2309980 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Mon, 22 Jan 2018 17:31:10 -0800 Subject: [PATCH 1/2] Limit output to changes by default #41 * Add command option: '-a/--all'. * Pass 'all' parameter to 'write_csv()', 'write_json()' and 'utils.deltas()'. * Fix failing tests, modify/add new tests for '-a/--all' option. Signed-off-by: John M. Horan --- src/deltacode/cli.py | 26 +++-- src/deltacode/utils.py | 11 +- .../cli/1_file_moved_all_not_selected.csv | 2 + tests/test_cli.py | 109 +++++++++++++++--- 4 files changed, 119 insertions(+), 29 deletions(-) create mode 100644 tests/data/cli/1_file_moved_all_not_selected.csv diff --git a/src/deltacode/cli.py b/src/deltacode/cli.py index db16b154..80b35047 100644 --- a/src/deltacode/cli.py +++ b/src/deltacode/cli.py @@ -39,10 +39,12 @@ # FIXME: update the function argument delta to deltacode -def write_csv(delta, result_file): +def write_csv(delta, result_file, all): """ Using the DeltaCode object, create a .csv file - containing the primary information from the Delta objects. + containing the primary information from the Delta objects. Omit all Delta + objects whose 'category' is 'unmodified' unless the user selects the + '-a'/'--all' option. """ with open(result_file, 'wb') as out: csv_out = csv.writer(out) @@ -55,18 +57,23 @@ def write_csv(delta, result_file): f.old_file.size if f.category == 'removed' else f.new_file.size, f.old_file.path if f.category == 'moved' else '') for d in delta.deltas for f in delta.deltas.get(d)]: - csv_out.writerow(row) + if all is True: + csv_out.writerow(row) + elif row[0] != 'unmodified': + csv_out.writerow(row) -def write_json(deltacode, outfile): +def write_json(deltacode, outfile, all): """ Using the DeltaCode object, create a .json file - containing the primary information from the Delta objects. + containing the primary information from the Delta objects. Omit all Delta + objects whose 'category' is 'unmodified' unless the user selects the + '-a'/'--all' option. """ results = OrderedDict([ ('deltacode_version', __version__), ('deltacode_stats', deltacode.get_stats()), - ('deltas', deltas(deltacode)), + ('deltas', deltas(deltacode, all)), ]) # TODO: add toggle for pretty printing @@ -80,7 +87,8 @@ def write_json(deltacode, outfile): @click.option('-o', '--old', required=True, prompt=False, type=click.Path(exists=True, readable=True), help='Identify the path to the "old" scan file') @click.option('-c', '--csv-file', prompt=False, type=click.Path(exists=False), help='Identify the path to the .csv output file') @click.option('-j', '--json-file', prompt=False, default='-', type=click.File(mode='wb', lazy=False), help='Identify the path to the .json output file') -def cli(new, old, csv_file, json_file): +@click.option('-a', '--all', is_flag=True, help="Include unmodified files as well as all changed files in the .json or .csv output. If not selected, only changed files are included.") +def cli(new, old, csv_file, json_file, all): """ Identify the changes that need to be made to the 'old' scan file (-o or -old) in order to generate the 'new' scan file (-n or @@ -93,7 +101,7 @@ def cli(new, old, csv_file, json_file): # output to csv if csv_file: - write_csv(deltacode, csv_file) + write_csv(deltacode, csv_file, all) # generate JSON output else: - write_json(deltacode, json_file) + write_json(deltacode, json_file, all) diff --git a/src/deltacode/utils.py b/src/deltacode/utils.py index cf5bc7f1..5e278404 100644 --- a/src/deltacode/utils.py +++ b/src/deltacode/utils.py @@ -31,13 +31,18 @@ from commoncode import paths -def deltas(deltacode): +def deltas(deltacode, all): """ - Return a generator of Delta dictionaries for JSON serialized ouput. + Return a generator of Delta dictionaries for JSON serialized ouput. Omit + all Delta objects whose 'category' is 'unmodified' unless the user selects + the '-a'/'--all' option. """ for category, deltas in deltacode.deltas.iteritems(): for delta in deltas: - yield delta.to_dict() + if all is True: + yield delta.to_dict() + elif delta.category != 'unmodified': + yield delta.to_dict() class AlignmentException(Exception): diff --git a/tests/data/cli/1_file_moved_all_not_selected.csv b/tests/data/cli/1_file_moved_all_not_selected.csv new file mode 100644 index 00000000..1a44f2d0 --- /dev/null +++ b/tests/data/cli/1_file_moved_all_not_selected.csv @@ -0,0 +1,2 @@ +Type of delta,Path,Name,Type,Size,Old Path +moved,b/a4.py,a4.py,file,200,a/a4.py diff --git a/tests/test_cli.py b/tests/test_cli.py index 1d035962..be6450ef 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -81,7 +81,7 @@ def test_write_csv_added(self): delta = DeltaCode(new_scan, old_scan) result_file = self.get_temp_file('.csv') - cli.write_csv(delta, result_file) + cli.write_csv(delta, result_file, True) expected_file = self.get_test_loc('cli/added1.csv') check_csvs(result_file, expected_file) @@ -91,7 +91,7 @@ def test_write_csv_modified(self): delta = DeltaCode(new_scan, old_scan) result_file = self.get_temp_file('.csv') - cli.write_csv(delta, result_file) + cli.write_csv(delta, result_file, True) expected_file = self.get_test_loc('cli/modified1.csv') check_csvs(result_file, expected_file) @@ -101,7 +101,7 @@ def test_write_csv_removed(self): delta = DeltaCode(new_scan, old_scan) result_file = self.get_temp_file('.csv') - cli.write_csv(delta, result_file) + cli.write_csv(delta, result_file, True) expected_file = self.get_test_loc('cli/removed1.csv') check_csvs(result_file, expected_file) @@ -111,7 +111,7 @@ def test_write_csv_renamed(self): delta = DeltaCode(new_scan, old_scan) result_file = self.get_temp_file('.csv') - cli.write_csv(delta, result_file) + cli.write_csv(delta, result_file, True) expected_file = self.get_test_loc('cli/renamed1.csv') check_csvs(result_file, expected_file) @@ -121,7 +121,7 @@ def test_write_csv_modified_new_license_added(self): delta = DeltaCode(new_scan, old_scan) result_file = self.get_temp_file('.csv') - cli.write_csv(delta, result_file) + cli.write_csv(delta, result_file, True) expected_file = self.get_test_loc('cli/modified_new_license_added.csv') check_csvs(result_file, expected_file) @@ -131,7 +131,7 @@ def test_write_csv_modified_new_license_added_low_score(self): delta = DeltaCode(new_scan, old_scan) result_file = self.get_temp_file('.csv') - cli.write_csv(delta, result_file) + cli.write_csv(delta, result_file, True) expected_file = self.get_test_loc('cli/modified_new_license_added_low_score.csv') check_csvs(result_file, expected_file) @@ -141,7 +141,7 @@ def test_write_csv_license_info_removed(self): delta = DeltaCode(new_scan, old_scan) result_file = self.get_temp_file('.csv') - cli.write_csv(delta, result_file) + cli.write_csv(delta, result_file, True) expected_file = self.get_test_loc('cli/license_info_removed.csv') check_csvs(result_file, expected_file) @@ -151,7 +151,7 @@ def test_write_csv_license_info_added(self): delta = DeltaCode(new_scan, old_scan) result_file = self.get_temp_file('.csv') - cli.write_csv(delta, result_file) + cli.write_csv(delta, result_file, True) expected_file = self.get_test_loc('cli/license_info_added.csv') check_csvs(result_file, expected_file) @@ -161,7 +161,7 @@ def test_write_csv_license_info_removed_below_cutoff_score(self): delta = DeltaCode(new_scan, old_scan) result_file = self.get_temp_file('.csv') - cli.write_csv(delta, result_file) + cli.write_csv(delta, result_file, True) expected_file = self.get_test_loc('cli/license_info_removed_below_cutoff_score.csv') check_csvs(result_file, expected_file) @@ -171,7 +171,7 @@ def test_write_csv_license_info_added_below_cutoff_score(self): delta = DeltaCode(new_scan, old_scan) result_file = self.get_temp_file('.csv') - cli.write_csv(delta, result_file) + cli.write_csv(delta, result_file, True) expected_file = self.get_test_loc('cli/license_info_added_below_cutoff_score.csv') check_csvs(result_file, expected_file) @@ -181,7 +181,7 @@ def test_write_csv_1_file_moved(self): delta = DeltaCode(new_scan, old_scan) result_file = self.get_temp_file('.csv') - cli.write_csv(delta, result_file) + cli.write_csv(delta, result_file, True) expected_file = self.get_test_loc('cli/1_file_moved.csv') check_csvs(result_file, expected_file) @@ -191,7 +191,7 @@ def test_write_csv_1_file_moved_and_1_copy(self): delta = DeltaCode(new_scan, old_scan) result_file = self.get_temp_file('.csv') - cli.write_csv(delta, result_file) + cli.write_csv(delta, result_file, True) expected_file = self.get_test_loc('cli/1_file_moved_and_1_copy.csv') check_csvs(result_file, expected_file) @@ -201,16 +201,41 @@ def test_write_csv_1_file_moved_and_added(self): delta = DeltaCode(new_scan, old_scan) result_file = self.get_temp_file('.csv') - cli.write_csv(delta, result_file) + cli.write_csv(delta, result_file, True) expected_file = self.get_test_loc('cli/1_file_moved_and_added.csv') check_csvs(result_file, expected_file) - def test_json_output_option_selected(self): + def test_json_output_option_selected_all_selected(self): + new_scan = self.get_test_loc('deltacode/scan_1_file_moved_new.json') + old_scan = self.get_test_loc('deltacode/scan_1_file_moved_old.json') + + result_file = self.get_temp_file('json') + + runner = CliRunner() + result = runner.invoke(cli.cli, ['-n', new_scan, '-o', old_scan, '-j', result_file, '-a']) + + assert result.exit_code == 0 + + json_result = json.load(open(result_file)) + stats = {'unmodified': 7, 'removed': 0, 'added': 0, 'moved': 1, 'modified': 0} + + assert json_result.get('deltacode_stats') == stats + + moved_expected = {'category': 'moved', 'name': 'a4.py', 'path': 'b/a4.py', 'old_path': 'a/a4.py', 'type': 'file', 'size': 200} + moved_result = [d for d in json_result.get('deltas') if d.get('category') == 'moved'].pop() + + assert moved_result == moved_expected + + unmodified_expected = {'category': 'unmodified', 'name': 'a3.py', 'path': 'a/a3.py', 'type': 'file', 'size': 200} + unmodified_result = [d for d in json_result.get('deltas') if d.get('category') == 'unmodified' and d.get('path') == 'a/a3.py'].pop() + + assert unmodified_result == unmodified_expected + + def test_json_output_option_selected_all_not_selected(self): new_scan = self.get_test_loc('deltacode/scan_1_file_moved_new.json') old_scan = self.get_test_loc('deltacode/scan_1_file_moved_old.json') result_file = self.get_temp_file('json') - expected_file = self.get_test_loc('cli/1_file_moved.json') runner = CliRunner() result = runner.invoke(cli.cli, ['-n', new_scan, '-o', old_scan, '-j', result_file]) @@ -227,20 +252,65 @@ def test_json_output_option_selected(self): assert moved_result == moved_expected - def test_csv_output_option_selected(self): + unmodified_result = [d for d in json_result.get('deltas') if d.get('category') == 'unmodified'] + + assert len(unmodified_result) == 0 + + def test_csv_output_option_selected_all_selected(self): new_scan = self.get_test_loc('deltacode/scan_1_file_moved_new.json') old_scan = self.get_test_loc('deltacode/scan_1_file_moved_old.json') result_file = self.get_temp_file('.csv') expected_file = self.get_test_loc('cli/1_file_moved.csv') + runner = CliRunner() + result = runner.invoke(cli.cli, ['-n', new_scan, '-o', old_scan, '-c', result_file, '-a']) + + assert result.exit_code == 0 + check_csvs(result_file, expected_file) + + def test_csv_output_option_selected_all_not_selected(self): + new_scan = self.get_test_loc('deltacode/scan_1_file_moved_new.json') + old_scan = self.get_test_loc('deltacode/scan_1_file_moved_old.json') + + result_file = self.get_temp_file('.csv') + expected_file = self.get_test_loc('cli/1_file_moved_all_not_selected.csv') + runner = CliRunner() result = runner.invoke(cli.cli, ['-n', new_scan, '-o', old_scan, '-c', result_file]) assert result.exit_code == 0 check_csvs(result_file, expected_file) - def test_no_output_option_selected(self): + def test_no_output_option_selected_all_selected(self): + new_scan = self.get_test_loc('deltacode/scan_1_file_moved_new.json') + old_scan = self.get_test_loc('deltacode/scan_1_file_moved_old.json') + + runner = CliRunner() + result = runner.invoke(cli.cli, ['-n', new_scan, '-o', old_scan, '-a']) + + assert result.exit_code == 0 + + assert '"added": 0' in result.output + assert '"modified": 0' in result.output + assert '"moved": 1' in result.output + assert '"removed": 0' in result.output + assert '"unmodified": 7' in result.output + + assert '"category": "moved"' in result.output + assert '"path": "b/a4.py"' in result.output + assert '"old_path": "a/a4.py"' in result.output + assert '"name": "a4.py"' in result.output + assert '"type": "file"' in result.output + assert '"size": 200' in result.output + + assert '"category": "unmodified"' in result.output + assert '"path": "a/a3.py"' in result.output + assert '"name": "a3.py"' in result.output + assert '"type": "file"' in result.output + assert '"size": 200' in result.output + + def test_no_output_option_selected_all_not_selected(self): new_scan = self.get_test_loc('deltacode/scan_1_file_moved_new.json') old_scan = self.get_test_loc('deltacode/scan_1_file_moved_old.json') @@ -262,6 +332,10 @@ def test_no_output_option_selected(self): assert '"type": "file"' in result.output assert '"size": 200' in result.output + assert '"category": "unmodified"' not in result.output + assert '"path": "a/a3.py"' not in result.output + assert '"name": "a3.py"' not in result.output + def test_help(self): runner = CliRunner() result = runner.invoke(cli.cli, ['--help']) @@ -269,6 +343,7 @@ def test_help(self): assert 'Usage: cli [OPTIONS]' in result.output assert 'Identify the changes that need to be made' in result.output assert 'If no file option is selected' in result.output + assert 'Include unmodified files' in result.output def test_empty(self): runner = CliRunner() From d5119f6fd6a5f553829f1b361cf1bff0e430958d Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Mon, 22 Jan 2018 18:24:35 -0800 Subject: [PATCH 2/2] Modify command name #41 * Change '-a/--all' to '-a/--all-delta-types'. * Add default of False to this parameter in 'write_csv()', 'write_json()' and 'utils.deltas()'. Signed-off-by: John M. Horan --- src/deltacode/cli.py | 16 ++++++++-------- src/deltacode/utils.py | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/deltacode/cli.py b/src/deltacode/cli.py index 80b35047..8dd23c85 100644 --- a/src/deltacode/cli.py +++ b/src/deltacode/cli.py @@ -39,7 +39,7 @@ # FIXME: update the function argument delta to deltacode -def write_csv(delta, result_file, all): +def write_csv(delta, result_file, all_delta_types=False): """ Using the DeltaCode object, create a .csv file containing the primary information from the Delta objects. Omit all Delta @@ -57,13 +57,13 @@ def write_csv(delta, result_file, all): f.old_file.size if f.category == 'removed' else f.new_file.size, f.old_file.path if f.category == 'moved' else '') for d in delta.deltas for f in delta.deltas.get(d)]: - if all is True: + if all_delta_types is True: csv_out.writerow(row) elif row[0] != 'unmodified': csv_out.writerow(row) -def write_json(deltacode, outfile, all): +def write_json(deltacode, outfile, all_delta_types=False): """ Using the DeltaCode object, create a .json file containing the primary information from the Delta objects. Omit all Delta @@ -73,7 +73,7 @@ def write_json(deltacode, outfile, all): results = OrderedDict([ ('deltacode_version', __version__), ('deltacode_stats', deltacode.get_stats()), - ('deltas', deltas(deltacode, all)), + ('deltas', deltas(deltacode, all_delta_types)), ]) # TODO: add toggle for pretty printing @@ -87,8 +87,8 @@ def write_json(deltacode, outfile, all): @click.option('-o', '--old', required=True, prompt=False, type=click.Path(exists=True, readable=True), help='Identify the path to the "old" scan file') @click.option('-c', '--csv-file', prompt=False, type=click.Path(exists=False), help='Identify the path to the .csv output file') @click.option('-j', '--json-file', prompt=False, default='-', type=click.File(mode='wb', lazy=False), help='Identify the path to the .json output file') -@click.option('-a', '--all', is_flag=True, help="Include unmodified files as well as all changed files in the .json or .csv output. If not selected, only changed files are included.") -def cli(new, old, csv_file, json_file, all): +@click.option('-a', '--all-delta-types', is_flag=True, help="Include unmodified files as well as all changed files in the .json or .csv output. If not selected, only changed files are included.") +def cli(new, old, csv_file, json_file, all_delta_types): """ Identify the changes that need to be made to the 'old' scan file (-o or -old) in order to generate the 'new' scan file (-n or @@ -101,7 +101,7 @@ def cli(new, old, csv_file, json_file, all): # output to csv if csv_file: - write_csv(deltacode, csv_file, all) + write_csv(deltacode, csv_file, all_delta_types) # generate JSON output else: - write_json(deltacode, json_file, all) + write_json(deltacode, json_file, all_delta_types) diff --git a/src/deltacode/utils.py b/src/deltacode/utils.py index 5e278404..cec50c56 100644 --- a/src/deltacode/utils.py +++ b/src/deltacode/utils.py @@ -31,7 +31,7 @@ from commoncode import paths -def deltas(deltacode, all): +def deltas(deltacode, all_delta_types=False): """ Return a generator of Delta dictionaries for JSON serialized ouput. Omit all Delta objects whose 'category' is 'unmodified' unless the user selects @@ -39,7 +39,7 @@ def deltas(deltacode, all): """ for category, deltas in deltacode.deltas.iteritems(): for delta in deltas: - if all is True: + if all_delta_types is True: yield delta.to_dict() elif delta.category != 'unmodified': yield delta.to_dict()