From 3c00c3c6f99df72e61002d04d26a583a94a275c4 Mon Sep 17 00:00:00 2001 From: Steven Esser Date: Thu, 21 Dec 2017 12:17:25 -0800 Subject: [PATCH 1/3] Use simplejson to output to JSON * simplejson allows us to consume a generator during file creation * using a generator for our calls to Delta.to_dict() significantly reduces memory usage of DeltaCode Signed-off-by: Steven Esser --- src/deltacode/cli.py | 26 +++++++++++++------------- src/deltacode/utils.py | 9 +++++++++ 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/deltacode/cli.py b/src/deltacode/cli.py index 1e16ea90..c61b13ec 100644 --- a/src/deltacode/cli.py +++ b/src/deltacode/cli.py @@ -31,9 +31,11 @@ import json import click +import simplejson from deltacode import DeltaCode from deltacode import __version__ +from deltacode.utils import deltas def generate_csv(delta, result_file): @@ -54,19 +56,20 @@ def generate_csv(delta, result_file): csv_out.writerow(row) -def generate_json(delta, result_file): +def write_json(deltacode, outfile): """ Using the DeltaCode object, create a .json file containing the primary information from the Delta objects. """ - output = OrderedDict([ + results = OrderedDict([ ('deltacode_version', __version__), - ('deltacode_stats', delta.get_stats()), - ('deltas', delta.to_dict()) + ('deltacode_stats', deltacode.get_stats()), + ('deltas', deltas(deltacode)), ]) - with open(result_file, 'w') as outfile: - json.dump(output, outfile, indent=4) + # TODO: add toggle for pretty printing + simplejson.dump(results, outfile, iterable_as_array=True, indent=2) + outfile.write('\n') @click.command() @@ -74,7 +77,7 @@ def generate_json(delta, result_file): @click.option('-n', '--new', required=True, prompt=False, type=click.Path(exists=True, readable=True), help='Identify the path to the "new" scan file') @click.option('-o', '--old', required=True, prompt=False, type=click.Path(exists=True, readable=True), help='Identify the path to the "old" scan file') @click.option('-c', '--csv-file', prompt=False, type=click.Path(exists=False), help='Identify the path to the .csv output file') -@click.option('-j', '--json-file', prompt=False, type=click.Path(exists=False), help='Identify the path to the .json output file') +@click.option('-j', '--json-file', prompt=False, default='-', type=click.File(mode='wb', lazy=False), help='Identify the path to the .json output file') def cli(new, old, csv_file, json_file): """ Identify the changes that need to be made to the 'old' @@ -84,14 +87,11 @@ def cli(new, old, csv_file, json_file): option is selected, print the JSON results to the console. """ # do the delta - delta = DeltaCode(new, old) + deltacode = DeltaCode(new, old) # output to csv if csv_file: - generate_csv(delta, csv_file) + generate_csv(deltacode, csv_file) # generate JSON output - elif json_file: - generate_json(delta, json_file) - # print to stdout else: - print(json.dumps(delta.to_dict())) + write_json(deltacode, json_file) diff --git a/src/deltacode/utils.py b/src/deltacode/utils.py index e469461e..89bcbb9e 100644 --- a/src/deltacode/utils.py +++ b/src/deltacode/utils.py @@ -31,6 +31,15 @@ from commoncode import paths +def deltas(deltacode): + """ + Return a generator of Delta dictionaries for JSON serialized ouput. + """ + for category, deltas in deltacode.deltas.iteritems(): + for delta in deltas: + yield delta.to_dict() + + class AlignmentException(Exception): """ Named exception for alignment errors. From 46a5f1274150d44cc0b159c4b51213328dab8985 Mon Sep 17 00:00:00 2001 From: Steven Esser Date: Thu, 21 Dec 2017 12:23:20 -0800 Subject: [PATCH 2/3] Update generate_csv to write_csv Signed-off-by: Steven Esser --- src/deltacode/cli.py | 5 +++-- tests/test_cli.py | 24 ++++++++++++------------ 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/deltacode/cli.py b/src/deltacode/cli.py index c61b13ec..767deb8f 100644 --- a/src/deltacode/cli.py +++ b/src/deltacode/cli.py @@ -38,7 +38,8 @@ from deltacode.utils import deltas -def generate_csv(delta, result_file): +# FIXME: update the function argument delta to deltacode +def write_csv(delta, result_file): """ Using the DeltaCode object, create a .csv file containing the primary information from the Delta objects. @@ -91,7 +92,7 @@ def cli(new, old, csv_file, json_file): # output to csv if csv_file: - generate_csv(deltacode, csv_file) + write_csv(deltacode, csv_file) # generate JSON output else: write_json(deltacode, json_file) diff --git a/tests/test_cli.py b/tests/test_cli.py index d48bdf26..f2585995 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -75,62 +75,62 @@ class TestCLI(FileBasedTesting): test_data_dir = os.path.join(os.path.dirname(__file__), 'data') - def test_generate_csv_added(self): + def test_write_csv_added(self): new_scan = self.get_test_loc('cli/new_added1.json') old_scan = self.get_test_loc('cli/old_added1.json') delta = DeltaCode(new_scan, old_scan) result_file = self.get_temp_file('.csv') - cli.generate_csv(delta, result_file) + cli.write_csv(delta, result_file) expected_file = self.get_test_loc('cli/added1.csv') check_csvs(result_file, expected_file) - def test_generate_csv_modified(self): + def test_write_csv_modified(self): new_scan = self.get_test_loc('cli/new_modified1.json') old_scan = self.get_test_loc('cli/old_modified1.json') delta = DeltaCode(new_scan, old_scan) result_file = self.get_temp_file('.csv') - cli.generate_csv(delta, result_file) + cli.write_csv(delta, result_file) expected_file = self.get_test_loc('cli/modified1.csv') check_csvs(result_file, expected_file) - def test_generate_csv_removed(self): + def test_write_csv_removed(self): new_scan = self.get_test_loc('cli/new_removed1.json') old_scan = self.get_test_loc('cli/old_removed1.json') delta = DeltaCode(new_scan, old_scan) result_file = self.get_temp_file('.csv') - cli.generate_csv(delta, result_file) + cli.write_csv(delta, result_file) expected_file = self.get_test_loc('cli/removed1.csv') check_csvs(result_file, expected_file) - def test_generate_csv_renamed(self): + def test_write_csv_renamed(self): new_scan = self.get_test_loc('cli/new_renamed1.json') old_scan = self.get_test_loc('cli/old_renamed1.json') delta = DeltaCode(new_scan, old_scan) result_file = self.get_temp_file('.csv') - cli.generate_csv(delta, result_file) + cli.write_csv(delta, result_file) expected_file = self.get_test_loc('cli/renamed1.csv') check_csvs(result_file, expected_file) - def test_generate_csv_modified_new_license_added(self): + def test_write_csv_modified_new_license_added(self): new_scan = self.get_test_loc('cli/scan_modified_new_license_added.json') old_scan = self.get_test_loc('cli/scan_modified_old_license_added.json') delta = DeltaCode(new_scan, old_scan) result_file = self.get_temp_file('.csv') - cli.generate_csv(delta, result_file) + cli.write_csv(delta, result_file) expected_file = self.get_test_loc('cli/modified_new_license_added.csv') check_csvs(result_file, expected_file) - def test_generate_csv_modified_new_license_added_low_score(self): + def test_write_csv_modified_new_license_added_low_score(self): new_scan = self.get_test_loc('cli/scan_modified_new_license_added_low_score.json') old_scan = self.get_test_loc('cli/scan_modified_old_license_added_low_score.json') delta = DeltaCode(new_scan, old_scan) result_file = self.get_temp_file('.csv') - cli.generate_csv(delta, result_file) + cli.write_csv(delta, result_file) expected_file = self.get_test_loc('cli/modified_new_license_added_low_score.csv') check_csvs(result_file, expected_file) From ad852ecbdaa051653ae22dd4b3c6d1f493b5d29c Mon Sep 17 00:00:00 2001 From: Steven Esser Date: Thu, 21 Dec 2017 15:12:16 -0800 Subject: [PATCH 3/3] Fix failing tests Signed-off-by: Steven Esser --- tests/test_cli.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index e11fe8a0..6426eef8 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -135,42 +135,42 @@ def test_write_csv_modified_new_license_added_low_score(self): expected_file = self.get_test_loc('cli/modified_new_license_added_low_score.csv') check_csvs(result_file, expected_file) - def test_generate_csv_license_info_removed(self): + def test_write_csv_license_info_removed(self): new_scan = self.get_test_loc('cli/scan_new_license_info_removed.json') old_scan = self.get_test_loc('cli/scan_old_license_info_removed.json') delta = DeltaCode(new_scan, old_scan) result_file = self.get_temp_file('.csv') - cli.generate_csv(delta, result_file) + cli.write_csv(delta, result_file) expected_file = self.get_test_loc('cli/license_info_removed.csv') check_csvs(result_file, expected_file) - def test_generate_csv_license_info_added(self): + def test_write_csv_license_info_added(self): new_scan = self.get_test_loc('cli/scan_new_license_info_added.json') old_scan = self.get_test_loc('cli/scan_old_license_info_added.json') delta = DeltaCode(new_scan, old_scan) result_file = self.get_temp_file('.csv') - cli.generate_csv(delta, result_file) + cli.write_csv(delta, result_file) expected_file = self.get_test_loc('cli/license_info_added.csv') check_csvs(result_file, expected_file) - def test_generate_csv_license_info_removed_below_cutoff_score(self): + def test_write_csv_license_info_removed_below_cutoff_score(self): new_scan = self.get_test_loc('cli/scan_new_license_info_removed_below_cutoff_score.json') old_scan = self.get_test_loc('cli/scan_old_license_info_removed_below_cutoff_score.json') delta = DeltaCode(new_scan, old_scan) result_file = self.get_temp_file('.csv') - cli.generate_csv(delta, result_file) + cli.write_csv(delta, result_file) expected_file = self.get_test_loc('cli/license_info_removed_below_cutoff_score.csv') check_csvs(result_file, expected_file) - def test_generate_csv_license_info_added_below_cutoff_score(self): + def test_write_csv_license_info_added_below_cutoff_score(self): new_scan = self.get_test_loc('cli/scan_new_license_info_added_below_cutoff_score.json') old_scan = self.get_test_loc('cli/scan_old_license_info_added_below_cutoff_score.json') delta = DeltaCode(new_scan, old_scan) result_file = self.get_temp_file('.csv') - cli.generate_csv(delta, result_file) + cli.write_csv(delta, result_file) expected_file = self.get_test_loc('cli/license_info_added_below_cutoff_score.csv') check_csvs(result_file, expected_file)