Skip to content

16 simplify DeltaCode output #21

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
steven-esser merged 4 commits into from
Nov 21, 2017
Merged

16 simplify DeltaCode output #21

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e55bcb7
Simplify DeltaCode output #16
johnmhoran Nov 18, 2017
515c4fd
Refactor generate_csv() and generate_json() # 16
johnmhoran Nov 18, 2017
054888e
Refactor JSON- and CSV-output functions #16
johnmhoran Nov 20, 2017
5a42628
Refactor generate_csv() #16
johnmhoran Nov 21, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 16 additions & 11 deletions src/deltacode/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ def determine_delta(self):
for path, old_files in old_index.items():
for old_file in old_files:
old_files_to_visit -= 1

if old_file.type != 'file':
continue

Expand All @@ -100,7 +100,7 @@ def determine_delta(self):
except KeyError:
deltas['removed'].append(Delta(None, old_file, 'removed'))
continue

# make sure everything is accounted for
assert new_files_to_visit == 0
assert old_files_to_visit == 0
Expand Down Expand Up @@ -130,7 +130,7 @@ def to_dict(self):
"""
if self.deltas == None:
return

return OrderedDict([
('added', [d.to_dict() for d in self.deltas.get('added')]),
('removed', [d.to_dict() for d in self.deltas.get('removed')]),
Expand All @@ -155,18 +155,23 @@ def to_dict(self):
if self.new_file == None and self.old_file == None:
return

if self.new_file == None:
if self.category == 'added':
return OrderedDict([
('category', 'added'),
('path', self.new_file.path)
])
elif self.category == 'removed':
return OrderedDict([
('new', None),
('old', self.old_file.to_dict()),
('category', 'removed'),
('path', self.old_file.path)
])
elif self.old_file == None:
elif self.category == 'modified':
return OrderedDict([
('new', self.new_file.to_dict()),
('old', None),
('category', 'modified'),
('path', self.old_file.path)
])
else:
return OrderedDict([
('new', self.new_file.to_dict()),
('old', self.old_file.to_dict()),
('category', 'unmodified'),
('path', self.old_file.path)
])
63 changes: 20 additions & 43 deletions src/deltacode/cli.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,64 +3,44 @@
#
from __future__ import absolute_import

from collections import OrderedDict

import csv
import json

import click

from deltacode import DeltaCode
from deltacode import __version__


def generate_csv(data, result_file):
def generate_csv(delta, result_file):
"""
Using the OrderedDict generated by DeltaCode.to_dict(), create a .csv file
Using the DeltaCode object and its to_dict() method, create a .csv file
containing the primary information from the Delta objects.
"""
category, new, new_filename, new_sha1, new_size, new_type, new_orig, old,\
old_filename, old_sha1, old_size, old_type, old_orig = '', '', '', '',\
'', '', '', '', '', '', '', '', ''
tuple = ()
tuple_list = []
deltas = data

for delta in deltas:
category = delta
for f in deltas[delta]:
new = '' if delta == 'removed' else f['new']['path']
new_filename = '' if delta == 'removed' else f['new']['name']
new_sha1 = '' if delta == 'removed' else f['new']['sha1']
new_size = '' if delta == 'removed' else f['new']['size']
new_type = '' if delta == 'removed' else f['new']['type']
new_orig = '' if delta == 'removed' else f['new']['original_path']
old = '' if delta == 'added' else f['old']['path']
old_filename = '' if delta == 'added' else f['old']['name']
old_sha1 = '' if delta == 'added' else f['old']['sha1']
old_size = '' if delta == 'added' else f['old']['size']
old_type = '' if delta == 'added' else f['old']['type']
old_orig = '' if delta == 'added' else f['old']['original_path']

tuple = (category, new, old, new_filename, old_filename, new_sha1,
old_sha1, new_size, old_size, new_type, old_type, new_orig, old_orig)
tuple_list.append(tuple)
output = delta.to_dict()

@steven-esser steven-esser Nov 20, 2017

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looking closer at our csv function, we should probably NOT call to_dict() in the csv function now that we are passing the whole delta object. This is just unnecessary overhead and will cause our program to run slower.


with open(result_file, 'wb') as out:
csv_out = csv.writer(out)
csv_out.writerow(['Type of delta', 'New scan path', 'Old scan path',
'new_filename', 'old_filename', 'new_sha1', 'old_sha1', 'new_size',
'old_size', 'new_type', 'old_type', 'new_original_path', 'old_original_path'])

for row in tuple_list:
csv_out.writerow(['Type of delta', 'Path'])
for row in [(f.get('category'), f.get('path')) for d in output for f in output.get(d)]:

@steven-esser steven-esser Nov 20, 2017

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This means we should convert this line to something like:

for row in [f.category, f.path for d in delta.deltas for f in delta.deltas.get(d)]:

@johnmhoran johnmhoran Nov 20, 2017

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@MaJuRG It seems to me that we have to call the Delta object's to_dict() method somewhere if we want to display the category and path values defined in that method. The following works:

for row in [(f.to_dict().get('category'), f.to_dict().get('path')) for d in delta.deltas for f in delta.deltas.get(d)]:

Is this what you had in mind? Let me know if you want me to push this first before you review.

@steven-esser steven-esser Nov 20, 2017

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@johnmhoran We should not have to do that. category and path are fields in our delta object that we can access normally.

@steven-esser steven-esser Nov 20, 2017 •
edited
Loading

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

to_dict() is only required for json output because dumping as a json requires it.

@steven-esser steven-esser Nov 20, 2017

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Think about how you would access the path value in a delta object. What line of code would that be? You certainly do not need to convert to dict before being able to access an object field value(s).

@johnmhoran johnmhoran Nov 20, 2017

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@MaJuRG We can access the path value of a new_file and/or an old_file if one or both exist, e.g., f.new_file.path. However, if we call a new_file for a delta where a file has been removed, we get a NoneType error, and similar result if we call an old_file where a file has been added. The Delta.to_dict() method handles these alternative scenarios, and I think we must as well if we're calling method with something like f.new_file.path.

. . . I'll be damned -- this seems to work:

for row in [(f.category, f.new_file.path if f.category == 'added' else f.old_file.path) for d in delta.deltas for f in delta.deltas.get(d)]:

Am I getting warm?

@steven-esser steven-esser Nov 21, 2017

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@johnmhoran Good point about the to_delta() handling the different cases.

Two things:

  1. You are on the right track. The line you posted above looks good, except (and this is kind of personal preference) that I would do it the opposite logic, just so we are using the new_file.path whenever able (to me this makes more sense from a user perspective):
for row in [(f.category, f.old_file.path if f.category == 'removed' else f.new_file.path) for d in delta.deltas for f in delta.deltas.get(d)]:
  1. IF you run into any bugs or strange output, you may just have to implement similar logic from delta.to_dict() into this function. But, if you find the above line(s) work well and handle everything, this is probably OK for now.

@johnmhoran johnmhoran Nov 21, 2017

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@MaJuRG Your approach works as well -- all 79 tests pass with both approaches. I'll commit and push shortly with your version. (I get and agree with your point re the user's perspective.)

csv_out.writerow(row)


def generate_json(data, result_file):
def generate_json(delta, result_file):
"""
Using the OrderedDict generated by DeltaCode.to_dict(), create a .json file
Using the DeltaCode object, create a .json file
containing the primary information from the Delta objects.
"""
# TODO: Add json file headers here
output = OrderedDict([
('deltacode_version', __version__),
('deltacode_stats', delta.get_stats()),
('deltas', delta.to_dict())
])

with open(result_file, 'w') as outfile:
json.dump(data, outfile, indent=4)
json.dump(output, outfile, indent=4)


@click.command()
Expand All @@ -77,18 +57,15 @@ def cli(new, old, csv_file, json_file):
.json file (-j or -json-file) at a user-designated location. If no file
option is selected, the JSON results are printed to the console.
"""

# do the delta
delta = DeltaCode(new, old)
data = delta.to_dict()


# output to csv
if csv_file:
generate_csv(data, csv_file)
generate_csv(delta, csv_file)
# generate JSON output
elif json_file:
generate_json(data, json_file)
generate_json(delta, json_file)
# print to stdout
else:
print(json.dumps(data, indent=4))
print(json.dumps(delta.to_dict()))
20 changes: 10 additions & 10 deletions tests/data/cli/added1.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
Type of delta,New scan path,Old scan path,new_filename,old_filename,new_sha1,old_sha1,new_size,old_size,new_type,old_type,new_original_path,old_original_path
added,a/a5.py,,a5.py,,a0c7dab0c0f07cc8ef8fac7729359b6997ed50cd,,200,,file,,codebase_01_1_file_added/a/a5.py,
unmodified,a/a3.py,a/a3.py,a3.py,a3.py,fd5d3589c825f448546d7dcec36da3e567d35fe9,fd5d3589c825f448546d7dcec36da3e567d35fe9,200,200,file,file,codebase_01_1_file_added/a/a3.py,codebase_01/a/a3.py
unmodified,b/b4.py,b/b4.py,b4.py,b4.py,98c9e6bed78b1513c28e666016cb35a50708c36e,98c9e6bed78b1513c28e666016cb35a50708c36e,200,200,file,file,codebase_01_1_file_added/b/b4.py,codebase_01/b/b4.py
unmodified,a/a2.py,a/a2.py,a2.py,a2.py,310797523e47db8481aeb06f1634317285115091,310797523e47db8481aeb06f1634317285115091,200,200,file,file,codebase_01_1_file_added/a/a2.py,codebase_01/a/a2.py
unmodified,b/b2.py,b/b2.py,b2.py,b2.py,3340d86b1da9323067db8022f86dc97cfccee1d0,3340d86b1da9323067db8022f86dc97cfccee1d0,200,200,file,file,codebase_01_1_file_added/b/b2.py,codebase_01/b/b2.py
unmodified,b/b1.py,b/b1.py,b1.py,b1.py,70f6ce80985578b5104db0abc578cf5a05e78f4b,70f6ce80985578b5104db0abc578cf5a05e78f4b,200,200,file,file,codebase_01_1_file_added/b/b1.py,codebase_01/b/b1.py
unmodified,b/b3.py,b/b3.py,b3.py,b3.py,e49d4463662414bee5ad2d2e5c1fbd704f33b84e,e49d4463662414bee5ad2d2e5c1fbd704f33b84e,200,200,file,file,codebase_01_1_file_added/b/b3.py,codebase_01/b/b3.py
unmodified,a/a4.py,a/a4.py,a4.py,a4.py,6f71666c46446c29d3f45feef5419ae76fb86a5b,6f71666c46446c29d3f45feef5419ae76fb86a5b,200,200,file,file,codebase_01_1_file_added/a/a4.py,codebase_01/a/a4.py
unmodified,a/a1.py,a/a1.py,a1.py,a1.py,84b647771481d39dd3a53f6dc210c26abac37748,84b647771481d39dd3a53f6dc210c26abac37748,200,200,file,file,codebase_01_1_file_added/a/a1.py,codebase_01/a/a1.py
Type of delta,Path
added,a/a5.py
unmodified,a/a3.py
unmodified,b/b4.py
unmodified,a/a2.py
unmodified,b/b2.py
unmodified,b/b1.py
unmodified,b/b3.py
unmodified,a/a4.py
unmodified,a/a1.py
18 changes: 9 additions & 9 deletions tests/data/cli/modified1.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
Type of delta,New scan path,Old scan path,new_filename,old_filename,new_sha1,old_sha1,new_size,old_size,new_type,old_type,new_original_path,old_original_path
modified,a/a4.py,a/a4.py,a4.py,a4.py,56aacf4e8ebcd7d8365e68cf11df933a237d33eb,6f71666c46446c29d3f45feef5419ae76fb86a5b,246,200,file,file,codebase_01_1_file_modified_not_renamed/a/a4.py,codebase_01/a/a4.py
unmodified,a/a3.py,a/a3.py,a3.py,a3.py,fd5d3589c825f448546d7dcec36da3e567d35fe9,fd5d3589c825f448546d7dcec36da3e567d35fe9,200,200,file,file,codebase_01_1_file_modified_not_renamed/a/a3.py,codebase_01/a/a3.py
unmodified,b/b4.py,b/b4.py,b4.py,b4.py,98c9e6bed78b1513c28e666016cb35a50708c36e,98c9e6bed78b1513c28e666016cb35a50708c36e,200,200,file,file,codebase_01_1_file_modified_not_renamed/b/b4.py,codebase_01/b/b4.py
unmodified,b/b1.py,b/b1.py,b1.py,b1.py,70f6ce80985578b5104db0abc578cf5a05e78f4b,70f6ce80985578b5104db0abc578cf5a05e78f4b,200,200,file,file,codebase_01_1_file_modified_not_renamed/b/b1.py,codebase_01/b/b1.py
unmodified,a/a2.py,a/a2.py,a2.py,a2.py,310797523e47db8481aeb06f1634317285115091,310797523e47db8481aeb06f1634317285115091,200,200,file,file,codebase_01_1_file_modified_not_renamed/a/a2.py,codebase_01/a/a2.py
unmodified,b/b2.py,b/b2.py,b2.py,b2.py,3340d86b1da9323067db8022f86dc97cfccee1d0,3340d86b1da9323067db8022f86dc97cfccee1d0,200,200,file,file,codebase_01_1_file_modified_not_renamed/b/b2.py,codebase_01/b/b2.py
unmodified,b/b3.py,b/b3.py,b3.py,b3.py,e49d4463662414bee5ad2d2e5c1fbd704f33b84e,e49d4463662414bee5ad2d2e5c1fbd704f33b84e,200,200,file,file,codebase_01_1_file_modified_not_renamed/b/b3.py,codebase_01/b/b3.py
unmodified,a/a1.py,a/a1.py,a1.py,a1.py,84b647771481d39dd3a53f6dc210c26abac37748,84b647771481d39dd3a53f6dc210c26abac37748,200,200,file,file,codebase_01_1_file_modified_not_renamed/a/a1.py,codebase_01/a/a1.py
Type of delta,Path
modified,a/a4.py
unmodified,a/a3.py
unmodified,b/b4.py
unmodified,b/b1.py
unmodified,a/a2.py
unmodified,b/b2.py
unmodified,b/b3.py
unmodified,a/a1.py
18 changes: 9 additions & 9 deletions tests/data/cli/removed1.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
Type of delta,New scan path,Old scan path,new_filename,old_filename,new_sha1,old_sha1,new_size,old_size,new_type,old_type,new_original_path,old_original_path
removed,,a/a4.py,,a4.py,,6f71666c46446c29d3f45feef5419ae76fb86a5b,,200,,file,,codebase_01/a/a4.py
unmodified,a/a3.py,a/a3.py,a3.py,a3.py,fd5d3589c825f448546d7dcec36da3e567d35fe9,fd5d3589c825f448546d7dcec36da3e567d35fe9,200,200,file,file,codebase_01_1_file_removed/a/a3.py,codebase_01/a/a3.py
unmodified,b/b4.py,b/b4.py,b4.py,b4.py,98c9e6bed78b1513c28e666016cb35a50708c36e,98c9e6bed78b1513c28e666016cb35a50708c36e,200,200,file,file,codebase_01_1_file_removed/b/b4.py,codebase_01/b/b4.py
unmodified,b/b1.py,b/b1.py,b1.py,b1.py,70f6ce80985578b5104db0abc578cf5a05e78f4b,70f6ce80985578b5104db0abc578cf5a05e78f4b,200,200,file,file,codebase_01_1_file_removed/b/b1.py,codebase_01/b/b1.py
unmodified,a/a2.py,a/a2.py,a2.py,a2.py,310797523e47db8481aeb06f1634317285115091,310797523e47db8481aeb06f1634317285115091,200,200,file,file,codebase_01_1_file_removed/a/a2.py,codebase_01/a/a2.py
unmodified,b/b2.py,b/b2.py,b2.py,b2.py,3340d86b1da9323067db8022f86dc97cfccee1d0,3340d86b1da9323067db8022f86dc97cfccee1d0,200,200,file,file,codebase_01_1_file_removed/b/b2.py,codebase_01/b/b2.py
unmodified,b/b3.py,b/b3.py,b3.py,b3.py,e49d4463662414bee5ad2d2e5c1fbd704f33b84e,e49d4463662414bee5ad2d2e5c1fbd704f33b84e,200,200,file,file,codebase_01_1_file_removed/b/b3.py,codebase_01/b/b3.py
unmodified,a/a1.py,a/a1.py,a1.py,a1.py,84b647771481d39dd3a53f6dc210c26abac37748,84b647771481d39dd3a53f6dc210c26abac37748,200,200,file,file,codebase_01_1_file_removed/a/a1.py,codebase_01/a/a1.py
Type of delta,Path
removed,a/a4.py
unmodified,a/a3.py
unmodified,b/b4.py
unmodified,b/b1.py
unmodified,a/a2.py
unmodified,b/b2.py
unmodified,b/b3.py
unmodified,a/a1.py
20 changes: 10 additions & 10 deletions tests/data/cli/renamed1.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
Type of delta,New scan path,Old scan path,new_filename,old_filename,new_sha1,old_sha1,new_size,old_size,new_type,old_type,new_original_path,old_original_path
added,a/a4_renamed_not_modified.py,,a4_renamed_not_modified.py,,6f71666c46446c29d3f45feef5419ae76fb86a5b,,200,,file,,codebase_01_1_file_renamed_not_modified/a/a4_renamed_not_modified.py,
removed,,a/a4.py,,a4.py,,6f71666c46446c29d3f45feef5419ae76fb86a5b,,200,,file,,codebase_01/a/a4.py
unmodified,a/a3.py,a/a3.py,a3.py,a3.py,fd5d3589c825f448546d7dcec36da3e567d35fe9,fd5d3589c825f448546d7dcec36da3e567d35fe9,200,200,file,file,codebase_01_1_file_renamed_not_modified/a/a3.py,codebase_01/a/a3.py
unmodified,b/b4.py,b/b4.py,b4.py,b4.py,98c9e6bed78b1513c28e666016cb35a50708c36e,98c9e6bed78b1513c28e666016cb35a50708c36e,200,200,file,file,codebase_01_1_file_renamed_not_modified/b/b4.py,codebase_01/b/b4.py
unmodified,b/b1.py,b/b1.py,b1.py,b1.py,70f6ce80985578b5104db0abc578cf5a05e78f4b,70f6ce80985578b5104db0abc578cf5a05e78f4b,200,200,file,file,codebase_01_1_file_renamed_not_modified/b/b1.py,codebase_01/b/b1.py
unmodified,b/b2.py,b/b2.py,b2.py,b2.py,3340d86b1da9323067db8022f86dc97cfccee1d0,3340d86b1da9323067db8022f86dc97cfccee1d0,200,200,file,file,codebase_01_1_file_renamed_not_modified/b/b2.py,codebase_01/b/b2.py
unmodified,b/b3.py,b/b3.py,b3.py,b3.py,e49d4463662414bee5ad2d2e5c1fbd704f33b84e,e49d4463662414bee5ad2d2e5c1fbd704f33b84e,200,200,file,file,codebase_01_1_file_renamed_not_modified/b/b3.py,codebase_01/b/b3.py
unmodified,a/a1.py,a/a1.py,a1.py,a1.py,84b647771481d39dd3a53f6dc210c26abac37748,84b647771481d39dd3a53f6dc210c26abac37748,200,200,file,file,codebase_01_1_file_renamed_not_modified/a/a1.py,codebase_01/a/a1.py
unmodified,a/a2.py,a/a2.py,a2.py,a2.py,310797523e47db8481aeb06f1634317285115091,310797523e47db8481aeb06f1634317285115091,200,200,file,file,codebase_01_1_file_renamed_not_modified/a/a2.py,codebase_01/a/a2.py
Type of delta,Path
added,a/a4_renamed_not_modified.py
removed,a/a4.py
unmodified,a/a3.py
unmodified,b/b4.py
unmodified,b/b1.py
unmodified,b/b2.py
unmodified,b/b3.py
unmodified,a/a1.py
unmodified,a/a2.py
14 changes: 4 additions & 10 deletions tests/test_cli.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,11 +58,8 @@ def test_generate_csv_added(self):
old_scan = self.get_test_loc('cli/old_added1.json')

delta = DeltaCode(new_scan, old_scan)
data = delta.to_dict()

result_file = self.get_temp_file('.csv')
cli.generate_csv(data, result_file)

cli.generate_csv(delta, result_file)
expected_file = self.get_test_loc('cli/added1.csv')
check_csvs(result_file, expected_file)

Expand All @@ -71,9 +68,8 @@ def test_generate_csv_modified(self):
old_scan = self.get_test_loc('cli/old_modified1.json')

delta = DeltaCode(new_scan, old_scan)
data = delta.to_dict()
result_file = self.get_temp_file('.csv')
cli.generate_csv(data, result_file)
cli.generate_csv(delta, result_file)
expected_file = self.get_test_loc('cli/modified1.csv')
check_csvs(result_file, expected_file)

Expand All @@ -82,9 +78,8 @@ def test_generate_csv_removed(self):
old_scan = self.get_test_loc('cli/old_removed1.json')

delta = DeltaCode(new_scan, old_scan)
data = delta.to_dict()
result_file = self.get_temp_file('.csv')
cli.generate_csv(data, result_file)
cli.generate_csv(delta, result_file)
expected_file = self.get_test_loc('cli/removed1.csv')
check_csvs(result_file, expected_file)

Expand All @@ -93,8 +88,7 @@ def test_generate_csv_renamed(self):
old_scan = self.get_test_loc('cli/old_renamed1.json')

delta = DeltaCode(new_scan, old_scan)
data = delta.to_dict()
result_file = self.get_temp_file('.csv')
cli.generate_csv(data, result_file)
cli.generate_csv(delta, result_file)
expected_file = self.get_test_loc('cli/renamed1.csv')
check_csvs(result_file, expected_file)
Loading