Skip to content

Commit 7aab5b6

Browse files
pombredannepombredanne
committed
#156 #157 : Added scan for emails, urls and authors (with copyrights)
Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>
1 parent 2d83104 commit 7aab5b6

5 files changed

Lines changed: 438 additions & 67 deletions

File tree

‎src/scancode/api.py‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,18 +55,51 @@ def get_copyrights(location):
5555
"""
5656
from cluecode.copyrights import detect_copyrights
5757

58-
for copyrights, _authors, years, holders, start_line, end_line in detect_copyrights(location):
58+
for copyrights, authors, _years, holders, start_line, end_line in detect_copyrights(location):
5959
if not copyrights:
6060
continue
6161
result = OrderedDict()
6262
# FIXME: we should call this copyright instead, and yield one item per statement
6363
result['statements'] = copyrights
6464
result['holders'] = holders
65+
result['authors'] = authors
6566
result['start_line'] = start_line
6667
result['end_line'] = end_line
6768
yield result
6869

6970

71+
def get_emails(location):
72+
"""
73+
Yield an iterable of dictionaries of emails detected in the file at
74+
location.
75+
"""
76+
from cluecode.finder import find_emails
77+
for email, line_num in find_emails(location):
78+
if not email:
79+
continue
80+
misc = OrderedDict()
81+
misc['email'] = email
82+
misc['start_line'] = line_num
83+
misc['end_line'] = line_num
84+
yield misc
85+
86+
87+
def get_urls(location):
88+
"""
89+
Yield an iterable of dictionaries of urls detected in the file at
90+
location.
91+
"""
92+
from cluecode.finder import find_urls
93+
for urls, line_num in find_urls(location):
94+
if not urls:
95+
continue
96+
misc = OrderedDict()
97+
misc['url'] = urls
98+
misc['start_line'] = line_num
99+
misc['end_line'] = line_num
100+
yield misc
101+
102+
70103
DEJACODE_LICENSE_URL = 'https://enterprise.dejacode.com/license_library/Demo/{}/'
71104

72105

‎src/scancode/cli.py‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,8 @@
4949
from scancode.api import get_licenses
5050
from scancode.api import get_file_infos
5151
from scancode.api import get_package_infos
52+
from scancode.api import get_emails
53+
from scancode.api import get_urls
5254

5355

5456
info_text = '''
@@ -216,6 +218,8 @@ class ScanCommand(utils.BaseCommand):
216218
@click.option('-c', '--copyright', is_flag=True, default=False, help='Scan <input> for copyrights. [default]')
217219
@click.option('-l', '--license', is_flag=True, default=False, help='Scan <input> for licenses. [default]')
218220
@click.option('-p', '--package', is_flag=True, default=False, help='Scan <input> for packages. [default]')
221+
@click.option('--email', is_flag=True, default=False, help='Scan <input> for emails.')
222+
@click.option('--url', is_flag=True, default=False, help='Scan <input> for urls.')
219223
@click.option('-i', '--info', is_flag=True, default=False, help='Scan <input> for files information.')
220224

221225
@click.option('-f', '--format', is_flag=False, default='json', show_default=True, metavar='<style>',
@@ -229,24 +233,24 @@ class ScanCommand(utils.BaseCommand):
229233
@click.option('--version', is_flag=True, is_eager=True, callback=print_version, help='Show the version and exit.')
230234

231235
def scancode(ctx, input, output_file, copyright, license, package, # @ReservedAssignment
232-
info, format, verbose, quiet, *args, **kwargs): # @ReservedAssignment
233-
"""scan the <input> file or directory for origin and license and save results to the <output_file>.
236+
email, url, info, format, verbose, quiet, *args, **kwargs): # @ReservedAssignment
237+
"""scan the <input> file or directory for origin clues and license and save results to the <output_file>.
234238
235239
The scan results are printed on terminal if <output_file> is not provided.
236240
"""
237-
possible_scans = [copyright, license, package, info]
241+
possible_scans = [copyright, license, package, email, url, info]
238242
# Default scan when no options is provided
239243
if not any(possible_scans):
240244
copyright = True # @ReservedAssignment
241245
license = True # @ReservedAssignment
242246
package = True
243247

244-
results = scan(input, copyright, license, package, info, verbose, quiet)
248+
results = scan(input, copyright, license, package, email, url, info, verbose, quiet)
245249
save_results(results, format, input, output_file)
246250

247251

248252
def scan(input_path, copyright=True, license=True, package=True, # @ReservedAssignment
249-
info=True, verbose=False, quiet=False): # @ReservedAssignment
253+
email=False, url=False, info=True, verbose=False, quiet=False): # @ReservedAssignment
250254
"""
251255
Do the scans proper, return results.
252256
"""
@@ -259,6 +263,8 @@ def scan(input_path, copyright=True, license=True, package=True, # @ReservedAss
259263
'copyrights': copyright and get_copyrights,
260264
'licenses': license and get_licenses,
261265
'packages': package and get_package_infos,
266+
'emails': email and get_emails,
267+
'urls': url and get_urls,
262268
'infos': info and get_file_infos,
263269
}
264270

@@ -270,14 +276,12 @@ def scan_start():
270276
"""Progress event displayed at start of scan"""
271277
return style('Scanning files...', fg='green')
272278

273-
274279
def scan_event(item):
275280
"""Progress event displayed each time a file is scanned"""
276281
if item:
277282
line = verbose and item or fileutils.file_name(item) or ''
278283
return 'Scanning: %(line)s' % locals()
279284

280-
281285
def scan_end():
282286
"""Progress event displayed at end of scan"""
283287
has_warnings = False
@@ -289,7 +293,6 @@ def scan_end():
289293
summary.append(style('Scanning done.', fg=summary_color, reset=True))
290294
return '\n'.join(summary)
291295

292-
293296
ignored = partial(ignore.is_ignored, ignores=ignore.ignores_VCS, unignores={})
294297
resources = fileutils.resource_iter(abs_input, ignored=ignored)
295298

@@ -340,7 +343,7 @@ def save_results(results, format, input, output_file): # @ReservedAssignment
340343
if format not in formats:
341344
# render using a user-provided custom format template
342345
if not os.path.isfile(format):
343-
click.secho('\nInvalid template passed.', err=True, fg='red')
346+
click.secho('\nInvalid template passed.', err=True, fg='red')
344347
else:
345348
output_file.write(as_template(results, template=format))
346349
elif format == 'html':

‎tests/scancode/data/info/all.expected.json‎

Lines changed: 45 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
"results": [
55
{
66
"location": "basic",
7+
"copyrights": [],
78
"licenses": [],
89
"infos": [
910
{
@@ -25,11 +26,11 @@
2526
"is_source": null,
2627
"is_script": null
2728
}
28-
],
29-
"copyrights": []
29+
]
3030
},
3131
{
3232
"location": "basic/dbase.fdt",
33+
"copyrights": [],
3334
"licenses": [],
3435
"infos": [
3536
{
@@ -51,11 +52,11 @@
5152
"is_source": null,
5253
"is_script": null
5354
}
54-
],
55-
"copyrights": []
55+
]
5656
},
5757
{
5858
"location": "basic/dir",
59+
"copyrights": [],
5960
"licenses": [],
6061
"infos": [
6162
{
@@ -77,11 +78,11 @@
7778
"is_source": null,
7879
"is_script": null
7980
}
80-
],
81-
"copyrights": []
81+
]
8282
},
8383
{
8484
"location": "basic/dir/e.tar",
85+
"copyrights": [],
8586
"licenses": [],
8687
"infos": [
8788
{
@@ -103,11 +104,11 @@
103104
"is_source": null,
104105
"is_script": null
105106
}
106-
],
107-
"copyrights": []
107+
]
108108
},
109109
{
110110
"location": "basic/dir/subdir",
111+
"copyrights": [],
111112
"licenses": [],
112113
"infos": [
113114
{
@@ -129,11 +130,11 @@
129130
"is_source": null,
130131
"is_script": null
131132
}
132-
],
133-
"copyrights": []
133+
]
134134
},
135135
{
136136
"location": "basic/dir/subdir/a.aif",
137+
"copyrights": [],
137138
"licenses": [],
138139
"infos": [
139140
{
@@ -155,11 +156,11 @@
155156
"is_source": null,
156157
"is_script": null
157158
}
158-
],
159-
"copyrights": []
159+
]
160160
},
161161
{
162162
"location": "basic/dir2",
163+
"copyrights": [],
163164
"licenses": [],
164165
"infos": [
165166
{
@@ -181,11 +182,11 @@
181182
"is_source": null,
182183
"is_script": null
183184
}
184-
],
185-
"copyrights": []
185+
]
186186
},
187187
{
188188
"location": "basic/dir2/subdir",
189+
"copyrights": [],
189190
"licenses": [],
190191
"infos": [
191192
{
@@ -207,11 +208,24 @@
207208
"is_source": null,
208209
"is_script": null
209210
}
210-
],
211-
"copyrights": []
211+
]
212212
},
213213
{
214214
"location": "basic/dir2/subdir/bcopy.s",
215+
"copyrights": [
216+
{
217+
"statements": [
218+
"Copyright (c) 1993 The Regents of the University of California."
219+
],
220+
"holders": [
221+
"Regents of the University of California.",
222+
"University of California."
223+
],
224+
"authors": [],
225+
"start_line": 22,
226+
"end_line": 23
227+
}
228+
],
215229
"licenses": [
216230
{
217231
"key": "bsd-original-uc",
@@ -248,23 +262,11 @@
248262
"is_source": true,
249263
"is_script": null
250264
}
251-
],
252-
"copyrights": [
253-
{
254-
"statements": [
255-
"Copyright (c) 1993 The Regents of the University of California."
256-
],
257-
"holders": [
258-
"Regents of the University of California.",
259-
"University of California."
260-
],
261-
"start_line": 22,
262-
"end_line": 23
263-
}
264265
]
265266
},
266267
{
267268
"location": "basic/dir2/subdir/config.conf",
269+
"copyrights": [],
268270
"licenses": [],
269271
"infos": [
270272
{
@@ -286,11 +288,23 @@
286288
"is_source": null,
287289
"is_script": null
288290
}
289-
],
290-
"copyrights": []
291+
]
291292
},
292293
{
293294
"location": "basic/main.c",
295+
"copyrights": [
296+
{
297+
"statements": [
298+
"Copyright (c) 2006, Jouni Malinen <j@w1.fi>"
299+
],
300+
"holders": [
301+
"Jouni Malinen"
302+
],
303+
"authors": [],
304+
"start_line": 2,
305+
"end_line": 3
306+
}
307+
],
294308
"licenses": [
295309
{
296310
"key": "gpl-2.0",
@@ -341,18 +355,6 @@
341355
"is_source": true,
342356
"is_script": null
343357
}
344-
],
345-
"copyrights": [
346-
{
347-
"statements": [
348-
"Copyright (c) 2006, Jouni Malinen <j@w1.fi>"
349-
],
350-
"holders": [
351-
"Jouni Malinen"
352-
],
353-
"start_line": 2,
354-
"end_line": 3
355-
}
356358
]
357359
}
358360
]

0 commit comments

Comments
 (0)