Skip to content

Commit 1c04c47

Browse files
AyanSinhaMahapatraAyanSinhaMahapatra
committed
Add new CLI option --config-file
Signed-off-by: Ayan Sinha Mahapatra <asmahapatra@aboutcode.org>
1 parent dc9d3b5 commit 1c04c47

5 files changed

Lines changed: 110 additions & 42 deletions

File tree

‎src/scancode/cli.py‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,14 @@
1717
import logging
1818
import os
1919
import platform
20+
import saneyaml
2021
import sys
2122
import traceback
2223

2324
from collections import defaultdict
2425
from functools import partial
2526
from multiprocessing import TimeoutError
27+
from pathlib import Path
2628
from time import sleep
2729
from time import time
2830

@@ -231,6 +233,15 @@ def default_processes():
231233
cls=PluggableCommandLineOption,
232234
)
233235

236+
@click.option('-c', '--config-file',
237+
type=click.File('r'),
238+
required=False,
239+
help='Path to the configuration file.',
240+
sort_order=11,
241+
help_group=cliutils.CORE_GROUP,
242+
cls=PluggableCommandLineOption,
243+
)
244+
234245
@click.option('--strip-root',
235246
is_flag=True,
236247
default=False,
@@ -405,6 +416,7 @@ def default_processes():
405416
def scancode(
406417
ctx,
407418
input, # NOQA
419+
config_file,
408420
ignore,
409421
strip_root,
410422
full_root,
@@ -517,6 +529,7 @@ def scancode(
517529
success, _results = run_scan(
518530
input=input,
519531
ignore=ignore,
532+
config_file=config_file,
520533
from_json=from_json,
521534
strip_root=strip_root,
522535
full_root=full_root,
@@ -558,6 +571,7 @@ def scancode(
558571

559572
def run_scan(
560573
input, #
574+
config_file=None,
561575
ignore=[],
562576
from_json=False,
563577
strip_root=False,
@@ -667,6 +681,10 @@ def echo_func(*_args, **_kwargs):
667681
include = [as_posixpath(path).rstrip('/') for path in abs_input]
668682
input = common_prefix # NOQA
669683

684+
config_ignores = load_configuration_file(config_file)
685+
if config_ignores:
686+
ignore = ignore + tuple(config_ignores)
687+
670688
# build mappings of all options to pass down to plugins
671689
standard_options = dict(
672690
input=input,
@@ -1108,6 +1126,34 @@ def echo_func(*_args, **_kwargs):
11081126
return success, results
11091127

11101128

1129+
def load_configuration_file(path):
1130+
"""
1131+
Load scancode configuration values from a file at `path`.
1132+
1133+
Currently only supports ignore path patterns specified with
1134+
"ignored_patterns". This should be compatible with scancode.io
1135+
configuration values whenever possible:
1136+
https://scancodeio.readthedocs.io/en/latest/project-configuration.html
1137+
"""
1138+
ignores = []
1139+
if not path:
1140+
return ignores
1141+
1142+
click.echo(f"Loading env from {path}")
1143+
try:
1144+
1145+
config_values = saneyaml.load(path.read())
1146+
ignores = config_values.get("ignored_patterns", [])
1147+
except (saneyaml.YAMLError, Exception):
1148+
msg = (
1149+
f'Failed to load configuration from "{path}". '
1150+
f"The file format is invalid."
1151+
)
1152+
raise ScancodeError(msg + '\n' + traceback.format_exc())
1153+
1154+
return ignores
1155+
1156+
11111157
def run_codebase_plugins(
11121158
stage,
11131159
plugins,

‎tests/scancode/data/help/help.txt‎

Lines changed: 23 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -136,27 +136,29 @@ Options:
136136
which are todo items and needs manual review.
137137

138138
core:
139-
--ignore <pattern> Ignore files matching <pattern>.
140-
--timeout <seconds> Stop an unfinished file scan after a timeout in
141-
seconds. [default: 120 seconds]
142-
-n, --processes INT Set the number of parallel processes to use. Disable
143-
parallel processing if 0. Also disable threading if
144-
-1. [default: (number of CPUs)-1]
145-
-q, --quiet Do not print summary or progress.
146-
-v, --verbose Print progress as file-by-file path instead of a
147-
progress bar. Print verbose scan counters.
148-
--from-json Load codebase from one or more <input> JSON scan
149-
file(s).
150-
--max-in-memory INTEGER Maximum number of files and directories scan details
151-
kept in memory during a scan. Additional files and
152-
directories scan details above this number are cached
153-
on-disk rather than in memory. Use 0 to use unlimited
154-
memory and disable on-disk caching. Use -1 to use
155-
only on-disk caching. [default: 10000]
156-
--max-depth INTEGER Maximum nesting depth of subdirectories to scan.
157-
Descend at most INTEGER levels of directories below
158-
and including the starting directory. Use 0 for no
159-
scan depth limit.
139+
--ignore <pattern> Ignore files matching <pattern>.
140+
--timeout <seconds> Stop an unfinished file scan after a timeout in
141+
seconds. [default: 120 seconds]
142+
-n, --processes INT Set the number of parallel processes to use.
143+
Disable parallel processing if 0. Also disable
144+
threading if -1. [default: (number of CPUs)-1]
145+
-c, --config-file FILENAME Path to the configuration file.
146+
-q, --quiet Do not print summary or progress.
147+
-v, --verbose Print progress as file-by-file path instead of a
148+
progress bar. Print verbose scan counters.
149+
--from-json Load codebase from one or more <input> JSON scan
150+
file(s).
151+
--max-in-memory INTEGER Maximum number of files and directories scan
152+
details kept in memory during a scan. Additional
153+
files and directories scan details above this
154+
number are cached on-disk rather than in memory.
155+
Use 0 to use unlimited memory and disable on-disk
156+
caching. Use -1 to use only on-disk caching.
157+
[default: 10000]
158+
--max-depth INTEGER Maximum nesting depth of subdirectories to scan.
159+
Descend at most INTEGER levels of directories
160+
below and including the starting directory. Use 0
161+
for no scan depth limit.
160162

161163
documentation:
162164
-h, --help Show this message and exit.

‎tests/scancode/data/help/help_linux.txt‎

Lines changed: 23 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -138,27 +138,29 @@ Options:
138138
which are todo items and needs manual review.
139139

140140
core:
141-
--ignore <pattern> Ignore files matching <pattern>.
142-
--timeout <seconds> Stop an unfinished file scan after a timeout in
143-
seconds. [default: 120 seconds]
144-
-n, --processes INT Set the number of parallel processes to use. Disable
145-
parallel processing if 0. Also disable threading if
146-
-1. [default: (number of CPUs)-1]
147-
-q, --quiet Do not print summary or progress.
148-
-v, --verbose Print progress as file-by-file path instead of a
149-
progress bar. Print verbose scan counters.
150-
--from-json Load codebase from one or more <input> JSON scan
151-
file(s).
152-
--max-in-memory INTEGER Maximum number of files and directories scan details
153-
kept in memory during a scan. Additional files and
154-
directories scan details above this number are cached
155-
on-disk rather than in memory. Use 0 to use unlimited
156-
memory and disable on-disk caching. Use -1 to use
157-
only on-disk caching. [default: 10000]
158-
--max-depth INTEGER Maximum nesting depth of subdirectories to scan.
159-
Descend at most INTEGER levels of directories below
160-
and including the starting directory. Use 0 for no
161-
scan depth limit.
141+
--ignore <pattern> Ignore files matching <pattern>.
142+
--timeout <seconds> Stop an unfinished file scan after a timeout in
143+
seconds. [default: 120 seconds]
144+
-n, --processes INT Set the number of parallel processes to use.
145+
Disable parallel processing if 0. Also disable
146+
threading if -1. [default: (number of CPUs)-1]
147+
-c, --config-file FILENAME Path to the configuration file.
148+
-q, --quiet Do not print summary or progress.
149+
-v, --verbose Print progress as file-by-file path instead of a
150+
progress bar. Print verbose scan counters.
151+
--from-json Load codebase from one or more <input> JSON scan
152+
file(s).
153+
--max-in-memory INTEGER Maximum number of files and directories scan
154+
details kept in memory during a scan. Additional
155+
files and directories scan details above this
156+
number are cached on-disk rather than in memory.
157+
Use 0 to use unlimited memory and disable on-disk
158+
caching. Use -1 to use only on-disk caching.
159+
[default: 10000]
160+
--max-depth INTEGER Maximum nesting depth of subdirectories to scan.
161+
Descend at most INTEGER levels of directories
162+
below and including the starting directory. Use 0
163+
for no scan depth limit.
162164

163165
documentation:
164166
-h, --help Show this message and exit.

‎tests/scancode/data/plugin_ignore/ignore.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
ignored_patterns:
2+
- '*.doc'
3+
- '*/test*'

‎tests/scancode/test_ignore.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -220,6 +220,21 @@ def test_scancode_multiple_ignores(self):
220220
scan_locs = [x['path'] for x in scan_result['files']]
221221
assert scan_locs == [u'user', u'user/src', u'user/src/test']
222222

223+
def test_scancode_ignore_files_from_config(self):
224+
test_dir = self.extract_test_tar('plugin_ignore/user.tgz')
225+
config_file = self.get_test_loc('plugin_ignore/ignore.yaml')
226+
result_file = self.get_temp_file('json')
227+
args = ['--copyright', '--strip-root', '--config-file', config_file, test_dir, '--json', result_file]
228+
run_scan_click(args)
229+
scan_result = load_json_result(result_file)
230+
assert scan_result['headers'][0]['extra_data']['files_count'] == 0
231+
scan_locs = [x['path'] for x in scan_result['files']]
232+
expected = [
233+
u'user',
234+
u'user/src',
235+
]
236+
assert scan_locs == expected
237+
223238
def test_scancode_codebase_attempt_to_access_an_ignored_resourced_cached_to_disk(self):
224239
test_dir = self.extract_test_tar('plugin_ignore/user.tgz')
225240
result_file = self.get_temp_file('json')

0 commit comments

Comments
 (0)