Skip to content

Commit e38f5a3

Browse files
TG1999JonoYang
TG1999
and
JonoYang
authored
Add inspect elf binaries pipeline (#353)
* Add inspect_elf_binaries Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com> * fix tests Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com> * Add additional tests Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com> * Fix tests Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com> * Assign variable names to expected and result data Signed-off-by: Jono Yang <jyang@nexb.com> --------- Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com> Signed-off-by: Jono Yang <jyang@nexb.com> Co-authored-by: Jono Yang <jyang@nexb.com>
1 parent 2de322a commit e38f5a3

4 files changed

Lines changed: 230 additions & 0 deletions

File tree

‎minecode/model_utils.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
'scan_single_package',
3232
'fingerprint_codebase',
3333
'collect_symbols',
34+
'inspect_elf_binaries',
3435
)
3536

3637

‎minecode/tests/test_indexing.py‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,14 @@ def setUp(self):
3232
version='20040705.181715'
3333
)
3434

35+
self.package2 = Package.objects.create(
36+
download_url='https://github.com/nexB/elf-inspector/raw/4333e1601229da87fa88961389d7397af6e027c4/tests/data/dwarf_and_elf/analyze.so.debug',
37+
type='generic',
38+
namespace='',
39+
name='debug',
40+
version='1.23'
41+
)
42+
3543
def test_indexing_index_package_files(self):
3644
scan_data_loc = self.get_test_loc('scancodeio/get_scan_data.json')
3745
with open(scan_data_loc, 'rb') as f:
@@ -98,3 +106,40 @@ def test_indexing_index_package(self):
98106
self.assertEqual(64, result.count())
99107
result = ExactFileIndex.objects.filter(package=self.package1)
100108
self.assertEqual(45, result.count())
109+
110+
def test_indexing_index_package_dwarf(self):
111+
scan_data_loc = self.get_test_loc('scancodeio/get_scan_data_dwarf.json')
112+
with open(scan_data_loc, 'rb') as f:
113+
scan_data = json.load(f)
114+
115+
scan_summary_loc = self.get_test_loc('scancodeio/scan_summary_dwarf.json')
116+
with open(scan_summary_loc, 'rb') as f:
117+
scan_summary = json.load(f)
118+
119+
project_extra_data = {}
120+
121+
# Set up ScannableURI
122+
scannable_uri = ScannableURI.objects.create(
123+
uri='https://github.com/nexB/elf-inspector/raw/4333e1601229da87fa88961389d7397af6e027c4/tests/data/dwarf_and_elf/analyze.so.debug',
124+
scan_status=ScannableURI.SCAN_COMPLETED,
125+
package=self.package2
126+
)
127+
128+
# Run test
129+
indexing.index_package(
130+
scannable_uri,
131+
self.package2,
132+
scan_data,
133+
scan_summary,
134+
project_extra_data,
135+
)
136+
137+
package = Package.objects.filter(id=self.package2.id)
138+
self.assertEqual(1, package.count())
139+
140+
result = Resource.objects.filter(package=self.package2)
141+
self.assertEqual(1, result.count())
142+
143+
extra_data = result.first().extra_data
144+
expected_extra_data = scan_data["files"][0]["extra_data"]
145+
self.assertEqual(expected_extra_data, extra_data)

‎minecode/tests/testfiles/scancodeio/get_scan_data_dwarf.json‎

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
{
2+
"headers": [
3+
{
4+
"tool_name": "scanpipe",
5+
"tool_version": "v34.0.0-5-g4391b4875",
6+
"other_tools": [
7+
"pkg:pypi/scancode-toolkit@32.0.8"
8+
],
9+
"notice": "Generated with ScanCode.io and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied.\nNo content created from ScanCode.io should be considered or used as legal advice.\nConsult an Attorney for any legal advice.\nScanCode.io is a free software code scanning tool from nexB Inc. and others\nlicensed under the Apache License version 2.0.\nScanCode is a trademark of nexB Inc.\nVisit https://github.com/nexB/scancode.io for support and download.\n",
10+
"uuid": "8386c243-4a46-488a-bc07-5616c9a0cfd7",
11+
"created_date": "2024-03-21T18:24:32.726Z",
12+
"notes": "",
13+
"settings": {},
14+
"input_sources": [
15+
{
16+
"uuid": "400f03db-295c-4303-b549-807145b806a1",
17+
"filename": "analyze.so.debug",
18+
"download_url": "https://github.com/nexB/elf-inspector/raw/4333e1601229da87fa88961389d7397af6e027c4/tests/data/dwarf_and_elf/analyze.so.debug",
19+
"is_uploaded": false,
20+
"tag": "",
21+
"size": 38412,
22+
"is_file": true,
23+
"exists": true
24+
}
25+
],
26+
"runs": [
27+
{
28+
"pipeline_name": "scan_single_package",
29+
"status": "success",
30+
"description": "Scan a single package archive (or package manifest file).",
31+
"uuid": "645a281f-01da-45ed-8679-7033c3922823",
32+
"created_date": "2024-03-21T18:24:32.728657Z",
33+
"scancodeio_version": "v34.0.0-5-g4391b4875",
34+
"task_id": "645a281f-01da-45ed-8679-7033c3922823",
35+
"task_start_date": "2024-03-21T18:24:32.752793Z",
36+
"task_end_date": "2024-03-21T18:24:36.884901Z",
37+
"task_exitcode": 0,
38+
"task_output": "",
39+
"log": "2024-03-21 18:24:32.75 Pipeline [scan_single_package] starting\n2024-03-21 18:24:32.75 Step [download_missing_inputs] starting\n2024-03-21 18:24:32.76 Fetching input from https://github.com/nexB/elf-inspector/raw/4333e1601229da87fa88961389d7397af6e027c4/tests/data/dwarf_and_elf/analyze.so.debug\n2024-03-21 18:24:33.16 Step [download_missing_inputs] completed in 0 seconds\n2024-03-21 18:24:33.17 Step [get_package_input] starting\n2024-03-21 18:24:33.17 Step [get_package_input] completed in 0 seconds\n2024-03-21 18:24:33.17 Step [collect_input_information] starting\n2024-03-21 18:24:33.18 Step [collect_input_information] completed in 0 seconds\n2024-03-21 18:24:33.18 Step [extract_input_to_codebase_directory] starting\n2024-03-21 18:24:33.19 Step [extract_input_to_codebase_directory] completed in 0 seconds\n2024-03-21 18:24:33.19 Step [extract_archives] starting\n2024-03-21 18:24:33.20 Step [extract_archives] completed in 0 seconds\n2024-03-21 18:24:33.20 Step [run_scan] starting\n2024-03-21 18:24:36.84 Step [run_scan] completed in 4 seconds\n2024-03-21 18:24:36.85 Step [load_inventory_from_toolkit_scan] starting\n2024-03-21 18:24:36.86 Step [load_inventory_from_toolkit_scan] completed in 0 seconds\n2024-03-21 18:24:36.86 Step [make_summary_from_scan_results] starting\n2024-03-21 18:24:36.88 Step [make_summary_from_scan_results] completed in 0 seconds\n2024-03-21 18:24:36.88 Pipeline completed in 4 seconds\n",
40+
"execution_time": 4
41+
},
42+
{
43+
"pipeline_name": "inspect_elf_binaries",
44+
"status": "success",
45+
"description": "Inspect ELF binaries and collect DWARF paths.",
46+
"uuid": "9ebedfb1-e38c-4066-90c6-960e517b0dc7",
47+
"created_date": "2024-03-21T18:24:38.836887Z",
48+
"scancodeio_version": "v34.0.0-5-g4391b4875",
49+
"task_id": "9ebedfb1-e38c-4066-90c6-960e517b0dc7",
50+
"task_start_date": "2024-03-21T18:24:38.870473Z",
51+
"task_end_date": "2024-03-21T18:24:38.917895Z",
52+
"task_exitcode": 0,
53+
"task_output": "",
54+
"log": "2024-03-21 18:24:38.87 Pipeline [inspect_elf_binaries] starting\n2024-03-21 18:24:38.87 Step [collect_dwarf_source_path_references] starting\n2024-03-21 18:24:38.91 Step [collect_dwarf_source_path_references] completed in 0 seconds\n2024-03-21 18:24:38.91 Pipeline completed in 0 seconds\n",
55+
"execution_time": 0
56+
},
57+
{
58+
"pipeline_name": "fingerprint_codebase",
59+
"status": "success",
60+
"description": "Calculate the directory fingerprints of a Project codebase",
61+
"uuid": "a38c0c55-f232-4acd-9143-be4d6680e17b",
62+
"created_date": "2024-03-21T18:28:33.744363Z",
63+
"scancodeio_version": "v34.0.0-5-g4391b4875",
64+
"task_id": "a38c0c55-f232-4acd-9143-be4d6680e17b",
65+
"task_start_date": "2024-03-21T18:28:33.777809Z",
66+
"task_end_date": "2024-03-21T18:28:33.797765Z",
67+
"task_exitcode": 0,
68+
"task_output": "",
69+
"log": "2024-03-21 18:28:33.77 Pipeline [fingerprint_codebase] starting\n2024-03-21 18:28:33.78 Step [fingerprint_codebase] starting\n2024-03-21 18:28:33.79 Step [fingerprint_codebase] completed in 0 seconds\n2024-03-21 18:28:33.79 Pipeline completed in 0 seconds\n",
70+
"execution_time": 0
71+
}
72+
],
73+
"extra_data": {
74+
"md5": "9fd9dafa0306323270636753c842cc19",
75+
"sha1": "0c2ec13722082672f1bb2949e8236564ac6ee36c",
76+
"size": 38412,
77+
"sha256": "f39785d98457d5191fd61218baf9ffa1e0238e799e6ac468157069c3b502252e",
78+
"sha512": "2177630bbb33e0ddaf82db46c7b888d18a524bc742c542258c95545383e601fb1523a59c79887a3f3dabda0fc4bdb37d2642a59357950bf462fae45ef5ab586e",
79+
"filename": "analyze.so.debug",
80+
"sha1_git": "603a7fdf47e38478784dd2b6b99fd710ca514ca0"
81+
}
82+
}],
83+
"packages": [
84+
],
85+
"dependencies": [
86+
],
87+
"files": [
88+
{
89+
"path": "analyze.so.debug",
90+
"type": "file",
91+
"name": "analyze.so.debug",
92+
"status": "",
93+
"tag": "",
94+
"extension": ".debug",
95+
"size": 38412,
96+
"md5": "9fd9dafa0306323270636753c842cc19",
97+
"sha1": "0c2ec13722082672f1bb2949e8236564ac6ee36c",
98+
"sha256": "f39785d98457d5191fd61218baf9ffa1e0238e799e6ac468157069c3b502252e",
99+
"sha512": "",
100+
"mime_type": "application/x-sharedlib",
101+
"file_type": "ELF 32-bit LSB shared object, ARM, EABI5 version 1 (SYSV), dynamically linked, BuildID[sha1]=aa1f96375250e783f253f60a90f6a7ea72c3dfe2, with debug_info, not stripped",
102+
"programming_language": "",
103+
"is_binary": true,
104+
"is_text": false,
105+
"is_archive": false,
106+
"is_media": false,
107+
"is_key_file": false,
108+
"detected_license_expression": "",
109+
"detected_license_expression_spdx": "",
110+
"license_detections": [],
111+
"license_clues": [],
112+
"percentage_of_license_text": 0.0,
113+
"compliance_alert": "",
114+
"copyrights": [],
115+
"holders": [],
116+
"authors": [],
117+
"package_data": [],
118+
"for_packages": [],
119+
"emails": [],
120+
"urls": [],
121+
"extra_data": {
122+
"compiled_paths": [
123+
"usr/src/debug/ImageMagick-6.7.5-6/filters/analyze.c"
124+
],
125+
"included_paths": [
126+
"filters/analyze.c",
127+
"magick/cache-view.h",
128+
"magick/color.h",
129+
"magick/colorspace.h",
130+
"magick/composite.h",
131+
"magick/compress.h",
132+
"magick/exception.h",
133+
"magick/gem.h",
134+
"magick/geometry.h",
135+
"magick/image.h",
136+
"magick/layer.h",
137+
"magick/list.h",
138+
"magick/locale_.h",
139+
"magick/magick-type.h",
140+
"magick/monitor.h",
141+
"magick/pixel.h",
142+
"magick/profile.h",
143+
"magick/property.h",
144+
"magick/quantum.h",
145+
"magick/resample.h",
146+
"magick/semaphore.h",
147+
"magick/timer.h",
148+
"usr/include/assert.h",
149+
"usr/include/bits/mathcalls.h",
150+
"usr/include/bits/types.h",
151+
"usr/include/libio.h",
152+
"usr/include/stdio.h",
153+
"usr/lib/gcc/armv5tel-redhat-linux-gnueabi/4.7.0/include/stddef.h"
154+
]
155+
}
156+
}],
157+
"relations": [
158+
]
159+
}

‎minecode/tests/testfiles/scancodeio/scan_summary_dwarf.json‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
{
2+
"declared_license_expression": null,
3+
"license_clarity_score": {
4+
"score": 0,
5+
"declared_license": false,
6+
"identification_precision": false,
7+
"has_license_text": false,
8+
"declared_copyrights": false,
9+
"conflicting_license_categories": false,
10+
"ambiguous_compound_licensing": true
11+
},
12+
"declared_holder": "",
13+
"primary_language": null,
14+
"other_license_expressions": [],
15+
"other_holders": [
16+
{
17+
"value": null,
18+
"count": 1
19+
}
20+
],
21+
"other_languages": [],
22+
"license_matches": {},
23+
"key_files": [],
24+
"key_files_packages": []
25+
}

0 commit comments

Comments
 (0)