Skip to content

Commit f3bba1a

Browse files
pombredannepombredanne
authored
Merge pull request #14 from nexB/python3-port
Complete Python3 port and improve ELF handling
2 parents f36c8d3 + d1d80ef commit f3bba1a

74 files changed

Lines changed: 6055 additions & 1223 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎binary-analysis/scancode-compiledcode/.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,4 @@
11
/build/
22
/dist/
3+
/.pytest_cache/
4+
/tmp/

‎binary-analysis/scancode-compiledcode/NOTICE‎

Lines changed: 40 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,43 @@
1-
Software license
2-
================
1+
ScanCode Plugins Licenses
2+
=================================
33

4-
Copyright (c) 2019 nexB Inc. and others. All rights reserved.
5-
http://nexb.com and https://github.com/nexB/scancode-toolkit/
6-
The ScanCode software is licensed under the Apache License version 2.0.
7-
Data generated with ScanCode require an acknowledgment.
4+
Copyright (c) nexB Inc. and others. All rights reserved.
85
ScanCode is a trademark of nexB Inc.
96

10-
You may not use this software except in compliance with the License.
11-
You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
12-
Unless required by applicable law or agreed to in writing, software distributed
13-
under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
14-
CONDITIONS OF ANY KIND, either express or implied. See the License for the
15-
specific language governing permissions and limitations under the License.
16-
17-
When you publish or redistribute any data created with ScanCode or any ScanCode
18-
derivative work, you must accompany this data with the following acknowledgment:
19-
20-
Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
21-
OR CONDITIONS OF ANY KIND, either express or implied. No content created from
22-
ScanCode should be considered or used as legal advice. Consult an Attorney
23-
for any legal advice.
24-
ScanCode is a free software code scanning tool from nexB Inc. and others.
25-
Visit https://github.com/nexB/scancode-toolkit/ for support and download.
7+
SPDX-License-Identifier: Apache-2.0
8+
9+
ScanCode software is licensed under the Apache License version 2.0.
10+
11+
See https://www.apache.org/licenses/LICENSE-2.0 for the Apache-2.0 license text.
12+
13+
See https://github.com/nexB/scancode-plugins for support or download.
14+
See https://aboutcode.org for more information about nexB OSS projects
15+
16+
17+
Third-party software licenses
18+
=============================
19+
20+
ScanCode plugins embed third-party free and open source software packages under
21+
various licenses including permissive, limited copyleft and copyleft licenses.
22+
The origin and license of each package is documented by a .ABOUT file included
23+
in the codetree and the releases archive.
24+
25+
The corresponding source code for pre-compiled third-party software is available
26+
for immediate download from the same release page where you obtained ScanCode,
27+
including:
28+
29+
- https://github.com/nexB/scancode-toolkit/
30+
- https://pypi.org/project/scancode-toolkit/
31+
- https://thirdparty.aboutcode.org/pypi/
32+
- https://github.com/nexB/thirdparty-packages/pypi/
33+
- https://github.com/nexB/scancode-plugins/
34+
- https://github.com/nexB/scancode-thirdparty-src/
35+
36+
You may also contact us to request the source code by email at info@nexb.com or
37+
by postal mail at:
38+
39+
nexB Inc., ScanCode open source code request
40+
4966 El Camino Real #119, Los Altos, CA 94022, USA
41+
42+
Please indicate in your communication the ScanCode tool and version for which
43+
you are requesting source code.

‎binary-analysis/scancode-compiledcode/README.rst‎

Lines changed: 12 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
A ScanCode scan plugin to get lkmclue, dwarf, gwt, cpp includes, code/comments lines generated code and elf info.
1+
CompiledCode is ScanCode scan plugin to get lkm clue, dwarfs, gwt, cpp includes,
2+
code/comments lines and various binary and compiled code info.
23

34
--cpp-includes:
45
--dwarf:
@@ -9,22 +10,16 @@ A ScanCode scan plugin to get lkmclue, dwarf, gwt, cpp includes, code/comments l
910
--makedepend:
1011
--codecommentlines:
1112

12-
To start the test case, please run:
13-
1. ./configure
14-
2. source bin/activate
15-
3. pip install -e plugins/scancode-dwarfdump-manylinux2014_x86_64 -e plugins/scancode-ctags-manylinux2014_x86_64 -e plugins/scancode-readelf-manylinux2014_x86_64 -e plugins/scancode-compiledcode
16-
4. pytest -vvs plugins/scancode-compiledcode/tests/test_lkmclue.py
17-
pytest -vvs plugins/scancode-compiledcode/tests/test_elf.py
18-
pytest -vvs plugins/scancode-compiledcode/tests/test_cpp_includes.py
19-
pytest -vvs plugins/scancode-compiledcode/tests/test_dwarf.py
20-
pytest -vvs plugins/scancode-compiledcode/tests/test_gwt.py
21-
pytest -vvs plugins/scancode-compiledcode/tests/test_makedepend.py
22-
pytest -vvs plugins/scancode-compiledcode/tests/test_javaclass.py
23-
pytest -vvs plugins/scancode-compiledcode/tests/test_generatedcode.py
24-
pytest -vvs plugins/scancode-compiledcode/tests/test_sourcecode.py
25-
pytest -vvs plugins/scancode-compiledcode/tests/test_dwarf.py
26-
pytest -vvs plugins/scancode-compiledcode/tests/test_dwarf2.py
27-
pytest -vvs plugins/scancode-compiledcode/tests/test_dwarf3.py
13+
To run tests::
2814

15+
./configure --dev
16+
source bin/activate
17+
pip install \
18+
-e plugins/scancode-dwarfdump-manylinux2014_x86_64 \
19+
-e plugins/scancode-ctags-manylinux2014_x86_64 \
20+
-e plugins/scancode-readelf-manylinux2014_x86_64 \
21+
-e plugins/scancode-compiledcode[testing,binary]
22+
23+
pytest -vvs plugins/scancode-compiledcode/tests
2924

3025
Note that in step3, the path depends on your OS versions, please update according to your real os enviroment.

‎binary-analysis/scancode-compiledcode/setup.py‎

Lines changed: 26 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,6 @@
11
#!/usr/bin/env python
22
# -*- encoding: utf-8 -*-
33

4-
from __future__ import absolute_import
5-
from __future__ import print_function
6-
74
from glob import glob
85
from os.path import basename
96
from os.path import join
@@ -12,18 +9,17 @@
129
from setuptools import find_packages
1310
from setuptools import setup
1411

15-
1612
desc = '''A ScanCode scan plugin to get lkmclue, dwarf, gwt, cpp includes, code/comments lines generated code and elf info.'''
1713

1814
setup(
19-
name='scancode-compiledcode',
20-
version='1.0.0',
21-
license='Apache-2.0 with ScanCode acknowledgment',
15+
name='compiledcode',
16+
version='2.0.0',
17+
license='Apache-2.0',
2218
description=desc,
2319
long_description=desc,
2420
author='nexB',
2521
author_email='info@aboutcode.org',
26-
url='https://github.com/nexB/scancode-toolkit/plugins/scancode-compiledcode',
22+
url='https://github.com/nexB/scancode-plugins/binary-analysis/scancode-compiledcode',
2723
packages=find_packages('src'),
2824
package_dir={'': 'src'},
2925
py_modules=[splitext(basename(path))[0] for path in glob('src/*.py')],
@@ -36,31 +32,38 @@
3632
'Intended Audience :: Developers',
3733
'License :: OSI Approved :: Apache Software License',
3834
'Programming Language :: Python',
39-
'Programming Language :: Python :: 2.7',
35+
'Programming Language :: Python :: 3',
4036
'Topic :: Utilities',
4137
],
4238
keywords=[
43-
'open source', 'scancode', 'dwarf', 'lkmclue', 'elf', 'cpp includes', 'gwt', 'generatedcode', 'codecommentlines'
39+
'open source', 'scancode', 'dwarf', 'lkmclue', 'elf', 'cpp includes', 'gwt',
4440
],
4541
install_requires=[
4642
'scancode-toolkit',
4743
'attr',
48-
'scancode-ctags',
49-
'scancode-dwarfdump',
50-
'scancode-readelf',
44+
'pyelftools',
5145
],
46+
47+
extra_requires={
48+
'binary': [
49+
'scancode-ctags',
50+
'scancode-dwarfdump',
51+
'scancode-readelf',
52+
],
53+
'testing': [
54+
'pytest',
55+
]
56+
},
5257
entry_points={
5358
'scancode_scan': [
54-
'scancode-lkmclue = lkmclue:LKMClueScanner',
55-
'scancode-elf = elf:ELFScanner',
56-
'scancode-cppincludes = cppincludes:CPPIncludesScanner',
57-
'scancode-dwarf = dwarf:DwarfScanner',
58-
'scancode-gwt = gwt:GWTScanner',
59-
'scancode-makedepend = makedepend:MakeDependScanner',
60-
'scancode-javaclass = javaclass:JavaClassScanner',
61-
'scancode-generatedcode = generatedcode:GeneratedCodeScanner',
62-
'scancode-codecommentlines = sourcecode:CodeCommentLinesScanner',
63-
59+
'scancode-lkmclue = compiledcode.lkmclue:LKMClueScanner',
60+
'scancode-elf = compiledcode.elf:ELFScanner',
61+
'scancode-cppincludes = compiledcode.cppincludes:CPPIncludesScanner',
62+
'scancode-dwarf = compiledcode.dwarf:DwarfScanner',
63+
'scancode-gwt = compiledcode.gwt:GWTScanner',
64+
'scancode-makedepend = compiledcode.makedepend:MakeDependScanner',
65+
'scancode-javaclass = compiledcode.javaclass:JavaClassScanner',
66+
'scancode-codecommentlines = compiledcode.sourcecode:CodeCommentLinesScanner',
6467
],
6568
}
6669
)

‎binary-analysis/scancode-compiledcode/src/compiledcode/__init__.py‎

Whitespace-only changes.

binary-analysis/scancode-compiledcode/src/cppincludes/__init__.py renamed to binary-analysis/scancode-compiledcode/src/compiledcode/cppincludes.py

Lines changed: 9 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,22 @@
1+
# -*- coding: utf-8 -*-
12
#
2-
# Copyright (c) 2019 nexB Inc. and others. All rights reserved.
3-
# http://nexb.com and https://github.com/nexB/scancode-toolkit/
4-
# The ScanCode software is licensed under the Apache License version 2.0.
5-
# Data generated with ScanCode require an acknowledgment.
3+
# Copyright (c) nexB Inc. and others. All rights reserved.
64
# ScanCode is a trademark of nexB Inc.
5+
# SPDX-License-Identifier: Apache-2.0
6+
# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
7+
# See https://github.com/nexB/scancode-toolkit for support or download.
8+
# See https://aboutcode.org for more information about nexB OSS projects.
79
#
8-
# You may not use this software except in compliance with the License.
9-
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
10-
# Unless required by applicable law or agreed to in writing, software distributed
11-
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
12-
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
13-
# specific language governing permissions and limitations under the License.
14-
#
15-
# When you publish or redistribute any data created with ScanCode or any ScanCode
16-
# derivative work, you must accompany this data with the following acknowledgment:
17-
#
18-
# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
19-
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
20-
# ScanCode should be considered or used as legal advice. Consult an Attorney
21-
# for any legal advice.
22-
# ScanCode is a free software code scanning tool from nexB Inc. and others.
23-
# Visit https://github.com/nexB/scancode-toolkit/ for support and download.
2410

2511
import re
26-
from functools import partial
27-
from itertools import chain
2812

2913
import attr
3014

31-
from textcode import analysis
15+
from commoncode.cliutils import SCAN_GROUP
16+
from commoncode.cliutils import PluggableCommandLineOption
3217
from plugincode.scan import ScanPlugin
3318
from plugincode.scan import scan_impl
34-
from commoncode.cliutils import PluggableCommandLineOption
35-
from commoncode.cliutils import SCAN_GROUP
19+
from textcode import analysis
3620
from typecode import contenttype
3721

3822

binary-analysis/scancode-compiledcode/src/dwarf/__init__.py renamed to binary-analysis/scancode-compiledcode/src/compiledcode/dwarf/__init__.py

Lines changed: 24 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,12 @@
1+
# -*- coding: utf-8 -*-
12
#
2-
# Copyright (c) 2019 nexB Inc. and others. All rights reserved.
3-
# http://nexb.com and https://github.com/nexB/scancode-toolkit/
4-
# The ScanCode software is licensed under the Apache License version 2.0.
5-
# Data generated with ScanCode require an acknowledgment.
3+
# Copyright (c) nexB Inc. and others. All rights reserved.
64
# ScanCode is a trademark of nexB Inc.
5+
# SPDX-License-Identifier: Apache-2.0
6+
# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
7+
# See https://github.com/nexB/scancode-plugins for support or download.
8+
# See https://aboutcode.org for more information about nexB OSS projects.
79
#
8-
# You may not use this software except in compliance with the License.
9-
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
10-
# Unless required by applicable law or agreed to in writing, software distributed
11-
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
12-
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
13-
# specific language governing permissions and limitations under the License.
14-
#
15-
# When you publish or redistribute any data created with ScanCode or any ScanCode
16-
# derivative work, you must accompany this data with the following acknowledgment:
17-
#
18-
# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
19-
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
20-
# ScanCode should be considered or used as legal advice. Consult an Attorney
21-
# for any legal advice.
22-
# ScanCode is a free software code scanning tool from nexB Inc. and others.
23-
# Visit https://github.com/nexB/scancode-toolkit/ for support and download.
24-
25-
from __future__ import absolute_import
26-
from __future__ import unicode_literals
2710

2811
import os
2912

@@ -39,8 +22,9 @@
3922
from commoncode.cliutils import SCAN_GROUP
4023
from typecode import contenttype
4124

42-
from dwarf import dwarf
43-
from dwarf import dwarf2
25+
from compiledcode.dwarf import dwarf
26+
from compiledcode.dwarf import dwarf2
27+
from compiledcode.dwarf import dwarfng
4428

4529

4630
@scan_impl
@@ -49,15 +33,17 @@ class DwarfScanner(ScanPlugin):
4933
Scan a dwarf infos for URLs.
5034
"""
5135
resource_attributes = dict(
52-
dwarf_source_path=attr.ib(default=attr.Factory(list), repr=False))
36+
dwarf_source_path=attr.ib(default=attr.Factory(list), repr=False)
37+
)
5338

5439
options = [
5540
PluggableCommandLineOption(('--dwarf',),
5641
is_flag=True, default=False,
5742
help='Collect source code path from compilation units found in '
5843
'ELF DWARFs.',
5944
help_group=SCAN_GROUP,
60-
sort_order=100),
45+
sort_order=100
46+
),
6147
]
6248

6349
def is_enabled(self, dwarf, **kwargs):
@@ -72,7 +58,17 @@ def get_dwarfs(location, **kwargs):
7258
Return a mapping with original_source_files and included_source_files or None.
7359
"""
7460
return dict(
75-
dwarf_source_path=list(dwarf_source_path(location)))
61+
# dwarf_source_path=list(dwarf_source_path(location))
62+
dwarf_source_path=list(dwarf_source_path_ng(location))
63+
)
64+
65+
66+
def dwarf_source_path_ng(location, **kwargs):
67+
"""
68+
Collect unique paths to compiled source code found in Elf binaries DWARF
69+
sections for D2D.
70+
"""
71+
return dwarfng.get_dwarf_cu_and_die_paths(location)
7672

7773

7874
def dwarf_source_path(location):
@@ -120,8 +116,6 @@ def get_dwarf1(location):
120116
# std and original path as separate annotations
121117
for p in d.original_source_files + d.included_source_files:
122118
yield p
123-
124-
125119

126120

127121
def get_dwarf2(location):

0 commit comments

Comments
 (0)