Support rust binary parsing

Support parsing rust binaries and getting:

1. Symbols
2. Packages and Dependencies

Reference: aboutcode-org/scancode.io#1435
Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

Author: AyanSinhaMahapatra

.readthedocs.yml

@@ -7,9 +7,9 @@ version: 2
 
 # Build in latest ubuntu/python
 build:
-  os: ubuntu-22.04
+  os: ubuntu-24.04
   tools:
-    python: "3.11"
+    python: "3.12"
 
 # Build PDF & ePub
 formats:

AUTHORS.rst

@@ -1,3 +1,5 @@
 The following organizations or individuals have contributed to this repo:
 
-- 
+- Ayan Sinha Mahapatra @AyanSinhaMahapatra
+- nexB Inc.
+

CHANGELOG.rst

@@ -2,7 +2,7 @@ Changelog
 =========
 
 
-v0.0.0
+v0.1.0
 ------
 
 *xxxx-xx-xx* -- Initial release.

requirements-dev.txt

@@ -0,0 +1 @@
+scancode-toolkit==32.3.0

requirements.txt

@@ -0,0 +1,4 @@
+# from blint v2.3.2
+# https://github.com/owasp-dep-scan/blint/blob/1e1250a4bf6c25eccba8970bd877901ee56070c7/poetry.lock
+lief==0.15.1
+symbolic==10.2.1

setup.cfg

@@ -1,12 +1,12 @@
 [metadata]
-name = skeleton
-license = Apache-2.0
+name = rust-inspector
+license = Apache-2.0 AND MIT
 
 # description must be on ONE line https://github.com/pypa/setuptools/issues/1390
-description = skeleton
+description = rust-inspector is a scancode plugin to extract symbols and dependencies found in Rust binaries.
 long_description = file:README.rst
 long_description_content_type = text/x-rst
-url = https://github.com/aboutcode-org/skeleton
+url = https://github.com/nexB/rust-inspector
 
 author = nexB. Inc. and others
 author_email = info@aboutcode.org
@@ -38,17 +38,27 @@ zip_safe = false
 
 setup_requires = setuptools_scm[toml] >= 4
 
-python_requires = >=3.8
+python_requires = >=3.9
 
 install_requires =
+    commoncode
+    plugincode
+    typecode
+    lief==0.15.1
+    symbolic==10.2.1
 
+[options.entry_points]
+
+scancode_scan =
+    rust_symbol = rust_inspector.plugin:RustSymbolScannerPlugin
 
 [options.packages.find]
 where = src
 
 
 [options.extras_require]
 testing =
+    scancode-toolkit
     pytest >= 6, != 7.0.0
     pytest-xdist >= 2
     aboutcode-toolkit >= 7.0.2
@@ -65,4 +75,3 @@ docs =
     sphinx-autobuild
     sphinx-rtd-dark-mode>=1.3.0
     sphinx-copybutton
-

src/README.rst

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# ScanCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/rust-inspector for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+

src/rust_inspector/__init__.py

@@ -0,0 +1,278 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# ScanCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/rust-inspector for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import os
+import lief
+import json
+import zlib
+
+from typecode import contenttype
+from typecode.contenttype import get_type
+
+from rust_inspector.blint_binary import parse_symbols
+from rust_inspector.config import SPLIT_CHARACTERS_RUST
+from rust_inspector.config import STANDARD_SYMBOLS_RUST
+
+
+def is_macho(location):
+    """
+    Return True if the file at ``location`` is in macOS/Darwin's Mach-O format, otherwise False.
+    """
+    t = get_type(location)
+    return t.filetype_file.lower().startswith("mach-o") or t.mimetype_file.lower().startswith(
+        "application/x-mach-binary"
+    )
+
+
+def is_executable_binary(location):
+    """
+    Return True if the file at ``location`` is an executable binary.
+    """
+    if not os.path.exists(location):
+        return False
+
+    if not os.path.isfile(location):
+        return False
+
+    typ = contenttype.Type(location)
+
+    if not (typ.is_elf or typ.is_winexe or is_macho(location=location)):
+        return False
+
+    return True
+
+
+def parse_rust_binary(location):
+    """
+    Get a parsed lief._lief.ELF.Binary object from parsing the Rust binary
+    present at `location`.
+    """
+    return lief.parse(location)
+
+
+def get_rust_packages_data(location):
+    """
+    Given a rust binary present at location, parse and return packages
+    data present in the binary.
+
+    This get packages data only if it is present in the rust binary, i.e.
+    if the rust binary was built using `cargo-auditable` which adds a section
+    to the rust binary with data on packages and dependencies.
+    See https://github.com/rust-secure-code/cargo-auditable for more info.
+
+    Code for parsing rust bianries to get package data is from
+    https://github.com/rust-secure-code/cargo-auditable/blob/master/PARSING.md
+    """
+    if not is_executable_binary(location):
+        return
+
+    parsed_binary = parse_rust_binary(location)
+
+    section_deps = None
+    for section in parsed_binary.sections:
+        if section.name == ".dep-v0":
+            section_deps = section
+            break
+
+    decompressed_data = zlib.decompress(section_deps.content)
+    packages_data = json.loads(decompressed_data)
+
+    return packages_data
+
+
+
+def might_have_rust_symbols(string_with_symbols):
+
+    if not string_with_symbols:
+        return False
+
+    if not len(string_with_symbols) > 1:
+        return False
+
+    # Commonly encountered strings which are not rust symbols
+    ignore_strings = ["GCC_except_table", "anonymous", "SAFESEH"]
+    if string_with_symbols in ignore_strings:
+        return False
+
+    # We have sometimes strings which are all uppercase alphabets, these are
+    # usually not symbols?
+    # Examples: `__CALLSITE::META`, `tracing_core::dispatcher::NO_SUBSCRIBER`,
+    # `encoding_rs::SHIFT_JIS_INIT::hd4a3d8ee69a4c88a` etc
+    string_without_underscore = string_with_symbols.replace("_", "")
+    is_all_uppercase = string_without_underscore.isupper() and string_without_underscore.isalpha()
+    if is_all_uppercase:
+        return False
+
+    # fully numberic strings are not rust symbols
+    if string_with_symbols.isnumeric():
+        return False
+
+    # TODO: also filter checksum like values (entropy/randomness?)
+    # TODO: also filter Uppercase Alphabets + Numbers
+    # Example: encoding_rs::UTF_16LE_INIT::h47f34b513cc70084
+
+    return True
+
+def remove_standard_symbols(rust_symbols):
+    return [
+        symbol
+        for symbol in rust_symbols
+        if symbol not in STANDARD_SYMBOLS_RUST
+    ]
+
+
+def split_strings_by_char(split_strings, split_char):
+
+    final_split_strings = []
+    for split_str in split_strings:
+        if split_char in split_str:
+            split_strings = split_str.split(split_char)
+            final_split_strings.extend(split_strings)
+        else:
+            final_split_strings.append(split_str)
+
+    return [
+        split_string
+        for split_string in final_split_strings
+        if split_string 
+    ]
+
+
+def split_strings_into_rust_symbols(strings_to_split, split_by_chars=SPLIT_CHARACTERS_RUST):
+
+    split_strings = []
+    split_strings_log = []
+    for split_char in split_by_chars:
+        if not split_strings:
+            split_strings = strings_to_split
+
+        split_strings = split_strings_by_char(split_strings, split_char)
+        split_strings_log.append(split_strings)
+
+    return split_strings
+
+
+def cleanup_symbols(split_symbols, include_stdlib=False, unique=True, sort_symbols=False):
+
+    rust_symbols = []
+    for split_string in split_symbols:
+        if might_have_rust_symbols(split_string):
+            rust_symbols.append(split_string)
+
+    if not include_stdlib:
+        rust_symbols = remove_standard_symbols(rust_symbols)
+
+    if unique:
+        rust_symbols = list(set(rust_symbols))
+
+    if sort_symbols:
+        rust_symbols = sorted(rust_symbols)
+
+    return rust_symbols
+
+
+def extract_strings_with_symbols(symbols_data, include_stdlib=False, unique=True, sort_symbols=False):
+
+    strings_with_symbols = []
+    
+    ignore_types = ["NOTYPE", "TLS"]
+
+    for symbol_data in symbols_data:
+
+        if not symbol_data.get("name"):
+            continue
+        
+        if symbol_data.get("type") in ignore_types:
+            continue
+
+        # These are usually like the following:
+        # - memcpy@GLIBC_2.14
+        # - UI_method_set_writer@OPENSSL_3.0.0
+        # So these doesn't have source symbols
+        if symbol_data.get("is_imported"):
+            continue
+
+        # These are usually like the following:
+        # `getrandom@GLIBC_2.25`, `__umodti3`, `_ITM_registerTMCloneTable`
+        # So these doesn't have source symbols
+        if symbol_data.get("binding") == 'WEAK':
+            continue
+
+        # file/module names are also source symbols as they
+        # are imported in source code files
+        if symbol_data.get("type") == "FILE":
+            file_string = symbol_data.get("name")
+            file_segments = file_string.split('.')
+            if not file_segments:
+                continue
+
+            # These are usually like following:
+            # cyclonedx_bom.4f845c900a9ac4e1-cgu.11
+            # only the first part are symbols
+            filename = file_segments[0]
+            strings_with_symbols.append(filename)
+
+        # Symbols which are Objects and Functions by type are collections of
+        # rust symbols and misc strings
+        if symbol_data.get("type") in ["OBJECT", "FUNC"]:
+            file_string = symbol_data.get("name")
+            strings_with_symbols.append(file_string)
+
+    split_symbols = split_strings_into_rust_symbols(strings_to_split=strings_with_symbols)
+    rust_symbols = cleanup_symbols(
+        split_symbols=split_symbols,
+        include_stdlib=include_stdlib,
+        unique=unique,
+        sort_symbols=sort_symbols,
+    )
+
+    return rust_symbols
+
+
+def collect_and_parse_rust_symbols(location, include_stdlib=False, sort_symbols=True, **kwargs):
+    """
+    Return a mapping of Rust symbols of interest for the Rust binary file at ``location``.
+    Return an empty mapping if there is no symbols or if this is not a binary.
+    Raise exceptions on errors.
+    """
+    if not is_executable_binary(location):
+        return
+
+    rust_data = parse_rust_binary(location=location)
+    return collect_and_parse_rust_symbols_from_data(
+        rust_data=rust_data,
+        include_stdlib=include_stdlib,
+        unique=True,
+        sort_symbols=sort_symbols,
+    )
+
+
+def collect_and_parse_rust_symbols_from_data(rust_data, include_stdlib=False, unique=True, sort_symbols=False, **kwargs):
+    """
+    Return a mapping of Rust symbols of interest for the mapping of Rust binary of ``rust_data``.
+    Return an empty mapping if there is no symbols or if this is not a binary.
+    Raise exceptions on errors.
+    """
+    if not rust_data:
+        return {}
+
+    # Extract and demangle symbol strings from rust binary parsed object
+    # These are collections of symbol and module strings with stdlib objects
+    extracted_symbols = parse_symbols(rust_data.symtab_symbols)
+
+    # Cleanup and get individual symbols which could be rust symbols
+    symbol_strings = extract_strings_with_symbols(
+        symbols_data=extracted_symbols,
+        include_stdlib=include_stdlib,
+        unique=unique,
+        sort_symbols=sort_symbols,
+    )
+
+    return dict(rust_symbols=symbol_strings)

src/rust_inspector/binary.py

@@ -0,0 +1,10 @@
+about_resource: blint_binary.py
+name: blint
+author: Prabhu Subramanian, Caroline Russell and others
+author_email: prabhu@appthreat.com, caroline@appthreat.dev
+homepage_url: https://github.com/owasp-dep-scan/blint
+download_url: https://github.com/owasp-dep-scan/blint/blob/1e1250a4bf6c25eccba8970bd877901ee56070c7/blint/lib/binary.py
+license_expression: mit
+copyright: Copyright (c) OWASP Foundation
+package_url: pkg:pypi/blint@2.3.2
+notes: only a subset of functions from binary.py is used, after minor modifications