Addon pipeline for symbol collection

.github/workflows/ci.yml

@@ -42,6 +42,9 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
+      - name: Install universal ctags
+        run: sudo apt-get install -y universal-ctags
+
       - name: Install dependencies
         run: make dev envfile
 

CHANGELOG.rst

@@ -11,6 +11,10 @@ v34.1.0 (unreleased)
   The docstring are converted from markdown to html for proper rendering.
   https://github.com/nexB/scancode.io/pull/1105
 
+- Add a new `CollectSymbols` pipeline (addon) for collecting codebase symbols using
+  Universal Ctags.
+  https://github.com/nexB/scancode.io/pull/1116
+
 v34.0.0 (2024-03-04)
 --------------------
 

Dockerfile

@@ -40,6 +40,7 @@ ENV PYTHONPATH $PYTHONPATH:$APP_DIR
 
 # OS requirements as per
 # https://scancode-toolkit.readthedocs.io/en/latest/getting-started/install.html
+# Also install universal-ctags for symbol collection.
 RUN apt-get update \
  && apt-get install -y --no-install-recommends \
        bzip2 \
@@ -58,6 +59,7 @@ RUN apt-get update \
        linux-image-amd64 \
        git \
        wait-for-it \
+       universal-ctags \
  && apt-get clean \
  && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 

docs/built-in-pipelines.rst

@@ -42,6 +42,14 @@ Analyse Docker Windows Image
     :members:
     :member-order: bysource
 
+.. _pipeline_collect_symbols:
+
+Collect Codebase Symbols (addon)
+---------------------------------
+.. autoclass:: scanpipe.pipelines.collect_symbols.CollectSymbols()
+    :members:
+    :member-order: bysource
+
 .. _pipeline_find_vulnerabilities:
 
 Find Vulnerabilities (addon)

docs/installation.rst

@@ -260,6 +260,15 @@ Make sure those are installed before attempting the ScanCode.io installation::
 See also `ScanCode-toolkit Prerequisites <https://scancode-toolkit.readthedocs.io/en/
 latest/getting-started/install.html#prerequisites>`_ for more details.
 
+For the :ref:`pipeline_collect_symbols` pipeline, `Universal Ctags <https://github.com/universal-ctags/ctags>`_ is needed.
+On **Linux** install it using::
+
+    sudo apt-get install universal-ctags
+
+On **MacOS** install Universal Ctags using Homebrew::
+
+    brew install universal-ctags
+
 Clone and Configure
 ^^^^^^^^^^^^^^^^^^^
 

scanpipe/pipelines/collect_symbols.py

@@ -0,0 +1,42 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https://github.com/nexB/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https://github.com/nexB/scancode.io for support and download.
+
+from scanpipe.pipelines import Pipeline
+from scanpipe.pipes import symbols
+
+
+class CollectSymbols(Pipeline):
+    """Collect symbols from codebase files and keep them in extra data field."""
+
+    download_inputs = False
+    is_addon = True
+
+    @classmethod
+    def steps(cls):
+        return (cls.collect_and_store_resource_symbols,)
+
+    def collect_and_store_resource_symbols(self):
+        """
+        Collect symbols from codebase files using Ctags and store
+        them in the extra data field.
+        """
+        symbols.collect_and_store_resource_symbols(self.project, self.log)

scanpipe/pipes/symbols.py

@@ -0,0 +1,67 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https://github.com/nexB/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https://github.com/nexB/scancode.io for support and download.
+
+from source_inspector import symbols_ctags
+
+from scanpipe.pipes import LoopProgress
+
+
+class UniversalCtagsNotFound(Exception):
+    pass
+
+
+def collect_and_store_resource_symbols(project, logger=None):
+    """
+    Collect symbols from codebase files using Ctags and store
+    them in the extra data field.
+    """
+    if not symbols_ctags.is_ctags_installed():
+        raise UniversalCtagsNotFound(
+            "``Universal Ctags`` not found."
+            "Install ``Universal Ctags`` to use this pipeline."
+        )
+
+    project_files = project.codebaseresources.files()
+
+    resources = project_files.filter(
+        is_binary=False,
+        is_archive=False,
+        is_media=False,
+    )
+
+    resources_count = resources.count()
+
+    resource_iterator = resources.iterator(chunk_size=2000)
+    progress = LoopProgress(resources_count, logger)
+
+    for resource in progress.iter(resource_iterator):
+        _collect_and_store_resource_symbols(resource)
+
+
+def _collect_and_store_resource_symbols(resource):
+    """
+    Collect symbols from a resource using Ctags and store
+    them in the extra data field.
+    """
+    symbols = symbols_ctags.collect_symbols(resource.location)
+    tags = [symbol["name"] for symbol in symbols if "name" in symbol]
+    resource.update_extra_data({"source_symbols": tags})

scanpipe/tests/data/javascript_collect_symbols.json

@@ -0,0 +1,96 @@
+{
+  "headers": [
+    {
+      "tool_name": "scanpipe",
+      "notice": "Generated with ScanCode.io and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied.\nNo content created from ScanCode.io should be considered or used as legal advice.\nConsult an Attorney for any legal advice.\nScanCode.io is a free software code scanning tool from nexB Inc. and others\nlicensed under the Apache License version 2.0.\nScanCode is a trademark of nexB Inc.\nVisit https://github.com/nexB/scancode.io for support and download.\n",
+      "input_sources": [],
+      "runs": [
+        {
+          "pipeline_name": "collect_symbols",
+          "status": "not_started",
+          "scancodeio_version": "",
+          "task_id": null,
+          "task_start_date": null,
+          "task_end_date": null,
+          "task_exitcode": null,
+          "task_output": "",
+          "execution_time": null
+        }
+      ],
+      "extra_data": {}
+    }
+  ],
+  "packages": [],
+  "dependencies": [],
+  "files": [
+    {
+      "path": "codefile",
+      "type": "directory",
+      "name": "codefile",
+      "status": "",
+      "tag": "",
+      "extension": "",
+      "md5": "",
+      "sha1": "",
+      "sha256": "",
+      "sha512": "",
+      "programming_language": "",
+      "is_binary": false,
+      "is_text": false,
+      "is_archive": false,
+      "is_media": false,
+      "is_key_file": false,
+      "detected_license_expression": "",
+      "detected_license_expression_spdx": "",
+      "license_detections": [],
+      "license_clues": [],
+      "percentage_of_license_text": null,
+      "copyrights": [],
+      "holders": [],
+      "authors": [],
+      "package_data": [],
+      "for_packages": [],
+      "emails": [],
+      "urls": [],
+      "extra_data": {}
+    },
+    {
+      "path": "codefile/main.js",
+      "type": "file",
+      "name": "main.js",
+      "status": "",
+      "tag": "",
+      "extension": ".js",
+      "md5": "5a6e6fa1e732b600d4c2260bc49ed73f",
+      "sha1": "d6bfcf7d1f8a00cc639b3a186a52453d37c52f61",
+      "sha256": "adf540c42cfd6b8413d7232fcd6e5df39fa990be6f280531f9ca05d92c6bc0d6",
+      "sha512": "",
+      "programming_language": "JavaScript",
+      "is_binary": false,
+      "is_text": true,
+      "is_archive": false,
+      "is_media": false,
+      "is_key_file": false,
+      "detected_license_expression": "",
+      "detected_license_expression_spdx": "",
+      "license_detections": [],
+      "license_clues": [],
+      "percentage_of_license_text": null,
+      "copyrights": [],
+      "holders": [],
+      "authors": [],
+      "package_data": [],
+      "for_packages": [],
+      "emails": [],
+      "urls": [],
+      "extra_data": {
+        "symbols": [
+          "passwordLength",
+          "generatePassword",
+          "charSet"
+        ]
+      }
+    }
+  ],
+  "relations": []
+}

scanpipe/tests/pipes/test_symbols.py

@@ -0,0 +1,54 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https://github.com/nexB/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https://github.com/nexB/scancode.io for support and download.
+
+from pathlib import Path
+
+from django.test import TestCase
+
+from scanpipe import pipes
+from scanpipe.models import Project
+from scanpipe.pipes import symbols
+from scanpipe.pipes.input import copy_input
+
+
+class ScanPipeSymbolsPipesTest(TestCase):
+    data_location = Path(__file__).parent.parent / "data"
+
+    def setUp(self):
+        self.project1 = Project.objects.create(name="Analysis")
+
+    def test_scanpipe_pipes_symbols_collect_and_store_resource_symbols(self):
+
+        dir = self.project1.codebase_path / "codefile"
+        dir.mkdir(parents=True)
+
+        file_location = self.data_location / "d2d-javascript" / "from" / "main.js"
+        copy_input(file_location, dir)
+
+        pipes.collect_and_create_codebase_resources(self.project1)
+
+        symbols.collect_and_store_resource_symbols(self.project1)
+
+        main_file = self.project1.codebaseresources.files()[0]
+        result_extra_data_symbols = main_file.extra_data.get("source_symbols")
+        expected_extra_data_symbols = ["generatePassword", "passwordLength", "charSet"]
+        self.assertCountEqual(expected_extra_data_symbols, result_extra_data_symbols)

scanpipe/tests/test_pipelines.py

@@ -1211,3 +1211,26 @@ def mock_request_post_return(url, data, headers, timeout):
         )
         self.assertIn("1 PURLs were already present in PurlDB index queue", run.log)
         self.assertIn("Couldn't index 1 unsupported PURLs", run.log)
+
+    def test_scanpipe_collect_symbols_pipeline_integration(self):
+        pipeline_name = "collect_symbols"
+        project1 = Project.objects.create(name="Analysis")
+
+        dir = project1.codebase_path / "codefile"
+        dir.mkdir(parents=True)
+
+        file_location = self.data_location / "d2d-javascript" / "from" / "main.js"
+        copy_input(file_location, dir)
+
+        pipes.collect_and_create_codebase_resources(project1)
+
+        run = project1.add_pipeline(pipeline_name)
+        pipeline = run.make_pipeline_instance()
+
+        exitcode, out = pipeline.execute()
+        self.assertEqual(0, exitcode, msg=out)
+
+        main_file = project1.codebaseresources.files()[0]
+        result_extra_data_symbols = main_file.extra_data.get("source_symbols")
+        expected_extra_data_symbols = ["generatePassword", "passwordLength", "charSet"]
+        self.assertCountEqual(expected_extra_data_symbols, result_extra_data_symbols)

setup.cfg

@@ -78,6 +78,7 @@ install_requires =
     # Inspectors
     python-inspector==0.11.0
     elf-inspector==0.0.1
+    source-inspector==0.2.0
     aboutcode-toolkit==10.1.0
     # Utilities
     XlsxWriter==3.2.0
@@ -128,6 +129,7 @@ scancodeio_pipelines =
     analyze_docker_image = scanpipe.pipelines.docker:Docker
     analyze_root_filesystem_or_vm_image = scanpipe.pipelines.root_filesystem:RootFS
     analyze_windows_docker_image = scanpipe.pipelines.docker_windows:DockerWindows
+    collect_symbols = scanpipe.pipelines.collect_symbols:CollectSymbols
     find_vulnerabilities = scanpipe.pipelines.find_vulnerabilities:FindVulnerabilities
     inspect_elf_binaries = scanpipe.pipelines.inspect_elf_binaries:InspectELFBinaries
     inspect_packages = scanpipe.pipelines.inspect_packages:InspectPackages