diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d98de713a..aa0b565be6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,6 +42,9 @@ jobs: with: python-version: ${{ matrix.python-version }} + - name: Install universal ctags + run: sudo apt-get install -y universal-ctags + - name: Install dependencies run: make dev envfile diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4e801df3fd..1653b71013 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,6 +11,10 @@ v34.1.0 (unreleased) The docstring are converted from markdown to html for proper rendering. https://github.com/nexB/scancode.io/pull/1105 +- Add a new `CollectSymbols` pipeline (addon) for collecting codebase symbols using + Universal Ctags. + https://github.com/nexB/scancode.io/pull/1116 + v34.0.0 (2024-03-04) -------------------- diff --git a/Dockerfile b/Dockerfile index 7c067cd9ed..8e9abe7d64 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,6 +40,7 @@ ENV PYTHONPATH $PYTHONPATH:$APP_DIR # OS requirements as per # https://scancode-toolkit.readthedocs.io/en/latest/getting-started/install.html +# Also install universal-ctags for symbol collection. RUN apt-get update \ && apt-get install -y --no-install-recommends \ bzip2 \ @@ -58,6 +59,7 @@ RUN apt-get update \ linux-image-amd64 \ git \ wait-for-it \ + universal-ctags \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/docs/built-in-pipelines.rst b/docs/built-in-pipelines.rst index 7ba1da89c4..8f0143a10f 100644 --- a/docs/built-in-pipelines.rst +++ b/docs/built-in-pipelines.rst @@ -42,6 +42,14 @@ Analyse Docker Windows Image :members: :member-order: bysource +.. _pipeline_collect_symbols: + +Collect Codebase Symbols (addon) +--------------------------------- +.. autoclass:: scanpipe.pipelines.collect_symbols.CollectSymbols() + :members: + :member-order: bysource + .. _pipeline_find_vulnerabilities: Find Vulnerabilities (addon) diff --git a/docs/installation.rst b/docs/installation.rst index 4b7939bdcd..4a9403f1b4 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -260,6 +260,15 @@ Make sure those are installed before attempting the ScanCode.io installation:: See also `ScanCode-toolkit Prerequisites `_ for more details. +For the :ref:`pipeline_collect_symbols` pipeline, `Universal Ctags `_ is needed. +On **Linux** install it using:: + + sudo apt-get install universal-ctags + +On **MacOS** install Universal Ctags using Homebrew:: + + brew install universal-ctags + Clone and Configure ^^^^^^^^^^^^^^^^^^^ diff --git a/scanpipe/pipelines/collect_symbols.py b/scanpipe/pipelines/collect_symbols.py new file mode 100644 index 0000000000..0204277dae --- /dev/null +++ b/scanpipe/pipelines/collect_symbols.py @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/nexB/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode.io for support and download. + +from scanpipe.pipelines import Pipeline +from scanpipe.pipes import symbols + + +class CollectSymbols(Pipeline): + """Collect symbols from codebase files and keep them in extra data field.""" + + download_inputs = False + is_addon = True + + @classmethod + def steps(cls): + return (cls.collect_and_store_resource_symbols,) + + def collect_and_store_resource_symbols(self): + """ + Collect symbols from codebase files using Ctags and store + them in the extra data field. + """ + symbols.collect_and_store_resource_symbols(self.project, self.log) diff --git a/scanpipe/pipes/symbols.py b/scanpipe/pipes/symbols.py new file mode 100644 index 0000000000..1feb15e638 --- /dev/null +++ b/scanpipe/pipes/symbols.py @@ -0,0 +1,67 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/nexB/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode.io for support and download. + +from source_inspector import symbols_ctags + +from scanpipe.pipes import LoopProgress + + +class UniversalCtagsNotFound(Exception): + pass + + +def collect_and_store_resource_symbols(project, logger=None): + """ + Collect symbols from codebase files using Ctags and store + them in the extra data field. + """ + if not symbols_ctags.is_ctags_installed(): + raise UniversalCtagsNotFound( + "``Universal Ctags`` not found." + "Install ``Universal Ctags`` to use this pipeline." + ) + + project_files = project.codebaseresources.files() + + resources = project_files.filter( + is_binary=False, + is_archive=False, + is_media=False, + ) + + resources_count = resources.count() + + resource_iterator = resources.iterator(chunk_size=2000) + progress = LoopProgress(resources_count, logger) + + for resource in progress.iter(resource_iterator): + _collect_and_store_resource_symbols(resource) + + +def _collect_and_store_resource_symbols(resource): + """ + Collect symbols from a resource using Ctags and store + them in the extra data field. + """ + symbols = symbols_ctags.collect_symbols(resource.location) + tags = [symbol["name"] for symbol in symbols if "name" in symbol] + resource.update_extra_data({"source_symbols": tags}) diff --git a/scanpipe/tests/data/javascript_collect_symbols.json b/scanpipe/tests/data/javascript_collect_symbols.json new file mode 100644 index 0000000000..3d3e2ffff3 --- /dev/null +++ b/scanpipe/tests/data/javascript_collect_symbols.json @@ -0,0 +1,96 @@ +{ + "headers": [ + { + "tool_name": "scanpipe", + "notice": "Generated with ScanCode.io and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied.\nNo content created from ScanCode.io should be considered or used as legal advice.\nConsult an Attorney for any legal advice.\nScanCode.io is a free software code scanning tool from nexB Inc. and others\nlicensed under the Apache License version 2.0.\nScanCode is a trademark of nexB Inc.\nVisit https://github.com/nexB/scancode.io for support and download.\n", + "input_sources": [], + "runs": [ + { + "pipeline_name": "collect_symbols", + "status": "not_started", + "scancodeio_version": "", + "task_id": null, + "task_start_date": null, + "task_end_date": null, + "task_exitcode": null, + "task_output": "", + "execution_time": null + } + ], + "extra_data": {} + } + ], + "packages": [], + "dependencies": [], + "files": [ + { + "path": "codefile", + "type": "directory", + "name": "codefile", + "status": "", + "tag": "", + "extension": "", + "md5": "", + "sha1": "", + "sha256": "", + "sha512": "", + "programming_language": "", + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_key_file": false, + "detected_license_expression": "", + "detected_license_expression_spdx": "", + "license_detections": [], + "license_clues": [], + "percentage_of_license_text": null, + "copyrights": [], + "holders": [], + "authors": [], + "package_data": [], + "for_packages": [], + "emails": [], + "urls": [], + "extra_data": {} + }, + { + "path": "codefile/main.js", + "type": "file", + "name": "main.js", + "status": "", + "tag": "", + "extension": ".js", + "md5": "5a6e6fa1e732b600d4c2260bc49ed73f", + "sha1": "d6bfcf7d1f8a00cc639b3a186a52453d37c52f61", + "sha256": "adf540c42cfd6b8413d7232fcd6e5df39fa990be6f280531f9ca05d92c6bc0d6", + "sha512": "", + "programming_language": "JavaScript", + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_key_file": false, + "detected_license_expression": "", + "detected_license_expression_spdx": "", + "license_detections": [], + "license_clues": [], + "percentage_of_license_text": null, + "copyrights": [], + "holders": [], + "authors": [], + "package_data": [], + "for_packages": [], + "emails": [], + "urls": [], + "extra_data": { + "symbols": [ + "passwordLength", + "generatePassword", + "charSet" + ] + } + } + ], + "relations": [] +} \ No newline at end of file diff --git a/scanpipe/tests/pipes/test_symbols.py b/scanpipe/tests/pipes/test_symbols.py new file mode 100644 index 0000000000..014c7af6a5 --- /dev/null +++ b/scanpipe/tests/pipes/test_symbols.py @@ -0,0 +1,54 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/nexB/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode.io for support and download. + +from pathlib import Path + +from django.test import TestCase + +from scanpipe import pipes +from scanpipe.models import Project +from scanpipe.pipes import symbols +from scanpipe.pipes.input import copy_input + + +class ScanPipeSymbolsPipesTest(TestCase): + data_location = Path(__file__).parent.parent / "data" + + def setUp(self): + self.project1 = Project.objects.create(name="Analysis") + + def test_scanpipe_pipes_symbols_collect_and_store_resource_symbols(self): + + dir = self.project1.codebase_path / "codefile" + dir.mkdir(parents=True) + + file_location = self.data_location / "d2d-javascript" / "from" / "main.js" + copy_input(file_location, dir) + + pipes.collect_and_create_codebase_resources(self.project1) + + symbols.collect_and_store_resource_symbols(self.project1) + + main_file = self.project1.codebaseresources.files()[0] + result_extra_data_symbols = main_file.extra_data.get("source_symbols") + expected_extra_data_symbols = ["generatePassword", "passwordLength", "charSet"] + self.assertCountEqual(expected_extra_data_symbols, result_extra_data_symbols) diff --git a/scanpipe/tests/test_pipelines.py b/scanpipe/tests/test_pipelines.py index 3490b40734..07fe221176 100644 --- a/scanpipe/tests/test_pipelines.py +++ b/scanpipe/tests/test_pipelines.py @@ -1211,3 +1211,26 @@ def mock_request_post_return(url, data, headers, timeout): ) self.assertIn("1 PURLs were already present in PurlDB index queue", run.log) self.assertIn("Couldn't index 1 unsupported PURLs", run.log) + + def test_scanpipe_collect_symbols_pipeline_integration(self): + pipeline_name = "collect_symbols" + project1 = Project.objects.create(name="Analysis") + + dir = project1.codebase_path / "codefile" + dir.mkdir(parents=True) + + file_location = self.data_location / "d2d-javascript" / "from" / "main.js" + copy_input(file_location, dir) + + pipes.collect_and_create_codebase_resources(project1) + + run = project1.add_pipeline(pipeline_name) + pipeline = run.make_pipeline_instance() + + exitcode, out = pipeline.execute() + self.assertEqual(0, exitcode, msg=out) + + main_file = project1.codebaseresources.files()[0] + result_extra_data_symbols = main_file.extra_data.get("source_symbols") + expected_extra_data_symbols = ["generatePassword", "passwordLength", "charSet"] + self.assertCountEqual(expected_extra_data_symbols, result_extra_data_symbols) diff --git a/setup.cfg b/setup.cfg index 27e0d53cf0..91a360f732 100644 --- a/setup.cfg +++ b/setup.cfg @@ -78,6 +78,7 @@ install_requires = # Inspectors python-inspector==0.11.0 elf-inspector==0.0.1 + source-inspector==0.2.0 aboutcode-toolkit==10.1.0 # Utilities XlsxWriter==3.2.0 @@ -128,6 +129,7 @@ scancodeio_pipelines = analyze_docker_image = scanpipe.pipelines.docker:Docker analyze_root_filesystem_or_vm_image = scanpipe.pipelines.root_filesystem:RootFS analyze_windows_docker_image = scanpipe.pipelines.docker_windows:DockerWindows + collect_symbols = scanpipe.pipelines.collect_symbols:CollectSymbols find_vulnerabilities = scanpipe.pipelines.find_vulnerabilities:FindVulnerabilities inspect_elf_binaries = scanpipe.pipelines.inspect_elf_binaries:InspectELFBinaries inspect_packages = scanpipe.pipelines.inspect_packages:InspectPackages