From 2183c6951501550f65218fd89658d8dfe8e2c480 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Mon, 11 Mar 2024 19:10:42 +0530 Subject: [PATCH 01/11] Add addon pipeline for symbol collection Signed-off-by: Keshav Priyadarshi --- scanpipe/pipelines/collect_symbols.py | 80 +++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 scanpipe/pipelines/collect_symbols.py diff --git a/scanpipe/pipelines/collect_symbols.py b/scanpipe/pipelines/collect_symbols.py new file mode 100644 index 0000000000..d819ee672d --- /dev/null +++ b/scanpipe/pipelines/collect_symbols.py @@ -0,0 +1,80 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/nexB/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode.io for support and download. + +from scanpipe.pipelines import Pipeline +from scanpipe.pipes import fetch + + +class CollectSymbols(Pipeline): + """Collect symbols from codebase files and keep them in extra data field.""" + + download_inputs = False + is_addon = True + + @classmethod + def steps(cls): + return (cls.collect_and_store_resource_symbols,) + + def collect_and_store_resource_symbols(self): + """ + Collect symbols from codebase files using Ctags and store + them in the extra data field. + """ + if not is_ctags_installed(): + self.log( + "``Universal Ctags`` missing." + "Install ``Universal Ctags`` to use this pipeline." + ) + return + + project_files = self.project.codebaseresources.files().filter( + is_binary=False, + is_archive=False, + is_media=False, + ) + + for file in project_files: + if symbols := extract_symbols_from_resource(file.location, self.log): + file.update_extra_data({"symbols": symbols}) + + +def extract_symbols_from_resource(location, logger): + """Given the location of a resource, use Universal Ctags to extract symbols.""" + command = ["ctags", "-f", "-", location] + + ctags_result = fetch.run_command_safely(command) + symbols = set(line.split()[0] for line in ctags_result.split("\n") if line) + + return list(symbols) + + +def is_ctags_installed(): + """Check if Universal Ctags is installed.""" + try: + result = fetch.run_command_safely(["ctags", "--version"]) + + if "universal ctags" in result.lower(): + return True + except FileNotFoundError: + pass + + return False From b10eafd66ec3c74863d783d03727da2ad38e1d65 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Tue, 12 Mar 2024 15:08:20 +0530 Subject: [PATCH 02/11] Update dockerfile to install universal-ctags Signed-off-by: Keshav Priyadarshi --- Dockerfile | 2 ++ setup.cfg | 1 + 2 files changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index 7c067cd9ed..8e9abe7d64 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,6 +40,7 @@ ENV PYTHONPATH $PYTHONPATH:$APP_DIR # OS requirements as per # https://scancode-toolkit.readthedocs.io/en/latest/getting-started/install.html +# Also install universal-ctags for symbol collection. RUN apt-get update \ && apt-get install -y --no-install-recommends \ bzip2 \ @@ -58,6 +59,7 @@ RUN apt-get update \ linux-image-amd64 \ git \ wait-for-it \ + universal-ctags \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/setup.cfg b/setup.cfg index 911cd01d3d..bc036d3f47 100644 --- a/setup.cfg +++ b/setup.cfg @@ -128,6 +128,7 @@ scancodeio_pipelines = analyze_docker_image = scanpipe.pipelines.docker:Docker analyze_root_filesystem_or_vm_image = scanpipe.pipelines.root_filesystem:RootFS analyze_windows_docker_image = scanpipe.pipelines.docker_windows:DockerWindows + collect_symbols = scanpipe.pipelines.collect_symbols:CollectSymbols find_vulnerabilities = scanpipe.pipelines.find_vulnerabilities:FindVulnerabilities inspect_elf_binaries = scanpipe.pipelines.inspect_elf_binaries:InspectELFBinaries inspect_packages = scanpipe.pipelines.inspect_packages:InspectPackages From ac0489e2274923c0669dea4ef472de2a28a8db4d Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Tue, 12 Mar 2024 15:10:10 +0530 Subject: [PATCH 03/11] Update docs to include Universal Ctags as system dependency Signed-off-by: Keshav Priyadarshi --- docs/installation.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/installation.rst b/docs/installation.rst index 4b7939bdcd..c86f2c30b5 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -260,6 +260,15 @@ Make sure those are installed before attempting the ScanCode.io installation:: See also `ScanCode-toolkit Prerequisites `_ for more details. +For the ``CollectSymbols`` pipeline, `Universal Ctags `_ is needed. +On **Linux** install it using:: + + sudo apt-get install universal-ctags + +On **MacOS** install Universal Ctags using Homebrew:: + + brew install universal-ctags + Clone and Configure ^^^^^^^^^^^^^^^^^^^ From e19a9c7347422f8c4893b3514d2fc6157c139723 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Tue, 12 Mar 2024 17:59:34 +0530 Subject: [PATCH 04/11] Add test for collect_symbols pipeline Signed-off-by: Keshav Priyadarshi --- .../data/javascript_collect_symbols.json | 96 +++++++++++++++++++ scanpipe/tests/test_pipelines.py | 23 +++++ 2 files changed, 119 insertions(+) create mode 100644 scanpipe/tests/data/javascript_collect_symbols.json diff --git a/scanpipe/tests/data/javascript_collect_symbols.json b/scanpipe/tests/data/javascript_collect_symbols.json new file mode 100644 index 0000000000..3d3e2ffff3 --- /dev/null +++ b/scanpipe/tests/data/javascript_collect_symbols.json @@ -0,0 +1,96 @@ +{ + "headers": [ + { + "tool_name": "scanpipe", + "notice": "Generated with ScanCode.io and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied.\nNo content created from ScanCode.io should be considered or used as legal advice.\nConsult an Attorney for any legal advice.\nScanCode.io is a free software code scanning tool from nexB Inc. and others\nlicensed under the Apache License version 2.0.\nScanCode is a trademark of nexB Inc.\nVisit https://github.com/nexB/scancode.io for support and download.\n", + "input_sources": [], + "runs": [ + { + "pipeline_name": "collect_symbols", + "status": "not_started", + "scancodeio_version": "", + "task_id": null, + "task_start_date": null, + "task_end_date": null, + "task_exitcode": null, + "task_output": "", + "execution_time": null + } + ], + "extra_data": {} + } + ], + "packages": [], + "dependencies": [], + "files": [ + { + "path": "codefile", + "type": "directory", + "name": "codefile", + "status": "", + "tag": "", + "extension": "", + "md5": "", + "sha1": "", + "sha256": "", + "sha512": "", + "programming_language": "", + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_key_file": false, + "detected_license_expression": "", + "detected_license_expression_spdx": "", + "license_detections": [], + "license_clues": [], + "percentage_of_license_text": null, + "copyrights": [], + "holders": [], + "authors": [], + "package_data": [], + "for_packages": [], + "emails": [], + "urls": [], + "extra_data": {} + }, + { + "path": "codefile/main.js", + "type": "file", + "name": "main.js", + "status": "", + "tag": "", + "extension": ".js", + "md5": "5a6e6fa1e732b600d4c2260bc49ed73f", + "sha1": "d6bfcf7d1f8a00cc639b3a186a52453d37c52f61", + "sha256": "adf540c42cfd6b8413d7232fcd6e5df39fa990be6f280531f9ca05d92c6bc0d6", + "sha512": "", + "programming_language": "JavaScript", + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_key_file": false, + "detected_license_expression": "", + "detected_license_expression_spdx": "", + "license_detections": [], + "license_clues": [], + "percentage_of_license_text": null, + "copyrights": [], + "holders": [], + "authors": [], + "package_data": [], + "for_packages": [], + "emails": [], + "urls": [], + "extra_data": { + "symbols": [ + "passwordLength", + "generatePassword", + "charSet" + ] + } + } + ], + "relations": [] +} \ No newline at end of file diff --git a/scanpipe/tests/test_pipelines.py b/scanpipe/tests/test_pipelines.py index 3490b40734..24b3fc5b33 100644 --- a/scanpipe/tests/test_pipelines.py +++ b/scanpipe/tests/test_pipelines.py @@ -1211,3 +1211,26 @@ def mock_request_post_return(url, data, headers, timeout): ) self.assertIn("1 PURLs were already present in PurlDB index queue", run.log) self.assertIn("Couldn't index 1 unsupported PURLs", run.log) + + def test_scanpipe_collect_symbols_pipeline_integration(self): + pipeline_name = "collect_symbols" + project1 = Project.objects.create(name="Analysis") + + dir = project1.codebase_path / "codefile" + dir.mkdir(parents=True) + + file_location = self.data_location / "d2d-javascript" / "from" / "main.js" + copy_input(file_location, dir) + + pipes.collect_and_create_codebase_resources(project1) + + run = project1.add_pipeline(pipeline_name) + pipeline = run.make_pipeline_instance() + + exitcode, out = pipeline.execute() + self.assertEqual(0, exitcode, msg=out) + + main_file = project1.codebaseresources.files()[0] + result_extra_data_symbols = main_file.extra_data.get("symbols") + expected_extra_data_symbols = ["generatePassword", "passwordLength", "charSet"] + self.assertCountEqual(expected_extra_data_symbols, result_extra_data_symbols) From 7b82ac3c94cce78cec9e4ef717f2374f13f042bd Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Tue, 12 Mar 2024 18:18:43 +0530 Subject: [PATCH 05/11] Update CI to install Universal Ctags Signed-off-by: Keshav Priyadarshi --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d98de713a..aa0b565be6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,6 +42,9 @@ jobs: with: python-version: ${{ matrix.python-version }} + - name: Install universal ctags + run: sudo apt-get install -y universal-ctags + - name: Install dependencies run: make dev envfile From df38f8fbddb375ab192297235cd47f8cd513aea6 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Fri, 15 Mar 2024 14:06:37 +0530 Subject: [PATCH 06/11] Use source-inspector for symbol collection Signed-off-by: Keshav Priyadarshi --- scanpipe/pipelines/collect_symbols.py | 33 +++++---------------------- setup.cfg | 1 + 2 files changed, 7 insertions(+), 27 deletions(-) diff --git a/scanpipe/pipelines/collect_symbols.py b/scanpipe/pipelines/collect_symbols.py index d819ee672d..bc442a7a29 100644 --- a/scanpipe/pipelines/collect_symbols.py +++ b/scanpipe/pipelines/collect_symbols.py @@ -20,8 +20,9 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/scancode.io for support and download. +from source_inpector import symbols_ctags + from scanpipe.pipelines import Pipeline -from scanpipe.pipes import fetch class CollectSymbols(Pipeline): @@ -39,7 +40,7 @@ def collect_and_store_resource_symbols(self): Collect symbols from codebase files using Ctags and store them in the extra data field. """ - if not is_ctags_installed(): + if not symbols_ctags.is_ctags_installed(): self.log( "``Universal Ctags`` missing." "Install ``Universal Ctags`` to use this pipeline." @@ -53,28 +54,6 @@ def collect_and_store_resource_symbols(self): ) for file in project_files: - if symbols := extract_symbols_from_resource(file.location, self.log): - file.update_extra_data({"symbols": symbols}) - - -def extract_symbols_from_resource(location, logger): - """Given the location of a resource, use Universal Ctags to extract symbols.""" - command = ["ctags", "-f", "-", location] - - ctags_result = fetch.run_command_safely(command) - symbols = set(line.split()[0] for line in ctags_result.split("\n") if line) - - return list(symbols) - - -def is_ctags_installed(): - """Check if Universal Ctags is installed.""" - try: - result = fetch.run_command_safely(["ctags", "--version"]) - - if "universal ctags" in result.lower(): - return True - except FileNotFoundError: - pass - - return False + symbols = symbols_ctags.collect_symbols(file.location) + tags = [symbol["name"] for symbol in symbols if symbol["_type"] == "tag"] + file.update_extra_data({"symbols": tags}) diff --git a/setup.cfg b/setup.cfg index bc036d3f47..43461e3000 100644 --- a/setup.cfg +++ b/setup.cfg @@ -78,6 +78,7 @@ install_requires = # Inspectors python-inspector==0.11.0 elf-inspector==0.0.1 + source-inspector==0.1.0 aboutcode-toolkit==10.1.0 # Utilities XlsxWriter==3.1.9 From 824683e6b381ea378e4f5e3258ac3a0b16aa995f Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Fri, 15 Mar 2024 20:41:50 +0530 Subject: [PATCH 07/11] Bump source-inspector to v0.2.0 Signed-off-by: Keshav Priyadarshi --- scanpipe/pipelines/collect_symbols.py | 4 ++-- setup.cfg | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scanpipe/pipelines/collect_symbols.py b/scanpipe/pipelines/collect_symbols.py index bc442a7a29..96489d624e 100644 --- a/scanpipe/pipelines/collect_symbols.py +++ b/scanpipe/pipelines/collect_symbols.py @@ -20,7 +20,7 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/scancode.io for support and download. -from source_inpector import symbols_ctags +from source_inspector import symbols_ctags from scanpipe.pipelines import Pipeline @@ -55,5 +55,5 @@ def collect_and_store_resource_symbols(self): for file in project_files: symbols = symbols_ctags.collect_symbols(file.location) - tags = [symbol["name"] for symbol in symbols if symbol["_type"] == "tag"] + tags = [symbol["name"] for symbol in symbols if "name" in symbol] file.update_extra_data({"symbols": tags}) diff --git a/setup.cfg b/setup.cfg index 43461e3000..f8d6447270 100644 --- a/setup.cfg +++ b/setup.cfg @@ -78,7 +78,7 @@ install_requires = # Inspectors python-inspector==0.11.0 elf-inspector==0.0.1 - source-inspector==0.1.0 + source-inspector==0.2.0 aboutcode-toolkit==10.1.0 # Utilities XlsxWriter==3.1.9 From 7e7922feac15829ed47ccb88b76c7d4d01d2f17a Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Mon, 18 Mar 2024 15:29:53 +0530 Subject: [PATCH 08/11] Add CollectSymbols in built-in-pipelines doc Signed-off-by: Keshav Priyadarshi --- docs/built-in-pipelines.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/built-in-pipelines.rst b/docs/built-in-pipelines.rst index 7ba1da89c4..8f0143a10f 100644 --- a/docs/built-in-pipelines.rst +++ b/docs/built-in-pipelines.rst @@ -42,6 +42,14 @@ Analyse Docker Windows Image :members: :member-order: bysource +.. _pipeline_collect_symbols: + +Collect Codebase Symbols (addon) +--------------------------------- +.. autoclass:: scanpipe.pipelines.collect_symbols.CollectSymbols() + :members: + :member-order: bysource + .. _pipeline_find_vulnerabilities: Find Vulnerabilities (addon) From 974775856449c36a0b7a4e672d29007fb558e549 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Mon, 18 Mar 2024 15:30:34 +0530 Subject: [PATCH 09/11] Use proper ref for CollectSymbols pipeline Signed-off-by: Keshav Priyadarshi --- docs/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.rst b/docs/installation.rst index c86f2c30b5..4a9403f1b4 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -260,7 +260,7 @@ Make sure those are installed before attempting the ScanCode.io installation:: See also `ScanCode-toolkit Prerequisites `_ for more details. -For the ``CollectSymbols`` pipeline, `Universal Ctags `_ is needed. +For the :ref:`pipeline_collect_symbols` pipeline, `Universal Ctags `_ is needed. On **Linux** install it using:: sudo apt-get install universal-ctags From 67ce75603e4b829ee62acc6a9a66ac6f8b55225c Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Mon, 18 Mar 2024 16:51:57 +0530 Subject: [PATCH 10/11] Move symbol collection to new pipe Signed-off-by: Keshav Priyadarshi --- scanpipe/pipelines/collect_symbols.py | 21 +-------- scanpipe/pipes/symbols.py | 67 +++++++++++++++++++++++++++ scanpipe/tests/pipes/test_symbols.py | 54 +++++++++++++++++++++ scanpipe/tests/test_pipelines.py | 2 +- 4 files changed, 124 insertions(+), 20 deletions(-) create mode 100644 scanpipe/pipes/symbols.py create mode 100644 scanpipe/tests/pipes/test_symbols.py diff --git a/scanpipe/pipelines/collect_symbols.py b/scanpipe/pipelines/collect_symbols.py index 96489d624e..0204277dae 100644 --- a/scanpipe/pipelines/collect_symbols.py +++ b/scanpipe/pipelines/collect_symbols.py @@ -20,9 +20,8 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/scancode.io for support and download. -from source_inspector import symbols_ctags - from scanpipe.pipelines import Pipeline +from scanpipe.pipes import symbols class CollectSymbols(Pipeline): @@ -40,20 +39,4 @@ def collect_and_store_resource_symbols(self): Collect symbols from codebase files using Ctags and store them in the extra data field. """ - if not symbols_ctags.is_ctags_installed(): - self.log( - "``Universal Ctags`` missing." - "Install ``Universal Ctags`` to use this pipeline." - ) - return - - project_files = self.project.codebaseresources.files().filter( - is_binary=False, - is_archive=False, - is_media=False, - ) - - for file in project_files: - symbols = symbols_ctags.collect_symbols(file.location) - tags = [symbol["name"] for symbol in symbols if "name" in symbol] - file.update_extra_data({"symbols": tags}) + symbols.collect_and_store_resource_symbols(self.project, self.log) diff --git a/scanpipe/pipes/symbols.py b/scanpipe/pipes/symbols.py new file mode 100644 index 0000000000..1feb15e638 --- /dev/null +++ b/scanpipe/pipes/symbols.py @@ -0,0 +1,67 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/nexB/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode.io for support and download. + +from source_inspector import symbols_ctags + +from scanpipe.pipes import LoopProgress + + +class UniversalCtagsNotFound(Exception): + pass + + +def collect_and_store_resource_symbols(project, logger=None): + """ + Collect symbols from codebase files using Ctags and store + them in the extra data field. + """ + if not symbols_ctags.is_ctags_installed(): + raise UniversalCtagsNotFound( + "``Universal Ctags`` not found." + "Install ``Universal Ctags`` to use this pipeline." + ) + + project_files = project.codebaseresources.files() + + resources = project_files.filter( + is_binary=False, + is_archive=False, + is_media=False, + ) + + resources_count = resources.count() + + resource_iterator = resources.iterator(chunk_size=2000) + progress = LoopProgress(resources_count, logger) + + for resource in progress.iter(resource_iterator): + _collect_and_store_resource_symbols(resource) + + +def _collect_and_store_resource_symbols(resource): + """ + Collect symbols from a resource using Ctags and store + them in the extra data field. + """ + symbols = symbols_ctags.collect_symbols(resource.location) + tags = [symbol["name"] for symbol in symbols if "name" in symbol] + resource.update_extra_data({"source_symbols": tags}) diff --git a/scanpipe/tests/pipes/test_symbols.py b/scanpipe/tests/pipes/test_symbols.py new file mode 100644 index 0000000000..014c7af6a5 --- /dev/null +++ b/scanpipe/tests/pipes/test_symbols.py @@ -0,0 +1,54 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/nexB/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode.io for support and download. + +from pathlib import Path + +from django.test import TestCase + +from scanpipe import pipes +from scanpipe.models import Project +from scanpipe.pipes import symbols +from scanpipe.pipes.input import copy_input + + +class ScanPipeSymbolsPipesTest(TestCase): + data_location = Path(__file__).parent.parent / "data" + + def setUp(self): + self.project1 = Project.objects.create(name="Analysis") + + def test_scanpipe_pipes_symbols_collect_and_store_resource_symbols(self): + + dir = self.project1.codebase_path / "codefile" + dir.mkdir(parents=True) + + file_location = self.data_location / "d2d-javascript" / "from" / "main.js" + copy_input(file_location, dir) + + pipes.collect_and_create_codebase_resources(self.project1) + + symbols.collect_and_store_resource_symbols(self.project1) + + main_file = self.project1.codebaseresources.files()[0] + result_extra_data_symbols = main_file.extra_data.get("source_symbols") + expected_extra_data_symbols = ["generatePassword", "passwordLength", "charSet"] + self.assertCountEqual(expected_extra_data_symbols, result_extra_data_symbols) diff --git a/scanpipe/tests/test_pipelines.py b/scanpipe/tests/test_pipelines.py index 24b3fc5b33..07fe221176 100644 --- a/scanpipe/tests/test_pipelines.py +++ b/scanpipe/tests/test_pipelines.py @@ -1231,6 +1231,6 @@ def test_scanpipe_collect_symbols_pipeline_integration(self): self.assertEqual(0, exitcode, msg=out) main_file = project1.codebaseresources.files()[0] - result_extra_data_symbols = main_file.extra_data.get("symbols") + result_extra_data_symbols = main_file.extra_data.get("source_symbols") expected_extra_data_symbols = ["generatePassword", "passwordLength", "charSet"] self.assertCountEqual(expected_extra_data_symbols, result_extra_data_symbols) From f0ed58a4e1ba64ce778ea19972387079f3e5bfc0 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Mon, 18 Mar 2024 17:01:06 +0530 Subject: [PATCH 11/11] Add CHANGELOG for CollectSymbols pipeline Signed-off-by: Keshav Priyadarshi --- CHANGELOG.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4e801df3fd..1653b71013 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,6 +11,10 @@ v34.1.0 (unreleased) The docstring are converted from markdown to html for proper rendering. https://github.com/nexB/scancode.io/pull/1105 +- Add a new `CollectSymbols` pipeline (addon) for collecting codebase symbols using + Universal Ctags. + https://github.com/nexB/scancode.io/pull/1116 + v34.0.0 (2024-03-04) --------------------