Create a ScanCode.io pipeline to integrate Grimoire

docker-compose.yml

@@ -47,6 +47,7 @@ services:
       - /etc/scancodeio/:/etc/scancodeio/
       - workspace:/var/scancodeio/workspace/
       - static:/var/scancodeio/static/
+      - healthycode_venv:/opt/healthycode
     restart: always
     depends_on:
       db:
@@ -67,6 +68,7 @@ services:
       - .env:/opt/scancodeio/.env
       - /etc/scancodeio/:/etc/scancodeio/
       - workspace:/var/scancodeio/workspace/
+      - healthycode_venv:/opt/healthycode
     restart: always
     depends_on:
       db:
@@ -97,10 +99,19 @@ services:
       - workspace:/var/scancodeio/workspace/
     restart: always
 
+  healthycode:
+    profiles:
+      - healthycode
+    image: ghcr.io/aboutcode-org/healthycode:v1.0.0
+    volumes:
+      - healthycode_venv:/opt/healthycode
+    restart: "no"
+
 volumes:
   db_data:
   redis_data:
   clamav_data:
   static:
   workspace:
   webroot:
+  healthycode_venv:

docker.env

@@ -7,3 +7,4 @@ SCANCODEIO_DB_HOST=db
 SCANCODEIO_RQ_REDIS_HOST=redis
 SCANCODEIO_ASYNC=True
 SCANCODEIO_WORKSPACE_LOCATION=/var/scancodeio/workspace/
+GRIMOIRELAB_METRICS_EXECUTABLE=/opt/healthycode/.venv/bin/grimoirelab-metrics

pyproject.toml

@@ -176,6 +176,7 @@ scan_codebase = "scanpipe.pipelines.scan_codebase:ScanCodebase"
 scan_for_virus = "scanpipe.pipelines.scan_for_virus:ScanForVirus"
 scan_maven_package = "scanpipe.pipelines.scan_maven_package:ScanMavenPackage"
 scan_single_package = "scanpipe.pipelines.scan_single_package:ScanSinglePackage"
+scan_repo_health = "scanpipe.pipelines.scan_repo_health:ScanRepoHealth"
 
 [tool.setuptools.packages.find]
 where = ["."]

scanpipe/pipelines/scan_repo_health.py

@@ -0,0 +1,196 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https://github.com/aboutcode-org/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https://github.com/aboutcode-org/scancode.io for support and download.
+import json
+import subprocess
+import urllib.parse
+from os import environ
+
+from scanpipe.pipelines import Pipeline
+from scanpipe.pipes import run_command_safely
+
+GRIMOIRELAB_METRICS_EXECUTABLE = environ.get("GRIMOIRELAB_METRICS_EXECUTABLE", "")
+GRIMOIRELAB_OPENSEARCH_INDEX = environ.get("GRIMOIRELAB_OPENSEARCH_INDEX", "")
+GRIMOIRELAB_OPENSEARCH_PASSWORD = environ.get("GRIMOIRELAB_OPENSEARCH_PASSWORD", "")
+GRIMOIRELAB_OPENSEARCH_URL = environ.get("GRIMOIRELAB_OPENSEARCH_URL", "")
+GRIMOIRELAB_OPENSEARCH_USERNAME = environ.get("GRIMOIRELAB_OPENSEARCH_USERNAME", "")
+GRIMOIRELAB_PASSWORD = environ.get("GRIMOIRELAB_PASSWORD", "")
+GRIMOIRELAB_URL = environ.get("GRIMOIRELAB_URL", "")
+GRIMOIRELAB_USERNAME = environ.get("GRIMOIRELAB_USERNAME", "")
+GRIMOIRELAB_ECOSYSTEM = environ.get("GRIMOIRELAB_ECOSYSTEM", "")
+GRIMOIRELAB_PROJECT = environ.get("GRIMOIRELAB_PROJECT", "")
+
+
+class ScanRepoHealth(Pipeline):
+    """Run a Repo Health scan to extract repository metrics and health score."""
+
+    results_url = "/project/{slug}/resources/?extra_data=grimoire_data"
+    download_inputs = False
+
+    @classmethod
+    def steps(cls):
+        return (
+            cls.get_repo_url_input,
+            cls.collect_and_store_grimoire_metric,
+            cls.format_metrics_output,
+        )
+
+    @classmethod
+    def get_availability(cls):
+        if not (
+            GRIMOIRELAB_METRICS_EXECUTABLE
+            and GRIMOIRELAB_OPENSEARCH_INDEX
+            and GRIMOIRELAB_OPENSEARCH_PASSWORD
+            and GRIMOIRELAB_OPENSEARCH_URL
+            and GRIMOIRELAB_OPENSEARCH_USERNAME
+            and GRIMOIRELAB_PASSWORD
+            and GRIMOIRELAB_URL
+            and GRIMOIRELAB_USERNAME
+            and GRIMOIRELAB_ECOSYSTEM
+            and GRIMOIRELAB_PROJECT
+        ):
+            return "Grimoirelab is not configured."
+
+    def get_repo_url_input(self):
+        """Validate and extract the repository URL from the project's input sources"""
+        if len(self.project.input_sources) != 1:
+            raise ValueError("Expected exactly one input source")
+
+        self.repo_url = self.project.input_sources[0]["download_url"]
+        if not is_valid_vcs_url(self.repo_url):
+            raise ValueError(
+                "Invalid input source: the pipeline accepts only a valid repository URL"
+            )
+
+        self.repo_url = self.repo_url.replace("git://", "https://")
+        if not self.repo_url.endswith(".git"):
+            self.repo_url += ".git"
+
+    def collect_and_store_grimoire_metric(self):
+        """
+        Run the grimoirelab-metrics command against the input source.
+        Save the generated metrics JSON to the project output directory.
+        """
+        self.metrics_output_path = self.project.get_output_file_path("metrics", "json")
+        command_args = [
+            GRIMOIRELAB_METRICS_EXECUTABLE,
+            self.repo_url,
+            "--grimoirelab-url",
+            GRIMOIRELAB_URL,
+            "--grimoirelab-user",
+            GRIMOIRELAB_USERNAME,
+            "--grimoirelab-password",
+            GRIMOIRELAB_PASSWORD,
+            "--grimoirelab-ecosystem",
+            GRIMOIRELAB_ECOSYSTEM,
+            "--grimoirelab-project",
+            GRIMOIRELAB_PROJECT,
+            "--opensearch-url",
+            GRIMOIRELAB_OPENSEARCH_URL,
+            "--opensearch-index",
+            GRIMOIRELAB_OPENSEARCH_INDEX,
+            "--opensearch-user",
+            GRIMOIRELAB_OPENSEARCH_USERNAME,
+            "--opensearch-password",
+            GRIMOIRELAB_OPENSEARCH_PASSWORD,
+            "--output",
+            str(self.metrics_output_path),
+        ]
+
+        try:
+            run_command_safely(command_args=command_args)
+            self.log("GrimoireLab metrics pipeline completed successfully")
+        except subprocess.SubprocessError:
+            raise RuntimeError("Grimoirelab-metrics client failure")
+        except FileNotFoundError:
+            raise FileNotFoundError(
+                "Grimoirelab-metrics not found. "
+                "Please ensure grimoirelab-metrics is correctly configured."
+            )
+
+    def format_metrics_output(self):
+        """
+        Format the GrimoireLab metrics output by extracting the repository URL,
+        score, and metrics from the generated JSON and overwriting it with a
+        simplified structure, and updating the project's extra data.
+        """
+        if not self.metrics_output_path.exists():
+            raise FileNotFoundError(
+                "GrimoireLab client did not return a valid metrics JSON file"
+            )
+
+        with open(self.metrics_output_path) as f:
+            data = json.load(f)
+
+        if not isinstance(data, dict):
+            raise ValueError("Invalid metrics JSON: Expected a JSON object.")
+
+        package_data = data.get("packages")
+        if not package_data or not isinstance(package_data, dict):
+            raise ValueError(
+                "Invalid metrics JSON: Missing or malformed 'packages' section."
+            )
+
+        packages = list(package_data.values())
+        if not packages:
+            raise ValueError("Invalid metrics JSON: 'packages' contains no data.")
+
+        target_package = packages[0]
+        repository = target_package.get("repository")
+        score = target_package.get("score")
+        metrics = target_package.get("metrics")
+
+        if repository is None or score is None or metrics is None:
+            raise ValueError(
+                f"Invalid metrics JSON. missing or null field(s): "
+                f"repository: {repository}, score: {score}, metrics: {metrics}"
+            )
+
+        result = {
+            "repository": repository,
+            "npm_health_score": score,
+            "health_metrics": metrics,
+        }
+
+        with open(self.metrics_output_path, "w") as f:
+            json.dump(result, f)
+
+        self.project.update_extra_data(result)
+
+
+def is_valid_vcs_url(url):
+    """Determine whether the URL string has the expected syntax of a VCS repository."""
+    if not isinstance(url, str) or not url:
+        return False
+
+    if any(char.isspace() for char in url):
+        return False
+
+    forbidden_chars = ["|", ";", "&", "`", "$(", ">", "<", "&&", "||"]
+    if any(char in url for char in forbidden_chars):
+        return False
+
+    parsed = urllib.parse.urlparse(url)
+    valid_schemes = {"https", "http", "git"}
+    if parsed.scheme in valid_schemes and parsed.netloc:
+        return True
+
+    return False

scanpipe/tests/data/grimorielab/expected-metrics.json

@@ -0,0 +1,45 @@
+{
+  "repository": "https://github.com/baidu/san.git",
+  "npm_health_score": {
+    "value": 0.027432660614030716,
+    "metadata": {
+      "ecosystem": "npm",
+      "model": "health",
+      "version": "0.1"
+    }
+  },
+  "health_metrics": {
+    "total_commits": 21,
+    "total_contributors": 5,
+    "total_organizations": 4,
+    "pony_factor": 1,
+    "elephant_factor": 1,
+    "recent_organizations": 4,
+    "recent_contributors": 5,
+    "recent_commits": 21,
+    "contributor_growth": -2,
+    "contributor_growth_rate": -0.5,
+    "active_branches": 4,
+    "days_since_last_commit": 54,
+    "casual_regular_contributors_rate": 0.6666666666666666,
+    "returning_contributors": 0,
+    "commits_over_periods_rate": 1.0,
+    "coefficient_of_variation": 1.5459698172140883,
+    "file_types_code": 29,
+    "file_types_binary": 0,
+    "file_types_other": 14,
+    "commit_size_added_lines": 7197,
+    "commit_size_removed_lines": 4963,
+    "message_size_total": 2212,
+    "message_size_mean": 105.33333333333333,
+    "message_size_median": 21,
+    "developer_categories_core": 2,
+    "developer_categories_regular": 1,
+    "developer_categories_casual": 2,
+    "commits_per_week": 0.4027397260273972,
+    "commits_per_month": 1.726027397260274,
+    "commits_per_year": 21.0,
+    "found_file_license": 1,
+    "found_file_adopters": 0
+  }
+}

scanpipe/tests/data/grimorielab/metrics.json

@@ -0,0 +1,72 @@
+{
+  "packages": {
+    "package0": {
+      "metrics": {
+        "total_commits": 21,
+        "total_contributors": 5,
+        "total_organizations": 4,
+        "pony_factor": 1,
+        "elephant_factor": 1,
+        "recent_organizations": 4,
+        "recent_contributors": 5,
+        "recent_commits": 21,
+        "contributor_growth": -2,
+        "contributor_growth_rate": -0.5,
+        "active_branches": 4,
+        "days_since_last_commit": 54,
+        "casual_regular_contributors_rate": 0.6666666666666666,
+        "returning_contributors": 0,
+        "commits_over_periods_rate": 1.0,
+        "coefficient_of_variation": 1.5459698172140883,
+        "file_types_code": 29,
+        "file_types_binary": 0,
+        "file_types_other": 14,
+        "commit_size_added_lines": 7197,
+        "commit_size_removed_lines": 4963,
+        "message_size_total": 2212,
+        "message_size_mean": 105.33333333333333,
+        "message_size_median": 21,
+        "developer_categories_core": 2,
+        "developer_categories_regular": 1,
+        "developer_categories_casual": 2,
+        "commits_per_week": 0.4027397260273972,
+        "commits_per_month": 1.726027397260274,
+        "commits_per_year": 21.0,
+        "found_file_license": 1,
+        "found_file_adopters": 0
+      },
+      "metadata": {
+        "first_commit": "82375f63b0404bd41bf27cdae26bed9a90aaaae2",
+        "last_commit": "7818ad38bda7c48cd4a296ce8568f23ec1b1d400",
+        "first_commit_date": "2025-09-24T19:02:16+00:00",
+        "last_commit_date": "2026-07-15T19:58:51+08:00"
+      },
+      "repository": "https://github.com/baidu/san.git",
+      "score": {
+        "value": 0.027432660614030716,
+        "metadata": {
+          "ecosystem": "npm",
+          "model": "health",
+          "version": "0.1"
+        }
+      }
+    }
+  },
+  "metadata": {
+    "version": "0.1.0",
+    "started_at": "2026-09-08T08:22:00.631055+00:00",
+    "finished_at": "2026-09-08T08:22:02.586554+00:00",
+    "configuration": {
+      "from_date": "2025-09-08T00:00:00",
+      "to_date": "2026-09-08T00:00:00",
+      "code_file_pattern": "\\.bazel$|\\.bazelrc$|\\.bzl$|\\.c$|\\.cc$|\\.cp$|\\.cpp$|\\.cs$\\|\\.cxx$|\\.c\\+\\+$|\\.go$|\\.h$|\\.hpp$|\\.js$|\\.mjs$|\\.java$|\\.pl$|\\.py$|\\.rs$|\\.sh$|\\.tf$|\\.ts$",
+      "binary_file_pattern": "\\.7z$|\\.a$|\\.abb$|\\.apk$|\\.app$|\\.appx$|\\.arc$|\\.bin$|\\.bz2$|\\.class$|\\.deb$|\\.dll$|\\.dmg$|\\.exe$|\\.gz$|\\.ipa$|\\.iso$|\\.jar$|\\.lib$|\\.msi$|\\.o$|\\.obj$|\\.rar$|\\.rpm$|\\.so$|\\.tar$|\\.xar$|\\.xz$|\\.zip$|\\.zst$|\\.Z$",
+      "pony_threshold": 0.5,
+      "elephant_threshold": 0.5,
+      "dev_categories_thresholds": [
+        0.8,
+        0.95
+      ]
+    }
+  }
+}