Skip to content

Commit 5a893d1

Browse files
ziadhanyziadhany
committed
Update grimoirelab and format output
Validate VCS URL Add a test Update Grimoirelab to use the minimal setting Signed-off-by: ziad hany <ziadhany2016@gmail.com>
1 parent c1075d4 commit 5a893d1

5 files changed

Lines changed: 291 additions & 81 deletions

File tree

‎scancodeio/settings.py‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -373,13 +373,3 @@
373373
GRIMOIRELAB_OPENSEARCH_INDEX = env.str("GRIMOIRELAB_OPENSEARCH_INDEX", default="")
374374
GRIMOIRELAB_OPENSEARCH_USERNAME = env.str("GRIMOIRELAB_OPENSEARCH_USERNAME", default="")
375375
GRIMOIRELAB_OPENSEARCH_PASSWORD = env.str("GRIMOIRELAB_OPENSEARCH_PASSWORD", default="")
376-
GRIMOIRELAB_FROM_DATE = env.str("GRIMOIRELAB_FROM_DATE", default="")
377-
GRIMOIRELAB_TO_DATE = env.str("GRIMOIRELAB_TO_DATE", default="")
378-
GRIMOIRELAB_REPOSITORY_TIMEOUT = env.str("GRIMOIRELAB_REPOSITORY_TIMEOUT", default="")
379-
GRIMOIRELAB_CODE_FILE_PATTERN = env.str("GRIMOIRELAB_CODE_FILE_PATTERN", default="")
380-
GRIMOIRELAB_BINARY_FILE_PATTERN = env.str("GRIMOIRELAB_BINARY_FILE_PATTERN", default="")
381-
GRIMOIRELAB_PONY_THRESHOLD = env.str("GRIMOIRELAB_PONY_THRESHOLD", default="")
382-
GRIMOIRELAB_ELEPHANT_THRESHOLD = env.str("GRIMOIRELAB_ELEPHANT_THRESHOLD", default="")
383-
GRIMOIRELAB_DEVELOPER_CATEGORIES_THRESHOLDS = env.str(
384-
"GRIMOIRELAB_DEVELOPER_CATEGORIES_THRESHOLDS", default=""
385-
).split(",")

‎scanpipe/pipelines/scan_repo_grimoirelab.py‎

Lines changed: 95 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -19,23 +19,16 @@
1919
#
2020
# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
2121
# Visit https://github.com/aboutcode-org/scancode.io for support and download.
22-
22+
import json
2323
import subprocess
24-
from datetime import date
24+
import urllib.parse
2525

26-
from scancodeio.settings import GRIMOIRELAB_BINARY_FILE_PATTERN
27-
from scancodeio.settings import GRIMOIRELAB_CODE_FILE_PATTERN
28-
from scancodeio.settings import GRIMOIRELAB_DEVELOPER_CATEGORIES_THRESHOLDS
29-
from scancodeio.settings import GRIMOIRELAB_ELEPHANT_THRESHOLD
30-
from scancodeio.settings import GRIMOIRELAB_FROM_DATE
3126
from scancodeio.settings import GRIMOIRELAB_METRICS_EXECUTABLE
3227
from scancodeio.settings import GRIMOIRELAB_OPENSEARCH_INDEX
3328
from scancodeio.settings import GRIMOIRELAB_OPENSEARCH_PASSWORD
3429
from scancodeio.settings import GRIMOIRELAB_OPENSEARCH_URL
3530
from scancodeio.settings import GRIMOIRELAB_OPENSEARCH_USERNAME
3631
from scancodeio.settings import GRIMOIRELAB_PASSWORD
37-
from scancodeio.settings import GRIMOIRELAB_PONY_THRESHOLD
38-
from scancodeio.settings import GRIMOIRELAB_REPOSITORY_TIMEOUT
3932
from scancodeio.settings import GRIMOIRELAB_URL
4033
from scancodeio.settings import GRIMOIRELAB_USERNAME
4134
from scanpipe.pipelines import Pipeline
@@ -49,57 +42,98 @@ class ScanRepoGrimoirelab(Pipeline):
4942

5043
@classmethod
5144
def steps(cls):
52-
return (cls.collect_and_store_grimoire_metric,)
45+
return (
46+
cls.collect_and_store_grimoire_metric,
47+
cls.format_metrics_output,
48+
)
5349

5450
def collect_and_store_grimoire_metric(self):
55-
for input_source in self.project.input_sources:
56-
repo_url = input_source["download_url"]
57-
metrics_output_path = self.project.get_output_file_path("metrics", "json")
58-
grimoirelab_to_date = date.today().isoformat()
59-
60-
command_args = [
61-
GRIMOIRELAB_METRICS_EXECUTABLE,
62-
repo_url,
63-
"--grimoirelab-url",
64-
GRIMOIRELAB_URL,
65-
"--grimoirelab-user",
66-
GRIMOIRELAB_USERNAME,
67-
"--grimoirelab-password",
68-
GRIMOIRELAB_PASSWORD,
69-
"--opensearch-url",
70-
GRIMOIRELAB_OPENSEARCH_URL,
71-
"--opensearch-index",
72-
GRIMOIRELAB_OPENSEARCH_INDEX,
73-
"--opensearch-user",
74-
GRIMOIRELAB_OPENSEARCH_USERNAME,
75-
"--opensearch-password",
76-
GRIMOIRELAB_OPENSEARCH_PASSWORD,
77-
"--from-date",
78-
GRIMOIRELAB_FROM_DATE,
79-
"--to-date",
80-
grimoirelab_to_date,
81-
"--repository-timeout",
82-
GRIMOIRELAB_REPOSITORY_TIMEOUT,
83-
"--code-file-pattern",
84-
GRIMOIRELAB_CODE_FILE_PATTERN,
85-
"--binary-file-pattern",
86-
GRIMOIRELAB_BINARY_FILE_PATTERN,
87-
"--pony-threshold",
88-
GRIMOIRELAB_PONY_THRESHOLD,
89-
"--elephant-threshold",
90-
GRIMOIRELAB_ELEPHANT_THRESHOLD,
91-
"--dev-categories-thresholds",
92-
*GRIMOIRELAB_DEVELOPER_CATEGORIES_THRESHOLDS,
93-
"--output",
94-
str(metrics_output_path),
95-
]
96-
97-
try:
98-
run_command_safely(command_args=command_args)
99-
self.log(f"Metrics successfully saved to {metrics_output_path}")
100-
except subprocess.CalledProcessError as e:
101-
raise RuntimeError(
102-
f"grimoirelab-metrics pipeline failed {e.returncode} for {repo_url}"
103-
)
104-
except subprocess.TimeoutExpired:
105-
raise RuntimeError("grimoirelab-metrics pipeline timed out")
51+
"""
52+
Run the grimoirelab-metrics command against the input source.
53+
Save the generated metrics JSON to the project output directory.
54+
"""
55+
if len(self.project.input_sources) != 1:
56+
raise ValueError("Expected exactly one input source")
57+
58+
repo_url = self.project.input_sources[0]["download_url"]
59+
if not is_valid_vcs_url(repo_url):
60+
raise ValueError(
61+
"Invalid input source: the pipeline accepts only a valid repository URL"
62+
)
63+
64+
self.metrics_output_path = self.project.get_output_file_path("metrics", "json")
65+
command_args = [
66+
GRIMOIRELAB_METRICS_EXECUTABLE,
67+
repo_url,
68+
"--grimoirelab-url",
69+
GRIMOIRELAB_URL,
70+
"--grimoirelab-user",
71+
GRIMOIRELAB_USERNAME,
72+
"--grimoirelab-password",
73+
GRIMOIRELAB_PASSWORD,
74+
"--opensearch-url",
75+
GRIMOIRELAB_OPENSEARCH_URL,
76+
"--opensearch-index",
77+
GRIMOIRELAB_OPENSEARCH_INDEX,
78+
"--opensearch-user",
79+
GRIMOIRELAB_OPENSEARCH_USERNAME,
80+
"--opensearch-password",
81+
GRIMOIRELAB_OPENSEARCH_PASSWORD,
82+
"--output",
83+
str(self.metrics_output_path),
84+
]
85+
86+
try:
87+
run_command_safely(command_args=command_args)
88+
self.log("GrimoireLab metrics pipeline completed successfully")
89+
except subprocess.SubprocessError:
90+
raise RuntimeError("Grimoirelab-metrics pipeline failed")
91+
except FileNotFoundError:
92+
raise FileNotFoundError(
93+
"Grimoirelab-metrics not found. "
94+
"Please ensure grimoirelab-metrics is correctly configured."
95+
)
96+
97+
def format_metrics_output(self):
98+
"""
99+
Format the GrimoireLab metrics output by extracting the repository URL,
100+
score, and metrics from the generated JSON and overwriting it with a
101+
simplified structure.
102+
"""
103+
with open(self.metrics_output_path) as f:
104+
data = json.load(f)
105+
106+
package = list(data["packages"].values())[0]
107+
108+
repository = package["repository"]
109+
score = package["score"]
110+
metrics = package["metrics"]
111+
112+
result = {
113+
"repository": repository,
114+
"npm_health_score": score,
115+
"metrics": metrics,
116+
}
117+
118+
with open(self.metrics_output_path, "w") as f:
119+
json.dump(result, f)
120+
121+
122+
def is_valid_vcs_url(url):
123+
"""Determine whether the URL is a valid VCS repository URL."""
124+
if not isinstance(url, str) or not url:
125+
return False
126+
127+
if any(char.isspace() for char in url):
128+
return False
129+
130+
forbidden_chars = ["|", ";", "&", "`", "$(", ">", "<", "&&", "||"]
131+
if any(char in forbidden_chars for char in url):
132+
return False
133+
134+
parsed = urllib.parse.urlparse(url)
135+
valid_schemes = {"https", "git", "ssh", "git+https", "git+ssh"}
136+
if parsed.scheme in valid_schemes and parsed.netloc:
137+
return True
138+
139+
return False

‎scanpipe/tests/data/grimorielab/expected-metrics.json‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
{
2+
"metrics": {
3+
"total_commits": 260,
4+
"total_contributors": 33,
5+
"total_organizations": 16,
6+
"pony_factor": 2,
7+
"elephant_factor": 1,
8+
"recent_organizations": 4,
9+
"recent_contributors": 6,
10+
"recent_commits": 69,
11+
"contributor_growth": 7,
12+
"contributor_growth_rate": 0.4666666666666667,
13+
"active_branches": 4,
14+
"days_since_last_commit": 0,
15+
"casual_regular_contributors_rate": 0.65,
16+
"returning_contributors": 4,
17+
"commits_over_periods_rate": 0.2653846153846154,
18+
"coefficient_of_variation": 0.745479473205275,
19+
"file_types_code": 143,
20+
"file_types_binary": 0,
21+
"file_types_other": 2124,
22+
"commit_size_added_lines": 38313,
23+
"commit_size_removed_lines": 62036,
24+
"message_size_total": 35352,
25+
"message_size_mean": 135.96923076923076,
26+
"message_size_median": 88,
27+
"developer_categories_core": 1,
28+
"developer_categories_regular": 19,
29+
"developer_categories_casual": 13,
30+
"commits_per_week": 1.4387351778656126,
31+
"commits_per_month": 6.16600790513834,
32+
"commits_per_year": 75.0197628458498,
33+
"found_file_license": 1,
34+
"found_file_adopters": 1
35+
},
36+
"repository": "https://github.com/chaoss/grimoirelab.git",
37+
"npm_health_score": 2.361341589536065e-72
38+
}

‎scanpipe/tests/data/grimorielab/metrics.json‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
{
2+
"packages": {
3+
"SPDXRef-Package-grimoirelab": {
4+
"metrics": {
5+
"total_commits": 260,
6+
"total_contributors": 33,
7+
"total_organizations": 16,
8+
"pony_factor": 2,
9+
"elephant_factor": 1,
10+
"recent_organizations": 4,
11+
"recent_contributors": 6,
12+
"recent_commits": 69,
13+
"contributor_growth": 7,
14+
"contributor_growth_rate": 0.4666666666666667,
15+
"active_branches": 4,
16+
"days_since_last_commit": 0,
17+
"casual_regular_contributors_rate": 0.65,
18+
"returning_contributors": 4,
19+
"commits_over_periods_rate": 0.2653846153846154,
20+
"coefficient_of_variation": 0.745479473205275,
21+
"file_types_code": 143,
22+
"file_types_binary": 0,
23+
"file_types_other": 2124,
24+
"commit_size_added_lines": 38313,
25+
"commit_size_removed_lines": 62036,
26+
"message_size_total": 35352,
27+
"message_size_mean": 135.96923076923076,
28+
"message_size_median": 88,
29+
"developer_categories_core": 1,
30+
"developer_categories_regular": 19,
31+
"developer_categories_casual": 13,
32+
"commits_per_week": 1.4387351778656126,
33+
"commits_per_month": 6.16600790513834,
34+
"commits_per_year": 75.0197628458498,
35+
"found_file_license": 1,
36+
"found_file_adopters": 1
37+
},
38+
"metadata": {
39+
"first_commit": "fc8754a69b50d1bb6a7a30fafc4fe69bda6e3a10",
40+
"last_commit": "c51f5deb930371f512394af3d95a1a27383a14c2",
41+
"first_commit_date": "2023-01-10T09:54:25+01:00",
42+
"last_commit_date": "2026-06-18T09:38:47+02:00"
43+
},
44+
"repository": "https://github.com/chaoss/grimoirelab.git",
45+
"score": 2.361341589536065e-72
46+
}
47+
},
48+
"metadata": {
49+
"version": "0.1.0",
50+
"started_at": "2026-08-01T02:02:39.612433+00:00",
51+
"finished_at": "2026-08-01T02:02:40.794247+00:00",
52+
"configuration": {
53+
"from_date": "2023-01-01T00:00:00",
54+
"to_date": "2026-06-19T00:00:00",
55+
"code_file_pattern": "\\.py$|\\.js$",
56+
"binary_file_pattern": "\\.exe$|\\.tar$",
57+
"pony_threshold": 0.5,
58+
"elephant_threshold": 0.5,
59+
"dev_categories_thresholds": [
60+
0.8,
61+
0.95
62+
]
63+
}
64+
}
65+
}

0 commit comments

Comments
 (0)