Reorganising benchmarks for simplified python packaging

examples/bench_ackley.py

@@ -0,0 +1,21 @@
+"""here."""
+
+from deephyper.hpo import CBO
+from deephyper_benchmark.benchmarks.cbbo import AckleyBenchmark
+
+
+def main():
+    """Run example."""
+    bench = AckleyBenchmark()
+
+    search = CBO(bench.problem, bench.run_function)
+    results = search.search(25)
+
+    print(results)
+
+    cumul_regret = bench.scorer.cumul_regret(results.objective)
+    print(cumul_regret)
+
+
+if __name__ == "__main__":
+    main()

examples/bench_branin.py

@@ -0,0 +1,21 @@
+"""here."""
+
+from deephyper.hpo import CBO
+from deephyper_benchmark.benchmarks.cbbo import BraninBenchmark
+
+
+def main():
+    """Run example."""
+    bench = BraninBenchmark()
+
+    search = CBO(bench.problem, bench.run_function)
+    results = search.search(25)
+
+    print(results)
+
+    cumul_regret = bench.scorer.cumul_regret(results.objective)
+    print(cumul_regret)
+
+
+if __name__ == "__main__":
+    main()

examples/bench_dtlz.py

@@ -1,50 +1,28 @@
-# Setup info-level logging
-import logging
-logging.basicConfig(
-    level=logging.INFO,
-    format="%(asctime)s - %(levelname)s - %(filename)s:%(funcName)s - " + \
-           "%(message)s",
-    force=True,
-)
-
-# Set DTLZ problem environment variables
-import os
-os.environ["DEEPHYPER_BENCHMARK_NDIMS"] = "5" # 5 vars
-os.environ["DEEPHYPER_BENCHMARK_NOBJS"] = "3" # 2 objs
-os.environ["DEEPHYPER_BENCHMARK_DTLZ_PROB"] = "2" # DTLZ2 problem
-os.environ["DEEPHYPER_BENCHMARK_DTLZ_OFFSET"] = "0.6" # [x_o, .., x_d]*=0.6
-
-# Load DTLZ benchmark suite, nothing to install
-import deephyper_benchmark as dhb
-dhb.load("DTLZ")
-
-
-# Necessary IF statement otherwise it will enter in a infinite loop
-# when loading the 'run' function from a subprocess
-if __name__ == "__main__":
-    from deephyper.problem import HpProblem
-    from deephyper.search.hps import CBO
-
-    # Run HPO-pipeline with default configuration of hyperparameters
-    from deephyper_benchmark.lib.dtlz import hpo
-    from deephyper.evaluator import RunningJob, Evaluator
-    config = hpo.problem.default_configuration
-    print(config)
-    res = hpo.run(RunningJob(parameters=config))
-    print(f"{res=}")
-
-    # define the evaluator to distribute the computation
-    evaluator = Evaluator.create(
-        hpo.run,
-        method="process",
-        method_kwargs={
-            "num_workers": 2,
-        },
-    )
-
-    # define your search and execute it
-    search = CBO(hpo.problem, evaluator)
-
-    # solve with 100 evals
-    results = search.search(max_evals=100)
+"""here."""
+
+import numpy as np
+from deephyper.hpo import CBO
+from deephyper_benchmark.benchmarks.dtlz import DTLZBenchmark
+
+
+def main():
+    """Run example."""
+    nobj = 2
+    bench = DTLZBenchmark(nobj=nobj)
+
+    search = CBO(bench.problem, bench.run_function, acq_optimizer="sampling")
+    results = search.search(25)
+
     print(results)
+
+    obj = results[[f"objective_{i}" for i in range(nobj)]].values
+    print(np.shape(obj))
+    hvi = bench.scorer.hypervolume_score(obj)
+    print(hvi)
+
+    hvi_iter = bench.scorer.hypervolume(obj)
+    print(hvi_iter)
+
+
+if __name__ == "__main__":
+    main()

pyproject.toml

@@ -6,7 +6,6 @@ build-backend = "hatchling.build"
 name = "deephyper-benchmark"
 version = "0.0.2"
 dependencies = [
-    # "deephyper",
     "deephyper>=0.9.3",
 ]
 requires-python = ">=3.10"
@@ -29,23 +28,13 @@ classifiers = [
     "Programming Language :: Python :: 3.13",
 ]
 
-[project.optional-dependencies]
-
-
-[project.scripts]
-
-
 [project.urls]
 Documentation = "http://deephyper.readthedocs.io"
 Changes = "https://github.com/deephyper/deephyper/releases"
 Forum = "https://github.com/deephyper/deephyper/discussions"
 GitHub = "https://github.com/deephyper/deephyper"
 Issues = "https://github.com/deephyper/deephyper/issues"
 
-[tool.pytest.ini_options]
-norecursedirs = ".git"
-
-
 [tool.coverage.paths]
 source = ["src/", "*/site-packages"]
 
@@ -56,9 +45,15 @@ source = ["deephyper"]
 [tool.coverage.report]
 exclude_lines = ["if __name__ == '__main__':"]
 
+[tool.pyright]
+pythonVersion = "3.13"
+venvPath = "."
+venv = ".venv"
+
+[tool.pytest.ini_options]
+norecursedirs = ".git"
+
 [tool.ruff]
-exclude = [
-]
 line-length = 100
 
 [tool.ruff.lint]

src/deephyper_benchmark/__init__.py

@@ -1,6 +1,13 @@
-"""DeepHyper-Benchmark package."""
+"""Public interface for deephyper_benchmark package."""
 
 from ._benchmark import Benchmark, HPOBenchmark
-from ._scorer import Scorer, HPOScorer
+from ._scorer import Scorer, HPOScorer, MultiObjHPOScorer
 
-__all__ = ["Benchmark", "HPOBenchmark", "Scorer", "HPOScorer"]
+
+__all__ = [
+    "Benchmark",
+    "HPOBenchmark",
+    "Scorer",
+    "HPOScorer",
+    "MultiObjHPOScorer",
+]

src/deephyper_benchmark/_benchmark.py

@@ -2,15 +2,12 @@
 
 
 class Benchmark(abc.ABC):
-    def __init__(self):
-        super().__init__()
-        self.refresh_settings()
-
-    def refresh_settings(self):
-        """Refresh benchmark settings."""
+    """Base class for benchmarks."""
 
 
 class HPOBenchmark(Benchmark):
+    """Base class for Hyperparameter optimization benchmarks."""
+
     @property
     @abc.abstractmethod
     def problem(self):

src/deephyper_benchmark/_scorer.py

@@ -1,13 +1,18 @@
 import abc
+
 import numpy as np
+from deephyper.skopt.moo import hypervolume
 
 
 class Scorer(abc.ABC):
     pass
 
 
 class HPOScorer(Scorer):
-    @abc.abstractmethod
+    def __init__(self, nparams: int, y_max: float):
+        self.nparams = nparams
+        self.y_max = y_max
+
     def simple_regret(self, y: np.ndarray) -> np.ndarray:
         """Compute the regret of a list of given solution.
 
@@ -17,8 +22,8 @@ def simple_regret(self, y: np.ndarray) -> np.ndarray:
         Returns:
             np.ndarray: An array of regret values.
         """
+        return self.y_max - y
 
-    @abc.abstractmethod
     def cumul_regret(self, y: np.ndarray) -> np.ndarray:
         """Compute the cumulative regret of an array of ordered given solution.
 
@@ -28,3 +33,51 @@ def cumul_regret(self, y: np.ndarray) -> np.ndarray:
         Returns:
             np.ndarray: An array of cumulative regret values.
         """
+        return np.cumsum(self.simple_regret(y))
+
+
+class MultiObjHPOScorer(Scorer):
+    def __init__(self, nobj: int):
+        self.nobj = nobj
+
+    def hypervolume_score(self, pts: np.ndarray):
+        """Calculate the hypervolume dominated by soln, wrt the Nadir point.
+
+        Args:
+            pts (np.ndarray): A 2d array of objective values.
+                Each row is an objective value in the solution set.
+
+        Returns:
+            float: The total hypervolume dominated by the current solution,
+            filtering out points worse than the Nadir point and using the
+            Nadir point as the reference.
+
+        """
+        if np.any(pts < 0):
+            filtered_pts = -pts.copy()
+        else:
+            filtered_pts = pts.copy()
+        nadir = self.nadir_point
+        for i in range(pts.shape[0]):
+            if np.any(filtered_pts[i, :] > nadir):
+                filtered_pts[i, :] = nadir
+        return hypervolume(filtered_pts, nadir)
+
+    @property
+    @abc.abstractmethod
+    def nadir_point(self):
+        pass
+
+    def hypervolume(self, y: np.ndarray) -> np.ndarray:
+        """Compute the regret of a list of given solution.
+
+        Args:
+            y (np.ndarray): An array of solutions.
+
+        Returns:
+            np.ndarray: An array of regret values.
+        """
+        scores = np.zeros(y.shape[0])
+        for i in range(1, y.shape[0] + 1):
+            scores[i - 1] = self.hypervolume_score(y[:i])
+        return scores

src/deephyper_benchmark/benchmarks/cbbo/README.md

@@ -0,0 +1,20 @@
+# C-BBO: Continuous Black-Box Optimization
+
+> **Warning**
+> Work in progress, this benchmark is not yet ready.
+
+Set of continuous function benchmarks. The `DEEPHYPER_BENCHMARK_NDIMS` environment variable defines the number of dimensions of the problem.
+
+| Function Name | Number of Dimensions  |                   Comment                   |
+| ------------- | --------------------- | ------------------------------------------- |
+| ackley        | $\infty$ (default 5)  | Many local minima and single global optimum |
+| branin        | 2                     | Three global optimum                        |
+| cossin        | 1                     | Many local minima, good for visualisation.  |
+| easom         | 2                     | Almost flat everywhere                      |
+| griewank      | $\infty$ (default 5)  |                                             |
+| hartmann6D    | 6                     |                                             |
+| levy          | $\infty$ (default 5)  |                                             |
+| michal        | $\infty$ (default 2)  |                                             |
+| rosen         | $\infty$ (default 5)  |                                             |
+| schwefel      | $\infty$ (default 5)  |                                             |
+| shekel        | 4                     | Many local minima with flat areas           |

src/deephyper_benchmark/benchmarks/cbbo/__init__.py

@@ -0,0 +1,25 @@
+"""Subpackage for the C-BBO benchmarks."""
+
+from .ackley import AckleyBenchmark
+from .branin import BraninBenchmark
+from .easom import EasomBenchmark
+from .griewank import GriewankBenchmark
+from .hartmann6D import Hartmann6DBenchmark
+from .levy import LevyBenchmark
+from .michal import MichalBenchmark
+from .rosen import RosenBenchmark
+from .schwefel import SchwefelBenchmark
+from .shekel import ShekelBenchmark
+
+__all__ = [
+    "AckleyBenchmark",
+    "BraninBenchmark",
+    "EasomBenchmark",
+    "GriewankBenchmark",
+    "Hartmann6DBenchmark",
+    "LevyBenchmark",
+    "MichalBenchmark",
+    "RosenBenchmark",
+    "SchwefelBenchmark",
+    "ShekelBenchmark",
+]