Coverage for moptipy / evaluation / ioh_analyzer.py: 79%
106 statements
« prev ^ index » next coverage.py v7.12.0, created at 2025-11-24 08:49 +0000
1"""
2Convert `moptipy` data to IOHanalyzer data.
4The IOHanalyzer (https://iohanalyzer.liacs.nl/) is a tool that can analyze
5the performance of iterative optimization heuristics in a wide variety of
6ways. It is available both for local installation as well as online for
7direct and free use (see, again, https://iohanalyzer.liacs.nl/). The
8IOHanalyzer supports many of the diagrams that our evaluation utilities
9provide - and several more. Here we provide the function
10:func:`moptipy_to_ioh_analyzer` which converts the data generated by the
11`moptipy` experimentation function
12:func:`~moptipy.api.experiment.run_experiment` to the format that the
13IOHanalyzer understands, as documented at
14https://iohprofiler.github.io/IOHanalyzer/data/.
16Notice that we here have implemented the meta data format version
17"0.3.2 and below", as described at https://iohprofiler.github.io/IOHanalyzer\
18/data/#iohexperimenter-version-032-and-below.
201. Carola Doerr, Furong Ye, Naama Horesh, Hao Wang, Ofer M. Shir, and Thomas
21 Bäck. Benchmarking Discrete Optimization Heuristics with IOHprofiler.
22 *Applied Soft Computing* 88(106027):1-21. March 2020.
23 doi: https://doi.org/10.1016/j.asoc.2019.106027},
242. Carola Doerr, Hao Wang, Furong Ye, Sander van Rijn, and Thomas Bäck.
25 *IOHprofiler: A Benchmarking and Profiling Tool for Iterative Optimization
26 Heuristics.* October 15, 2018. New York, NY, USA: Cornell University,
27 Cornell Tech. arXiv:1810.05281v1 [cs.NE] 11 Oct 2018.
28 https://arxiv.org/pdf/1810.05281.pdf
293. Hao Wang, Diederick Vermetten, Furong Ye, Carola Doerr, and Thomas Bäck.
30 IOHanalyzer: Detailed Performance Analyses for Iterative Optimization
31 Heuristics. *ACM Transactions on Evolutionary Learning and Optimization*
32 2(1)[3]:1-29. March 2022.doi: https://doi.org/10.1145/3510426.
334. Jacob de Nobel and Furong Ye and Diederick Vermetten and Hao Wang and
34 Carola Doerr and Thomas Bäck. *IOHexperimenter: Benchmarking Platform for
35 Iterative Optimization Heuristics.* November 2021. New York, NY, USA:
36 Cornell University, Cornell Tech. arXiv:2111.04077v2 [cs.NE] 17 Apr 2022.
37 https://arxiv.org/pdf/2111.04077.pdf
385. Data Format: Iterative Optimization Heuristics Profiler.
39 https://iohprofiler.github.io/IOHanalyzer/data/
40"""
42import argparse
43import contextlib
44from typing import Any, Callable, Final
46import numpy as np
47from pycommons.io.console import logger
48from pycommons.io.path import Path, directory_path
49from pycommons.strings.string_conv import float_to_str
50from pycommons.types import check_int_range, type_error
52from moptipy.evaluation.base import F_NAME_RAW, TIME_UNIT_FES, check_f_name
53from moptipy.evaluation.progress import from_logs
54from moptipy.utils.help import moptipy_argparser
57def __prefix(s: str) -> str:
58 """
59 Return `xxx` if `s` is of the form `xxx_i` and `i` is `int`.
61 :param s: the function name
62 :return: the dimension
63 """
64 idx = s.rfind("_")
65 if idx > 0:
66 with contextlib.suppress(ValueError):
67 i = int(s[idx + 1:])
68 if i > 0:
69 return s[:idx].strip()
70 return s
73def __int_suffix(s: str) -> int:
74 """
75 Return `i` if `s` is of the form `xxx_i` and `i` is `int`.
77 This function tries to check if the name
79 :param s: the function name
80 :return: the dimension
81 """
82 idx = s.rfind("_")
83 if idx > 0:
84 with contextlib.suppress(ValueError):
85 i = int(s[idx + 1:])
86 if i > 0:
87 return i
88 return 1
91def __npstr(a: Any) -> str:
92 """
93 Convert numpy numbers to strings.
95 :param a: the input
96 :returns: a string
97 """
98 return str(int(a)) if isinstance(a, np.integer) \
99 else float_to_str(float(a))
102def moptipy_to_ioh_analyzer(
103 results_dir: str,
104 dest_dir: str,
105 inst_name_to_func_id: Callable[[str], str] = __prefix,
106 inst_name_to_dimension: Callable[[str], int] = __int_suffix,
107 inst_name_to_inst_id: Callable[[str], int] = lambda _: 1,
108 suite: str = "moptipy",
109 f_name: str = F_NAME_RAW,
110 f_standard: dict[str, int | float] | None = None) -> None:
111 """
112 Convert moptipy log data to IOHanalyzer log data.
114 :param results_dir: the directory where we can find the results in moptipy
115 format
116 :param dest_dir: the directory where we would write the IOHanalyzer style
117 data
118 :param inst_name_to_func_id: convert the instance name to a function ID
119 :param inst_name_to_dimension: convert an instance name to a function
120 dimension
121 :param inst_name_to_inst_id: convert the instance name an instance ID,
122 which must be a positive integer number
123 :param suite: the suite name
124 :param f_name: the objective name
125 :param f_standard: a dictionary mapping instances to standard values
126 """
127 source: Final[Path] = directory_path(results_dir)
128 dest: Final[Path] = Path(dest_dir)
129 dest.ensure_dir_exists()
130 logger(f"converting the moptipy log files in {source!r} to "
131 f"IOHprofiler data in {dest!r}. First we load the data.")
133 if (f_standard is not None) and (not isinstance(f_standard, dict)):
134 raise type_error(f_standard, "f_standard", dict)
135 if not isinstance(suite, str):
136 raise type_error(suite, "suite", str)
137 if (len(suite) <= 0) or (" " in suite):
138 raise ValueError(f"invalid suite name {suite!r}")
139 if not callable(inst_name_to_func_id):
140 raise type_error(
141 inst_name_to_func_id, "inst_name_to_func_id", call=True)
142 if not callable(inst_name_to_dimension):
143 raise type_error(
144 inst_name_to_dimension, "inst_name_to_dimension", call=True)
145 if not callable(inst_name_to_inst_id):
146 raise type_error(
147 inst_name_to_inst_id, "inst_name_to_inst_id", call=True)
149 # the data
150 data: Final[dict[str, dict[str, dict[int, list[
151 tuple[int, np.ndarray, np.ndarray]]]]]] = {}
153 for progress in from_logs(
154 source, time_unit=TIME_UNIT_FES, f_name=check_f_name(f_name),
155 f_standard=f_standard, only_improvements=True):
156 algo: dict[str, dict[int, list[tuple[int, np.ndarray, np.ndarray]]]]
157 if progress.algorithm in data:
158 algo = data[progress.algorithm]
159 else:
160 data[progress.algorithm] = algo = {}
161 func_id: str = inst_name_to_func_id(progress.instance)
162 if not isinstance(func_id, str):
163 raise type_error(func_id, "function id", str)
164 if (len(func_id) <= 0) or ("_" in func_id):
165 raise ValueError(f"invalid function id {func_id!r}.")
166 func: dict[int, list[tuple[int, np.ndarray, np.ndarray]]]
167 if func_id in algo:
168 func = algo[func_id]
169 else:
170 algo[func_id] = func = {}
171 dim: int = check_int_range(
172 inst_name_to_dimension(progress.instance), "dimension", 1)
173 iid: int = check_int_range(
174 inst_name_to_inst_id(progress.instance), "instance id", 1)
175 res: tuple[int, np.ndarray, np.ndarray] = (
176 iid, progress.time, progress.f)
177 if dim in func:
178 func[dim].append(res)
179 else:
180 func[dim] = [res]
182 if len(data) <= 0:
183 raise ValueError("did not find any data!")
184 logger(f"finished loading data from {len(data)} algorithms, "
185 "now writing output.")
187 for algo_name in sorted(data.keys()):
188 algo = data[algo_name]
189 algo_dir: Path = dest.resolve_inside(algo_name)
190 algo_dir.ensure_dir_exists()
191 logger(f"writing output for {len(algo)} functions of "
192 f"algorithm {algo_name!r}.")
193 for func_id in sorted(algo.keys()):
194 func_dir: Path = algo_dir.resolve_inside(f"data_f{func_id}")
195 func_dir.ensure_dir_exists()
196 func = algo[func_id]
197 logger(f"writing output for algorithm {algo_name!r} and "
198 f"function {func_id!r}, got {len(func)} dimensions.")
200 func_name = f"IOHprofiler_f{func_id}"
201 with algo_dir.resolve_inside(
202 f"{func_name}.info").open_for_write() as info:
203 for dimi in sorted(func.keys()):
204 dim_path = func_dir.resolve_inside(
205 f"{func_name}_DIM{dimi}.dat")
206 info.write(f"suite = {suite!r}, funcId = {func_id!r}, "
207 f"DIM = {dimi}, algId = {algo_name!r}\n")
208 info.write("%\n")
209 info.write(dim_path[len(algo_dir) + 1:])
210 with dim_path.open_for_write() as dat:
211 for per_dim in sorted(
212 func[dimi], key=lambda x:
213 (x[0], x[2][-1], x[1][-1])):
214 info.write(f", {per_dim[0]}:")
215 fes = per_dim[1]
216 f = per_dim[2]
217 info.write(__npstr(fes[-1]))
218 info.write("|")
219 info.write(__npstr(f[-1]))
220 dat.write(
221 '"function evaluation" "best-so-far f(x)"\n')
222 for i, ff in enumerate(f):
223 dat.write(
224 f"{__npstr(fes[i])} {__npstr(ff)}\n")
225 dat.write("\n")
226 info.write("\n")
227 del data
228 logger("finished converting moptipy data to IOHprofiler data.")
231# Run conversion if executed as script
232if __name__ == "__main__":
233 parser: Final[argparse.ArgumentParser] = moptipy_argparser(
234 __file__,
235 "Convert experimental results from the moptipy to the "
236 "IOHanalyzer format.",
237 "The experiment execution API of moptipy creates an output "
238 "folder structure with clearly specified log files that can be"
239 " evaluated with our experimental data analysis API. The "
240 "IOHprofiler tool chain offers another format (specified in "
241 "https://iohprofiler.github.io/IOHanalyzer/data/). With this "
242 "tool here, you can convert from the moptipy to the "
243 "IOHprofiler format.")
244 parser.add_argument(
245 "source", help="the directory with moptipy log files", type=Path,
246 nargs="?", default="./results")
247 parser.add_argument(
248 "dest", help="the directory to write the IOHanalyzer data to",
249 type=Path, nargs="?", default="./IOHanalyzer")
250 args: Final[argparse.Namespace] = parser.parse_args()
251 moptipy_to_ioh_analyzer(args.source, args.dest)