Coverage for moptipy / evaluation / end_statistics.py: 79%
671 statements
« prev ^ index » next coverage.py v7.12.0, created at 2025-11-24 08:49 +0000
1"""
2SampleStatistics aggregated over multiple instances of `EndResult`.
4The :mod:`~moptipy.evaluation.end_results` records hold the final result of
5a run of an optimization algorithm on a problem instance. Often, we do not
6want to compare these single results directly, but instead analyze summary
7statistics, such as the mean best objective value found. For this purpose,
8:class:`EndStatistics` exists. It summarizes the singular results from the
9runs into a record with the most important statistics.
10"""
11import argparse
12import os.path
13from dataclasses import dataclass
14from itertools import starmap
15from math import ceil, inf, isfinite
16from typing import Callable, Final, Generator, Iterable, Iterator, cast
18from pycommons.ds.sequences import reiterable
19from pycommons.io.console import logger
20from pycommons.io.csv import (
21 SCOPE_SEPARATOR,
22 csv_column,
23 csv_column_or_none,
24 csv_scope,
25 csv_select_scope,
26 csv_select_scope_or_none,
27 csv_str_or_none,
28 csv_val_or_none,
29)
30from pycommons.io.csv import CsvReader as CsvReaderBase
31from pycommons.io.csv import CsvWriter as CsvWriterBase
32from pycommons.io.path import Path, file_path, write_lines
33from pycommons.math.sample_statistics import (
34 KEY_MEAN_ARITH,
35 KEY_STDDEV,
36 SampleStatistics,
37 from_samples,
38 from_single_value,
39)
40from pycommons.math.sample_statistics import CsvReader as StatReader
41from pycommons.math.sample_statistics import CsvWriter as StatWriter
42from pycommons.math.sample_statistics import getter as stat_getter
43from pycommons.strings.string_conv import (
44 num_or_none_to_str,
45 str_to_num,
46)
47from pycommons.types import (
48 check_int_range,
49 type_error,
50 type_name_of,
51)
53from moptipy.api.logging import (
54 KEY_ALGORITHM,
55 KEY_BEST_F,
56 KEY_GOAL_F,
57 KEY_INSTANCE,
58 KEY_LAST_IMPROVEMENT_FE,
59 KEY_LAST_IMPROVEMENT_TIME_MILLIS,
60 KEY_MAX_FES,
61 KEY_MAX_TIME_MILLIS,
62 KEY_TOTAL_FES,
63 KEY_TOTAL_TIME_MILLIS,
64)
65from moptipy.evaluation._utils import (
66 _check_max_time_millis,
67)
68from moptipy.evaluation.base import (
69 DESC_ALGORITHM,
70 DESC_ENCODING,
71 DESC_INSTANCE,
72 DESC_OBJECTIVE_FUNCTION,
73 F_NAME_RAW,
74 F_NAME_SCALED,
75 KEY_ENCODING,
76 KEY_N,
77 KEY_OBJECTIVE_FUNCTION,
78 MultiRunData,
79 motipy_footer_bottom_comments,
80)
81from moptipy.evaluation.end_results import (
82 DESC_BEST_F,
83 DESC_GOAL_F,
84 DESC_LAST_IMPROVEMENT_FE,
85 DESC_LAST_IMPROVEMENT_TIME_MILLIS,
86 DESC_MAX_FES,
87 DESC_MAX_TIME_MILLIS,
88 DESC_TOTAL_FES,
89 DESC_TOTAL_TIME_MILLIS,
90 EndResult,
91)
92from moptipy.evaluation.end_results import from_csv as end_results_from_csv
93from moptipy.evaluation.end_results import from_logs as end_results_from_logs
94from moptipy.utils.help import moptipy_argparser
95from moptipy.utils.math import try_int, try_int_div
97#: The key for the best F.
98KEY_BEST_F_SCALED: Final[str] = KEY_BEST_F + "scaled"
99#: The key for the number of successful runs.
100KEY_N_SUCCESS: Final[str] = "successN"
101#: The key for the success FEs.
102KEY_SUCCESS_FES: Final[str] = "successFEs"
103#: The key for the success time millis.
104KEY_SUCCESS_TIME_MILLIS: Final[str] = "successTimeMillis"
105#: The key for the ERT in FEs.
106KEY_ERT_FES: Final[str] = "ertFEs"
107#: The key for the ERT in milliseconds.
108KEY_ERT_TIME_MILLIS: Final[str] = "ertTimeMillis"
111@dataclass(frozen=True, init=False, order=False, eq=False)
112class EndStatistics(MultiRunData):
113 """
114 Statistics over end results of one or multiple algorithm*instance setups.
116 If one algorithm*instance is used, then `algorithm` and `instance` are
117 defined. Otherwise, only the parameter which is the same over all recorded
118 runs is defined.
119 """
121 #: The statistics about the best encountered result.
122 best_f: SampleStatistics
123 #: The statistics about the last improvement FE.
124 last_improvement_fe: SampleStatistics
125 #: The statistics about the last improvement time.
126 last_improvement_time_millis: SampleStatistics
127 #: The statistics about the total number of FEs.
128 total_fes: SampleStatistics
129 #: The statistics about the total time.
130 total_time_millis: SampleStatistics
131 #: The goal objective value.
132 goal_f: SampleStatistics | int | float | None
133 #: best_f / goal_f if goal_f is consistently defined and always positive.
134 best_f_scaled: SampleStatistics | None
135 #: The number of successful runs, if goal_f != None, else None.
136 n_success: int | None
137 #: The FEs to success, if n_success > 0, None otherwise.
138 success_fes: SampleStatistics | None
139 #: The time to success, if n_success > 0, None otherwise.
140 success_time_millis: SampleStatistics | None
141 #: The ERT if FEs, while is inf if n_success=0, None if goal_f is None,
142 #: and finite otherwise.
143 ert_fes: int | float | None
144 #: The ERT if milliseconds, while is inf if n_success=0, None if goal_f
145 #: is None, and finite otherwise.
146 ert_time_millis: int | float | None
147 #: The budget in FEs, if every run had one; None otherwise.
148 max_fes: SampleStatistics | int | None
149 #: The budget in milliseconds, if every run had one; None otherwise.
150 max_time_millis: SampleStatistics | int | None
152 def __init__(self,
153 algorithm: str | None,
154 instance: str | None,
155 objective: str | None,
156 encoding: str | None,
157 n: int,
158 best_f: SampleStatistics,
159 last_improvement_fe: SampleStatistics,
160 last_improvement_time_millis: SampleStatistics,
161 total_fes: SampleStatistics,
162 total_time_millis: SampleStatistics,
163 goal_f: float | int | SampleStatistics | None,
164 best_f_scaled: SampleStatistics | None,
165 n_success: int | None,
166 success_fes: SampleStatistics | None,
167 success_time_millis: SampleStatistics | None,
168 ert_fes: int | float | None,
169 ert_time_millis: int | float | None,
170 max_fes: SampleStatistics | int | None,
171 max_time_millis: SampleStatistics | int | None):
172 """
173 Create the end statistics of an experiment-setup combination.
175 :param algorithm: the algorithm name, if all runs are with the same
176 algorithm
177 :param instance: the instance name, if all runs are on the same
178 instance
179 :param objective: the objective name, if all runs are on the same
180 objective function, `None` otherwise
181 :param encoding: the encoding name, if all runs are on the same
182 encoding and an encoding was actually used, `None` otherwise
183 :param n: the total number of runs
184 :param best_f: statistics about the best achieved result
185 :param last_improvement_fe: statistics about the last improvement FE
186 :param last_improvement_time_millis: statistics about the last
187 improvement time
188 :param total_fes: statistics about the total FEs
189 :param total_time_millis: statistics about the total runtime in
190 milliseconds
191 :param goal_f: if the goal objective value is not defined sometimes,
192 this will be `None`. If it is always defined and always the same,
193 then this will be that value. If different goal values exist, then
194 this is the `SampleStatistics` record about them
195 :param best_f_scaled: if `goal_f` is not `None` and greater than zero,
196 then here we provide statistics about `best_f` divided by the
197 corresponding `goal_f`
198 :param n_success: the number of successful runs is only defined if
199 `goal_f` is not `None` and counts the number of runs that reach or
200 surpass their corresponding `goal_f`
201 :param success_fes: if `goal_f` is not `None`,
202 then this holds statistics about the last improvement FE of only
203 the successful runs
204 :param success_time_millis: if `goal_f` is not `None`, then this holds
205 statistics about the last improvement times of only the successful
206 runs
207 :param ert_fes: if `goal_f` is always defined, then this is the
208 empirically estimated running time to solve the problem in FEs if
209 `n_success>0` and `inf` otherwise
210 :param ert_time_millis: if `goal_f` is always defined, then this is
211 the empirically estimated running time to solve the problem in
212 milliseconds if `n_success>0` and `inf` otherwise
213 :param max_fes: the budget in FEs, if any
214 :param max_time_millis: the budget in terms of milliseconds
215 """
216 super().__init__(algorithm, instance, objective, encoding, n)
218 if not isinstance(best_f, SampleStatistics):
219 raise type_error(best_f, "best_f", SampleStatistics)
220 object.__setattr__(self, "best_f", best_f)
221 if best_f.n != n:
222 raise ValueError(f"best_f.n={best_f.n} != n={n}")
224 if not isinstance(last_improvement_fe, SampleStatistics):
225 raise type_error(last_improvement_fe, "last_improvement_fe",
226 SampleStatistics)
227 if last_improvement_fe.n != n:
228 raise ValueError(
229 f"last_improvement_fe.n={last_improvement_fe.n} != n={n}")
230 check_int_range(
231 last_improvement_fe.minimum, "last_improvement_fe.minimum",
232 1, 1_000_000_000_000_000)
233 check_int_range(
234 last_improvement_fe.maximum, "last_improvement_fe.maximum",
235 last_improvement_fe.minimum, 1_000_000_000_000_000)
236 object.__setattr__(self, "last_improvement_fe", last_improvement_fe)
238 if not isinstance(last_improvement_time_millis, SampleStatistics):
239 raise type_error(last_improvement_time_millis,
240 "last_improvement_time_millis", SampleStatistics)
241 if last_improvement_time_millis.n != n:
242 raise ValueError("last_improvement_time_millis.n="
243 f"{last_improvement_time_millis.n} != n={n}")
244 check_int_range(
245 last_improvement_time_millis.minimum,
246 "last_improvement_time_millis.minimum",
247 0, 100_000_000_000)
248 check_int_range(
249 last_improvement_time_millis.maximum,
250 "last_improvement_time_millis.maximum",
251 last_improvement_time_millis.minimum, 100_000_000_000)
252 object.__setattr__(self, "last_improvement_time_millis",
253 last_improvement_time_millis)
255 if not isinstance(total_fes, SampleStatistics):
256 raise type_error(total_fes, "total_fes", SampleStatistics)
257 if total_fes.n != n:
258 raise ValueError(
259 f"total_fes.n={total_fes.n} != n={n}")
260 check_int_range(
261 total_fes.minimum, "total_fes.minimum",
262 last_improvement_fe.minimum, 1_000_000_000_000_000)
263 check_int_range(
264 total_fes.maximum, "total_fes.maximum",
265 max(total_fes.minimum, last_improvement_fe.maximum),
266 1_000_000_000_000_000)
267 object.__setattr__(self, "total_fes", total_fes)
269 if not isinstance(total_time_millis, SampleStatistics):
270 raise type_error(total_time_millis, "total_time_millis",
271 SampleStatistics)
272 if total_time_millis.n != n:
273 raise ValueError(
274 f"total_time_millis.n={total_time_millis.n} != n={n}")
275 check_int_range(
276 total_time_millis.minimum, "total_time_millis.minimum",
277 last_improvement_time_millis.minimum, 100_000_000_000)
278 check_int_range(
279 total_time_millis.maximum, "total_time_millis.maximum",
280 max(total_time_millis.minimum,
281 last_improvement_time_millis.maximum),
282 100_000_000_000)
283 object.__setattr__(self, "total_time_millis", total_time_millis)
285 if goal_f is None:
286 if best_f_scaled is not None:
287 raise ValueError(
288 "If goal_f is None, best_f_scaled must also be None, "
289 f"but is {type(best_f_scaled)}.")
290 if n_success is not None:
291 raise ValueError(
292 "If goal_f is None, n_success must also be None, "
293 f"but is {type(n_success)}.")
294 if success_fes is not None:
295 raise ValueError(
296 "If success_fes is None, best_f_scaled must also be None, "
297 f"but is {type(success_fes)}.")
298 if success_time_millis is not None:
299 raise ValueError(
300 "If success_time_millis is None, best_f_scaled "
301 "must also be None, "
302 f"but is {type(success_time_millis)}.")
303 if ert_fes is not None:
304 raise ValueError(
305 "If goal_f is None, ert_fes must also be None, "
306 f"but is {type(ert_fes)}.")
307 if ert_time_millis is not None:
308 raise ValueError(
309 "If goal_f is None, ert_time_millis must also be None, "
310 f"but is {type(ert_time_millis)}.")
311 else: # goal_f is not None
312 if isinstance(goal_f, SampleStatistics):
313 if goal_f.n != n:
314 raise ValueError(f"goal_f.n={goal_f.n} != n={n}")
315 goal_f = goal_f.compact(False)
316 if isinstance(goal_f, float):
317 goal_f = None if goal_f <= (-inf) else try_int(goal_f)
318 elif not isinstance(goal_f, int | SampleStatistics):
319 raise type_error(goal_f, "goal_f", (
320 int, float, SampleStatistics))
322 if best_f_scaled is not None:
323 goal_f_min: Final[int | float] = \
324 goal_f.minimum if isinstance(goal_f, SampleStatistics) \
325 else goal_f
326 if goal_f_min <= 0:
327 raise ValueError(
328 f"best_f_scaled must be None if minimum goal_f "
329 f"({goal_f_min}) of goal_f {goal_f} is not positive,"
330 f" but is {best_f_scaled}.")
331 if not isinstance(best_f_scaled, SampleStatistics):
332 raise type_error(best_f_scaled, "best_f_scaled",
333 SampleStatistics)
334 if best_f_scaled.n != n:
335 raise ValueError(
336 f"best_f_scaled.n={best_f_scaled.n} != n={n}")
337 if best_f_scaled.minimum < 0:
338 raise ValueError(
339 "best_f_scaled cannot be negative, but encountered "
340 f"{best_f_scaled.minimum}.")
342 check_int_range(n_success, "n_success")
343 if not isinstance(ert_fes, int | float):
344 raise type_error(ert_fes, "ert_fes", (int, float))
345 if not isinstance(ert_time_millis, int | float):
346 raise type_error(ert_time_millis, "ert_time_millis",
347 (int, float))
349 if n_success > 0:
350 if not isinstance(success_fes, SampleStatistics):
351 raise type_error(success_fes,
352 "if n_success>0, then success_fes",
353 SampleStatistics)
354 if success_fes.n != n_success:
355 raise ValueError(f"success_fes.n={success_fes.n} != "
356 f"n_success={n_success}")
357 check_int_range(
358 success_fes.minimum, "success_fes.minimum",
359 last_improvement_fe.minimum, 1_000_000_000_000_000)
360 check_int_range(
361 success_fes.maximum, "success_fes.maximum",
362 success_fes.minimum, last_improvement_fe.maximum)
363 if not isinstance(success_time_millis, SampleStatistics):
364 raise type_error(
365 success_time_millis,
366 "if n_success>0, then success_time_millis",
367 SampleStatistics)
368 if success_time_millis.n != n_success:
369 raise ValueError(
370 f"success_time_millis.n={success_time_millis.n} != "
371 f"n_success={n_success}")
372 check_int_range(
373 success_time_millis.minimum,
374 "success_time_millis.minimum",
375 last_improvement_time_millis.minimum, 100_000_000_000)
376 check_int_range(
377 success_time_millis.maximum,
378 "success_time_millis.maximum",
379 success_time_millis.minimum,
380 last_improvement_time_millis.maximum)
381 ert_fes = try_int(ert_fes)
382 if ert_fes < success_fes.minimum:
383 raise ValueError(
384 "ert_fes must be >= "
385 f"{success_fes.minimum}, but is {ert_fes}.")
386 ert_fe_max = ceil(total_fes.mean_arith * n)
387 if ert_fes > ert_fe_max:
388 raise ValueError(
389 "ert_fes must be <= "
390 f"{ert_fe_max}, but is {ert_fes}.")
392 ert_time_millis = try_int(ert_time_millis)
393 if ert_time_millis < success_time_millis.minimum:
394 raise ValueError(
395 "ert_time_millis must be >= "
396 f"{success_time_millis.minimum}, but "
397 f"is {ert_time_millis}.")
398 ert_time_max = ceil(total_time_millis.mean_arith * n)
399 if ert_time_millis > ert_time_max:
400 raise ValueError(
401 "ert_time_millis must be <= "
402 f"{ert_time_max}, but is {ert_time_millis}.")
403 else:
404 if success_fes is not None:
405 raise ValueError(
406 "If n_success<=0, then success_fes must be None, "
407 f"but it's a {type_name_of(success_fes)}.")
408 if success_time_millis is not None:
409 raise ValueError(
410 "If n_success<=0, then success_time_millis must be "
411 f"None, but it is a "
412 f"{type_name_of(success_time_millis)}.")
413 if ert_fes < inf:
414 raise ValueError(
415 "If n_success<=0, then ert_fes must "
416 f"be inf, but it's {ert_fes}.")
417 if ert_time_millis < inf:
418 raise ValueError(
419 "If n_success<=0, then ert_time_millis must "
420 f"be inf, but it's {ert_time_millis}.")
422 object.__setattr__(self, "goal_f", goal_f)
423 object.__setattr__(self, "best_f_scaled", best_f_scaled)
424 object.__setattr__(self, "n_success", n_success)
425 object.__setattr__(self, "success_fes", success_fes)
426 object.__setattr__(self, "success_time_millis", success_time_millis)
427 object.__setattr__(self, "ert_fes", ert_fes)
428 object.__setattr__(self, "ert_time_millis", ert_time_millis)
430 if isinstance(max_fes, SampleStatistics):
431 if max_fes.n != n:
432 raise ValueError(f"max_fes.n={max_fes.n} != n={n}")
433 max_fes_f: int | float | SampleStatistics = max_fes.compact(
434 needs_n=False)
435 if isinstance(max_fes_f, float):
436 raise type_error(max_fes_f, "max_fes", (
437 int, SampleStatistics, None))
438 max_fes = max_fes_f
439 if isinstance(max_fes, int):
440 if (max_fes < total_fes.maximum) or (max_fes < 0):
441 raise ValueError(f"0<max_fes must be >= "
442 f"{total_fes.maximum}, but is {max_fes}.")
443 elif isinstance(max_fes, SampleStatistics):
444 if (max_fes.minimum < total_fes.minimum) or (
445 max_fes.minimum <= 0):
446 raise ValueError(
447 f"0<max_fes.minimum must be >= {total_fes.minimum},"
448 f" but is {max_fes.minimum}.")
449 if max_fes.maximum < total_fes.maximum:
450 raise ValueError(
451 f"max_fes.maximum must be >= {total_fes.maximum},"
452 f" but is {max_fes.maximum}.")
453 elif max_fes is not None:
454 raise type_error(max_fes, "max_fes", (int, SampleStatistics, None))
455 object.__setattr__(self, "max_fes", max_fes)
457 if isinstance(max_time_millis, SampleStatistics):
458 if max_time_millis.n != n:
459 raise ValueError(
460 f"max_time_millis.n={max_time_millis.n} != n={n}")
461 max_time_millis_f: int | float | SampleStatistics = (
462 max_time_millis.compact(False))
463 if isinstance(max_time_millis_f, float):
464 raise type_error(max_time_millis_f, "max_time_millis", (
465 int, SampleStatistics, None))
466 if isinstance(max_time_millis, int):
467 _check_max_time_millis(max_time_millis,
468 total_fes.minimum,
469 total_time_millis.maximum)
470 elif isinstance(max_time_millis, SampleStatistics):
471 _check_max_time_millis(max_time_millis.minimum,
472 total_fes.minimum,
473 total_time_millis.minimum)
474 _check_max_time_millis(max_time_millis.maximum,
475 total_fes.minimum,
476 total_time_millis.maximum)
477 elif max_time_millis is not None:
478 raise type_error(max_time_millis, "max_time_millis",
479 (int, SampleStatistics, None))
480 object.__setattr__(self, "max_time_millis", max_time_millis)
482 def get_n(self) -> int:
483 """
484 Get the number of runs.
486 :returns: the number of runs.
487 """
488 if not isinstance(self, EndStatistics):
489 raise type_error(self, "self", EndStatistics)
490 return self.n
492 def get_best_f(self) -> SampleStatistics:
493 """
494 Get the statistics about the best objective value reached.
496 :returns: the statistics about the best objective value reached
497 """
498 if not isinstance(self, EndStatistics):
499 raise type_error(self, "self", EndStatistics)
500 return self.best_f
502 def get_last_improvement_fe(self) -> SampleStatistics:
503 """
504 Get the statistics about the last improvement FE.
506 :returns: the statistics about the last improvement FE
507 """
508 if not isinstance(self, EndStatistics):
509 raise type_error(self, "self", EndStatistics)
510 return self.last_improvement_fe
512 def get_last_improvement_time_millis(self) -> SampleStatistics:
513 """
514 Get the statistics about the last improvement time millis.
516 :returns: the statistics about the last improvement time millis
517 """
518 if not isinstance(self, EndStatistics):
519 raise type_error(self, "self", EndStatistics)
520 return self.last_improvement_time_millis
522 def get_total_fes(self) -> SampleStatistics:
523 """
524 Get the statistics about the total FEs.
526 :returns: the statistics about the total FEs
527 """
528 if not isinstance(self, EndStatistics):
529 raise type_error(self, "self", EndStatistics)
530 return self.total_fes
532 def get_total_time_millis(self) -> SampleStatistics:
533 """
534 Get the statistics about the total time millis.
536 :returns: the statistics about the total time millis
537 """
538 if not isinstance(self, EndStatistics):
539 raise type_error(self, "self", EndStatistics)
540 return self.total_time_millis
542 def get_goal_f(self) -> SampleStatistics | int | float | None:
543 """
544 Get the statistics about the goal objective value.
546 :returns: the statistics about the goal objective value
547 """
548 if not isinstance(self, EndStatistics):
549 raise type_error(self, "self", EndStatistics)
550 return self.goal_f
552 def get_best_f_scaled(self) -> SampleStatistics | None:
553 """
554 Get the statistics about the scaled best objective value.
556 :returns: the statistics about the scaled best objective value
557 """
558 if not isinstance(self, EndStatistics):
559 raise type_error(self, "self", EndStatistics)
560 return self.best_f_scaled
562 def get_n_success(self) -> int | None:
563 """
564 Get the number of successful runs.
566 :returns: the number of successful runs.
567 """
568 if not isinstance(self, EndStatistics):
569 raise type_error(self, "self", EndStatistics)
570 return self.n_success
572 def get_success_fes(self) -> SampleStatistics | None:
573 """
574 Get the statistics about the FEs until success of the successful runs.
576 :returns: the statistics about the FEs until success of the successful
577 runs
578 """
579 if not isinstance(self, EndStatistics):
580 raise type_error(self, "self", EndStatistics)
581 return self.success_fes
583 def get_success_time_millis(self) -> SampleStatistics | None:
584 """
585 Get the statistics about the ms until success of the successful runs.
587 :returns: the statistics about the ms until success of the successful
588 runs
589 """
590 if not isinstance(self, EndStatistics):
591 raise type_error(self, "self", EndStatistics)
592 return self.success_time_millis
594 def get_ert_fes(self) -> int | float | None:
595 """
596 Get the expected FEs until success.
598 :returns: the statistics about the expected FEs until success.
599 """
600 if not isinstance(self, EndStatistics):
601 raise type_error(self, "self", EndStatistics)
602 return self.ert_fes
604 def get_ert_time_millis(self) -> int | float | None:
605 """
606 Get the expected milliseconds until success.
608 :returns: the statistics about the expected milliseconds until
609 success.
610 """
611 if not isinstance(self, EndStatistics):
612 raise type_error(self, "self", EndStatistics)
613 return self.ert_time_millis
615 def get_max_fes(self) -> SampleStatistics | int | None:
616 """
617 Get the statistics about the maximum permitted FEs.
619 :returns: the statistics about the maximum permitted FEs
620 """
621 if not isinstance(self, EndStatistics):
622 raise type_error(self, "self", EndStatistics)
623 return self.max_fes
625 def get_max_time_millis(self) -> SampleStatistics | int | None:
626 """
627 Get the statistics about the maximum permitted runtime in ms.
629 :returns: the statistics about the maximum permitted runtime in ms
630 """
631 if not isinstance(self, EndStatistics):
632 raise type_error(self, "self", EndStatistics)
633 return self.max_time_millis
636def create(source: Iterable[EndResult]) -> EndStatistics:
637 """
638 Create an `EndStatistics` Record from an Iterable of `EndResult`.
640 :param source: the source
641 :return: the statistics
642 :rtype: EndStatistics
643 """
644 if not isinstance(source, Iterable):
645 raise type_error(source, "source", Iterable)
647 n: int = 0
648 best_f: list[int | float] = []
649 last_improvement_fe: list[int] = []
650 last_improvement_time_millis: list[int] = []
651 total_fes: list[int] = []
652 total_time_millis: list[int] = []
653 max_fes: list[int] | None = []
654 max_fes_same: bool = True
655 max_time_millis: list[int] | None = []
656 max_time_same: bool = True
657 goal_f: list[int | float] | None = []
658 goal_f_same: bool = True
659 best_f_scaled: list[float] | None = []
660 n_success: int | None = 0
661 success_fes: list[int] | None = []
662 success_times: list[int] | None = []
664 fes: int = 0
665 time: int = 0
666 algorithm: str | None = None
667 instance: str | None = None
668 objective: str | None = None
669 encoding: str | None = None
671 for er in source:
672 if not isinstance(er, EndResult):
673 raise type_error(er, "end result", EndResult)
674 if n == 0:
675 algorithm = er.algorithm
676 instance = er.instance
677 objective = er.objective
678 encoding = er.encoding
679 else:
680 if algorithm != er.algorithm:
681 algorithm = None
682 if instance != er.instance:
683 instance = None
684 if objective != er.objective:
685 objective = None
686 if encoding != er.encoding:
687 encoding = None
688 n += 1
689 best_f.append(er.best_f)
690 last_improvement_fe.append(er.last_improvement_fe)
691 last_improvement_time_millis.append(
692 er.last_improvement_time_millis)
693 total_fes.append(er.total_fes)
694 total_time_millis.append(er.total_time_millis)
695 if er.max_fes is None:
696 max_fes = None
697 elif max_fes is not None:
698 if n > 1:
699 max_fes_same = max_fes_same \
700 and (max_fes[-1] == er.max_fes)
701 max_fes.append(er.max_fes)
702 if er.max_time_millis is None:
703 max_time_millis = None
704 elif max_time_millis is not None:
705 if n > 1:
706 max_time_same = \
707 max_time_same \
708 and (max_time_millis[-1] == er.max_time_millis)
709 max_time_millis.append(er.max_time_millis)
711 if er.goal_f is None:
712 goal_f = None
713 best_f_scaled = None
714 n_success = None
715 success_fes = None
716 success_times = None
717 elif goal_f is not None:
718 if n > 1:
719 goal_f_same = goal_f_same and (goal_f[-1] == er.goal_f)
720 goal_f.append(er.goal_f)
722 if er.goal_f <= 0:
723 best_f_scaled = None
724 elif best_f_scaled is not None:
725 best_f_scaled.append(er.best_f / er.goal_f)
727 if er.best_f <= er.goal_f:
728 n_success += 1
729 success_fes.append(er.last_improvement_fe)
730 success_times.append(er.last_improvement_time_millis)
731 fes += er.last_improvement_fe
732 time += er.last_improvement_time_millis
733 else:
734 fes += er.total_fes
735 time += er.total_time_millis
736 if n <= 0:
737 raise ValueError("There must be at least one end result record.")
739 return EndStatistics(
740 algorithm,
741 instance,
742 objective,
743 encoding,
744 n,
745 from_samples(best_f),
746 from_samples(last_improvement_fe),
747 from_samples(last_improvement_time_millis),
748 from_samples(total_fes),
749 from_samples(total_time_millis),
750 None if (goal_f is None)
751 else (goal_f[0] if goal_f_same else from_samples(goal_f)),
752 None if (best_f_scaled is None)
753 else from_samples(best_f_scaled),
754 n_success,
755 None if (n_success is None) or (n_success <= 0)
756 else from_samples(success_fes),
757 None if (n_success is None) or (n_success <= 0)
758 else from_samples(success_times),
759 None if (n_success is None)
760 else (inf if (n_success <= 0) else try_int_div(fes, n_success)),
761 None if (n_success is None) else
762 (inf if (n_success <= 0) else try_int_div(time, n_success)),
763 None if max_fes is None else
764 (max_fes[0] if max_fes_same else from_samples(max_fes)),
765 None if max_time_millis is None
766 else (max_time_millis[0] if max_time_same
767 else from_samples(max_time_millis)))
770def from_end_results(source: Iterable[EndResult],
771 join_all_algorithms: bool = False,
772 join_all_instances: bool = False,
773 join_all_objectives: bool = False,
774 join_all_encodings: bool = False) \
775 -> Generator[EndStatistics, None, None]:
776 """
777 Aggregate statistics over a stream of end results.
779 :param source: the stream of end results
780 :param join_all_algorithms: should the statistics be aggregated
781 over all algorithms
782 :param join_all_instances: should the statistics be aggregated
783 over all algorithms
784 :param join_all_objectives: should the statistics be aggregated over
785 all objectives?
786 :param join_all_encodings: should statistics be aggregated over all
787 encodings
788 :returns: iterates over the generated end statistics records
789 """
790 if not isinstance(source, Iterable):
791 raise type_error(source, "source", Iterable)
792 if not isinstance(join_all_algorithms, bool):
793 raise type_error(join_all_algorithms,
794 "join_all_algorithms", bool)
795 if not isinstance(join_all_instances, bool):
796 raise type_error(join_all_instances, "join_all_instances", bool)
797 if not isinstance(join_all_objectives, bool):
798 raise type_error(join_all_objectives, "join_all_objectives", bool)
799 if not isinstance(join_all_encodings, bool):
800 raise type_error(join_all_encodings, "join_all_encodings", bool)
802 if (join_all_algorithms and join_all_instances
803 and join_all_objectives and join_all_encodings):
804 yield create(source)
805 return
807 sorter: dict[tuple[str, str, str, str], list[EndResult]] = {}
808 for er in source:
809 if not isinstance(er, EndResult):
810 raise type_error(source, "end results from source",
811 EndResult)
812 key = ("" if join_all_algorithms else er.algorithm,
813 "" if join_all_instances else er.instance,
814 "" if join_all_objectives else er.objective,
815 "" if join_all_encodings else (
816 "" if er.encoding is None else er.encoding))
817 if key in sorter:
818 lst = sorter[key]
819 else:
820 lst = []
821 sorter[key] = lst
822 lst.append(er)
824 if len(sorter) <= 0:
825 raise ValueError("source must not be empty")
827 if len(sorter) > 1:
828 for key in sorted(sorter.keys()):
829 yield create(sorter[key])
830 else:
831 yield create(next(iter(sorter.values()))) #: pylint: disable=R1708
834def to_csv(data: EndStatistics | Iterable[EndStatistics],
835 file: str) -> Path:
836 """
837 Store a set of :class:`EndStatistics` in a CSV file.
839 :param data: the data to store
840 :param file: the file to generate
841 :return: the path to the generated CSV file
842 """
843 path: Final[Path] = Path(file)
844 logger(f"Writing end result statistics to CSV file {path!r}.")
845 path.ensure_parent_dir_exists()
846 with path.open_for_write() as wt:
847 write_lines(CsvWriter.write(
848 (data, ) if isinstance(data, EndStatistics) else data), wt)
850 logger(f"Done writing end result statistics to CSV file {path!r}.")
851 return path
854def from_csv(file: str) -> Generator[EndStatistics, None, None]:
855 """
856 Parse a CSV file and collect all encountered :class:`EndStatistics`.
858 :param file: the file to parse
859 :returns: the iterator with the results
860 """
861 path: Final[Path] = file_path(file)
862 logger(f"Begin reading end result statistics from CSV file {path!r}.")
863 with path.open_for_read() as rd:
864 yield from CsvReader.read(rows=rd)
865 logger("Finished reading end result statistics from CSV "
866 f"file {path!r}.")
869#: the internal getters that can work directly
870__PROPERTIES: Final[Callable[[str], Callable[[
871 EndStatistics], SampleStatistics | int | float | None] | None]] = {
872 KEY_N: EndStatistics.get_n,
873 KEY_N_SUCCESS: EndStatistics.get_n_success,
874 KEY_ERT_FES: EndStatistics.get_ert_fes,
875 KEY_ERT_TIME_MILLIS: EndStatistics.get_ert_time_millis,
876 KEY_GOAL_F: EndStatistics.get_goal_f,
877 KEY_MAX_TIME_MILLIS: EndStatistics.get_max_time_millis,
878 KEY_MAX_FES: EndStatistics.get_max_fes,
879 KEY_BEST_F: EndStatistics.get_best_f,
880 F_NAME_RAW: EndStatistics.get_best_f,
881 KEY_LAST_IMPROVEMENT_FE: EndStatistics.get_last_improvement_fe,
882 "last improvement FE": EndStatistics.get_last_improvement_fe,
883 KEY_LAST_IMPROVEMENT_TIME_MILLIS:
884 EndStatistics.get_last_improvement_time_millis,
885 "last improvement ms": EndStatistics.get_last_improvement_time_millis,
886 KEY_BEST_F_SCALED: EndStatistics.get_best_f_scaled,
887 KEY_SUCCESS_FES: EndStatistics.get_success_fes,
888 KEY_SUCCESS_TIME_MILLIS: EndStatistics.get_success_time_millis,
889 F_NAME_SCALED: EndStatistics.get_best_f_scaled,
890 KEY_TOTAL_FES: EndStatistics.get_total_fes,
891 "fes": EndStatistics.get_total_fes,
892 KEY_TOTAL_TIME_MILLIS: EndStatistics.get_total_time_millis,
893 "ms": EndStatistics.get_total_time_millis,
894 "f": EndStatistics.get_best_f,
895 "budgetFEs": EndStatistics.get_max_fes,
896 "budgetMS": EndStatistics.get_max_time_millis,
897}.get
899#: the success keys
900__SUCCESS_KEYS: Final[Callable[[str], bool]] = {
901 KEY_SUCCESS_FES, KEY_SUCCESS_TIME_MILLIS,
902}.__contains__
904#: the internal static getters
905__STATIC: Final[dict[str, Callable[[EndStatistics], int | float | None]]] = {
906 KEY_N: EndStatistics.get_n,
907 KEY_N_SUCCESS: EndStatistics.get_n_success,
908 KEY_ERT_FES: EndStatistics.get_ert_fes,
909 KEY_ERT_TIME_MILLIS: EndStatistics.get_ert_time_millis,
910}
913def getter(dimension: str) -> Callable[[EndStatistics], int | float | None]:
914 """
915 Create a function that obtains the given dimension from EndStatistics.
917 :param dimension: the dimension
918 :returns: a callable that returns the value corresponding to the
919 dimension
920 """
921 dimension = str.strip(dimension)
922 direct: Callable[[EndStatistics], int | float | None] = \
923 __STATIC.get(dimension)
924 if direct is not None:
925 return direct
927 names: Final[list[str]] = str.split(str.strip(dimension), SCOPE_SEPARATOR)
928 n_names: Final[int] = list.__len__(names)
929 if not (0 < n_names < 3):
930 raise ValueError(
931 f"Invalid name combination {dimension!r} -> {names!r}.")
932 getter_1: Final[Callable[[
933 EndStatistics], int | float | SampleStatistics | None] | None] = \
934 __PROPERTIES(names[0])
935 if getter_1 is None:
936 raise ValueError(f"Invalid dimension {names[0]!r} in {dimension!r}.")
937 getter_2: Final[Callable[[
938 SampleStatistics], int | float | None]] = \
939 stat_getter(names[1] if n_names > 1 else KEY_MEAN_ARITH)
941 if getter_2 is stat_getter(KEY_STDDEV): # it is sd
942 n_prop: Final[Callable[[EndStatistics], int | None]] = \
943 EndStatistics.get_n_success if __SUCCESS_KEYS(
944 names[0]) else EndStatistics.get_n
946 def __combo_sd(
947 data: EndStatistics, __g1=getter_1, __g2=getter_2,
948 __n=n_prop) -> int | float | None:
949 val: int | float | SampleStatistics | None = __g1(data)
950 if val is None:
951 return None
952 if isinstance(val, int | float):
953 n = __n(data)
954 return None if (n is None) or (n <= 0) else 0
955 return __g2(val)
956 direct = cast("Callable[[EndStatistics], int | float | None]",
957 __combo_sd)
958 else: # any other form of mean or statistic
960 def __combo_no_sd(data: EndStatistics,
961 __g1=getter_1, __g2=getter_2) -> int | float | None:
962 val: int | float | SampleStatistics | None = __g1(data)
963 if (val is None) or (isinstance(val, int | float)):
964 return val
965 return __g2(val)
966 direct = cast("Callable[[EndStatistics], int | float | None]",
967 __combo_no_sd)
969 __STATIC[dimension] = direct
970 return direct
973def _to_csv_writer(
974 data: Iterable[EndStatistics],
975 get_func: Callable[
976 [EndStatistics], SampleStatistics | int | float | None],
977 n_func: Callable[[EndStatistics], int],
978 scope: str | None = None,
979 what_short: str | None = None,
980 what_long: str | None = None) -> StatWriter | None:
981 """
982 Get a CSV Writer for the given data subset.
984 :param data: the data iterator
985 :param get_func: the getter for the value
986 :param n_func: the n-getter
987 :param scope: the scope to use
988 :param what_short: the short description
989 :param what_long: the long description
990 :returns: the writer, if there was any associated data
991 """
992 refined: list[tuple[SampleStatistics | int | float | None, int]] = [
993 v for v in ((get_func(es), n_func(es)) for es in data)
994 if v[0] is not None]
995 if list.__len__(refined) <= 0:
996 return None
997 return StatWriter(data=starmap(from_single_value, refined),
998 scope=scope, n_not_needed=True, what_short=what_short,
999 what_long=what_long)
1002class CsvWriter(CsvWriterBase[EndStatistics]):
1003 """A class for CSV writing of :class:`EndStatistics`."""
1005 def __init__(self, data: Iterable[EndStatistics],
1006 scope: str | None = None) -> None:
1007 """
1008 Initialize the csv writer.
1010 :param scope: the prefix to be pre-pended to all columns
1011 :param data: the data to write
1012 """
1013 data = reiterable(data)
1014 super().__init__(data, scope)
1015 checker: int = 127
1016 has_algorithm: bool = False
1017 has_instance: bool = False
1018 has_objective: bool = False
1019 has_encoding: bool = False
1020 has_n_success: bool = False
1021 has_ert_fes: bool = False
1022 has_ert_time_millis: bool = False
1023 for es in data:
1024 if es.algorithm is not None:
1025 has_algorithm = True
1026 checker &= ~1
1027 if es.instance is not None:
1028 has_instance = True
1029 checker &= ~2
1030 if es.objective is not None:
1031 has_objective = True
1032 checker &= ~4
1033 if es.encoding is not None:
1034 has_encoding = True
1035 checker &= ~8
1036 if es.n_success is not None:
1037 has_n_success = True
1038 checker &= ~16
1039 if es.ert_fes is not None:
1040 has_ert_fes = True
1041 checker &= ~32
1042 if es.ert_time_millis is not None:
1043 has_ert_time_millis = True
1044 checker &= ~64
1045 if checker == 0:
1046 break
1048 #: do we put the algorithm column?
1049 self.__has_algorithm: Final[bool] = has_algorithm
1050 #: do we put the instance column?
1051 self.__has_instance: Final[bool] = has_instance
1052 #: do we put the objective column?
1053 self.__has_objective: Final[bool] = has_objective
1054 #: do we put the encoding column?
1055 self.__has_encoding: Final[bool] = has_encoding
1056 #: do we put the n_success column?
1057 self.__has_n_success: Final[bool] = has_n_success
1058 #: do we put the ert-fes column?
1059 self.__has_ert_fes: Final[bool] = has_ert_fes
1060 #: do we put the ert time millis column?
1061 self.__has_ert_time_millis: Final[bool] = has_ert_time_millis
1063 self.__goal_f: Final[StatWriter | None] = _to_csv_writer(
1064 data, EndStatistics.get_goal_f, EndStatistics.get_n,
1065 csv_scope(scope, KEY_GOAL_F), KEY_GOAL_F,
1066 "the goal objective value after which the runs can stop")
1067 self.__best_f_scaled: Final[StatWriter | None] = _to_csv_writer(
1068 data, EndStatistics.get_best_f_scaled, EndStatistics.get_n,
1069 csv_scope(scope, KEY_BEST_F_SCALED), KEY_BEST_F_SCALED,
1070 f"best objective value reached ({KEY_BEST_F}), divided by"
1071 f" the goal objective value ({KEY_GOAL_F})")
1072 self.__success_fes: Final[StatWriter | None] = _to_csv_writer(
1073 data, EndStatistics.get_success_fes, EndStatistics.get_n_success,
1074 csv_scope(scope, KEY_SUCCESS_FES), KEY_SUCCESS_FES,
1075 f"the FEs needed to reach {KEY_GOAL_F} for the successful runs")
1076 self.__success_time_millis: Final[StatWriter | None] = _to_csv_writer(
1077 data, EndStatistics.get_success_time_millis,
1078 EndStatistics.get_n_success, csv_scope(
1079 scope, KEY_SUCCESS_TIME_MILLIS), KEY_SUCCESS_TIME_MILLIS,
1080 f"the milliseconds needed to reach {KEY_GOAL_F} for the "
1081 "successful runs")
1082 self.__max_fes: Final[StatWriter | None] = _to_csv_writer(
1083 data, EndStatistics.get_max_fes, EndStatistics.get_n,
1084 csv_scope(scope, KEY_MAX_FES), KEY_MAX_FES,
1085 "the maximum number of FEs in the computational budget")
1086 self.__max_time_millis: Final[StatWriter | None] = _to_csv_writer(
1087 data, EndStatistics.get_max_time_millis, EndStatistics.get_n,
1088 csv_scope(scope, KEY_MAX_TIME_MILLIS), KEY_MAX_TIME_MILLIS,
1089 "the maximum milliseconds per run in the computational budget")
1091 #: the best objective value reached
1092 self.__best_f: Final[StatWriter] = StatWriter(
1093 data=map(EndStatistics.get_best_f, data),
1094 scope=csv_scope(scope, KEY_BEST_F),
1095 n_not_needed=True, what_short=KEY_BEST_F,
1096 what_long="the best objective value reached per run")
1097 #: the FE when the last improvement happened
1098 self.__life: Final[StatWriter] = StatWriter(
1099 data=map(EndStatistics.get_last_improvement_fe, data),
1100 scope=csv_scope(scope, KEY_LAST_IMPROVEMENT_FE),
1101 n_not_needed=True, what_short=KEY_LAST_IMPROVEMENT_FE,
1102 what_long="the FE when the last improvement happened in a run")
1103 #: the milliseconds when the last improvement happened
1104 self.__lims: Final[StatWriter] = StatWriter(
1105 data=map(EndStatistics.get_last_improvement_time_millis, data),
1106 scope=csv_scope(
1107 scope, KEY_LAST_IMPROVEMENT_TIME_MILLIS),
1108 n_not_needed=True, what_short=KEY_LAST_IMPROVEMENT_TIME_MILLIS,
1109 what_long="the millisecond when the last "
1110 "improvement happened in a run")
1111 #: the total FEs
1112 self.__total_fes: Final[StatWriter] = StatWriter(
1113 data=map(EndStatistics.get_total_fes, data),
1114 scope=csv_scope(scope, KEY_TOTAL_FES),
1115 n_not_needed=True, what_short=KEY_TOTAL_FES,
1116 what_long="the total FEs consumed by the runs")
1117 #: the total milliseconds
1118 self.__total_ms: Final[StatWriter] = StatWriter(
1119 data=map(EndStatistics.get_total_time_millis, data),
1120 scope=csv_scope(scope, KEY_TOTAL_TIME_MILLIS),
1121 n_not_needed=True, what_short=KEY_TOTAL_TIME_MILLIS,
1122 what_long="the total millisecond consumed by a run")
1124 def get_column_titles(self) -> Iterator[str]:
1125 """
1126 Get the column titles.
1128 :returns: the column titles
1129 """
1130 p: Final[str] = self.scope
1131 if self.__has_algorithm:
1132 yield csv_scope(p, KEY_ALGORITHM)
1133 if self.__has_instance:
1134 yield csv_scope(p, KEY_INSTANCE)
1135 if self.__has_objective:
1136 yield csv_scope(p, KEY_OBJECTIVE_FUNCTION)
1137 if self.__has_encoding:
1138 yield csv_scope(p, KEY_ENCODING)
1139 yield csv_scope(p, KEY_N)
1140 yield from self.__best_f.get_column_titles()
1141 yield from self.__life.get_column_titles()
1142 yield from self.__lims.get_column_titles()
1143 yield from self.__total_fes.get_column_titles()
1144 yield from self.__total_ms.get_column_titles()
1145 if self.__goal_f is not None:
1146 yield from self.__goal_f.get_column_titles()
1147 if self.__best_f_scaled is not None:
1148 yield from self.__best_f_scaled.get_column_titles()
1149 if self.__has_n_success:
1150 yield csv_scope(p, KEY_N_SUCCESS)
1151 if self.__success_fes is not None:
1152 yield from self.__success_fes.get_column_titles()
1153 if self.__success_time_millis is not None:
1154 yield from self.__success_time_millis.get_column_titles()
1155 if self.__has_ert_fes:
1156 yield csv_scope(p, KEY_ERT_FES)
1157 if self.__has_ert_time_millis:
1158 yield csv_scope(p, KEY_ERT_TIME_MILLIS)
1159 if self.__max_fes is not None:
1160 yield from self.__max_fes.get_column_titles()
1161 if self.__max_time_millis is not None:
1162 yield from self.__max_time_millis.get_column_titles()
1164 def get_row(self, data: EndStatistics) -> Iterable[str]:
1165 """
1166 Render a single end result record to a CSV row.
1168 :param data: the end result record
1169 :returns: the row strings
1170 """
1171 if self.__has_algorithm:
1172 yield "" if data.algorithm is None else data.algorithm
1173 if self.__has_instance:
1174 yield "" if data.instance is None else data.instance
1175 if self.__has_objective:
1176 yield "" if data.objective is None else data.objective
1177 if self.__has_encoding:
1178 yield "" if data.encoding is None else data.encoding
1179 yield str(data.n)
1180 yield from self.__best_f.get_row(data.best_f)
1181 yield from self.__life.get_row(data.last_improvement_fe)
1182 yield from self.__lims.get_row(data.last_improvement_time_millis)
1183 yield from self.__total_fes.get_row(data.total_fes)
1184 yield from self.__total_ms.get_row(data.total_time_millis)
1185 if self.__goal_f is not None:
1186 yield from self.__goal_f.get_optional_row(data.goal_f, data.n)
1187 if self.__best_f_scaled is not None:
1188 yield from self.__best_f_scaled.get_optional_row(
1189 data.best_f_scaled, data.n)
1190 if self.__has_n_success:
1191 yield str(data.n_success)
1192 if self.__success_fes is not None:
1193 yield from self.__success_fes.get_optional_row(
1194 data.success_fes, data.n_success)
1195 if self.__success_time_millis is not None:
1196 yield from self.__success_time_millis.get_optional_row(
1197 data.success_time_millis, data.n_success)
1198 if self.__has_ert_fes:
1199 yield num_or_none_to_str(data.ert_fes)
1200 if self.__has_ert_time_millis:
1201 yield num_or_none_to_str(data.ert_time_millis)
1202 if self.__max_fes is not None:
1203 yield from self.__max_fes.get_optional_row(data.max_fes, data.n)
1204 if self.__max_time_millis is not None:
1205 yield from self.__max_time_millis.get_optional_row(
1206 data.max_time_millis, data.n)
1208 def get_header_comments(self) -> Iterable[str]:
1209 """
1210 Get any possible header comments.
1212 :returns: the header comments
1213 """
1214 return ("Experiment End Results Statistics",
1215 "See the description at the bottom of the file.")
1217 def get_footer_comments(self) -> Iterable[str]:
1218 """
1219 Get any possible footer comments.
1221 :param dest: the destination
1222 """
1223 yield ""
1224 scope: Final[str | None] = self.scope
1226 yield ("This file presents statistics gathered over multiple runs "
1227 "of optimization algorithms applied to problem instances.")
1228 if scope:
1229 yield ("All end result statistics records start with prefix "
1230 f"{scope}{SCOPE_SEPARATOR}.")
1231 if self.__has_algorithm:
1232 yield f"{csv_scope(scope, KEY_ALGORITHM)}: {DESC_ALGORITHM}"
1233 if self.__has_instance:
1234 yield f"{csv_scope(scope, KEY_INSTANCE)}: {DESC_INSTANCE}"
1235 if self.__has_objective:
1236 yield (f"{csv_scope(scope, KEY_OBJECTIVE_FUNCTION)}:"
1237 f" {DESC_OBJECTIVE_FUNCTION}")
1238 if self.__has_encoding:
1239 yield f"{csv_scope(scope, KEY_ENCODING)}: {DESC_ENCODING}"
1240 yield (f"{csv_scope(scope, KEY_N)}: the number of runs that were "
1241 f"performed for the given setup.")
1243 yield from self.__best_f.get_footer_comments()
1244 yield f"In summary {csv_scope(scope, KEY_BEST_F)} is {DESC_BEST_F}."
1246 yield from self.__life.get_footer_comments()
1247 yield (f"In summary {csv_scope(scope, KEY_LAST_IMPROVEMENT_FE)} "
1248 f"is {DESC_LAST_IMPROVEMENT_FE}.")
1250 yield from self.__lims.get_footer_comments()
1251 yield ("In summary "
1252 f"{csv_scope(scope, KEY_LAST_IMPROVEMENT_TIME_MILLIS)} "
1253 f"is {DESC_LAST_IMPROVEMENT_TIME_MILLIS}.")
1255 yield from self.__total_fes.get_footer_comments()
1256 yield (f"In summary {csv_scope(scope, KEY_TOTAL_FES)} "
1257 f"is {DESC_TOTAL_FES}.")
1259 yield from self.__total_ms.get_footer_comments()
1260 yield (f"In summary {csv_scope(scope, KEY_TOTAL_TIME_MILLIS)} "
1261 f"is {DESC_TOTAL_TIME_MILLIS}.")
1263 if self.__goal_f is not None:
1264 yield from self.__goal_f.get_footer_comments()
1265 yield (f"In summary {csv_scope(scope, KEY_GOAL_F)} is"
1266 f" {DESC_GOAL_F}.")
1268 if self.__best_f_scaled is not None:
1269 yield from self.__best_f_scaled.get_footer_comments()
1270 yield (f"In summary {csv_scope(scope, KEY_BEST_F_SCALED)} "
1271 "describes the best objective value reached ("
1272 f"{csv_scope(scope, KEY_BEST_F)}) divided by the goal "
1273 f"objective value ({csv_scope(scope, KEY_GOAL_F)}).")
1275 if self.__has_n_success:
1276 yield (f"{csv_scope(scope, KEY_N_SUCCESS)} is the number of "
1277 "runs that reached goal objective value "
1278 f"{csv_scope(scope, KEY_GOAL_F)}. Obviously, "
1279 f"0<={csv_scope(scope, KEY_N_SUCCESS)}<="
1280 f"{csv_scope(scope, KEY_N)}.")
1281 if self.__success_fes is not None:
1282 yield from self.__success_fes.get_footer_comments()
1283 yield (f"{csv_scope(scope, KEY_SUCCESS_FES)} offers statistics "
1284 "about the number of FEs that the 0<="
1285 f"{csv_scope(scope, KEY_N_SUCCESS)}<="
1286 f"{csv_scope(scope, KEY_N)} successful runs needed to "
1287 "reach the goal objective value "
1288 f"{csv_scope(scope, KEY_GOAL_F)}.")
1290 if self.__success_time_millis is not None:
1291 yield from self.__success_fes.get_footer_comments()
1292 yield (f"{csv_scope(scope, KEY_SUCCESS_TIME_MILLIS)} offers "
1293 "statistics about the number of milliseconds of clock time"
1294 f" that the 0<={csv_scope(scope, KEY_N_SUCCESS)}<="
1295 f"{csv_scope(scope, KEY_N)} successful runs needed to "
1296 "reach the goal objective value "
1297 f"{csv_scope(scope, KEY_GOAL_F)}.")
1299 if self.__has_ert_fes:
1300 yield (f"{csv_scope(scope, KEY_ERT_FES)} is the empirical "
1301 "estimate of the number of FEs to solve the problem. It "
1302 "can be approximated by dividing the sum of "
1303 f"{csv_scope(scope, KEY_TOTAL_FES)} over all runs by the "
1304 f"number {csv_scope(scope, KEY_N_SUCCESS)} of successful "
1305 "runs.")
1307 if self.__has_ert_time_millis:
1308 yield (f"{csv_scope(scope, KEY_ERT_TIME_MILLIS)} is the empirical"
1309 " estimate of the number of FEs to solve the problem. It "
1310 "can be approximated by dividing the sum of "
1311 f"{csv_scope(scope, KEY_TOTAL_TIME_MILLIS)} over all runs "
1312 f"by the number {csv_scope(scope, KEY_N_SUCCESS)} of "
1313 "successful runs.")
1315 if self.__max_fes is not None:
1316 yield from self.__max_fes.get_footer_comments()
1317 yield (f"In summary {csv_scope(scope, KEY_MAX_FES)} is"
1318 f" {DESC_MAX_FES}.")
1319 if self.__max_time_millis is not None:
1320 yield from self.__max_time_millis.get_footer_comments()
1321 yield (f"In summary {csv_scope(scope, KEY_MAX_TIME_MILLIS)} is"
1322 f" {DESC_MAX_TIME_MILLIS}.")
1324 def get_footer_bottom_comments(self) -> Iterable[str]:
1325 """
1326 Get the footer bottom comments.
1328 :returns: the footer comments
1329 """
1330 yield from motipy_footer_bottom_comments(
1331 self, ("The end statistics data is produced using module "
1332 "moptipy.evaluation.end_statistics."))
1333 yield from StatWriter.get_footer_bottom_comments(self.__best_f)
1336class CsvReader(CsvReaderBase[EndStatistics]):
1337 """A csv parser for end results."""
1339 def __init__(self, columns: dict[str, int]) -> None:
1340 """
1341 Create a CSV parser for :class:`EndResult`.
1343 :param columns: the columns
1344 """
1345 super().__init__(columns)
1346 #: the index of the algorithm column, if any
1347 self.__idx_algorithm: Final[int | None] = csv_column_or_none(
1348 columns, KEY_ALGORITHM)
1349 #: the index of the instance column, if any
1350 self.__idx_instance: Final[int | None] = csv_column_or_none(
1351 columns, KEY_INSTANCE)
1352 #: the index of the objective column, if any
1353 self.__idx_objective: Final[int | None] = csv_column_or_none(
1354 columns, KEY_OBJECTIVE_FUNCTION)
1355 #: the index of the encoding column, if any
1356 self.__idx_encoding: Final[int | None] = csv_column_or_none(
1357 columns, KEY_ENCODING)
1359 #: the index of the `N` column, i.e., where the number of runs is
1360 #: stored
1361 self.idx_n: Final[int] = csv_column(columns, KEY_N, True)
1363 n_key: Final[tuple[tuple[str, int]]] = ((KEY_N, self.idx_n), )
1364 #: the reader for the best-objective-value-reached statistics
1365 self.__best_f: Final[StatReader] = csv_select_scope(
1366 StatReader, columns, KEY_BEST_F, n_key)
1367 #: the reader for the last improvement FE statistics
1368 self.__life: Final[StatReader] = csv_select_scope(
1369 StatReader, columns, KEY_LAST_IMPROVEMENT_FE, n_key)
1370 #: the reader for the last improvement millisecond index statistics
1371 self.__lims: Final[StatReader] = csv_select_scope(
1372 StatReader, columns, KEY_LAST_IMPROVEMENT_TIME_MILLIS, n_key)
1373 #: the reader for the total FEs statistics
1374 self.__total_fes: Final[StatReader] = csv_select_scope(
1375 StatReader, columns, KEY_TOTAL_FES, n_key)
1376 #: the reader for the total milliseconds consumed statistics
1377 self.__total_ms: Final[StatReader] = csv_select_scope(
1378 StatReader, columns, KEY_TOTAL_TIME_MILLIS, n_key)
1380 #: the reader for the goal objective value statistics, if any
1381 self.__goal_f: Final[StatReader | None] = csv_select_scope_or_none(
1382 StatReader, columns, KEY_GOAL_F, n_key)
1383 #: the reader for the best-f / goal-f statistics, if any
1384 self.__best_f_scaled: Final[StatReader | None] = \
1385 csv_select_scope_or_none(
1386 StatReader, columns, KEY_BEST_F_SCALED, n_key)
1388 #: the index of the column where the number of successful runs is
1389 #: stored
1390 self.__idx_n_success: Final[int | None] = csv_column_or_none(
1391 columns, KEY_N_SUCCESS)
1392 succ_key: Final[tuple[tuple[str, int], ...]] = () \
1393 if self.__idx_n_success is None else (
1394 (KEY_N, self.__idx_n_success), )
1395 #: the reader for the success FE data, if any
1396 self.__success_fes: Final[StatReader | None] = \
1397 None if self.__idx_n_success is None else \
1398 csv_select_scope_or_none(
1399 StatReader, columns, KEY_SUCCESS_FES, succ_key)
1400 #: the reader for the success time milliseconds data, if any
1401 self.__success_time_millis: Final[StatReader | None] = \
1402 None if self.__idx_n_success is None else \
1403 csv_select_scope_or_none(
1404 StatReader, columns, KEY_SUCCESS_TIME_MILLIS, succ_key)
1406 #: the index of the expected FEs until success
1407 self.__idx_ert_fes: Final[int | None] = csv_column_or_none(
1408 columns, KEY_ERT_FES)
1409 #: the index of the expected milliseconds until success
1410 self.__idx_ert_time_millis: Final[int | None] = csv_column_or_none(
1411 columns, KEY_ERT_TIME_MILLIS)
1413 #: the columns with the maximum FE-based budget statistics
1414 self.__max_fes: Final[StatReader | None] = csv_select_scope_or_none(
1415 StatReader, columns, KEY_MAX_FES, n_key)
1416 #: the columns with the maximum time-based budget statistics
1417 self.__max_time_millis: Final[StatReader | None] = \
1418 csv_select_scope_or_none(
1419 StatReader, columns, KEY_MAX_TIME_MILLIS, n_key)
1421 def parse_row(self, data: list[str]) -> EndStatistics:
1422 """
1423 Parse a row of data.
1425 :param data: the data row
1426 :return: the end result statistics
1427 """
1428 return EndStatistics(
1429 algorithm=csv_str_or_none(data, self.__idx_algorithm),
1430 instance=csv_str_or_none(data, self.__idx_instance),
1431 objective=csv_str_or_none(data, self.__idx_objective),
1432 encoding=csv_str_or_none(data, self.__idx_encoding),
1433 n=int(data[self.idx_n]),
1434 best_f=self.__best_f.parse_row(data),
1435 last_improvement_fe=self.__life.parse_row(data),
1436 last_improvement_time_millis=self.__lims.parse_row(data),
1437 total_fes=self.__total_fes.parse_row(data),
1438 total_time_millis=self.__total_ms.parse_row(data),
1439 goal_f=StatReader.parse_optional_row(self.__goal_f, data),
1440 best_f_scaled=StatReader.parse_optional_row(
1441 self.__best_f_scaled, data),
1442 n_success=csv_val_or_none(data, self.__idx_n_success, int),
1443 success_fes=StatReader.parse_optional_row(
1444 self.__success_fes, data),
1445 success_time_millis=StatReader.parse_optional_row(
1446 self.__success_time_millis, data),
1447 ert_fes=csv_val_or_none(data, self.__idx_ert_fes, str_to_num),
1448 ert_time_millis=csv_val_or_none(
1449 data, self.__idx_ert_time_millis, str_to_num),
1450 max_fes=StatReader.parse_optional_row(self.__max_fes, data),
1451 max_time_millis=StatReader.parse_optional_row(
1452 self.__max_time_millis, data),
1453 )
1456@dataclass(frozen=True, init=False, order=False, eq=False)
1457class __PvEndStatistics(EndStatistics):
1458 """Aggregated end statistics."""
1460 #: the value of the parameter over which it is aggregated
1461 pv: int | float
1463 def __init__(self, es: EndStatistics, pv: int | float):
1464 """
1465 Create the end statistics of an experiment-setup combination.
1467 :param es: the original end statistics object
1468 :param pv: the parameter value
1469 """
1470 super().__init__(
1471 es.algorithm, es.instance, es.objective, es.encoding, es.n,
1472 es.best_f, es.last_improvement_fe,
1473 es.last_improvement_time_millis, es.total_fes,
1474 es.total_time_millis, es.goal_f, es.best_f_scaled, es.n_success,
1475 es.success_fes, es.success_time_millis, es.ert_fes,
1476 es.ert_time_millis, es.max_fes, es.max_time_millis)
1477 if not isinstance(pv, (int | float)):
1478 raise type_error(pv, "pv", (int, float))
1479 if not isfinite(pv):
1480 raise ValueError(f"got {pv=}")
1481 object.__setattr__(self, "pv", pv)
1483 def get_param_value(self) -> int | float:
1484 """
1485 Get the parameter value.
1487 :return: the parameter value
1488 """
1489 return self.pv
1492def aggregate_over_parameter(
1493 data: Iterable[EndResult],
1494 param_value: Callable[[EndResult], int | float],
1495 join_all_algorithms: bool = False,
1496 join_all_instances: bool = False,
1497 join_all_objectives: bool = False,
1498 join_all_encodings: bool = False) -> tuple[
1499 Callable[[EndStatistics], int | float], Iterable[EndStatistics]]:
1500 """
1501 Aggregate a stream of data into groups based on a parameter.
1503 :param data: the source data
1504 :param param_value: the function obtaining a parameter value
1505 :param join_all_algorithms: should the statistics be aggregated
1506 over all algorithms
1507 :param join_all_instances: should the statistics be aggregated
1508 over all algorithms
1509 :param join_all_objectives: should the statistics be aggregated over
1510 all objectives?
1511 :param join_all_encodings: should statistics be aggregated over all
1512 encodings
1513 """
1514 param_map: Final[dict[int | float, list[EndResult]]] = {}
1515 for er in data:
1516 pv = param_value(er)
1517 if not isinstance(pv, (int | float)):
1518 raise type_error(pv, f"param_value{er}", (int, float))
1519 if not isfinite(pv):
1520 raise ValueError(f"got {pv} = param({er})")
1521 if pv in param_map:
1522 param_map[pv].append(er)
1523 else:
1524 param_map[pv] = [er]
1525 if dict.__len__(param_map) <= 0:
1526 raise ValueError("Did not encounter any data.")
1528 stats: Final[list[EndStatistics]] = []
1529 for pv in sorted(param_map.keys()):
1530 for ess in from_end_results(
1531 param_map[pv], join_all_algorithms, join_all_instances,
1532 join_all_objectives, join_all_encodings):
1533 stats.append(__PvEndStatistics(ess, pv))
1534 return cast("Callable[[EndStatistics], int | float]",
1535 __PvEndStatistics.get_param_value), tuple(stats)
1538# Run end-results to stat file if executed as script
1539if __name__ == "__main__":
1540 parser: Final[argparse.ArgumentParser] = moptipy_argparser(
1541 __file__, "Build an end-results statistics CSV file.",
1542 "This program creates a CSV file with basic statistics on the "
1543 "end-of-run state of experiments conducted with moptipy. It "
1544 "therefore either parses a directory structure with log files "
1545 "(if src identifies a directory) or a end results CSV file (if"
1546 " src identifies a file). In the former case, the directory "
1547 "will follow the form 'algorithm/instance/log_file' with one "
1548 "log file per run. In the latter case, it will be a file "
1549 "generated by the end_results.py tool of moptipy. The output "
1550 "of this tool is a CSV file where the columns are separated by"
1551 " ';' and the rows contain the statistics.")
1552 def_src: str = "./evaluation/end_results.txt"
1553 if not os.path.isfile(def_src):
1554 def_src = "./results"
1555 parser.add_argument(
1556 "source", nargs="?", default=def_src,
1557 help="either the directory with moptipy log files or the path to the "
1558 "end-results CSV file", type=Path)
1559 parser.add_argument(
1560 "dest", type=Path, nargs="?",
1561 default="./evaluation/end_statistics.txt",
1562 help="the path to the end results statistics CSV file to be created")
1563 parser.add_argument(
1564 "--join_algorithms",
1565 help="compute statistics over all algorithms, i.e., the statistics"
1566 " are not separated by algorithm but all algorithms are treated "
1567 "as one", action="store_true")
1568 parser.add_argument(
1569 "--join_instances",
1570 help="compute statistics over all instances, i.e., the statistics"
1571 " are not separated by instance but all instances are treated "
1572 "as one", action="store_true")
1573 parser.add_argument(
1574 "--join_objectives",
1575 help="compute statistics over all objective functions, i.e., the "
1576 "statistics are not separated by objective functions but all "
1577 "objectives functions are treated as one", action="store_true")
1578 parser.add_argument(
1579 "--join_encodings",
1580 help="compute statistics over all encodings, i.e., the statistics"
1581 " are not separated by encodings but all encodings are treated "
1582 "as one", action="store_true")
1583 args: Final[argparse.Namespace] = parser.parse_args()
1585 src_path: Final[Path] = args.source
1586 end_results: Iterable[EndResult]
1587 if src_path.is_file():
1588 logger(f"{src_path!r} identifies as file, load as end-results csv")
1589 end_results = end_results_from_csv(src_path)
1590 else:
1591 logger(f"{src_path!r} identifies as directory, load it as log files")
1592 end_results = end_results_from_logs(src_path)
1594 end_stats: Final[list[EndStatistics]] = []
1595 to_csv(from_end_results(
1596 source=end_results,
1597 join_all_algorithms=args.join_algorithms,
1598 join_all_instances=args.join_instances,
1599 join_all_objectives=args.join_objectives,
1600 join_all_encodings=args.join_encodings), args.dest)