Coverage for moptipy/evaluation/end_results.py: 85%
477 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-07-18 11:24 +0000
1"""
2Record for EndResult as well as parsing, serialization, and parsing.
4When doing experiments with `moptipy`, you apply algorithm setups to problem
5instances. For each `setup x instance` combination, you may conduct a series
6of repetitions (so-called runs) with different random seeds. Each single run
7of an algorithm setup on a problem instances can produce a separate log file.
8From each log file, we can load a :class:`EndResult` instance, which
9represents, well, the end result of the run, i.e., information such as the
10best solution quality reached, when it was reached, and the termination
11criterion. These end result records then can be the basis for, e.g., computing
12summary statistics via :mod:`~moptipy.evaluation.end_statistics` or for
13plotting the end result distribution via
14:mod:`~moptipy.evaluation.plot_end_results`.
15"""
16import argparse
17from dataclasses import dataclass
18from math import inf, isfinite
19from typing import Any, Callable, Final, Generator, Iterable, TypeVar, cast
21from pycommons.ds.sequences import reiterable
22from pycommons.io.console import logger
23from pycommons.io.csv import (
24 CSV_SEPARATOR,
25 SCOPE_SEPARATOR,
26 csv_column,
27 csv_column_or_none,
28 csv_scope,
29 csv_str_or_none,
30 csv_val_or_none,
31 pycommons_footer_bottom_comments,
32)
33from pycommons.io.csv import CsvReader as CsvReaderBase
34from pycommons.io.csv import CsvWriter as CsvWriterBase
35from pycommons.io.path import Path, file_path, write_lines
36from pycommons.strings.chars import NEWLINE
37from pycommons.strings.string_conv import (
38 int_or_none_to_str,
39 num_or_none_to_str,
40 num_to_str,
41 str_to_num,
42)
43from pycommons.types import (
44 check_int_range,
45 check_to_int_range,
46 type_error,
47)
49from moptipy.api.logging import (
50 KEY_ALGORITHM,
51 KEY_BEST_F,
52 KEY_GOAL_F,
53 KEY_INSTANCE,
54 KEY_LAST_IMPROVEMENT_FE,
55 KEY_LAST_IMPROVEMENT_TIME_MILLIS,
56 KEY_MAX_FES,
57 KEY_MAX_TIME_MILLIS,
58 KEY_RAND_SEED,
59 KEY_TOTAL_FES,
60 KEY_TOTAL_TIME_MILLIS,
61 PROGRESS_CURRENT_F,
62 PROGRESS_FES,
63 PROGRESS_TIME_MILLIS,
64 SCOPE_SEARCH_SPACE,
65 SCOPE_SOLUTION_SPACE,
66 SECTION_PROGRESS,
67)
68from moptipy.evaluation._utils import (
69 _check_max_time_millis,
70)
71from moptipy.evaluation.base import (
72 DESC_ALGORITHM,
73 DESC_ENCODING,
74 DESC_INSTANCE,
75 DESC_OBJECTIVE_FUNCTION,
76 F_NAME_NORMALIZED,
77 F_NAME_RAW,
78 F_NAME_SCALED,
79 KEY_ENCODING,
80 KEY_OBJECTIVE_FUNCTION,
81 PerRunData,
82 motipy_footer_bottom_comments,
83)
84from moptipy.evaluation.log_parser import SetupAndStateParser
85from moptipy.utils.help import moptipy_argparser
86from moptipy.utils.math import try_float_div, try_int, try_int_div
88#: a description of the random seed
89DESC_RAND_SEED: Final[str] = (
90 "the value of the seed of the random number generator used in the run. "
91 f"Random seeds are in 0..{((1 << (8 * 8)) - 1)} and the random "
92 f"number generators are those from numpy.")
93#: the description of best-F
94DESC_BEST_F: Final[str] = (
95 " the best (smallest) objective value ever encountered during the run ("
96 "regardless whether the algorithm later forgot it again or not).")
97#: the description of the last improvement FE
98DESC_LAST_IMPROVEMENT_FE: Final[str] = (
99 "the objective function evaluation (FE) when the last improving move took"
100 " place. 1 FE corresponds to the construction and evaluation "
101 "of one solution. The first FE has index 1. With 'last "
102 "improving move' we mean the last time when a solution was "
103 "discovered that was better than all previous solutions. This "
104 "time / FE index is the one when the solution with objective "
105 f"value {KEY_BEST_F} was discovered.")
106#: the description of the last improvement time milliseconds
107DESC_LAST_IMPROVEMENT_TIME_MILLIS: Final[str] = (
108 "the clock time in milliseconds after the begin of the run when "
109 "the last improving search move took place.")
110#: the description of the total FEs
111DESC_TOTAL_FES: Final[str] = (
112 "the total number of objective function evaluations (FEs) that were "
113 "performed during the run.")
114#: the total consumed time in milliseconds
115DESC_TOTAL_TIME_MILLIS: Final[str] = (
116 "the clock time in milliseconds that has passed between the begin of the "
117 "run and the end of the run.")
118#: the description of the goal objective value
119DESC_GOAL_F: Final[str] = (
120 "the goal objective value. A run will stop as soon as a solution was"
121 "discovered which has an objective value less than or equal to "
122 f"{KEY_GOAL_F}. In other words, as soon as {KEY_BEST_F} reaches or dips "
123 f"under {KEY_GOAL_F}, the algorithm will stop. If {KEY_GOAL_F} is not "
124 "reached, the run will continue until other budget limits are exhausted. "
125 "If a lower bound for the objective function is known, this is often used"
126 " as a goal objective value. If o goal objective value is specified, this"
127 " field is empty.")
128#: a description of the budget as the maximum objective function evaluation
129DESC_MAX_FES: Final[str] = (
130 "the maximum number of permissible FEs per run. As soon as this limit is "
131 f"reached, the run will stop. In other words, {KEY_TOTAL_FES} will never "
132 f"be more than {KEY_MAX_FES}. A run may stop earlier if some other "
133 "termination criterion is reached, but never later.")
134#: a description of the budget in terms of maximum runtime
135DESC_MAX_TIME_MILLIS: Final[str] = (
136 "the maximum number of milliseconds of clock time that a run is permitted"
137 " to use as computational budget before being terminated. This limit is "
138 "more of a soft limit, as we cannot physically stop a run at arbitrary "
139 "points without causing mayhem. Thus, it may be that some runs consume "
140 "slightly more runtime than this limit. But the rule is that the "
141 "algorithm gets told to stop (via should_terminate() becoming True) as "
142 f"soon as this time has elapsed. But generally, {KEY_TOTAL_TIME_MILLIS}<="
143 f"{KEY_MAX_TIME_MILLIS} approximately holds.")
144#: the CSV separator replacement
145__REPLACE_CSVSEP: Final[str] = "[SEP]"
146#: the newline replacement
147__REPLACE_NEWLINE: Final[str] = "[NEWLINE]"
148#: a description of the point in the search space
149DESC_SEARCH_SPACE: Final[str] = (
150 "the point in the search space corresponding to the result, if any."
151 " Here, any newline character or line break is replaced with "
152 f"{__REPLACE_NEWLINE!r} and any occurrence of {CSV_SEPARATOR!r} is "
153 f"replaced with {__REPLACE_CSVSEP!r}.")
154#: a description of the point in the solution space
155DESC_SOLUTION_SPACE: Final[str] = (
156 "the point in the candidate solution corresponding to the result, if any."
157 " Here, any newline character or line break is replaced with "
158 f"{__REPLACE_NEWLINE!r} and any occurrence of {CSV_SEPARATOR!r} is "
159 f"replaced with {__REPLACE_CSVSEP!r}.")
162@dataclass(frozen=True, init=False, order=False, eq=False)
163class EndResult(PerRunData):
164 """
165 An immutable end result record of one run of one algorithm on one problem.
167 This record provides the information of the outcome of one application of
168 one algorithm to one problem instance in an immutable way.
169 """
171 #: The best objective value encountered.
172 best_f: int | float
174 #: The index of the function evaluation when best_f was reached.
175 last_improvement_fe: int
177 #: The time when best_f was reached.
178 last_improvement_time_millis: int
180 #: The total number of performed FEs.
181 total_fes: int
183 #: The total time consumed by the run.
184 total_time_millis: int
186 #: The goal objective value if provided
187 goal_f: int | float | None
189 #: The (optional) maximum permitted FEs.
190 max_fes: int | None
192 #: The (optional) maximum runtime.
193 max_time_millis: int | None
195 #: The (optional) point in the search space
196 x: str | None
197 #: The (optional) point in the solution space
198 y: str | None
200 def __init__(self,
201 algorithm: str,
202 instance: str,
203 objective: str,
204 encoding: str | None,
205 rand_seed: int,
206 best_f: int | float,
207 last_improvement_fe: int,
208 last_improvement_time_millis: int,
209 total_fes: int,
210 total_time_millis: int,
211 goal_f: int | float | None = None,
212 max_fes: int | None = None,
213 max_time_millis: int | None = None,
214 x: str | None = None,
215 y: str | None = None):
216 """
217 Create a consistent instance of :class:`EndResult`.
219 :param algorithm: the algorithm name
220 :param instance: the instance name
221 :param objective: the name of the objective function
222 :param encoding: the name of the encoding that was used, if any, or
223 `None` if no encoding was used
224 :param rand_seed: the random seed
225 :param best_f: the best reached objective value
226 :param last_improvement_fe: the FE when best_f was reached
227 :param last_improvement_time_millis: the time when best_f was reached
228 :param total_fes: the total FEs
229 :param total_time_millis: the total runtime
230 :param goal_f: the goal objective value, if provide
231 :param max_fes: the optional maximum FEs
232 :param max_time_millis: the optional maximum runtime
233 :param x: the point in the search space, if any
234 :param y: the point in the solution space, if any
236 :raises TypeError: if any parameter has a wrong type
237 :raises ValueError: if the parameter values are inconsistent
238 """
239 super().__init__(algorithm, instance, objective, encoding, rand_seed)
240 object.__setattr__(self, "best_f", try_int(best_f))
241 object.__setattr__(
242 self, "last_improvement_fe", check_int_range(
243 last_improvement_fe, "last_improvement_fe",
244 1, 1_000_000_000_000_000))
245 object.__setattr__(
246 self, "last_improvement_time_millis", check_int_range(
247 last_improvement_time_millis, "last_improvement_time_millis",
248 0, 100_000_000_000))
249 object.__setattr__(
250 self, "total_fes", check_int_range(
251 total_fes, "total_fes", last_improvement_fe,
252 1_000_000_000_000_000))
253 object.__setattr__(
254 self, "total_time_millis", check_int_range(
255 total_time_millis, "total_time_millis",
256 last_improvement_time_millis, 100_000_000_000))
258 if goal_f is not None:
259 goal_f = None if goal_f <= -inf else try_int(goal_f)
260 object.__setattr__(self, "goal_f", goal_f)
262 if max_fes is not None:
263 check_int_range(max_fes, "max_fes", total_fes,
264 1_000_000_000_000_000_000)
265 object.__setattr__(self, "max_fes", max_fes)
267 if max_time_millis is not None:
268 check_int_range(
269 max_time_millis, "max_time_millis", 1, 100_000_000_000)
270 _check_max_time_millis(max_time_millis,
271 total_fes,
272 total_time_millis)
273 object.__setattr__(self, "max_time_millis", max_time_millis)
275 use_x = x
276 if use_x is not None:
277 use_x = str.strip(use_x)
278 if str.__len__(use_x) <= 0:
279 raise ValueError(f"x cannot be {x!r}.")
280 object.__setattr__(self, "x", use_x)
282 use_y = y
283 if use_y is not None:
284 use_y = str.strip(use_y)
285 if str.__len__(use_y) <= 0:
286 raise ValueError(f"y cannot be {y!r}.")
287 object.__setattr__(self, "y", use_y)
289 def _tuple(self) -> tuple[Any, ...]:
290 """
291 Get the tuple representation of this object used in comparisons.
293 :return: the comparison-relevant data of this object in a tuple
294 """
295 return (self.__class__.__name__,
296 "" if self.algorithm is None else self.algorithm,
297 "" if self.instance is None else self.instance,
298 "" if self.objective is None else self.objective,
299 "" if self.encoding is None else self.encoding,
300 1, self.rand_seed, "", "",
301 inf if self.goal_f is None else self.goal_f,
302 inf if self.max_fes is None else self.max_fes,
303 inf if self.max_time_millis is None else self.max_time_millis,
304 self.best_f, self.last_improvement_fe,
305 self.last_improvement_time_millis, self.total_fes,
306 self.total_time_millis)
308 def success(self) -> bool:
309 """
310 Check if a run is successful.
312 This method returns `True` if and only if `goal_f` is defined and
313 `best_f <= goal_f` (and `False` otherwise).
315 :return: `True` if and only if `best_f<=goal_f`
316 """
317 return False if self.goal_f is None else self.best_f <= self.goal_f
319 def get_best_f(self) -> int | float:
320 """
321 Get the best objective value reached.
323 :returns: the best objective value reached
324 """
325 if not isinstance(self, EndResult):
326 raise type_error(self, "self", EndResult)
327 return self.best_f
329 def get_last_improvement_fe(self) -> int:
330 """
331 Get the index of the function evaluation when `best_f` was reached.
333 :returns: the index of the function evaluation when `best_f` was
334 reached
335 """
336 if not isinstance(self, EndResult):
337 raise type_error(self, "self", EndResult)
338 return self.last_improvement_fe
340 def get_last_improvement_time_millis(self) -> int:
341 """
342 Get the milliseconds when `best_f` was reached.
344 :returns: the milliseconds when `best_f` was reached
345 """
346 if not isinstance(self, EndResult):
347 raise type_error(self, "self", EndResult)
348 return self.last_improvement_time_millis
350 def get_total_fes(self) -> int:
351 """
352 Get the total number of performed FEs.
354 :returns: the total number of performed FEs
355 """
356 if not isinstance(self, EndResult):
357 raise type_error(self, "self", EndResult)
358 return self.total_fes
360 def get_total_time_millis(self) -> int:
361 """
362 Get the total time consumed by the run.
364 :returns: the total time consumed by the run
365 """
366 if not isinstance(self, EndResult):
367 raise type_error(self, "self", EndResult)
368 return self.total_time_millis
370 def get_goal_f(self) -> int | float | None:
371 """
372 Get the goal objective value, if any.
374 :returns: the goal objective value, if any
375 """
376 if not isinstance(self, EndResult):
377 raise type_error(self, "self", EndResult)
378 return self.goal_f
380 def get_max_fes(self) -> int | None:
381 """
382 Get the maximum number of FEs permissible.
384 :returns: the maximum number of FEs permissible
385 """
386 if not isinstance(self, EndResult):
387 raise type_error(self, "self", EndResult)
388 return self.max_fes
390 def get_max_time_millis(self) -> int | None:
391 """
392 Get the maximum permissible milliseconds permitted.
394 :returns: the maximum permissible milliseconds permitted
395 """
396 if not isinstance(self, EndResult):
397 raise type_error(self, "self", EndResult)
398 return self.max_time_millis
400 def get_normalized_best_f(self) -> int | float | None:
401 """
402 Get the normalized f.
404 :returns: the normalized f
405 """
406 g: Final[int | float | None] = EndResult.get_goal_f(self)
407 if (g is None) or (g <= 0):
408 return None
409 return try_float_div(self.best_f - g, g)
411 def get_scaled_best_f(self) -> int | float | None:
412 """
413 Get the normalized f.
415 :returns: the normalized f
416 """
417 g: Final[int | float | None] = EndResult.get_goal_f(self)
418 if (g is None) or (g <= 0):
419 return None
420 return try_float_div(self.best_f, g)
422 def get_fes_per_time_milli(self) -> int | float:
423 """
424 Get the fes per time milliseconds.
426 :returns: the fes per time milliseconds
427 """
428 return try_int_div(EndResult.get_total_fes(self), max(
429 1, EndResult.get_total_time_millis(self)))
432#: A set of getters for accessing variables of the end result
433__PROPERTIES: Final[Callable[[str], Callable[[
434 EndResult], int | float | None]]] = {
435 KEY_LAST_IMPROVEMENT_FE: EndResult.get_last_improvement_fe,
436 "last improvement FE": EndResult.get_last_improvement_fe,
437 KEY_LAST_IMPROVEMENT_TIME_MILLIS:
438 EndResult.get_last_improvement_time_millis,
439 "last improvement ms": EndResult.get_last_improvement_time_millis,
440 KEY_TOTAL_FES: EndResult.get_total_fes,
441 "fes": EndResult.get_total_fes,
442 KEY_TOTAL_TIME_MILLIS: EndResult.get_total_time_millis,
443 "ms": EndResult.get_total_time_millis,
444 KEY_GOAL_F: EndResult.get_goal_f,
445 F_NAME_RAW: EndResult.get_best_f,
446 KEY_BEST_F: EndResult.get_best_f,
447 "f": EndResult.get_best_f,
448 F_NAME_SCALED: EndResult.get_scaled_best_f,
449 "bestFscaled": EndResult.get_scaled_best_f,
450 F_NAME_NORMALIZED: EndResult.get_normalized_best_f,
451 "bestFnormalized": EndResult.get_normalized_best_f,
452 KEY_MAX_FES: EndResult.get_max_fes,
453 "budgetFEs": EndResult.get_max_fes,
454 KEY_MAX_TIME_MILLIS: EndResult.get_max_time_millis,
455 "budgetMS": EndResult.get_max_time_millis,
456 "fesPerTimeMilli": EndResult.get_fes_per_time_milli,
457}.get
460def getter(dimension: str) -> Callable[[EndResult], int | float | None]:
461 """
462 Produce a function that obtains the given dimension from EndResults.
464 The following dimensions are supported:
466 1. `lastImprovementFE`: :attr:`~EndResult.last_improvement_fe`
467 2. `lastImprovementTimeMillis`:
468 :attr:`~EndResult.last_improvement_time_millis`
469 3. `totalFEs`: :attr:`~EndResult.total_fes`
470 4. `totalTimeMillis`: :attr:`~EndResult.total_time_millis`
471 5. `goalF`: :attr:`~EndResult.goal_f`
472 6. `plainF`, `bestF`: :attr:`~EndResult.best_f`
473 7. `scaledF`: :attr:`~EndResult.best_f`/:attr:`~EndResult.goal_f`
474 8. `normalizedF`: (:attr:`~EndResult.best_f`-attr:`~EndResult.goal_f`)/
475 :attr:`~EndResult.goal_f`
476 9. `maxFEs`: :attr:`~EndResult.max_fes`
477 10. `maxTimeMillis`: :attr:`~EndResult.max_time_millis`
478 11. `fesPerTimeMilli`: :attr:`~EndResult.total_fes`
479 /:attr:`~EndResult.total_time_millis`
481 :param dimension: the dimension
482 :returns: a callable that returns the value corresponding to the
483 dimension from its input value, which must be an :class:`EndResult`
484 """
485 result: Callable[[EndResult], int | float] | None = __PROPERTIES(
486 str.strip(dimension))
487 if result is None:
488 raise ValueError(f"Unknown EndResult dimension {dimension!r}.")
489 return result
492def to_csv(results: Iterable[EndResult], file: str) -> Path:
493 """
494 Write a sequence of end results to a file in CSV format.
496 :param results: the end results
497 :param file: the path
498 :return: the path of the file that was written
499 """
500 path: Final[Path] = Path(file)
501 logger(f"Writing end results to CSV file {path!r}.")
502 path.ensure_parent_dir_exists()
503 with path.open_for_write() as wt:
504 write_lines(CsvWriter.write(results), wt)
505 logger(f"Done writing end results to CSV file {path!r}.")
506 return path
509def from_csv(file: str,
510 filterer: Callable[[EndResult], bool]
511 = lambda _: True) -> Generator[EndResult, None, None]:
512 """
513 Parse a given CSV file to get :class:`EndResult` Records.
515 :param file: the path to parse
516 :param filterer: an optional filter function
517 """
518 path: Final[Path] = file_path(file)
519 logger(f"Now reading CSV file {path!r}.")
520 with path.open_for_read() as rd:
521 for r in CsvReader.read(rd):
522 if filterer(r):
523 yield r
524 logger(f"Done reading CSV file {path!r}.")
527def _point_to_csv(s: str | None) -> str:
528 """
529 Convert a solution or search space point to CSV.
531 :param s: the solution or point in the search space
532 :return: the CSV string
533 :raises ValueError: if the original string contains a character sequence
534 that cannot be converted to a CSV string
535 """
536 if s is None:
537 return ""
538 if __REPLACE_CSVSEP in s:
539 raise ValueError(f"Cannot serialize {s!r} to CSV because "
540 f"it contains {__REPLACE_CSVSEP!r}.")
541 if __REPLACE_NEWLINE in s:
542 raise ValueError(f"Cannot serialize {s!r} to CSV because "
543 f"it contains {__REPLACE_NEWLINE!r}.")
544 for nl in NEWLINE:
545 s = str.replace(s, nl, __REPLACE_NEWLINE)
546 return str.replace(s, CSV_SEPARATOR, __REPLACE_CSVSEP)
549def _csv_to_point(s: str | None) -> str | None:
550 """
551 Convert a CSV string to a solution or search space point.
553 :param s: the CSV string
554 :return: the CSV string
555 """
556 if s is None:
557 return None
558 s = str.strip(s)
559 if str.__len__(s) <= 0:
560 return None
561 return str.replace(str.replace(s, __REPLACE_NEWLINE, "\n"),
562 __REPLACE_CSVSEP, CSV_SEPARATOR)
565class CsvWriter(CsvWriterBase):
566 """A class for CSV writing of :class:`EndResult`."""
568 def __init__(self, data: Iterable[EndResult],
569 scope: str | None = None) -> None:
570 """
571 Initialize the csv writer.
573 :param data: the data
574 :param scope: the prefix to be pre-pended to all columns
575 """
576 data = reiterable(data)
577 super().__init__(data, scope)
578 no_encoding: bool = True
579 no_max_fes: bool = True
580 no_max_ms: bool = True
581 no_goal_f: bool = True
582 no_x: bool = True
583 no_y: bool = True
584 check: int = 6
585 for er in data:
586 if no_encoding and (er.encoding is not None):
587 no_encoding = False
588 check -= 1
589 if check <= 0:
590 break
591 if no_max_fes and (er.max_fes is not None):
592 no_max_fes = False
593 check -= 1
594 if check <= 0:
595 break
596 if no_max_ms and (er.max_time_millis is not None):
597 no_max_ms = False
598 check -= 1
599 if check <= 0:
600 break
601 if no_goal_f and (er.goal_f is not None) and (
602 isfinite(er.goal_f)):
603 no_goal_f = False
604 check -= 1
605 if check <= 0:
606 break
607 if no_x and (er.x is not None):
608 no_x = False
609 check -= 1
610 if check <= 0:
611 break
612 if no_y and (er.y is not None):
613 no_y = False
614 check -= 1
615 if check <= 0:
616 break
618 #: do we need the encoding?
619 self.__needs_encoding: Final[bool] = not no_encoding
620 #: do we need the max FEs?
621 self.__needs_max_fes: Final[bool] = not no_max_fes
622 #: do we need the max millis?
623 self.__needs_max_ms: Final[bool] = not no_max_ms
624 #: do we need the goal F?
625 self.__needs_goal_f: Final[bool] = not no_goal_f
626 #: do we need an `x` column?
627 self.__needs_x: Final[bool] = not no_x
628 #: do we need an `y` column?
629 self.__needs_y: Final[bool] = not no_y
631 def get_column_titles(self) -> Iterable[str]:
632 """
633 Get the column titles.
635 :returns: the column titles
636 """
637 p: Final[str] = self.scope
638 data: list[str] = [
639 KEY_ALGORITHM, KEY_INSTANCE, KEY_OBJECTIVE_FUNCTION]
640 if self.__needs_encoding:
641 data.append(KEY_ENCODING)
642 data.extend((KEY_RAND_SEED, KEY_BEST_F, KEY_LAST_IMPROVEMENT_FE,
643 KEY_LAST_IMPROVEMENT_TIME_MILLIS, KEY_TOTAL_FES,
644 KEY_TOTAL_TIME_MILLIS))
646 if self.__needs_goal_f:
647 data.append(KEY_GOAL_F)
648 if self.__needs_max_fes:
649 data.append(KEY_MAX_FES)
650 if self.__needs_max_ms:
651 data.append(KEY_MAX_TIME_MILLIS)
652 if self.__needs_x:
653 data.append(SCOPE_SEARCH_SPACE)
654 if self.__needs_y:
655 data.append(SCOPE_SOLUTION_SPACE)
656 return (csv_scope(p, q) for q in data)
658 def get_row(self, data: EndResult) -> Iterable[str]:
659 """
660 Render a single end result record to a CSV row.
662 :param data: the end result record
663 :returns: the row iterator
664 """
665 yield data.algorithm
666 yield data.instance
667 yield data.objective
668 if self.__needs_encoding:
669 yield data.encoding or ""
670 yield hex(data.rand_seed)
671 yield num_to_str(data.best_f)
672 yield str(data.last_improvement_fe)
673 yield str(data.last_improvement_time_millis)
674 yield str(data.total_fes)
675 yield str(data.total_time_millis)
676 if self.__needs_goal_f:
677 yield num_or_none_to_str(data.goal_f)
678 if self.__needs_max_fes:
679 yield int_or_none_to_str(data.max_fes)
680 if self.__needs_max_ms:
681 yield int_or_none_to_str(data.max_time_millis)
682 if self.__needs_x:
683 yield _point_to_csv(data.x)
684 if self.__needs_y:
685 yield _point_to_csv(data.y)
687 def get_header_comments(self) -> Iterable[str]:
688 """
689 Get any possible header comments.
691 :returns: the header comments
692 """
693 return ("Experiment End Results",
694 "See the description at the bottom of the file.")
696 def get_footer_comments(self) -> Iterable[str]:
697 """
698 Get any possible footer comments.
700 :returns: the footer comments
701 """
702 yield ""
703 scope: Final[str | None] = self.scope
704 yield ("Records describing the end results of single runs ("
705 "single executions) of algorithms applied to optimization "
706 "problems.")
707 yield ("Each run is characterized by an algorithm setup, a problem "
708 "instance, and a random seed.")
709 if scope:
710 yield ("All end result records start with prefix "
711 f"{scope}{SCOPE_SEPARATOR}.")
712 yield f"{csv_scope(scope, KEY_ALGORITHM)}: {DESC_ALGORITHM}"
713 yield f"{csv_scope(scope, KEY_INSTANCE)}: {DESC_INSTANCE}"
714 yield (f"{csv_scope(scope, KEY_OBJECTIVE_FUNCTION)}:"
715 f" {DESC_OBJECTIVE_FUNCTION}")
716 if self.__needs_encoding:
717 yield f"{csv_scope(scope, KEY_ENCODING)}: {DESC_ENCODING}"
718 yield f"{csv_scope(scope, KEY_RAND_SEED)}: {DESC_RAND_SEED}"
719 yield f"{csv_scope(scope, KEY_BEST_F)}: {DESC_BEST_F}"
720 yield (f"{csv_scope(scope, KEY_LAST_IMPROVEMENT_FE)}: "
721 f"{DESC_LAST_IMPROVEMENT_FE}")
722 yield (f"{csv_scope(scope, KEY_LAST_IMPROVEMENT_TIME_MILLIS)}: "
723 f"{DESC_LAST_IMPROVEMENT_TIME_MILLIS}")
724 yield f"{csv_scope(scope, KEY_TOTAL_FES)}: {DESC_TOTAL_FES}"
725 yield (f"{csv_scope(scope, KEY_TOTAL_TIME_MILLIS)}: "
726 f"{DESC_TOTAL_TIME_MILLIS}")
727 if self.__needs_goal_f:
728 yield f"{csv_scope(scope, KEY_GOAL_F)}: {DESC_GOAL_F}"
729 if self.__needs_max_fes:
730 yield f"{csv_scope(scope, KEY_MAX_FES)}: {DESC_MAX_FES}"
731 if self.__needs_max_ms:
732 yield (f"{csv_scope(scope, KEY_MAX_TIME_MILLIS)}: "
733 f"{DESC_MAX_TIME_MILLIS}")
734 if self.__needs_x:
735 yield (f"{csv_scope(scope, SCOPE_SEARCH_SPACE)}: "
736 f"{DESC_SEARCH_SPACE}")
737 if self.__needs_y:
738 yield (f"{csv_scope(scope, SCOPE_SOLUTION_SPACE)}: "
739 f"{SCOPE_SOLUTION_SPACE}")
741 def get_footer_bottom_comments(self) -> Iterable[str]:
742 """
743 Get the footer bottom comments.
745 :returns: the footer comments
746 """
747 yield from motipy_footer_bottom_comments(
748 self, ("The end results data is produced using module "
749 "moptipy.evaluation.end_results."))
750 yield from pycommons_footer_bottom_comments(self)
753class CsvReader(CsvReaderBase):
754 """A csv parser for end results."""
756 def __init__(self, columns: dict[str, int]) -> None:
757 """
758 Create a CSV parser for :class:`EndResult`.
760 :param columns: the columns
761 """
762 super().__init__(columns)
763 #: the index of the algorithm column, if any
764 self.__idx_algorithm: Final[int] = csv_column(columns, KEY_ALGORITHM)
765 #: the index of the instance column, if any
766 self.__idx_instance: Final[int] = csv_column(columns, KEY_INSTANCE)
767 #: the index of the objective function column, if any
768 self.__idx_objective: Final[int] = csv_column(
769 columns, KEY_OBJECTIVE_FUNCTION)
770 #: the index of the encoding column, if any
771 self.__idx_encoding = csv_column_or_none(columns, KEY_ENCODING)
773 #: the index of the random seed column
774 self.__idx_seed: Final[int] = csv_column(columns, KEY_RAND_SEED)
775 #: the column with the last improvement FE
776 self.__idx_li_fe: Final[int] = csv_column(
777 columns, KEY_LAST_IMPROVEMENT_FE)
778 #: the column with the last improvement time milliseconds
779 self.__idx_li_ms: Final[int] = csv_column(
780 columns, KEY_LAST_IMPROVEMENT_TIME_MILLIS)
781 #: the column with the best obtained objective value
782 self.__idx_best_f: Final[int] = csv_column(columns, KEY_BEST_F)
783 #: the column with the total time in FEs
784 self.__idx_tt_fe: Final[int] = csv_column(columns, KEY_TOTAL_FES)
785 #: the column with the total time in milliseconds
786 self.__idx_tt_ms: Final[int] = csv_column(
787 columns, KEY_TOTAL_TIME_MILLIS)
789 #: the column with the goal objective value, if any
790 self.__idx_goal_f: Final[int | None] = csv_column_or_none(
791 columns, KEY_GOAL_F)
792 #: the column with the maximum FEs, if any such budget constraint was
793 #: defined
794 self.__idx_max_fes: Final[int | None] = csv_column_or_none(
795 columns, KEY_MAX_FES)
796 #: the column with the maximum runtime in milliseconds, if any such
797 #: budget constraint was defined
798 self.__idx_max_ms: Final[int | None] = csv_column_or_none(
799 columns, KEY_MAX_TIME_MILLIS)
800 #: the column with the search space element
801 self.__idx_x: Final[int | None] = csv_column_or_none(
802 columns, SCOPE_SEARCH_SPACE)
803 #: the column with the solution space element
804 self.__idx_y: Final[int | None] = csv_column_or_none(
805 columns, SCOPE_SOLUTION_SPACE)
807 def parse_row(self, data: list[str]) -> EndResult:
808 """
809 Parse a row of data.
811 :param data: the data row
812 :return: the end result statistics
813 """
814 return EndResult(
815 data[self.__idx_algorithm], # algorithm
816 data[self.__idx_instance], # instance
817 data[self.__idx_objective], # objective
818 csv_str_or_none(data, self.__idx_encoding), # encoding
819 int(data[self.__idx_seed], base=0), # rand seed
820 str_to_num(data[self.__idx_best_f]), # best_f
821 int(data[self.__idx_li_fe]), # last_improvement_fe
822 int(data[self.__idx_li_ms]), # last_improvement_time_millis
823 int(data[self.__idx_tt_fe]), # total_fes
824 int(data[self.__idx_tt_ms]), # total_time_millis
825 csv_val_or_none(data, self.__idx_goal_f, str_to_num),
826 csv_val_or_none(data, self.__idx_max_fes, int), # max_fes
827 csv_val_or_none(data, self.__idx_max_ms, int), # max_time_ms
828 csv_val_or_none(data, self.__idx_x, _csv_to_point), # x
829 csv_val_or_none(data, self.__idx_y, _csv_to_point)) # y
832#: the type variable for data to be read from the directories
833T = TypeVar("T", bound=EndResult)
836class EndResultLogParser[T](SetupAndStateParser[T]):
837 """The internal log parser class."""
839 def _parse_file(self, file: Path) -> T:
840 """
841 Get the parsing result.
843 :returns: the :class:`EndResult` instance
844 """
845 super()._parse_file(file)
846 return cast("T", EndResult(self.algorithm,
847 self.instance,
848 self.objective,
849 self.encoding,
850 self.rand_seed,
851 self.best_f,
852 self.last_improvement_fe,
853 self.last_improvement_time_millis,
854 self.total_fes,
855 self.total_time_millis,
856 self.goal_f,
857 self.max_fes,
858 self.max_time_millis))
861def _join_goals(vlimit, vgoal, select): # noqa
862 if vlimit is None:
863 return vgoal
864 if vgoal is None:
865 return vlimit
866 return select(vlimit, vgoal)
869class __EndResultProgressLogParser(SetupAndStateParser[EndResult]):
870 """The internal log parser class for virtual end results."""
872 def __init__(
873 self,
874 max_fes: int | Callable[[str, str], int | None] | None,
875 max_time_millis: int | Callable[[str, str], int | None] | None,
876 goal_f: int | float | Callable[
877 [str, str], int | float | None] | None,
878 path_filter: Callable[[Path], bool] | None = None):
879 """
880 Create the internal log parser.
882 :param max_fes: the maximum FEs, or `None` if unspecified
883 :param max_time_millis: the maximum runtime in milliseconds, or
884 `None` if unspecified
885 :param goal_f: the goal objective value, or `None` if unspecified
886 :param path_filter: the path filter
887 """
888 super().__init__(path_filter)
889 self.__src_limit_ms: Final[
890 int | Callable[[str, str], int | None] | None] = max_time_millis
891 self.__src_limit_fes: Final[
892 int | Callable[[str, str], int | None] | None] = max_fes
893 self.__src_limit_f: Final[
894 int | float | Callable[
895 [str, str], int | float | None] | None] = goal_f
897 self.__limit_ms: int | float = inf
898 self.__limit_ms_n: int | None = None
899 self.__limit_fes: int | float = inf
900 self.__limit_fes_n: int | None = None
901 self.__limit_f: int | float = -inf
902 self.__limit_f_n: int | float | None = None
904 self.__stop_fes: int | None = None
905 self.__stop_ms: int | None = None
906 self.__stop_f: int | float | None = None
907 self.__stop_li_fe: int | None = None
908 self.__stop_li_ms: int | None = None
909 self.__hit_goal: bool = False
910 self.__state: int = 0
912 def _parse_file(self, file: Path) -> EndResult:
913 super()._parse_file(file)
914 if self.__state != 2:
915 raise ValueError(
916 "Illegal state, log file must have a "
917 f"{SECTION_PROGRESS!r} section.")
918 self.__state = 0
919 l_hit_goal = self.__hit_goal
920 stop_fes: int = self.__stop_fes
921 stop_ms: int = self.__stop_ms
922 if not l_hit_goal:
923 stop_ms = max(stop_ms, cast("int", min(
924 self.total_time_millis, self.__limit_ms)))
925 ul_fes = self.total_fes
926 if stop_ms < self.total_time_millis:
927 ul_fes -= 1
928 stop_fes = max(stop_fes, cast("int", min(
929 ul_fes, self.__limit_fes)))
931 return EndResult(
932 algorithm=self.algorithm,
933 instance=self.instance,
934 objective=self.objective,
935 encoding=self.encoding,
936 rand_seed=self.rand_seed,
937 best_f=self.__stop_f,
938 last_improvement_fe=self.__stop_li_fe,
939 last_improvement_time_millis=self.__stop_li_ms,
940 total_fes=stop_fes,
941 total_time_millis=stop_ms,
942 goal_f=_join_goals(self.__limit_f_n, self.goal_f, max),
943 max_fes=_join_goals(self.__limit_fes_n, self.max_fes, min),
944 max_time_millis=_join_goals(
945 self.__limit_ms_n, self.max_time_millis, min))
947 def _end_parse_file(self, file: Path) -> None:
948 """
949 Cleanup.
951 :param file: the file that was parsed.
952 """
953 self.__stop_fes = None
954 self.__stop_ms = None
955 self.__stop_f = None
956 self.__stop_li_fe = None
957 self.__stop_li_ms = None
958 self.__limit_fes_n = None
959 self.__limit_fes = inf
960 self.__limit_ms_n = None
961 self.__limit_ms = inf
962 self.__limit_f_n = None
963 self.__limit_f = -inf
964 self.__hit_goal = False
965 super()._end_parse_file(file)
967 def _start_parse_file(self, file: Path) -> None:
968 super()._start_parse_file(file)
969 a: Final[str | None] = self.algorithm
970 i: Final[str | None] = self.instance
972 fes = self.__src_limit_fes(a, i) if (a and i and callable(
973 self.__src_limit_fes)) else (
974 self.__src_limit_fes if isinstance(self.__src_limit_fes, int)
975 else None)
976 self.__limit_fes_n = None if fes is None else \
977 check_int_range(fes, "limit_fes", 1, 1_000_000_000_000_000)
978 self.__limit_fes = inf if self.__limit_fes_n is None \
979 else self.__limit_fes_n
981 time = self.__src_limit_ms(a, i) if (a and i and callable(
982 self.__src_limit_ms)) else (
983 self.__src_limit_ms if isinstance(self.__src_limit_ms, int)
984 else None)
985 self.__limit_ms_n = None if time is None else \
986 check_int_range(time, "l_limit_ms", 1, 1_000_000_000_000)
987 self.__limit_ms = inf if self.__limit_ms_n is None \
988 else self.__limit_ms_n
990 self.__limit_f_n = self.__src_limit_f(a, i) if (a and i and callable(
991 self.__src_limit_f)) else (
992 self.__src_limit_f if isinstance(self.__src_limit_f, int | float)
993 else None)
994 if self.__limit_f_n is not None:
995 if not isinstance(self.__limit_f_n, int | float):
996 raise type_error(self.__limit_f_n, "limit_f", (
997 int, float))
998 if not isfinite(self.__limit_f_n):
999 if self.__limit_f_n <= -inf:
1000 self.__limit_f_n = None
1001 else:
1002 raise ValueError(
1003 f"invalid limit f={self.__limit_f_n} for "
1004 f"{self.algorithm} on {self.instance}")
1005 self.__limit_f = -inf if self.__limit_f_n is None \
1006 else self.__limit_f_n
1008 def _start_section(self, title: str) -> bool:
1009 if title == SECTION_PROGRESS:
1010 if self.__state != 0:
1011 raise ValueError(f"Already did section {title}.")
1012 self.__state = 1
1013 return True
1014 return super()._start_section(title)
1016 def _needs_more_lines(self) -> bool:
1017 return (self.__state < 2) or super()._needs_more_lines()
1019 def _lines(self, lines: list[str]) -> bool:
1020 if self.__state != 1:
1021 return super()._lines(lines)
1022 self.__state = 2
1024 n_rows = len(lines)
1025 if n_rows < 2:
1026 raise ValueError("lines must contain at least two elements,"
1027 f"but contains {n_rows}.")
1029 columns = [c.strip() for c in lines[0].split(CSV_SEPARATOR)]
1030 fe_col: Final[int] = columns.index(PROGRESS_FES)
1031 ms_col: Final[int] = columns.index(PROGRESS_TIME_MILLIS)
1032 f_col: Final[int] = columns.index(PROGRESS_CURRENT_F)
1033 current_fes: int = -1
1034 current_ms: int = -1
1035 current_f: int | float = inf
1036 current_li_fe: int | None = None
1037 current_li_ms: int | None = None
1038 stop_fes: int | None = None
1039 stop_ms: int | None = None
1040 stop_f: int | float | None = None
1041 stop_li_fe: int | None = None
1042 stop_li_ms: int | None = None
1043 limit_fes: Final[int | float] = self.__limit_fes
1044 l_limit_ms: Final[int | float] = self.__limit_ms
1045 limit_f: Final[int | float] = self.__limit_f
1047 for line in lines[1:]:
1048 values = line.split(CSV_SEPARATOR)
1049 current_fes = check_to_int_range(
1050 values[fe_col], "fes", current_fes, 1_000_000_000_000_000)
1051 current_ms = check_to_int_range(
1052 values[ms_col], "ms", current_ms, 1_000_000_000_00)
1053 f: int | float = str_to_num(values[f_col])
1054 if (current_fes <= limit_fes) and (current_ms <= l_limit_ms):
1055 if f < current_f: # can only update best within budget
1056 current_f = f
1057 current_li_fe = current_fes
1058 current_li_ms = current_ms
1059 stop_ms = current_ms
1060 stop_fes = current_fes
1061 stop_f = current_f
1062 stop_li_fe = current_li_fe
1063 stop_li_ms = current_li_ms
1064 if (current_fes >= limit_fes) or (current_ms >= l_limit_ms) or \
1065 (current_f <= limit_f):
1066 self.__hit_goal = True
1067 break # we can stop parsing the stuff
1069 if (stop_fes is None) or (stop_ms is None) or (stop_f is None) \
1070 or (current_fes <= 0) or (not isfinite(current_f)):
1071 raise ValueError(
1072 "Illegal state, no fitting data point found: stop_fes="
1073 f"{stop_fes}, stop_ms={stop_ms}, stop_f={stop_f}, "
1074 f"current_fes={current_fes}, current_ms={current_ms}, "
1075 f"current_f={current_f}.")
1077 if current_fes >= limit_fes:
1078 stop_fes = max(stop_fes, min(
1079 cast("int", limit_fes), current_fes))
1080 elif current_ms > l_limit_ms:
1081 stop_fes = max(stop_fes, current_fes - 1)
1082 else:
1083 stop_fes = max(stop_fes, current_fes)
1085 if current_ms >= l_limit_ms:
1086 stop_ms = max(stop_ms, min(cast("int", l_limit_ms), current_ms))
1087 else:
1088 stop_ms = max(stop_ms, current_ms)
1090 self.__stop_fes = stop_fes
1091 self.__stop_ms = stop_ms
1092 self.__stop_f = stop_f
1093 self.__stop_li_fe = stop_li_fe
1094 self.__stop_li_ms = stop_li_ms
1095 return self._needs_more_lines()
1098def from_logs(
1099 path: str, max_fes: int | Callable[
1100 [str, str], int | None] | None = None,
1101 max_time_millis: int | Callable[
1102 [str, str], int | None] | None = None,
1103 goal_f: int | float | Callable[
1104 [str, str], int | float | None] | None = None,
1105 path_filter: Callable[[Path], bool] | None = None) \
1106 -> Generator[EndResult, None, None]:
1107 """
1108 Parse a given path and yield all end results found.
1110 If `path` identifies a file with suffix `.txt`, then this file is
1111 parsed. The appropriate :class:`EndResult` is created and yielded.
1112 If `path` identifies a directory, then this directory is parsed
1113 recursively for each log file found, one record is yielded.
1115 Via the parameters `max_fes`, `max_time_millis`, and `goal_f`, you can
1116 set virtual limits for the objective function evaluations, the maximum
1117 runtime, and the objective value. The :class:`EndResult` records will
1118 then not represent the actual final state of the runs but be
1119 synthesized from the logged progress information. This, of course,
1120 requires such information to be present. It will also raise a
1121 `ValueError` if the goals are invalid, e.g., if a runtime limit is
1122 specified that is before the first logged points.
1124 There is one caveat when specifying `max_time_millis`: Let's say that
1125 the log files only log improvements. Then you might have a log point
1126 for 7000 FEs, 1000ms, and f=100. The next log point could be 8000 FEs,
1127 1200ms, and f=90. Now if your time limit specified is 1100ms, we know
1128 that the end result is f=100 (because f=90 was reached too late) and
1129 that the total runtime is 1100ms, as this is the limit you specified
1130 and it was also reached. But we do not know the number of consumed
1131 FEs. We know you consumed at least 7000 FEs, but you did not consume
1132 8000 FEs. It would be wrong to claim that 7000 FEs were consumed,
1133 since it could have been more. We therefore set a virtual end point at
1134 7999 FEs. In terms of performance metrics such as the
1135 :mod:`~moptipy.evaluation.ert`, this would be the most conservative
1136 choice in that it does not over-estimate the speed of the algorithm.
1137 It can, however, lead to very big deviations from the actual values.
1138 For example, if your algorithm quickly converged to a local optimum
1139 and there simply is no log point that exceeds the virtual time limit
1140 but the original run had a huge FE-based budget while your virtual
1141 time limit was small, this could lead to an estimate of millions of
1142 FEs taking part within seconds...
1144 :param path: the path to parse
1145 :param max_fes: the maximum FEs, a callable to compute the maximum
1146 FEs from the algorithm and instance name, or `None` if unspecified
1147 :param max_time_millis: the maximum runtime in milliseconds, a
1148 callable to compute the maximum runtime from the algorithm and
1149 instance name, or `None` if unspecified
1150 :param goal_f: the goal objective value, a callable to compute the
1151 goal objective value from the algorithm and instance name, or
1152 `None` if unspecified
1153 :param path_filter: a filter allowing us to skip paths or files. If
1154 this :class:`Callable` returns `True`, the file or directory is
1155 considered for parsing. If it returns `False`, it is skipped.
1156 """
1157 need_goals: bool = False
1158 if max_fes is not None:
1159 if not callable(max_fes):
1160 max_fes = check_int_range(
1161 max_fes, "max_fes", 1, 1_000_000_000_000_000)
1162 need_goals = True
1163 if max_time_millis is not None:
1164 if not callable(max_time_millis):
1165 max_time_millis = check_int_range(
1166 max_time_millis, "max_time_millis", 1, 1_000_000_000_000)
1167 need_goals = True
1168 if goal_f is not None:
1169 if callable(goal_f):
1170 need_goals = True
1171 else:
1172 if not isinstance(goal_f, int | float):
1173 raise type_error(goal_f, "goal_f", (int, float, None))
1174 if isfinite(goal_f):
1175 need_goals = True
1176 elif goal_f <= -inf:
1177 goal_f = None
1178 else:
1179 raise ValueError(f"goal_f={goal_f} is not permissible.")
1180 if need_goals:
1181 return __EndResultProgressLogParser(
1182 max_fes, max_time_millis, goal_f, path_filter).parse(path)
1183 return EndResultLogParser(path_filter).parse(path)
1186# Run log files to end results if executed as script
1187if __name__ == "__main__":
1188 parser: Final[argparse.ArgumentParser] = moptipy_argparser(
1189 __file__,
1190 "Convert log files obtained with moptipy to the end results CSV "
1191 "format that can be post-processed or exported to other tools.",
1192 "This program recursively parses a folder hierarchy created by"
1193 " the moptipy experiment execution facility. This folder "
1194 "structure follows the scheme of algorithm/instance/log_file "
1195 "and has one log file per run. As result of the parsing, one "
1196 "CSV file (where columns are separated by ';') is created with"
1197 " one row per log file. This row contains the end-of-run state"
1198 " loaded from the log file. Whereas the log files may store "
1199 "the complete progress of one run of one algorithm on one "
1200 "problem instance as well as the algorithm configuration "
1201 "parameters, instance features, system settings, and the final"
1202 " results, the end results CSV file will only represent the "
1203 "final result quality, when it was obtained, how long the runs"
1204 " took, etc. This information is much denser and smaller and "
1205 "suitable for importing into other tools such as Excel or for "
1206 "postprocessing.")
1207 parser.add_argument(
1208 "source", nargs="?", default="./results",
1209 help="the location of the experimental results, i.e., the root folder "
1210 "under which to search for log files", type=Path)
1211 parser.add_argument(
1212 "dest", help="the path to the end results CSV file to be created",
1213 type=Path, nargs="?", default="./evaluation/end_results.txt")
1214 parser.add_argument(
1215 "--maxFEs", help="the maximum permitted FEs",
1216 type=int, nargs="?", default=None)
1217 parser.add_argument(
1218 "--maxTime", help="the maximum permitted time in milliseconds",
1219 type=int, nargs="?", default=None)
1220 parser.add_argument(
1221 "--goalF", help="the goal objective value",
1222 type=str_to_num, nargs="?", default=None)
1223 args: Final[argparse.Namespace] = parser.parse_args()
1225 to_csv(from_logs(args.source, args.maxFEs, args.maxTime, args.goalF),
1226 args.dest)