Coverage for moptipy/evaluation/mo_end_results.py: 78%
240 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-07-18 11:24 +0000
1"""A set of end results from a multi-objective run."""
3import argparse
4from dataclasses import dataclass
5from itertools import chain
6from typing import Any, Callable, Final, Generator, Iterable, cast
8from pycommons.ds.sequences import reiterable
9from pycommons.io.console import logger
10from pycommons.io.csv import (
11 CSV_SEPARATOR,
12 csv_column,
13 csv_scope,
14 csv_val_or_none,
15)
16from pycommons.io.parser import Parser
17from pycommons.io.path import Path, file_path, write_lines
18from pycommons.strings.string_conv import (
19 num_to_str,
20 str_to_num,
21)
22from pycommons.types import type_error
24from moptipy.api.logging import (
25 PREFIX_SECTION_ARCHIVE,
26 SECTION_ARCHIVE_QUALITY,
27 SECTION_PROGRESS,
28 SUFFIX_SECTION_ARCHIVE_X,
29 SUFFIX_SECTION_ARCHIVE_Y,
30)
31from moptipy.evaluation.end_results import CsvReader as CsvReaderBase
32from moptipy.evaluation.end_results import CsvWriter as CsvWriterBase
33from moptipy.evaluation.end_results import EndResult
34from moptipy.evaluation.end_results import EndResultLogParser as _Erlp
35from moptipy.utils.help import moptipy_argparser
36from moptipy.utils.logger import (
37 SECTION_END,
38 SECTION_START,
39)
40from moptipy.utils.math import try_int
43@dataclass(frozen=True, init=False, order=False, eq=False)
44class MOEndResult(EndResult):
45 """A multi-objective end result record."""
47 #: The objective values for the subordinate objective functions.
48 fs: tuple[int | float, ...]
50 def __init__(self,
51 algorithm: str,
52 instance: str,
53 objective: str,
54 encoding: str | None,
55 rand_seed: int,
56 best_f: int | float,
57 last_improvement_fe: int,
58 last_improvement_time_millis: int,
59 total_fes: int,
60 total_time_millis: int,
61 goal_f: int | float | None,
62 max_fes: int | None,
63 max_time_millis: int | None,
64 fs: tuple[int | float, ...],
65 x: str | None = None,
66 y: str | None = None) -> None:
67 """
68 Create the multi-objective end result record.
70 :param algorithm: the algorithm name
71 :param instance: the instance name
72 :param objective: the name of the objective function
73 :param encoding: the name of the encoding that was used, if any, or
74 `None` if no encoding was used
75 :param rand_seed: the random seed
76 :param best_f: the best reached objective value
77 :param last_improvement_fe: the FE when best_f was reached
78 :param last_improvement_time_millis: the time when best_f was reached
79 :param total_fes: the total FEs
80 :param total_time_millis: the total runtime
81 :param goal_f: the goal objective value, if provide
82 :param max_fes: the optional maximum FEs
83 :param max_time_millis: the optional maximum runtime
84 :param fs: the objective value vector
85 :param x: the optional point in the search space
86 :param y: the optional point in the solution space
88 :raises TypeError: if any parameter has a wrong type
89 :raises ValueError: if the parameter values are inconsistent
90 """
91 super().__init__(
92 algorithm=algorithm,
93 instance=instance,
94 objective=objective,
95 encoding=encoding,
96 rand_seed=rand_seed,
97 best_f=best_f,
98 last_improvement_fe=last_improvement_fe,
99 last_improvement_time_millis=last_improvement_time_millis,
100 total_fes=total_fes,
101 total_time_millis=total_time_millis,
102 goal_f=goal_f,
103 max_fes=max_fes,
104 max_time_millis=max_time_millis,
105 x=x,
106 y=y)
107 fsc = tuple.__len__(fs)
108 if fsc <= 0:
109 raise ValueError("Number of objectives must be greater than 0.")
110 fsu: list[int | float] = []
111 changed: bool = False
112 for val in fs:
113 val2 = try_int(val)
114 if val2 is not val:
115 changed = True
116 fsu.append(val2)
117 object.__setattr__(self, "fs", tuple(fsu) if changed else fs)
119 def _tuple(self) -> tuple[Any, ...]:
120 """
121 Get the comparison tuple.
123 :return: the comparison tuple
124 """
125 cr = list(super()._tuple())
126 cr.extend(self.fs)
127 return tuple(cr)
130def to_csv(results: Iterable[EndResult], file: str) -> Path:
131 """
132 Write a sequence of end results to a file in CSV format.
134 :param results: the end results
135 :param file: the path
136 :return: the path of the file that was written
137 """
138 path: Final[Path] = Path(file)
139 logger(f"Writing end results to CSV file {path!r}.")
140 path.ensure_parent_dir_exists()
141 with path.open_for_write() as wt:
142 write_lines(CsvWriter.write(results), wt)
143 logger(f"Done writing end results to CSV file {path!r}.")
144 return path
147def from_csv(file: str,
148 filterer: Callable[[EndResult], bool]
149 = lambda _: True) -> Generator[
150 EndResult | MOEndResult, None, None]:
151 """
152 Parse a given CSV file to get :class:`MOEndResult` Records.
154 :param file: the path to parse
155 :param filterer: an optional filter function
156 """
157 path: Final[Path] = file_path(file)
158 logger(f"Now reading CSV file {path!r}.")
159 with path.open_for_read() as rd:
160 for r in CsvReader.read(rd):
161 if filterer(r):
162 yield r
163 logger(f"Done reading CSV file {path!r}.")
166def from_logs(path: str, parse_x: bool = True, parse_y: bool = True) \
167 -> Generator[EndResult | MOEndResult, None, None]:
168 """
169 Parse a given path and yield all (multi-objective) end results found.
171 If `path` identifies a file with suffix `.txt`, then this file is
172 parsed. The appropriate :class:`moptipy.evaluation.end_results.EndResult`
173 or :class:`MOEndResult` is created and yielded.
174 If `path` identifies a directory, then this directory is parsed
175 recursively for each log file found, one record is yielded.
177 :param path: the path to parse
178 :param parse_x: should we parse the points in the search space, too?
179 :param parse_y: should we parse the points in the solution space, too?
180 """
181 for group in __MOEndResultLogParser(parse_x, parse_y).parse(path):
182 yield from group
185class CsvWriter(CsvWriterBase):
186 """A class for CSV writing of `EndResult` records."""
188 def __init__(self, data: Iterable[EndResult],
189 scope: str | None = None) -> None:
190 """
191 Initialize the csv writer.
193 :param data: the data
194 :param scope: the prefix to be pre-pended to all columns
195 """
196 data = reiterable(data)
197 super().__init__(data, scope)
199 #: do we need the encoding?
200 self.__fcols: Final[int] = max(
201 tuple.__len__(er.fs) if isinstance(
202 er, MOEndResult) else 0 for er in data)
204 def get_column_titles(self) -> Iterable[str]:
205 """
206 Get the column titles.
208 :returns: the column titles
209 """
210 p: Final[str | None] = self.scope
211 return chain(super().get_column_titles(), (
212 csv_scope(p, x) for x in (
213 f"f{i}" for i in range(self.__fcols))))
215 def get_row(self, data: EndResult) -> Iterable[str]:
216 """
217 Render a single end result record to a CSV row.
219 :param data: the end result record
220 :returns: the row iterator
221 """
222 yield from super().get_row(data)
223 if isinstance(data, MOEndResult):
224 yield from map(num_to_str, data.fs)
226 def get_header_comments(self) -> Iterable[str]:
227 """
228 Get any possible header comments.
230 :returns: the header comments
231 """
232 return ("Multi-Objective Experiment End Results",
233 "See the description at the bottom of the file.")
235 def get_footer_comments(self) -> Iterable[str]:
236 """
237 Get any possible footer comments.
239 :returns: the footer comments
240 """
241 yield from super().get_footer_comments()
242 for i in range(self.__fcols):
243 yield (f"f{i}: the objective value computed with "
244 f"the {i + 1}-th objective function.")
247class CsvReader(CsvReaderBase):
248 """A csv parser for end results."""
250 def __init__(self, columns: dict[str, int]) -> None:
251 """
252 Create a CSV parser for `EndResult` records.
254 :param columns: the columns
255 """
256 super().__init__(columns)
257 i: int = 0
258 fcols: Final[list[int]] = []
259 while True:
260 colname: str = f"f{i}"
261 try:
262 fcols.append(csv_column(columns, colname))
263 except KeyError:
264 break
265 i += 1
266 #: the objective value columns
267 self.__fcols: Final[tuple[int, ...]] = tuple(fcols)
269 def parse_row(self, data: list[str]) -> EndResult | MOEndResult:
270 """
271 Parse a row of data.
273 :param data: the data row
274 :return: the end result statistics
275 """
276 res = super().parse_row(data)
277 vals: Final[list[int | float]] = []
278 for col in self.__fcols:
279 v = csv_val_or_none(data, col, str_to_num)
280 if v is None:
281 break
282 vals.append(v)
283 if list.__len__(vals) <= 0:
284 return res
285 return MOEndResult(
286 algorithm=res.algorithm,
287 instance=res.instance,
288 objective=res.objective,
289 encoding=res.encoding,
290 rand_seed=res.rand_seed,
291 best_f=res.best_f,
292 last_improvement_fe=res.last_improvement_fe,
293 last_improvement_time_millis=res.last_improvement_time_millis,
294 total_fes=res.total_fes,
295 total_time_millis=res.total_time_millis,
296 goal_f=res.goal_f,
297 max_fes=res.max_fes,
298 max_time_millis=res.max_time_millis,
299 fs=tuple(vals),
300 x=res.x,
301 y=res.y)
304class __MOEndResultLogParser(Parser[Iterable[EndResult]]):
305 """The internal log parser class."""
307 def __init__(self, parse_x: bool = True, parse_y: bool = True) -> None:
308 """
309 Parse the log files.
311 :param parse_x: whether to parse the x values
312 :param parse_y: whether to parse the y values
313 """
314 super().__init__()
315 if not isinstance(parse_x, bool):
316 raise type_error(parse_x, "parse_x", bool)
317 if not isinstance(parse_y, bool):
318 raise type_error(parse_y, "parse_y", bool)
319 #: shall we parse the points in the search space?
320 self.__parse_x: Final[bool] = parse_x
321 #: shall we parse the points in the solution space?
322 self.__parse_y: Final[bool] = parse_y
324 def _parse_file(self, file: Path) -> Iterable[EndResult]:
325 """
326 Get the parsing result.
328 :returns: the `EndResult` instance
329 """
330 self._progress_logger(
331 f"Beginning multi-objective parsing of file {file!r}.")
332 o: Final[EndResult] = _Erlp().parse_file(file)
333 if not isinstance(o, EndResult):
334 raise type_error(o, f"parse({file!r})", EndResult)
336 with file.open_for_read() as reader:
337 lines: tuple[str, ...] = tuple(map(str.strip, str.splitlines(
338 reader.read())))
339 count = tuple.__len__(lines)
340 if count <= 2:
341 raise ValueError(
342 f"Inconsistent number {count} of lines in file {file!r}")
344 # first, we process the archive to find all the retained points
345 archive: Final[list[tuple[int | float, ...]]] = []
346 begin: str = f"{SECTION_START}{SECTION_ARCHIVE_QUALITY}"
347 end: str = f"{SECTION_END}{SECTION_ARCHIVE_QUALITY}"
348 state: int = 0
349 for line in lines:
350 if line == begin:
351 if state != 0:
352 raise ValueError(f"Inconsistent begin state in "
353 f"file {file!r} vs. {begin!r}/{end!r}.")
354 state = 1
355 continue
356 if line == end:
357 if state != 2:
358 raise ValueError(f"Inconsistent end state in "
359 f"file {file!r} vs. {begin!r}/{end!r}.")
360 state = 3
361 break
362 if state == 1:
363 if line.startswith("f"):
364 state = 2
365 continue
366 state = 2
367 if state != 2:
368 continue
369 try:
370 archive.append(tuple(map(str_to_num, map(str.strip, str.split(
371 line, CSV_SEPARATOR)))))
372 except ValueError as ve:
373 raise ValueError(
374 f"Error when parsing line {line!r} of file {file!r} in "
375 f"{SECTION_ARCHIVE_QUALITY}.") from ve
376 if state != 3:
377 return (o, )
379 # now, we find the progress to find out when the solutions emerged
380 progress: Final[list[tuple[int | float, ...]]] = []
381 begin = f"{SECTION_START}{SECTION_PROGRESS}"
382 end = f"{SECTION_END}{SECTION_PROGRESS}"
383 state = 0
384 for line in lines:
385 if line == begin:
386 if state != 0:
387 raise ValueError(f"Inconsistent begin state in "
388 f"file {file!r} vs. {begin!r}/{end!r}.")
389 state = 1
390 continue
391 if line == end:
392 if state != 2:
393 raise ValueError(f"Inconsistent end state in "
394 f"file {file!r} vs. {begin!r}/{end!r}.")
395 state = 3
396 break
397 if state == 1:
398 if line.startswith("fes"):
399 state = 2
400 continue
401 state = 2
402 if state != 2:
403 continue
404 try:
405 progress.append(tuple(map(str_to_num, map(str.strip, str.split(
406 line, CSV_SEPARATOR)))))
407 except ValueError as ve:
408 raise ValueError(
409 f"Error when parsing line {line!r} of file {file!r} "
410 f"in {SECTION_PROGRESS}.") from ve
411 if state not in {0, 3}:
412 raise ValueError(f"Inconsistent state {state} in "
413 f"file {file!r} vs. {begin!r}/{end!r}.")
415 count = list.__len__(archive)
416 if count < 1:
417 raise ValueError(f"No solution archived in file {file!r}.")
419 # Now we try to find the solutions and points matching them
420 x: dict[int, str] = {}
421 y: dict[int, str] = {}
422 if self.__parse_x or self.__parse_y:
423 current: list[str] = []
424 current_id: int = 0
425 state = 0
426 arc_start: Final[str] = f"{SECTION_START}{PREFIX_SECTION_ARCHIVE}"
427 arc_end: Final[str] = f"{SECTION_END}{PREFIX_SECTION_ARCHIVE}"
429 for line in lines:
430 if str.startswith(line, arc_start):
431 if state != 0:
432 raise ValueError(
433 f"Inconsistent start state in {file!r}, "
434 f"encountered {line!r}.")
435 if str.endswith(line, SUFFIX_SECTION_ARCHIVE_X):
436 current_id = int(line[len(arc_start):-len(
437 SUFFIX_SECTION_ARCHIVE_X)])
438 if current_id in x:
439 raise ValueError(
440 f"Encountered archive X {current_id} twice "
441 f"in {file!r}.")
442 state = 1
443 current.clear()
444 continue
445 if str.endswith(line, SUFFIX_SECTION_ARCHIVE_Y):
446 current_id = int(line[len(arc_start):-len(
447 SUFFIX_SECTION_ARCHIVE_Y)])
448 if current_id in y:
449 raise ValueError(
450 f"Encountered archive Y {current_id} twice "
451 f"in {file!r}.")
452 state = 2
453 current.clear()
454 continue
455 if state in {1, 2}:
456 if str.startswith(line, arc_end):
457 if state == 1:
458 if self.__parse_x:
459 x[current_id] = "\n".join(current)
460 elif (state == 2) and self.__parse_y:
461 y[current_id] = "\n".join(current)
462 current.clear()
463 state = 0
464 continue
465 current.append(line)
467 if state != 0:
468 raise ValueError(f"Inconsistent state in file {file!r}.")
470 dim: Final[int] = tuple.__len__(archive[0])
471 for solution in archive:
472 if tuple.__len__(solution) != dim:
473 raise ValueError(
474 f"Inconsistent archive dimension of {solution} in file "
475 f"{file!r}, should be {dim}.")
477 for time in progress:
478 if tuple.__len__(time) != dim + 2:
479 raise ValueError(
480 "Inconsistent progress dimension of record "
481 f"{time} in {file!r}, should be {dim + 2}.")
483 out: list[MOEndResult] = []
484 for i, solution in enumerate(archive):
485 found: tuple[int | float, ...] | None = None
486 for time in progress:
487 if time[-dim:] == solution:
488 found = time
489 break
490 out.append(MOEndResult(
491 algorithm=o.algorithm,
492 instance=o.instance,
493 objective=o.objective,
494 encoding=o.encoding,
495 rand_seed=o.rand_seed,
496 best_f=solution[0],
497 last_improvement_fe=o.last_improvement_fe
498 if found is None else cast("int", found[0]),
499 last_improvement_time_millis=o.last_improvement_time_millis
500 if found is None else cast("int", found[1]),
501 total_fes=o.total_fes,
502 total_time_millis=o.total_time_millis,
503 goal_f=o.goal_f,
504 max_fes=o.max_fes,
505 max_time_millis=o.max_time_millis,
506 fs=solution[1:],
507 x=x.get(i),
508 y=y.get(i)))
509 self._progress_logger(f"Done parsing file {file!r} multi-objectively.")
510 return out
513# Run log files to end results if executed as script
514if __name__ == "__main__":
515 parser: Final[argparse.ArgumentParser] = moptipy_argparser(
516 __file__,
517 "Convert multi-objective log files obtained with moptipy to the "
518 "end results CSV format that can be post-processed or exported to "
519 "other tools.",
520 "This program recursively parses a folder hierarchy created by"
521 " the moptipy multi-objective experiment execution facility. "
522 "This folder structure follows the scheme of algorithm/instance/"
523 "log_file and has one log file per run. As result of the parsing, "
524 "one CSV file (where columns are separated by ';') is created with"
525 " one row per log file. This row contains the end-of-run state"
526 " loaded from the log file. Whereas the log files may store "
527 "the complete progress of one run of one algorithm on one "
528 "problem instance as well as the algorithm configuration "
529 "parameters, instance features, system settings, and the final"
530 " results, the end results CSV file will only represent the "
531 "final result quality, when it was obtained, how long the runs"
532 " took, etc. This information is much denser and smaller and "
533 "suitable for importing into other tools such as Excel or for "
534 "postprocessing.")
535 parser.add_argument(
536 "source", nargs="?", default="./results",
537 help="the location of the experimental results, i.e., the root folder "
538 "under which to search for log files", type=Path)
539 parser.add_argument(
540 "dest", help="the path to the end results CSV file to be created",
541 type=Path, nargs="?", default="./evaluation/end_results.txt")
542 args: Final[argparse.Namespace] = parser.parse_args()
544 to_csv(from_logs(args.source), args.dest)