Coverage for moptipy / evaluation / tabulate_end_results.py: 69%
373 statements
« prev ^ index » next coverage.py v7.12.0, created at 2025-11-24 08:49 +0000
1"""Provides function :func:`tabulate_end_results` to tabulate end results."""
3from math import inf, isfinite, isnan, nan
4from typing import Any, Callable, Final, Iterable, cast
6from pycommons.io.csv import SCOPE_SEPARATOR
7from pycommons.io.path import Path
8from pycommons.math.sample_statistics import (
9 KEY_MAXIMUM,
10 KEY_MEAN_ARITH,
11 KEY_MEAN_GEOM,
12 KEY_MEDIAN,
13 KEY_MINIMUM,
14 KEY_STDDEV,
15)
16from pycommons.types import type_error
18from moptipy.api.logging import (
19 KEY_ALGORITHM,
20 KEY_BEST_F,
21 KEY_GOAL_F,
22 KEY_INSTANCE,
23 KEY_LAST_IMPROVEMENT_FE,
24 KEY_LAST_IMPROVEMENT_TIME_MILLIS,
25 KEY_TOTAL_FES,
26 KEY_TOTAL_TIME_MILLIS,
27)
28from moptipy.evaluation.base import F_NAME_RAW, F_NAME_SCALED
29from moptipy.evaluation.end_results import EndResult
30from moptipy.evaluation.end_statistics import KEY_BEST_F_SCALED, EndStatistics
31from moptipy.evaluation.end_statistics import (
32 from_end_results as es_from_end_results,
33)
34from moptipy.evaluation.end_statistics import getter as es_getter
35from moptipy.utils.formatted_string import FormattedStr
36from moptipy.utils.lang import Lang
37from moptipy.utils.markdown import Markdown
38from moptipy.utils.number_renderer import (
39 DEFAULT_NUMBER_RENDERER,
40 NumberRenderer,
41)
42from moptipy.utils.table import Table
43from moptipy.utils.text_format import TextFormatDriver
45#: the lower bound key
46__KEY_LOWER_BOUND: Final[str] = "lower_bound"
47#: the lower bound short key
48__KEY_LOWER_BOUND_SHORT: Final[str] = "lower_bound_short"
51def default_column_namer(col: str) -> str:
52 """
53 Get the default name for columns.
55 :param col: the column identifier
56 :returns: the column name
57 """
58 if not isinstance(col, str):
59 raise type_error(col, "column name", str)
60 if col == KEY_INSTANCE:
61 return "I"
62 if col == KEY_ALGORITHM:
63 return Lang.translate("setup")
64 if col in {__KEY_LOWER_BOUND, __KEY_LOWER_BOUND_SHORT, KEY_GOAL_F}:
65 return "lb(f)"
66 if col in "summary":
67 return Lang.translate(col)
69 if SCOPE_SEPARATOR not in col:
70 raise ValueError(
71 f"statistic {col!r} should contain {SCOPE_SEPARATOR!r}.")
72 key, stat = col.split(SCOPE_SEPARATOR)
73 if (len(key) <= 0) or (len(stat) <= 0):
74 raise ValueError(f"invalid statistic {col!r}.")
76 if key == KEY_BEST_F_SCALED:
77 key = F_NAME_SCALED
78 elif key == KEY_BEST_F:
79 key = F_NAME_RAW
81 # now fix statistics part
82 if stat == KEY_MEAN_ARITH:
83 stat = "mean"
84 elif stat == KEY_STDDEV:
85 stat = "sd"
86 elif stat == KEY_MEAN_GEOM:
87 stat = "gmean"
88 elif stat == KEY_MEDIAN:
89 stat = "med"
90 elif stat == KEY_MINIMUM:
91 stat = Lang.translate("best") \
92 if key in {F_NAME_RAW, F_NAME_SCALED} else "min"
93 elif stat == KEY_MAXIMUM:
94 stat = Lang.translate("worst") \
95 if key in {F_NAME_RAW, F_NAME_SCALED} else "max"
96 else:
97 raise ValueError(f"unknown statistic {stat!r} for {col!r}.")
99 if key == F_NAME_RAW:
100 return stat
101 if key == F_NAME_SCALED:
102 return f"{stat}1"
103 if key == KEY_LAST_IMPROVEMENT_FE:
104 key = "fes"
105 elif key == KEY_TOTAL_FES:
106 key = "FEs"
107 elif key == KEY_LAST_IMPROVEMENT_TIME_MILLIS:
108 key = "t"
109 elif key == KEY_TOTAL_TIME_MILLIS:
110 key = "T"
111 else:
112 raise ValueError(f"unknown key {key!r}.")
113 return f"{stat}({key})"
116def command_column_namer(
117 col: str, put_dollars: bool = True,
118 summary_name: Callable[[bool], str] = lambda _: r"\summary",
119 setup_name: Callable[[bool], str] = lambda _: r"\setup") \
120 -> str:
121 """
122 Get the command-based names for columns, but in command format.
124 This function returns LaTeX-style commands for the column headers.
126 :param col: the column identifier
127 :param put_dollars: surround the command with `$`
128 :param summary_name: the name function for the key "summary"
129 :param setup_name: the name function for the key `KEY_ALGORITHM`
130 :returns: the column name
131 """
132 if not isinstance(col, str):
133 raise type_error(col, "column name", str)
134 if not isinstance(put_dollars, bool):
135 raise type_error(put_dollars, "put_dollars", bool)
136 if not callable(summary_name):
137 raise type_error(summary_name, "summary_name", call=True)
138 if not callable(setup_name):
139 raise type_error(setup_name, "setup_name", call=True)
140 if col == KEY_INSTANCE:
141 return r"$\instance$"
142 if col == KEY_ALGORITHM:
143 return setup_name(put_dollars)
144 if col in {__KEY_LOWER_BOUND, __KEY_LOWER_BOUND_SHORT, KEY_GOAL_F}:
145 return r"$\lowerBound(\objf)$" if \
146 put_dollars else r"\lowerBound(\objf)"
147 if col == "summary":
148 return summary_name(put_dollars)
150 if SCOPE_SEPARATOR not in col:
151 raise ValueError(
152 f"statistic {col!r} should contain {SCOPE_SEPARATOR!r}.")
153 key, stat = col.split(SCOPE_SEPARATOR)
154 if (len(key) <= 0) or (len(stat) <= 0):
155 raise ValueError(f"invalid statistic {col!r}.")
157 if key == KEY_BEST_F_SCALED:
158 key = F_NAME_SCALED
159 elif key == KEY_BEST_F:
160 key = F_NAME_RAW
162 # now fix statistics part
163 if stat == KEY_MEAN_ARITH:
164 stat = "mean"
165 elif stat == KEY_STDDEV:
166 stat = "stddev"
167 elif stat == KEY_MEAN_GEOM:
168 stat = "geomean"
169 elif stat == KEY_MEDIAN:
170 stat = "median"
171 elif stat == KEY_MINIMUM:
172 stat = "min"
173 elif stat == KEY_MAXIMUM:
174 stat = "max"
175 else:
176 raise ValueError(f"unknown statistic {stat!r} for {col!r}.")
178 if key == F_NAME_RAW:
179 key = "BestF"
180 elif key == F_NAME_SCALED:
181 key = "BestFscaled"
182 elif key == KEY_LAST_IMPROVEMENT_FE:
183 key = "LIFE"
184 elif key == KEY_TOTAL_FES:
185 key = "TotalFEs"
186 elif key == KEY_LAST_IMPROVEMENT_TIME_MILLIS:
187 key = "LIMS"
188 elif key == KEY_TOTAL_TIME_MILLIS:
189 key = "TotalMS"
190 else:
191 raise ValueError(f"unknown key {key!r}.")
192 return f"$\\{stat}{key}$" if put_dollars else f"\\{stat}{key}"
195def __finite_max(data: Iterable[int | float | None]) \
196 -> int | float:
197 """
198 Compute the finite maximum of a data column.
200 :param data: the data to iterate over
201 :returns: the finite maximum, or `nan` if none can be found or if there
202 is only a single value
203 """
204 if not isinstance(data, Iterable):
205 raise type_error(data, "data", Iterable)
206 maxi: int | float = -inf
207 count: int = 0
208 for d in data:
209 count += 1
210 if d is None:
211 continue
212 if isfinite(d) and (d > maxi):
213 maxi = d
214 return maxi if (count > 1) and isfinite(maxi) else nan
217def __finite_min(data: Iterable[int | float | None]) \
218 -> int | float:
219 """
220 Compute the finite minimum of a data column.
222 :param data: the data to iterate over
223 :returns: the finite minimum, or `nan` if none can be found
224 """
225 if not isinstance(data, Iterable):
226 raise type_error(data, "data", Iterable)
227 mini: int | float = inf
228 count: int = 0
229 for d in data:
230 count += 1
231 if d is None:
232 continue
233 if isfinite(d) and (d < mini):
234 mini = d
235 return mini if (count > 1) and isfinite(mini) else nan
238def __nan(data: Iterable[int | float | None]) -> float:
239 """
240 Get `nan`.
242 :param data: ignored
243 """
244 if not isinstance(data, Iterable):
245 raise type_error(data, "data", Iterable)
246 return nan
249def default_column_best(col: str) ->\
250 Callable[[Iterable[int | float | None]], int | float]:
251 """
252 Get a function to compute the best value in a column.
254 The returned function can compute the best value in a column. If no value
255 is best, it should return `nan`.
257 :param col: the column name string
258 :returns: a function that can compute the best value per column
259 """
260 if not isinstance(col, str):
261 raise type_error(col, "column name", str)
263 if col in {KEY_INSTANCE, KEY_ALGORITHM, __KEY_LOWER_BOUND,
264 __KEY_LOWER_BOUND_SHORT, KEY_GOAL_F}:
265 return __nan
267 if SCOPE_SEPARATOR not in col:
268 raise ValueError(
269 f"statistic {col!r} should contain {SCOPE_SEPARATOR!r}.")
270 key, stat = col.split(SCOPE_SEPARATOR)
271 if (len(key) <= 0) or (len(stat) <= 0):
272 raise ValueError(f"invalid statistic {col!r}.")
274 if stat == "sd":
275 return __finite_min
277 if key in {KEY_BEST_F_SCALED, F_NAME_SCALED, KEY_BEST_F, F_NAME_RAW}:
278 return __finite_min
279 if key in {KEY_LAST_IMPROVEMENT_TIME_MILLIS, KEY_LAST_IMPROVEMENT_FE,
280 KEY_TOTAL_FES, KEY_TOTAL_TIME_MILLIS}:
281 return __finite_max
283 return __nan
286#: the number renderer for times
287__TIME_NUMBER_RENDERER: Final[NumberRenderer] = \
288 DEFAULT_NUMBER_RENDERER.derive(
289 int_to_float_threshold=999_999,
290 get_float_format=lambda _, ma, __, ___, itft:
291 "{:.0f}" if ma <= itft else "{:.1e}")
294def default_number_renderer(col: str) -> NumberRenderer:
295 """
296 Get the number renderer for the specified column.
298 Time columns are rendered with less precision.
300 :param col: the column name
301 :returns: the number renderer
302 """
303 if not isinstance(col, str):
304 raise type_error(col, "column name", str)
305 if col not in {KEY_INSTANCE, KEY_ALGORITHM, __KEY_LOWER_BOUND,
306 __KEY_LOWER_BOUND_SHORT, KEY_GOAL_F}:
307 if SCOPE_SEPARATOR not in col:
308 raise ValueError(
309 f"statistic {col!r} should contain {SCOPE_SEPARATOR!r}.")
310 key, stat = col.split(SCOPE_SEPARATOR)
311 if (len(key) <= 0) or (len(stat) <= 0):
312 raise ValueError(f"invalid statistic {col!r}.")
313 if key in {KEY_LAST_IMPROVEMENT_TIME_MILLIS, KEY_LAST_IMPROVEMENT_FE,
314 KEY_TOTAL_FES, KEY_TOTAL_TIME_MILLIS}:
315 return __TIME_NUMBER_RENDERER
316 return DEFAULT_NUMBER_RENDERER
319def __getter(s: str) -> Callable[[EndStatistics], int | float | None]:
320 """
321 Obtain a getter for the end statistics.
323 :param s: the name
324 :returns: the getter
325 """
326 if not isinstance(s, str):
327 raise type_error(s, "getter name", str)
328 getter = es_getter(s)
330 def __fixed(e: EndStatistics, g=getter, n=s) -> int | float | None:
331 res = g(e)
332 if res is None:
333 return None
334 if not isinstance(res, int | float):
335 raise type_error(res, f"result of getter {n!r} for statistic {e}",
336 (int, float))
337 return res
338 return __fixed
341#: the default algorithm-instance statistics
342DEFAULT_ALGORITHM_INSTANCE_STATISTICS: Final[tuple[
343 str, str, str, str, str, str]] = (
344 f"{KEY_BEST_F}{SCOPE_SEPARATOR}{KEY_MINIMUM}",
345 f"{KEY_BEST_F}{SCOPE_SEPARATOR}{KEY_MEAN_ARITH}",
346 f"{KEY_BEST_F}{SCOPE_SEPARATOR}{KEY_STDDEV}",
347 f"{KEY_BEST_F_SCALED}{SCOPE_SEPARATOR}{KEY_MEAN_ARITH}",
348 f"{KEY_LAST_IMPROVEMENT_FE}{SCOPE_SEPARATOR}{KEY_MEAN_ARITH}",
349 f"{KEY_LAST_IMPROVEMENT_TIME_MILLIS}{SCOPE_SEPARATOR}"
350 f"{KEY_MEAN_ARITH}")
352#: the default algorithm summary statistics
353DEFAULT_ALGORITHM_SUMMARY_STATISTICS: Final[tuple[
354 str, str, str, str, str, str]] = (
355 f"{KEY_BEST_F_SCALED}{SCOPE_SEPARATOR}{KEY_MINIMUM}",
356 f"{KEY_BEST_F_SCALED}{SCOPE_SEPARATOR}{KEY_MEAN_GEOM}",
357 f"{KEY_BEST_F_SCALED}{SCOPE_SEPARATOR}{KEY_MAXIMUM}",
358 f"{KEY_BEST_F_SCALED}{SCOPE_SEPARATOR}{KEY_STDDEV}",
359 f"{KEY_LAST_IMPROVEMENT_FE}{SCOPE_SEPARATOR}{KEY_MEAN_ARITH}",
360 f"{KEY_LAST_IMPROVEMENT_TIME_MILLIS}{SCOPE_SEPARATOR}"
361 f"{KEY_MEAN_ARITH}")
364def tabulate_end_results(
365 end_results: Iterable[EndResult],
366 file_name: str = "table",
367 dir_name: str = ".",
368 algorithm_instance_statistics: Iterable[str] =
369 DEFAULT_ALGORITHM_INSTANCE_STATISTICS,
370 algorithm_summary_statistics: Iterable[str | None] | None =
371 DEFAULT_ALGORITHM_SUMMARY_STATISTICS,
372 text_format_driver: TextFormatDriver | Callable[[], TextFormatDriver]
373 = Markdown.instance,
374 algorithm_sort_key: Callable[[str], Any] = lambda a: a,
375 instance_sort_key: Callable[[str], Any] = lambda i: i,
376 col_namer: Callable[[str], str] = default_column_namer,
377 col_best: Callable[[str], Callable[
378 [Iterable[int | float | None]],
379 int | float]] = default_column_best,
380 col_renderer: Callable[[str], NumberRenderer] =
381 default_number_renderer,
382 put_lower_bound: bool = True,
383 lower_bound_getter: Callable[[EndStatistics],
384 int | float | None] | None =
385 __getter(KEY_GOAL_F),
386 lower_bound_name: str | None = __KEY_LOWER_BOUND,
387 use_lang: bool = True,
388 instance_namer: Callable[[str], str] = lambda x: x,
389 algorithm_namer: Callable[[str], str] = lambda x: x) -> Path:
390 r"""
391 Tabulate the statistics about the end results of an experiment.
393 A two-part table is produced. In the first part, it presents summary
394 statistics about each instance-algorithm combination, sorted by instance.
395 In the second part, it presents summary statistics of the algorithms over
396 all instances. The following default columns are provided:
398 1. Part 1: Algorithm-Instance statistics
399 - `I`: the instance name
400 - `lb(f)`: the lower bound of the objective value of
401 the instance
402 - `setup`: the name of the algorithm or algorithm setup
403 - `best`: the best objective value reached by any run on that instance
404 - `mean`: the arithmetic mean of the best objective values reached
405 over all runs
406 - `sd`: the standard deviation of the best objective values reached
407 over all runs
408 - `mean1`: the arithmetic mean of the best objective values reached
409 over all runs, divided by the lower bound (or goal objective value)
410 - `mean(FE/ms)`: the arithmetic mean of objective function evaluations
411 performed per millisecond, over all runs
412 - `mean(t)`: the arithmetic mean of the time in milliseconds when the
413 last improving move of a run was applied, over all runs
415 2. Part 2: Algorithm Summary Statistics
416 - `setup`: the name of the algorithm or algorithm setup
417 - `best1`: the minimum of the best objective values reached divided by
418 the lower bound (or goal objective value) over all runs
419 - `gmean1`: the geometric mean of the best objective values reached
420 divided by the lower bound (or goal objective value) over all runs
421 - `worst1`: the maximum of the best objective values reached divided
422 by the lower bound (or goal objective value) over all runs
423 - `sd1`: the standard deviation of the best objective values reached
424 divided by the lower bound (or goal objective value) over all runs
425 - `gmean(FE/ms)`: the geometric mean of objective function evaluations
426 performed per millisecond, over all runs
427 - `gmean(t)`: the geometric mean of the time in milliseconds when the
428 last improving move of a run was applied, over all runs
430 You can freely configure which columns you want for each part and whether
431 you want to have the second part included. Also, for each group of values,
432 the best one is marked in bold face.
434 Depending on the parameter `text_format_driver`, the tables can be
435 rendered in different formats, such as
436 :py:class:`~moptipy.utils.markdown.Markdown`,
437 :py:class:`~moptipy.utils.latex.LaTeX`, and
438 :py:class:`~moptipy.utils.html.HTML`.
440 :param end_results: the end results data
441 :param file_name: the base file name
442 :param dir_name: the base directory
443 :param algorithm_instance_statistics: the statistics to print
444 :param algorithm_summary_statistics: the summary statistics to print per
445 algorithm
446 :param text_format_driver: the text format driver
447 :param algorithm_sort_key: a function returning sort keys for algorithms
448 :param instance_sort_key: a function returning sort keys for instances
449 :param col_namer: the column namer function
450 :param col_best: the column-best getter function
451 :param col_renderer: the number renderer for the column
452 :param put_lower_bound: should we put the lower bound or goal objective
453 value?
454 :param lower_bound_getter: the getter for the lower bound
455 :param lower_bound_name: the name key for the lower bound to be passed
456 to `col_namer`
457 :param use_lang: should we use the language to define the filename?
458 :param instance_namer: the name function for instances receives an
459 instance ID and returns an instance name; default=identity function
460 :param algorithm_namer: the name function for algorithms receives an
461 algorithm ID and returns an instance name; default=identity function
462 :returns: the path to the file with the tabulated end results
463 """
464 # Before doing anything, let's do some type checking on the parameters.
465 # I want to ensure that this function is called correctly before we begin
466 # to actually process the data. It is better to fail early than to deliver
467 # some incorrect results.
468 if not isinstance(end_results, Iterable):
469 raise type_error(end_results, "end_results", Iterable)
470 if not isinstance(file_name, str):
471 raise type_error(file_name, "file_name", str)
472 if not isinstance(dir_name, str):
473 raise type_error(dir_name, "dir_name", str)
474 if not isinstance(algorithm_instance_statistics, Iterable):
475 raise type_error(algorithm_instance_statistics,
476 "algorithm_instance_statistics", Iterable)
477 if (algorithm_summary_statistics is not None)\
478 and (not isinstance(algorithm_instance_statistics, Iterable)):
479 raise type_error(algorithm_summary_statistics,
480 "algorithm_summary_statistics", (Iterable, None))
481 if not isinstance(put_lower_bound, bool):
482 raise type_error(put_lower_bound, "put_lower_bound", bool)
483 if put_lower_bound:
484 if not callable(lower_bound_getter):
485 raise type_error(lower_bound_getter, "lower_bound_getter",
486 call=True)
487 if not isinstance(lower_bound_name, str):
488 raise type_error(lower_bound_name, "lower_bound_name", str)
489 if callable(text_format_driver):
490 text_format_driver = text_format_driver()
491 if not isinstance(text_format_driver, TextFormatDriver):
492 raise type_error(text_format_driver, "text_format_driver",
493 TextFormatDriver, True)
494 if not callable(col_namer):
495 raise type_error(col_namer, "col_namer", call=True)
496 if not callable(col_best):
497 raise type_error(col_best, "col_best", call=True)
498 if not callable(col_renderer):
499 raise type_error(col_renderer, "col_renderer", call=True)
500 if not isinstance(use_lang, bool):
501 raise type_error(use_lang, "use_lang", bool)
502 if not callable(algorithm_namer):
503 raise type_error(algorithm_namer, "algorithm_namer", call=True)
504 if not callable(instance_namer):
505 raise type_error(instance_namer, "instance_namer", call=True)
507 # quick protection of renderer
508 def __col_renderer(col: str, __fwd=col_renderer) -> NumberRenderer:
509 res = __fwd(col)
510 if not isinstance(res, NumberRenderer):
511 raise type_error(res, f"col_renderer({col!r})", NumberRenderer)
512 return res
514 # get the getters
515 algo_inst_getters: Final[list[Callable[[EndStatistics],
516 int | float | None]]] = \
517 [__getter(d) for d in algorithm_instance_statistics]
518 n_algo_inst_getters: Final[int] = len(algo_inst_getters)
519 if n_algo_inst_getters <= 0:
520 raise ValueError("algorithm-instance dimensions must not be empty.")
521 algo_getters: Final[list[Callable[[EndStatistics],
522 int | float | None] | None] | None] = \
523 (None if (algorithm_summary_statistics is None)
524 else [None if (d is None) else __getter(d)
525 for d in cast("Iterable", algorithm_summary_statistics)])
526 if algo_getters is not None:
527 if len(algo_getters) != n_algo_inst_getters:
528 raise ValueError(
529 f"there are {n_algo_inst_getters} algorithm-instance columns,"
530 f" but {len(algo_getters)} algorithms summary columns.")
531 if all(g is None for g in algo_getters):
532 raise ValueError(
533 "if all elements of algorithm_summary_statistics are None, "
534 "then specify algorithm_summary_statistics=None")
536 # gather the statistics for each algorithm-instance combination
537 algo_inst_list: Final[list[EndStatistics]] = list(es_from_end_results(
538 end_results))
539 if len(algo_inst_list) <= 0:
540 raise ValueError("no algorithm-instance combinations?")
541 # get the sorted lists of algorithms and instances
542 insts: Final[list[str]] = sorted({s.instance for s in algo_inst_list},
543 key=instance_sort_key)
544 n_insts: Final[int] = len(insts)
545 if n_insts <= 0:
546 raise ValueError("no instance found?")
547 inst_names: Final[list[str]] = [instance_namer(inst) for inst in insts]
548 algos: Final[list[str]] = sorted({s.algorithm for s in algo_inst_list},
549 key=algorithm_sort_key)
550 n_algos: Final[int] = len(algos)
551 if n_algos <= 0:
552 raise ValueError("no algos found?")
553 algo_names: Final[list[str]] = [algorithm_namer(algo) for algo in algos]
555 # finalize the data dictionaries: d[inst][algo] = stats
556 algo_inst_dict: Final[dict[str, dict[str, EndStatistics]]] = {}
557 for e in algo_inst_list:
558 if e.instance not in algo_inst_dict:
559 algo_inst_dict[e.instance] = {}
560 algo_inst_dict[e.instance][e.algorithm] = e
561 for ina in insts:
562 if len(algo_inst_dict[ina]) != n_algos:
563 raise ValueError(
564 f"expected {n_algos} entries for instance {ina!r}, but "
565 f"got only {len(algo_inst_dict[ina])}, namely "
566 f"{algo_inst_dict[ina].keys()} instead of {algos}.")
568 # compute the per-instance lower bounds if we need them
569 lower_bounds: list[str] | None
570 if put_lower_bound:
571 lb: list[int | float | None] = []
572 for inst in insts:
573 bounds = list({lower_bound_getter(d)
574 for d in algo_inst_dict[inst].values()})
575 if len(bounds) != 1:
576 raise ValueError(f"inconsistent lower bounds {bounds} for "
577 f"instance {inst!r}.")
578 lb.append(bounds[0])
579 lower_bounds = cast("list[str]", __col_renderer(
580 __KEY_LOWER_BOUND).render(lb))
581 del lb
582 else:
583 lower_bounds = None
584 del algo_inst_list
586 # gather the algorithm summary statistics
587 algo_dict: Final[dict[str, EndStatistics] | None] = {} \
588 if (n_insts > 1) and (algo_getters is not None) else None
589 if algo_dict is not None:
590 for es in es_from_end_results(end_results, join_all_instances=True):
591 if es.algorithm in algo_dict:
592 raise ValueError(f"already encountered {es.algorithm}?")
593 algo_dict[es.algorithm] = es
594 del end_results
595 if len(algo_dict) != n_algos:
596 raise ValueError(f"there are {n_algos} algorithms, but in the "
597 f"summary, only {len(algo_dict)} appear?")
599 # set up column titles
600 def __fix_name(s: str, nn=col_namer) -> str:
601 if not isinstance(s, str):
602 raise type_error(s, "column name", str)
603 if len(s) <= 0:
604 raise ValueError("string must not be empty!")
605 na = nn(s)
606 if not isinstance(na, str):
607 raise type_error(na, f"name computed for {s}", str)
608 if len(na) <= 0:
609 raise ValueError(f"name computed for {s} cannot be empty.")
610 return na
612 algo_inst_cols: Final[list[str]] = \
613 [__fix_name(s) for s in algorithm_instance_statistics]
614 if len(algo_inst_cols) <= 0:
615 raise ValueError("no algorithm_instance columns?")
616 algo_cols: list[str | None] | None = \
617 None if algo_dict is None else \
618 [(None if s is None else __fix_name(s))
619 for s in cast("Iterable", algorithm_summary_statistics)]
620 if algo_cols == algo_inst_cols:
621 algo_cols = None # no need to repeat header if it is the same
623 # set up the column definitions
624 algo_inst_cols.insert(0, __fix_name(KEY_ALGORITHM))
625 if put_lower_bound:
626 algo_inst_cols.insert(0, __fix_name(lower_bound_name))
627 algo_inst_cols.insert(0, __fix_name(KEY_INSTANCE))
629 if algo_cols is not None:
630 algo_cols.insert(0, __fix_name(KEY_ALGORITHM))
631 if put_lower_bound:
632 algo_cols.insert(0, None)
633 algo_cols.insert(0, None)
635 col_def: Final[str] = ("lrl" if put_lower_bound else "ll") \
636 + ("r" * n_algo_inst_getters)
638 # get the data columns of all columns and convert to strings
639 # format: column -> columns data
640 # we first need to get all the data at once to allow for a uniform
641 # formatting via numbers_to_strings
642 algo_inst_data_raw: Final[list[list[int | float | None]]] =\
643 [[None if getter is None else getter(algo_inst_dict[inst][algo])
644 for inst in insts for algo in algos]
645 for getter in algo_inst_getters]
646 algo_inst_strs_raw: Final[list[list[str | None]]] = [
647 cast("list[str | None]",
648 __col_renderer(ais).render(algo_inst_data_raw[i]))
649 for i, ais in enumerate(algorithm_instance_statistics)]
651 # now break the data into sections
652 # format: column -> per-instance section -> section data
653 # after we break the data in sections, we can mark the per-section bests
654 # and we can flush the data to the table section-wise
655 algo_inst_data: Final[list[list[list[int | float | None]]]] = \
656 [[col[c * n_algos:(c + 1) * n_algos] for c in range(n_insts)]
657 for col in algo_inst_data_raw]
658 del algo_inst_data_raw
659 algo_inst_strs: Final[list[list[list[str | None]]]] = \
660 [[col[c * n_algos:(c + 1) * n_algos] for c in range(n_insts)]
661 for col in algo_inst_strs_raw]
662 del algo_inst_strs_raw
664 # now format the data, i.e., compute the per-section best value
665 # of each column and mark it with bold face
666 for col_idx, stat in enumerate(algorithm_instance_statistics):
667 col_n = algo_inst_data[col_idx]
668 col_s = algo_inst_strs[col_idx]
669 best_getter = col_best(stat)
670 if not callable(best_getter):
671 raise type_error(best_getter, f"result of col_best for {stat}",
672 call=True)
673 for chunk_idx, chunk_n in enumerate(col_n):
674 chunk_s = col_s[chunk_idx]
675 best = best_getter(chunk_n)
676 if (best is not None) and (not isnan(best)):
677 for idx, val in enumerate(chunk_n):
678 if val == best:
679 chunk_s[idx] = FormattedStr.add_format(
680 chunk_s[idx], bold=True)
681 del algo_inst_data
682 del algorithm_instance_statistics
684 # now we pre-pend the instance and algorithm information
685 algo_names_formatted: list[str] = [
686 FormattedStr.add_format(algo, code=True) for algo in algo_names]
687 algo_inst_strs.insert(0, [algo_names_formatted] * n_insts)
688 if put_lower_bound:
689 algo_inst_strs.insert(0, [[b] for b in lower_bounds])
690 algo_inst_strs.insert(0, [[
691 FormattedStr.add_format(inst, code=True)] for inst in inst_names])
692 del lower_bounds
693 del insts
695 algo_strs: list[list[str | None]] | None = None
696 if (algo_dict is not None) and (algorithm_summary_statistics is not None):
697 # get the data columns of the algorithm summaries
698 # format: column -> columns data
699 algo_data: Final[list[list[int | float | None]]] = \
700 [[None if getter is None else getter(algo_dict[algo])
701 for algo in algos]
702 for getter in algo_getters]
703 algo_strs = [cast("list[str]", __col_renderer(ass).render(
704 algo_data[i])) for i, ass in enumerate(
705 algorithm_summary_statistics)]
707 # now format the data, i.e., compute the per-section best value
708 # of each column and mark it with bold face
709 for col_idx, stat in enumerate(algorithm_summary_statistics):
710 if stat is None:
711 continue
712 acol_n = algo_data[col_idx]
713 acol_s = algo_strs[col_idx]
714 best_getter = col_best(stat)
715 if not callable(best_getter):
716 raise type_error(
717 best_getter, f"result of col_best for {stat}", call=True)
718 best = best_getter(acol_n)
719 if (best is not None) and (not isnan(best)):
720 for idx, val in enumerate(acol_n):
721 if val == best:
722 acol_s[idx] = FormattedStr.add_format(
723 acol_s[idx], bold=True)
724 del algo_data
725 algo_strs.insert(0, algo_names_formatted)
726 if put_lower_bound:
727 algo_strs.insert(0, [])
728 algo_strs.insert(0, [__fix_name("summary")] * n_algos)
730 # write the table
731 dest: Final[Path] = text_format_driver.filename(
732 file_name, dir_name, use_lang)
733 with dest.open_for_write() as wd, \
734 Table(wd, col_def, text_format_driver) as table:
735 with table.header() as head:
736 head.full_row(algo_inst_cols)
737 for i in range(n_insts):
738 with table.section() as sec:
739 sec.cols([col[i] for col in algo_inst_strs])
740 if algo_strs is not None:
741 with table.section() as sec:
742 if algo_cols is not None:
743 with sec.header() as head:
744 head.full_row(algo_cols)
745 sec.cols(algo_strs)
747 return dest