Coverage for moptipy / evaluation / plot_ecdf.py: 74%
207 statements
« prev ^ index » next coverage.py v7.12.0, created at 2025-11-24 08:49 +0000
1"""
2Plot a set of ECDF or ERT-ECDF objects into one figure.
4The empirical cumulative distribution function (ECDF, see
5:mod:`~moptipy.evaluation.ecdf`) is a function that shows the fraction of runs
6that were successful in attaining a certain goal objective value over the
7time. The combination of ERT and ECDF is discussed in
8:mod:`~moptipy.evaluation.ertecdf`.
101. Nikolaus Hansen, Anne Auger, Steffen Finck, Raymond Ros. *Real-Parameter
11 Black-Box Optimization Benchmarking 2010: Experimental Setup.*
12 Research Report RR-7215, INRIA. 2010. inria-00462481.
13 https://hal.inria.fr/inria-00462481/document/
142. Dave Andrew Douglas Tompkins and Holger H. Hoos. UBCSAT: An Implementation
15 and Experimentation Environment for SLS Algorithms for SAT and MAX-SAT. In
16 *Revised Selected Papers from the Seventh International Conference on
17 Theory and Applications of Satisfiability Testing (SAT'04),* May 10-13,
18 2004, Vancouver, BC, Canada, pages 306-320. Lecture Notes in Computer
19 Science (LNCS), volume 3542. Berlin, Germany: Springer-Verlag GmbH.
20 ISBN: 3-540-27829-X. doi: https://doi.org/10.1007/11527695_24.
213. Holger H. Hoos and Thomas Stützle. Evaluating Las Vegas Algorithms -
22 Pitfalls and Remedies. In Gregory F. Cooper and Serafín Moral, editors,
23 *Proceedings of the 14th Conference on Uncertainty in Artificial
24 Intelligence (UAI'98)*, July 24-26, 1998, Madison, WI, USA, pages 238-245.
25 San Francisco, CA, USA: Morgan Kaufmann Publishers Inc.
26 ISBN: 1-55860-555-X.
27"""
28from math import inf, isfinite
29from typing import Any, Callable, Final, Iterable, cast
31import numpy as np
32from matplotlib.artist import Artist # type: ignore
33from matplotlib.axes import Axes # type: ignore
34from matplotlib.figure import Figure # type: ignore
35from pycommons.types import type_error
37import moptipy.utils.plot_defaults as pd
38import moptipy.utils.plot_utils as pu
39from moptipy.evaluation.axis_ranger import AxisRanger
40from moptipy.evaluation.base import get_algorithm, sort_key
41from moptipy.evaluation.ecdf import Ecdf, get_goal, goal_to_str
42from moptipy.evaluation.styler import Styler
43from moptipy.utils.lang import Lang
46def plot_ecdf(ecdfs: Iterable[Ecdf],
47 figure: Axes | Figure,
48 x_axis: AxisRanger | Callable[[str], AxisRanger]
49 = AxisRanger.for_axis,
50 y_axis: AxisRanger | Callable[[str], AxisRanger]
51 = AxisRanger.for_axis,
52 legend: bool = True,
53 distinct_colors_func: Callable[[int], Any] =
54 pd.distinct_colors,
55 distinct_line_dashes_func: Callable[[int], Any] =
56 pd.distinct_line_dashes,
57 importance_to_line_width_func: Callable[[int], float] =
58 pd.importance_to_line_width,
59 importance_to_alpha_func: Callable[[int], float] =
60 pd.importance_to_alpha,
61 importance_to_font_size_func: Callable[[int], float] =
62 pd.importance_to_font_size,
63 x_grid: bool = True,
64 y_grid: bool = True,
65 x_label: str | Callable[[str], str] | None =
66 lambda x: x if isinstance(x, str) else x[0],
67 x_label_inside: bool = True,
68 y_label: str | Callable[[str], str] | None =
69 Lang.translate_func("ECDF"),
70 y_label_inside: bool = True,
71 algorithm_priority: float = 5.0,
72 goal_priority: float = 0.333,
73 algorithm_sort_key: Callable[[str], Any] = lambda x: x,
74 goal_sort_key: Callable[[str], Any] = lambda x: x,
75 algorithm_namer: Callable[[str], str] = lambda x: x,
76 color_algorithms_as_fallback_group: bool = True) -> Axes:
77 """
78 Plot a set of ECDF functions into one chart.
80 :param ecdfs: the iterable of ECDF functions
81 :param figure: the figure to plot in
82 :param x_axis: the x_axis ranger
83 :param y_axis: the y_axis ranger
84 :param legend: should we plot the legend?
85 :param distinct_colors_func: the function returning the palette
86 :param distinct_line_dashes_func: the function returning the line styles
87 :param importance_to_line_width_func: the function converting importance
88 values to line widths
89 :param importance_to_alpha_func: the function converting importance
90 values to alphas
91 :param importance_to_font_size_func: the function converting importance
92 values to font sizes
93 :param x_grid: should we have a grid along the x-axis?
94 :param y_grid: should we have a grid along the y-axis?
95 :param x_label: a callable returning the label for the x-axis, a label
96 string, or `None` if no label should be put
97 :param x_label_inside: put the x-axis label inside the plot (so that
98 it does not consume additional vertical space)
99 :param y_label: a callable returning the label for the y-axis, a label
100 string, or `None` if no label should be put
101 :param y_label_inside: put the y-axis label inside the plot (so that
102 it does not consume additional horizontal space)
103 :param algorithm_priority: the style priority for algorithms
104 :param goal_priority: the style priority for goal values
105 :param algorithm_namer: the name function for algorithms receives an
106 algorithm ID and returns an instance name; default=identity function
107 :param color_algorithms_as_fallback_group: if only a single group of data
108 was found, use algorithms as group and put them in the legend
109 :param algorithm_sort_key: the sort key function for algorithms
110 :param goal_sort_key: the sort key function for goals
111 :returns: the axes object to allow you to add further plot elements
112 """
113 # Before doing anything, let's do some type checking on the parameters.
114 # I want to ensure that this function is called correctly before we begin
115 # to actually process the data. It is better to fail early than to deliver
116 # some incorrect results.
117 if not isinstance(ecdfs, Iterable):
118 raise type_error(ecdfs, "ecdfs", Iterable)
119 if not isinstance(figure, Axes | Figure):
120 raise type_error(figure, "figure", (Axes, Figure))
121 if not isinstance(legend, bool):
122 raise type_error(legend, "legend", bool)
123 if not callable(distinct_colors_func):
124 raise type_error(
125 distinct_colors_func, "distinct_colors_func", call=True)
126 if not callable(distinct_line_dashes_func):
127 raise type_error(
128 distinct_line_dashes_func, "distinct_line_dashes_func", call=True)
129 if not callable(distinct_line_dashes_func):
130 raise type_error(importance_to_line_width_func,
131 "importance_to_line_width_func", call=True)
132 if not callable(importance_to_alpha_func):
133 raise type_error(
134 importance_to_alpha_func, "importance_to_alpha_func", call=True)
135 if not callable(importance_to_font_size_func):
136 raise type_error(importance_to_font_size_func,
137 "importance_to_font_size_func", call=True)
138 if not isinstance(x_grid, bool):
139 raise type_error(x_grid, "x_grid", bool)
140 if not isinstance(y_grid, bool):
141 raise type_error(y_grid, "y_grid", bool)
142 if not ((x_label is None) or callable(x_label)
143 or isinstance(x_label, str)):
144 raise type_error(x_label, "x_label", (str, None), call=True)
145 if not isinstance(x_label_inside, bool):
146 raise type_error(x_label_inside, "x_label_inside", bool)
147 if not ((y_label is None) or callable(y_label)
148 or isinstance(y_label, str)):
149 raise type_error(y_label, "y_label", (str, None), call=True)
150 if not isinstance(y_label_inside, bool):
151 raise type_error(y_label_inside, "y_label_inside", bool)
152 if not isinstance(algorithm_priority, float):
153 raise type_error(algorithm_priority, "algorithm_priority", float)
154 if not isfinite(algorithm_priority):
155 raise ValueError(f"algorithm_priority cannot be {algorithm_priority}.")
156 if not isfinite(goal_priority):
157 raise ValueError(f"goal_priority cannot be {goal_priority}.")
158 if not callable(algorithm_namer):
159 raise type_error(algorithm_namer, "algorithm_namer", call=True)
160 if not callable(algorithm_sort_key):
161 raise type_error(algorithm_sort_key, "algorithm_sort_key", call=True)
162 if not callable(goal_sort_key):
163 raise type_error(goal_sort_key, "goal_sort_key", call=True)
165 # First, we try to find groups of data to plot together in the same
166 # color/style. We distinguish progress objects from statistical runs.
167 goals: Final[Styler] = Styler(key_func=get_goal,
168 namer=goal_to_str,
169 priority=goal_priority,
170 name_sort_function=goal_sort_key)
171 algorithms: Final[Styler] = Styler(key_func=get_algorithm,
172 namer=algorithm_namer,
173 none_name=Lang.translate("all_algos"),
174 priority=algorithm_priority,
175 name_sort_function=algorithm_sort_key)
176 f_dim: str | None = None
177 t_dim: str | None = None
178 source: list[Ecdf] = cast("list[Ecdf]", ecdfs) \
179 if isinstance(ecdfs, list) else list(ecdfs)
180 del ecdfs
182 x_labels: set[str] = set()
184 # First pass: find out the goals and algorithms
185 for ee in source:
186 if not isinstance(ee, Ecdf):
187 raise type_error(ee, "data source", Ecdf)
188 goals.add(ee)
189 algorithms.add(ee)
190 x_labels.add(ee.time_label())
192 # Validate that we have consistent time and objective units.
193 if f_dim is None:
194 f_dim = ee.f_name
195 elif f_dim != ee.f_name:
196 raise ValueError(
197 f"F-units {f_dim} and {ee.f_name} do not fit!")
199 if t_dim is None:
200 t_dim = ee.time_unit
201 elif t_dim != ee.time_unit:
202 raise ValueError(
203 f"Time units {t_dim} and {ee.time_unit} do not fit!")
205 if f_dim is None:
206 raise ValueError("f_dim cannot be None")
207 if t_dim is None:
208 raise ValueError("t_dim cannot be None")
209 if (source is None) or (len(source) <= 0):
210 raise ValueError(f"source cannot be {source}.")
212 # determine the style groups
213 groups: list[Styler] = []
214 goals.finalize()
215 algorithms.finalize()
217 # pick the right sorting order
218 sf: Callable[[Ecdf], Any] = sort_key
219 if (goals.count > 1) and (algorithms.count == 1):
220 def __x1(r: Ecdf, ssf=goal_sort_key) -> Any:
221 return ssf(goal_to_str(r.goal_f))
222 sf = __x1
223 elif (goals.count == 1) and (algorithms.count > 1):
224 def __x2(r: Ecdf, ssf=algorithm_sort_key) -> Any:
225 return ssf(r.algorithm)
226 sf = __x2
227 elif (goals.count > 1) and (algorithms.count > 1):
228 def __x3(r: Ecdf, sgs=goal_sort_key, sas=algorithm_sort_key,
229 ag=algorithm_priority > goal_priority) -> tuple[Any, Any]:
230 k1 = sgs(goal_to_str(r.goal_f))
231 k2 = sas(r.algorithm)
232 return (k2, k1) if ag else (k1, k2)
233 sf = __x3
235 source.sort(key=sf)
237 def __set_importance(st: Styler) -> None:
238 none = 1
239 not_none = 0
240 none_lw = importance_to_line_width_func(none)
241 not_none_lw = importance_to_line_width_func(not_none)
242 st.set_line_width(lambda p: [none_lw if i <= 0 else not_none_lw
243 for i in range(p)])
244 none_a = importance_to_alpha_func(none)
245 not_none_a = importance_to_alpha_func(not_none)
246 st.set_line_alpha(lambda p: [none_a if i <= 0 else not_none_a
247 for i in range(p)])
249 if goals.count > 1:
250 groups.append(goals)
251 if algorithms.count > 1:
252 groups.append(algorithms)
254 if len(groups) > 0:
255 groups.sort()
256 groups[0].set_line_color(distinct_colors_func)
258 if len(groups) > 1:
259 groups[1].set_line_dash(distinct_line_dashes_func)
260 elif color_algorithms_as_fallback_group:
261 algorithms.set_line_color(distinct_colors_func)
262 groups.append(algorithms)
264 # If we only have <= 2 groups, we can mark None and not-None values with
265 # different importance.
266 if goals.has_none and (goals.count > 1):
267 __set_importance(goals)
268 elif algorithms.has_none and (algorithms.count > 1):
269 __set_importance(algorithms)
271 # we will collect all lines to plot in plot_list
272 plot_list: list[dict] = []
274 # set up the axis rangers
275 if callable(x_axis):
276 x_axis = x_axis(t_dim)
277 if not isinstance(x_axis, AxisRanger):
278 raise type_error(x_axis, "x_axis", AxisRanger)
280 if callable(y_axis):
281 y_axis = y_axis("ecdf")
282 if not isinstance(y_axis, AxisRanger):
283 raise type_error(y_axis, "y_axis", AxisRanger)
285 # first we collect all ecdf object
286 max_time: int | float = -inf
287 max_ecdf: int | float = -inf
288 max_ecdf_is_at_max_time: bool = False
289 for ee in source:
290 style = pd.create_line_style()
291 for g in groups:
292 g.add_line_style(ee, style)
293 x = ee.ecdf[:, 0]
294 style["x"] = x
295 x_axis.register_array(x)
296 y = ee.ecdf[:, 1]
297 y_axis.register_array(y)
298 style["y"] = y
299 plot_list.append(style)
301 # We need to detect the special case that the maximum time is at
302 # the maximum ECDF value. In this case, we will later need to extend
303 # the visible area of the x-axis.
304 if len(x) < 2:
305 continue
306 fy = y[-2]
307 ft = x[-2]
308 if isfinite(ft):
309 if fy >= max_ecdf:
310 if fy > max_ecdf:
311 max_ecdf_is_at_max_time = (ft >= max_time)
312 max_ecdf = fy
313 else:
314 max_ecdf_is_at_max_time = max_ecdf_is_at_max_time \
315 or (ft >= max_time)
316 elif ft > max_time:
317 max_ecdf_is_at_max_time = False
318 max_time = max(max_time, ft)
319 del source
321 font_size_0: Final[float] = importance_to_font_size_func(0)
323 # If the maximum of any ECDF is located directly at the end of the
324 # x-axis, we need to slightly extend the axis to make it visible.
325 if max_ecdf_is_at_max_time:
326 x_axis.pad_detected_range(pad_max=True)
328 # set up the graphics area
329 axes: Final[Axes] = pu.get_axes(figure)
330 axes.tick_params(axis="x", labelsize=font_size_0)
331 axes.tick_params(axis="y", labelsize=font_size_0)
333 # draw the grid
334 if x_grid or y_grid:
335 grid_lwd = importance_to_line_width_func(-1)
336 if x_grid:
337 axes.grid(axis="x", color=pd.GRID_COLOR, linewidth=grid_lwd)
338 if y_grid:
339 axes.grid(axis="y", color=pd.GRID_COLOR, linewidth=grid_lwd)
341 max_x: float = x_axis.get_pinf_replacement()
342 min_x: float | None = x_axis.get_0_replacement() \
343 if x_axis.log_scale else None
345 # plot the lines
346 for line in plot_list:
347 x = line["x"]
348 changed = False
349 if np.isposinf(x[-1]):
350 x = x.copy()
351 x[-1] = max_x
352 changed = True
353 if (x[0] <= 0) and (min_x is not None):
354 if not changed:
355 changed = True
356 x = x.copy()
357 x[0] = min_x
358 if changed:
359 line["x"] = x
360 axes.step(where="post", **line)
361 del plot_list
363 x_axis.apply(axes, "x")
364 y_axis.apply(axes, "y")
366 if legend:
367 handles: list[Artist] = []
369 for g in groups:
370 g.add_to_legend(handles.append)
371 g.has_style = False
373 if algorithms.has_style:
374 algorithms.add_to_legend(handles.append)
375 if goals.has_style:
376 goals.add_to_legend(handles.append)
378 axes.legend(loc="upper left",
379 handles=handles,
380 labelcolor=[art.color if hasattr(art, "color")
381 else pd.COLOR_BLACK for art in handles],
382 fontsize=font_size_0)
384 pu.label_axes(axes=axes,
385 x_label=" ".join([x_label(x) for x in sorted(x_labels)])
386 if callable(x_label) else x_label,
387 x_label_inside=x_label_inside,
388 x_label_location=1,
389 y_label=y_label(f_dim) if callable(y_label) else y_label,
390 y_label_inside=y_label_inside,
391 y_label_location=0,
392 font_size=font_size_0)
393 return axes