Coverage for moptipy / api / experiment.py: 89%
114 statements
« prev ^ index » next coverage.py v7.13.5, created at 2026-03-29 10:36 +0000
1"""
2The experiment execution API.
4Via the function :func:`run_experiment`, you can execute a complex experiment
5where multiple optimization algorithms are applied to multiple problem
6instances, where log files with the results and progress information about the
7runs are collected, and where multiprocessing is used to parallelize the
8experiment execution.
9Experiments are replicable, as random seeds are automatically generated based
10on problem instance names in a replicable fashion.
12The log files are structured according to the documentation in
13https://thomasweise.github.io/moptipy/#file-names-and-folder-structure
14and their contents follow the specification given in
15https://thomasweise.github.io/moptipy/#log-file-sections.
16"""
17import copy
18import gc
19import os.path
20from os import getpid
21from typing import ( # pylint: disable=W0611
22 Any,
23 Callable,
24 Final,
25 Iterable,
26 cast,
27) # pylint: disable=W0611
29from numpy.random import Generator, default_rng
30from pycommons.io.console import logger
31from pycommons.io.path import Path
32from pycommons.types import check_int_range, type_error
34from moptipy.api.execution import Execution
35from moptipy.api.logging import FILE_SUFFIX
36from moptipy.api.process import Process
37from moptipy.utils.nputils import rand_seeds_from_str
38from moptipy.utils.strings import sanitize_name, sanitize_names
39from moptipy.utils.sys_info import get_sys_info
42def __run_experiment(base_dir: Path,
43 experiments: list[list[Callable]],
44 n_runs: list[int],
45 thread_id: str,
46 perform_warmup: bool,
47 warmup_fes: int,
48 perform_pre_warmup: bool,
49 pre_warmup_fes: int,
50 on_completion: Callable[[
51 Any, Path, Process], None]) -> None:
52 """
53 Execute a single thread of experiments.
55 :param base_dir: the base directory
56 :param experiments: the stream of experiment setups
57 :param n_runs: the list of runs
58 :param thread_id: the thread id
59 :param perform_warmup: should we perform a warm-up per instance?
60 :param warmup_fes: the number of the FEs for the warm-up runs
61 :param perform_pre_warmup: should we do one warmup run for each
62 instance before we begin with the actual experiments?
63 :param pre_warmup_fes: the FEs for the pre-warmup runs
64 :param on_completion: a function to be called for every completed run,
65 receiving the instance, the path to the log file (before it is
66 created) and the :class:`~moptipy.api.process.Process` of the run
67 as parameters
68 """
69 random: Final[Generator] = default_rng()
70 for warmup in ((True, False) if perform_pre_warmup else (False, )):
71 wss: str
72 if warmup:
73 wss = "pre-warmup"
74 else:
75 wss = "warmup"
76 if perform_pre_warmup:
77 gc.collect() # do full garbage collection after pre-warmups
78 gc.collect() # one more, to be double-safe
79 gc.freeze() # whatever survived now, keep it permanently
81 for runs in ((1, ) if warmup else n_runs): # for each number of runs
82 if not warmup:
83 logger(f"now doing {runs} runs.", thread_id)
84 random.shuffle(cast("list", experiments)) # shuffle
86 for setup in experiments: # for each setup
87 instance = setup[0]() # load instance
88 if instance is None:
89 raise TypeError("None is not an instance.")
90 inst_name = sanitize_name(str(instance))
92 exp = setup[1](instance) # setup algorithm for instance
93 if not isinstance(exp, Execution):
94 raise type_error(exp, "result of setup callable",
95 Execution)
96 # noinspection PyProtectedMember
97 algo_name = sanitize_name(str(exp._algorithm))
99 cd = Path(os.path.join(base_dir, algo_name, inst_name))
100 cd.ensure_dir_exists()
102 # generate sequence of seeds
103 seeds: list[int] = [0] if warmup else \
104 rand_seeds_from_str(string=inst_name, n_seeds=runs)
105 random.shuffle(seeds)
106 needs_warmup = warmup or perform_warmup
107 for seed in seeds: # for every run
109 filename = sanitize_names(
110 [algo_name, inst_name, hex(seed)])
111 if warmup:
112 log_file = filename
113 else:
114 log_file = Path(
115 os.path.join(cd, filename + FILE_SUFFIX))
116 if log_file.ensure_file_exists():
117 continue # run already done
119 exp.set_rand_seed(seed)
121 if needs_warmup: # perform warmup run
122 needs_warmup = False
123 cpy: Execution = copy.copy(exp)
124 cpy.set_max_fes(
125 pre_warmup_fes if warmup else warmup_fes, True)
126 cpy.set_max_time_millis(3600000, True)
127 cpy.set_log_file(None)
128 cpy.set_log_improvements(False)
129 cpy.set_log_all_fes(False)
130 cpy.set_improvement_logger(None)
131 logger(f"{wss} for {filename!r}.", thread_id)
132 with cpy.execute():
133 pass
134 del cpy
136 if warmup:
137 continue
139 exp.set_log_file(log_file)
140 logger(filename, thread_id)
141 with exp.execute() as process: # run the experiment
142 on_completion(instance, cast(
143 "Path", log_file), process)
146def __no_complete(_: Any, __: Path, ___: Process) -> None:
147 """Do nothing."""
150def run_experiment(
151 base_dir: str, instances: Iterable[Callable[[], Any]],
152 setups: Iterable[Callable[[Any], Execution]],
153 n_runs: int | Iterable[int] = 11,
154 perform_warmup: bool = True, warmup_fes: int = 20,
155 perform_pre_warmup: bool = True, pre_warmup_fes: int = 20,
156 on_completion: Callable[[Any, Path, Process], None] = __no_complete) \
157 -> Path:
158 """
159 Run an experiment and store the log files into the given folder.
161 This function will automatically run an experiment, i.e., apply a set
162 `setups` of algorithm setups to a set `instances` of problem instances for
163 `n_runs` each. It will collect log files and store them into an
164 appropriate folder structure under the path `base_dir`. It will
165 automatically draw random seeds for all algorithm runs using
166 :func:`moptipy.utils.nputils.rand_seeds_from_str` based on the names of
167 the problem instances to solve. This yields replicable experiments, i.e.,
168 running the experiment program twice will yield exactly the same runs in
169 exactly the same file structure (give and take clock-time dependent
170 issues, which obviously cannot be controlled in a deterministic fashion).
172 :param base_dir: the base directory where to store the results
173 :param instances: an iterable of callables, each of which should return an
174 object representing a problem instance, whose `__str__` representation
175 is a valid name
176 :param setups: an iterable of callables, each receiving an instance (as
177 returned by instances) as input and producing an
178 :class:`moptipy.api.execution.Execution` as output
179 :param n_runs: the number of runs per algorithm-instance combination
180 :param perform_warmup: should we perform a warm-up for each instance?
181 If this parameter is `True`, then before the very first run of a
182 thread on an instance, we will execute the algorithm for just a few
183 function evaluations without logging and discard the results. The
184 idea is that during this warm-up, things such as JIT compilation or
185 complicated parsing can take place. While this cannot mitigate time
186 measurement problems for JIT compilations taking place late in runs,
187 it can at least somewhat solve the problem of delayed first FEs caused
188 by compilation and parsing.
189 :param warmup_fes: the number of the FEs for the warm-up runs
190 :param perform_pre_warmup: should we do one warmup run for each
191 instance before we begin with the actual experiments? This complements
192 the warmups defined by `perform_warmup`. It could be that, for some
193 reason, JIT or other activities may lead to stalls between multiple
194 processes when code is encountered for the first time. This may or may
195 not still cause strange timing issues even if `perform_warmup=True`.
196 We therefore can do one complete round of warmups before starting the
197 actual experiment. After that, we perform one garbage collection run
198 and then freeze all objects surviving it to prevent them from future
199 garbage collection runs. All processes that execute the experiment in
200 parallel will complete their pre-warmup and only after all of them have
201 completed it, the actual experiment will begin. I am not sure whether
202 this makes sense or not, but it also would not hurt.
203 :param pre_warmup_fes: the FEs for the pre-warmup runs
204 :param on_completion: a function to be called for every completed run,
205 receiving the instance, the path to the log file (before it is
206 created) and the :class:`~moptipy.api.process.Process` of the run
207 as parameters
209 :returns: the canonicalized path to `base_dir`
210 """
211 if not isinstance(instances, Iterable):
212 raise type_error(instances, "instances", Iterable)
213 if not isinstance(setups, Iterable):
214 raise type_error(setups, "setups", Iterable)
215 if not isinstance(perform_warmup, bool):
216 raise type_error(perform_warmup, "perform_warmup", bool)
217 if not isinstance(perform_pre_warmup, bool):
218 raise type_error(perform_pre_warmup, "perform_pre_warmup", bool)
219 check_int_range(warmup_fes, "warmup_fes", 1, 1_000_000)
220 check_int_range(pre_warmup_fes, "pre_warmup_fes", 1, 1_000_000)
222 instances = list(instances)
223 if list.__len__(instances) <= 0:
224 raise ValueError("Instance enumeration is empty.")
225 for instance in instances:
226 if not callable(instance):
227 raise type_error(instance, "all instances", call=True)
229 if str.__len__(get_sys_info()) <= 0:
230 raise ValueError("empty system info?")
232 setups = list(setups)
233 if list.__len__(setups) <= 0:
234 raise ValueError("Setup enumeration is empty.")
235 for setup in setups:
236 if not callable(setup):
237 raise type_error(setup, "all setups", call=True)
239 experiments: Final[list[list[Callable]]] = \
240 [[ii, ss] for ii in instances for ss in setups]
242 del instances
243 del setups
245 if list.__len__(experiments) <= 0:
246 raise ValueError("No experiments found?")
248 n_runs = [n_runs] if isinstance(n_runs, int) else list(n_runs)
249 if list.__len__(n_runs) <= 0:
250 raise ValueError("No number of runs provided?")
251 last = 0
252 for run in n_runs:
253 last = check_int_range(run, "n_runs", last + 1)
255 use_dir: Final[Path] = Path(base_dir)
256 use_dir.ensure_dir_exists()
258 thread_id: Final[str] = f"@{getpid():x}"
259 logger("beginning experiment execution.", thread_id)
260 __run_experiment(base_dir=use_dir,
261 experiments=experiments,
262 n_runs=n_runs,
263 thread_id=thread_id,
264 perform_warmup=perform_warmup,
265 warmup_fes=warmup_fes,
266 perform_pre_warmup=perform_pre_warmup,
267 pre_warmup_fes=pre_warmup_fes,
268 on_completion=on_completion)
269 logger("finished experiment execution.", thread_id)
270 return use_dir