Coverage for moptipy / api / experiment.py: 88%
113 statements
« prev ^ index » next coverage.py v7.13.4, created at 2026-02-14 07:09 +0000
1"""
2The experiment execution API.
4Via the function :func:`run_experiment`, you can execute a complex experiment
5where multiple optimization algorithms are applied to multiple problem
6instances, where log files with the results and progress information about the
7runs are collected, and where multiprocessing is used to parallelize the
8experiment execution.
9Experiments are replicable, as random seeds are automatically generated based
10on problem instance names in a replicable fashion.
12The log files are structured according to the documentation in
13https://thomasweise.github.io/moptipy/#file-names-and-folder-structure
14and their contents follow the specification given in
15https://thomasweise.github.io/moptipy/#log-file-sections.
16"""
17import copy
18import gc
19import os.path
20from os import getpid
21from typing import ( # pylint: disable=W0611
22 Any,
23 Callable,
24 Final,
25 Iterable,
26 cast,
27) # pylint: disable=W0611
29from numpy.random import Generator, default_rng
30from pycommons.io.console import logger
31from pycommons.io.path import Path
32from pycommons.types import check_int_range, type_error
34from moptipy.api.execution import Execution
35from moptipy.api.logging import FILE_SUFFIX
36from moptipy.api.process import Process
37from moptipy.utils.nputils import rand_seeds_from_str
38from moptipy.utils.strings import sanitize_name, sanitize_names
39from moptipy.utils.sys_info import get_sys_info
42def __run_experiment(base_dir: Path,
43 experiments: list[list[Callable]],
44 n_runs: list[int],
45 thread_id: str,
46 perform_warmup: bool,
47 warmup_fes: int,
48 perform_pre_warmup: bool,
49 pre_warmup_fes: int,
50 on_completion: Callable[[
51 Any, Path, Process], None]) -> None:
52 """
53 Execute a single thread of experiments.
55 :param base_dir: the base directory
56 :param experiments: the stream of experiment setups
57 :param n_runs: the list of runs
58 :param thread_id: the thread id
59 :param perform_warmup: should we perform a warm-up per instance?
60 :param warmup_fes: the number of the FEs for the warm-up runs
61 :param perform_pre_warmup: should we do one warmup run for each
62 instance before we begin with the actual experiments?
63 :param pre_warmup_fes: the FEs for the pre-warmup runs
64 :param on_completion: a function to be called for every completed run,
65 receiving the instance, the path to the log file (before it is
66 created) and the :class:`~moptipy.api.process.Process` of the run
67 as parameters
68 """
69 random: Final[Generator] = default_rng()
70 for warmup in ((True, False) if perform_pre_warmup else (False, )):
71 wss: str
72 if warmup:
73 wss = "pre-warmup"
74 else:
75 wss = "warmup"
76 if perform_pre_warmup:
77 gc.collect() # do full garbage collection after pre-warmups
78 gc.collect() # one more, to be double-safe
79 gc.freeze() # whatever survived now, keep it permanently
81 for runs in ((1, ) if warmup else n_runs): # for each number of runs
82 if not warmup:
83 logger(f"now doing {runs} runs.", thread_id)
84 random.shuffle(cast("list", experiments)) # shuffle
86 for setup in experiments: # for each setup
87 instance = setup[0]() # load instance
88 if instance is None:
89 raise TypeError("None is not an instance.")
90 inst_name = sanitize_name(str(instance))
92 exp = setup[1](instance) # setup algorithm for instance
93 if not isinstance(exp, Execution):
94 raise type_error(exp, "result of setup callable",
95 Execution)
96 # noinspection PyProtectedMember
97 algo_name = sanitize_name(str(exp._algorithm))
99 cd = Path(os.path.join(base_dir, algo_name, inst_name))
100 cd.ensure_dir_exists()
102 # generate sequence of seeds
103 seeds: list[int] = [0] if warmup else \
104 rand_seeds_from_str(string=inst_name, n_seeds=runs)
105 random.shuffle(seeds)
106 needs_warmup = warmup or perform_warmup
107 for seed in seeds: # for every run
109 filename = sanitize_names(
110 [algo_name, inst_name, hex(seed)])
111 if warmup:
112 log_file = filename
113 else:
114 log_file = Path(
115 os.path.join(cd, filename + FILE_SUFFIX))
116 if log_file.ensure_file_exists():
117 continue # run already done
119 exp.set_rand_seed(seed)
121 if needs_warmup: # perform warmup run
122 needs_warmup = False
123 cpy: Execution = copy.copy(exp)
124 cpy.set_max_fes(
125 pre_warmup_fes if warmup else warmup_fes, True)
126 cpy.set_max_time_millis(3600000, True)
127 cpy.set_log_file(None)
128 cpy.set_log_improvements(False)
129 cpy.set_log_all_fes(False)
130 logger(f"{wss} for {filename!r}.", thread_id)
131 with cpy.execute():
132 pass
133 del cpy
135 if warmup:
136 continue
138 exp.set_log_file(log_file)
139 logger(filename, thread_id)
140 with exp.execute() as process: # run the experiment
141 on_completion(instance, cast(
142 "Path", log_file), process)
145def __no_complete(_: Any, __: Path, ___: Process) -> None:
146 """Do nothing."""
149def run_experiment(
150 base_dir: str, instances: Iterable[Callable[[], Any]],
151 setups: Iterable[Callable[[Any], Execution]],
152 n_runs: int | Iterable[int] = 11,
153 perform_warmup: bool = True, warmup_fes: int = 20,
154 perform_pre_warmup: bool = True, pre_warmup_fes: int = 20,
155 on_completion: Callable[[Any, Path, Process], None] = __no_complete) \
156 -> Path:
157 """
158 Run an experiment and store the log files into the given folder.
160 This function will automatically run an experiment, i.e., apply a set
161 `setups` of algorithm setups to a set `instances` of problem instances for
162 `n_runs` each. It will collect log files and store them into an
163 appropriate folder structure under the path `base_dir`. It will
164 automatically draw random seeds for all algorithm runs using
165 :func:`moptipy.utils.nputils.rand_seeds_from_str` based on the names of
166 the problem instances to solve. This yields replicable experiments, i.e.,
167 running the experiment program twice will yield exactly the same runs in
168 exactly the same file structure (give and take clock-time dependent
169 issues, which obviously cannot be controlled in a deterministic fashion).
171 :param base_dir: the base directory where to store the results
172 :param instances: an iterable of callables, each of which should return an
173 object representing a problem instance, whose `__str__` representation
174 is a valid name
175 :param setups: an iterable of callables, each receiving an instance (as
176 returned by instances) as input and producing an
177 :class:`moptipy.api.execution.Execution` as output
178 :param n_runs: the number of runs per algorithm-instance combination
179 :param perform_warmup: should we perform a warm-up for each instance?
180 If this parameter is `True`, then before the very first run of a
181 thread on an instance, we will execute the algorithm for just a few
182 function evaluations without logging and discard the results. The
183 idea is that during this warm-up, things such as JIT compilation or
184 complicated parsing can take place. While this cannot mitigate time
185 measurement problems for JIT compilations taking place late in runs,
186 it can at least somewhat solve the problem of delayed first FEs caused
187 by compilation and parsing.
188 :param warmup_fes: the number of the FEs for the warm-up runs
189 :param perform_pre_warmup: should we do one warmup run for each
190 instance before we begin with the actual experiments? This complements
191 the warmups defined by `perform_warmup`. It could be that, for some
192 reason, JIT or other activities may lead to stalls between multiple
193 processes when code is encountered for the first time. This may or may
194 not still cause strange timing issues even if `perform_warmup=True`.
195 We therefore can do one complete round of warmups before starting the
196 actual experiment. After that, we perform one garbage collection run
197 and then freeze all objects surviving it to prevent them from future
198 garbage collection runs. All processes that execute the experiment in
199 parallel will complete their pre-warmup and only after all of them have
200 completed it, the actual experiment will begin. I am not sure whether
201 this makes sense or not, but it also would not hurt.
202 :param pre_warmup_fes: the FEs for the pre-warmup runs
203 :param on_completion: a function to be called for every completed run,
204 receiving the instance, the path to the log file (before it is
205 created) and the :class:`~moptipy.api.process.Process` of the run
206 as parameters
208 :returns: the canonicalized path to `base_dir`
209 """
210 if not isinstance(instances, Iterable):
211 raise type_error(instances, "instances", Iterable)
212 if not isinstance(setups, Iterable):
213 raise type_error(setups, "setups", Iterable)
214 if not isinstance(perform_warmup, bool):
215 raise type_error(perform_warmup, "perform_warmup", bool)
216 if not isinstance(perform_pre_warmup, bool):
217 raise type_error(perform_pre_warmup, "perform_pre_warmup", bool)
218 check_int_range(warmup_fes, "warmup_fes", 1, 1_000_000)
219 check_int_range(pre_warmup_fes, "pre_warmup_fes", 1, 1_000_000)
221 instances = list(instances)
222 if list.__len__(instances) <= 0:
223 raise ValueError("Instance enumeration is empty.")
224 for instance in instances:
225 if not callable(instance):
226 raise type_error(instance, "all instances", call=True)
228 if str.__len__(get_sys_info()) <= 0:
229 raise ValueError("empty system info?")
231 setups = list(setups)
232 if list.__len__(setups) <= 0:
233 raise ValueError("Setup enumeration is empty.")
234 for setup in setups:
235 if not callable(setup):
236 raise type_error(setup, "all setups", call=True)
238 experiments: Final[list[list[Callable]]] = \
239 [[ii, ss] for ii in instances for ss in setups]
241 del instances
242 del setups
244 if list.__len__(experiments) <= 0:
245 raise ValueError("No experiments found?")
247 n_runs = [n_runs] if isinstance(n_runs, int) else list(n_runs)
248 if list.__len__(n_runs) <= 0:
249 raise ValueError("No number of runs provided?")
250 last = 0
251 for run in n_runs:
252 last = check_int_range(run, "n_runs", last + 1)
254 use_dir: Final[Path] = Path(base_dir)
255 use_dir.ensure_dir_exists()
257 thread_id: Final[str] = f"@{getpid():x}"
258 logger("beginning experiment execution.", thread_id)
259 __run_experiment(base_dir=use_dir,
260 experiments=experiments,
261 n_runs=n_runs,
262 thread_id=thread_id,
263 perform_warmup=perform_warmup,
264 warmup_fes=warmup_fes,
265 perform_pre_warmup=perform_pre_warmup,
266 pre_warmup_fes=pre_warmup_fes,
267 on_completion=on_completion)
268 logger("finished experiment execution.", thread_id)
269 return use_dir