Coverage for moptipy/evaluation/base.py: 99%

1"""Some internal helper functions and base classes."""

3from dataclasses import dataclass

4from typing import Any, Final, Iterable

6from pycommons.io.path import Path

7from pycommons.types import check_int_range, type_error

9from moptipy.api.logging import FILE_SUFFIX

10from moptipy.utils.nputils import rand_seed_check

11from moptipy.utils.strings import (

12 sanitize_name,

13 sanitize_names,

14)

15from moptipy.version import __version__ as moptipy_version

17#: The key for the total number of runs.

18KEY_N: Final[str] = "n"

19#: a key for the objective function name

20KEY_OBJECTIVE_FUNCTION: Final[str] = "objective"

21#: a key for the encoding name

22KEY_ENCODING: Final[str] = "encoding"

24#: The unit of the time axis if time is measured in milliseconds.

25TIME_UNIT_MILLIS: Final[str] = "ms"

26#: The unit of the time axis of time is measured in FEs

27TIME_UNIT_FES: Final[str] = "FEs"

29#: The name of the raw objective values data.

30F_NAME_RAW: Final[str] = "plainF"

31#: The name of the scaled objective values data.

32F_NAME_SCALED: Final[str] = "scaledF"

33#: The name of the normalized objective values data.

34F_NAME_NORMALIZED: Final[str] = "normalizedF"

37def check_time_unit(time_unit: Any) -> str:

38 """

39 Check that the time unit is OK.

41 :param time_unit: the time unit

42 :return: the time unit string

44 >>> check_time_unit("FEs")

45 'FEs'

46 >>> check_time_unit("ms")

47 'ms'

48 >>> try:

49 ... check_time_unit(1)

50 ... except TypeError as te:

51 ... print(te)

52 time_unit should be an instance of str but is int, namely 1.

53 >>> try:

54 ... check_time_unit("blabedibla")

55 ... except ValueError as ve:

56 ... print(ve)

57 Invalid time unit 'blabedibla', only 'FEs' and 'ms' are permitted.

58 """

59 if not isinstance(time_unit, str):

60 raise type_error(time_unit, "time_unit", str)

61 if time_unit in {TIME_UNIT_FES, TIME_UNIT_MILLIS}:

62 return time_unit

63 raise ValueError(

64 f"Invalid time unit {time_unit!r}, only {TIME_UNIT_FES!r} "

65 f"and {TIME_UNIT_MILLIS!r} are permitted.")

68def check_f_name(f_name: Any) -> str:

69 """

70 Check whether an objective value name is valid.

72 :param f_name: the name of the objective function dimension

73 :return: the name of the objective function dimension

75 >>> check_f_name("plainF")

76 'plainF'

77 >>> check_f_name("scaledF")

78 'scaledF'

79 >>> check_f_name("normalizedF")

80 'normalizedF'

81 >>> try:

82 ... check_f_name(1.0)

83 ... except TypeError as te:

84 ... print(te)

85 f_name should be an instance of str but is float, namely 1.0.

86 >>> try:

87 ... check_f_name("oops")

88 ... except ValueError as ve:

89 ... print(ve)

90 Invalid f name 'oops', only 'plainF', 'scaledF', and 'normalizedF' \

91are permitted.

92 """

93 if not isinstance(f_name, str):

94 raise type_error(f_name, "f_name", str)

95 if f_name in {F_NAME_RAW, F_NAME_SCALED, F_NAME_NORMALIZED}:

96 return f_name

97 raise ValueError(

98 f"Invalid f name {f_name!r}, only {F_NAME_RAW!r}, "

99 f"{F_NAME_SCALED!r}, and {F_NAME_NORMALIZED!r} are permitted.")

100

101

102def _set_name(dest: object, name: str, what: str,

103 none_allowed: bool = False,

104 empty_to_none: bool = True) -> None:

105 """

106 Check and set a name.

107

108 :param dest: the destination

109 :param name: the name to set

110 :param what: the name's type

111 :param none_allowed: is `None` allowed?

112 :param empty_to_none: If both `none_allowed` and `empty_to_none` are

113 `True`, then empty strings are converted to `None`

114

115 >>> class TV:

116 ... algorithm: str

117 ... instance: str | None

118 >>> t = TV()

119 >>> _set_name(t, "bla", "algorithm", False)

120 >>> t.algorithm

121 'bla'

122 >>> _set_name(t, "xbla", "instance", True)

123 >>> t.instance

124 'xbla'

125 >>> _set_name(t, None, "instance", True)

126 >>> print(t.instance)

127 None

128 >>> t.instance = "x"

129 >>> _set_name(t, " ", "instance", True)

130 >>> print(t.instance)

131 None

132 >>> try:

133 ... _set_name(t, 1, "algorithm")

134 ... except TypeError as te:

135 ... print(te)

136 algorithm name should be an instance of str but is int, namely 1.

137 >>> t.algorithm

138 'bla'

139 >>> try:

140 ... _set_name(t, " ", "algorithm")

141 ... except ValueError as ve:

142 ... print(ve)

143 algorithm name cannot be empty of just consist of white space, but \

144' ' does.

145 >>> t.algorithm

146 'bla'

147 >>> try:

148 ... _set_name(t, "a a", "instance")

149 ... except ValueError as ve:

150 ... print(ve)

151 Invalid instance name 'a a'.

152 >>> print(t.instance)

153 None

154 >>> try:

155 ... _set_name(t, " ", "instance", True, False)

156 ... except ValueError as ve:

157 ... print(ve)

158 instance name cannot be empty of just consist of white space, but \

159' ' does.

160 >>> print(t.instance)

161 None

162 """

163 use_name = name

164 if isinstance(name, str):

165 use_name = use_name.strip()

166 if len(use_name) <= 0:

167 if empty_to_none and none_allowed:

168 use_name = None

169 else:

170 raise ValueError(f"{what} name cannot be empty of just cons"

171 f"ist of white space, but {name!r} does.")

172 elif use_name != sanitize_name(use_name):

173 raise ValueError(f"Invalid {what} name {name!r}.")

174 elif not ((name is None) and none_allowed):

175 raise type_error(name, f"{what} name",

176 (str, None) if none_allowed else str)

177 object.__setattr__(dest, what, use_name)

178

179

180class EvaluationDataElement:

181 """A base class for all the data classes in this module."""

182

183 def _tuple(self) -> tuple[Any, ...]:

184 """

185 Create a tuple with all the data of this data class for comparison.

186

187 All the relevant data of an instance of this class is stored in a

188 tuple. The tuple is then used in the dunder methods for comparisons.

189 The returned tuple *must* be based on the scheme

190 `tuple[str, str, str, str, str, int, int, str, str]`.

191 They can be shorter than this and they can be longer, but they must

192 adhere to this basic scheme:

193

194 1. class name

195 2. algorithm name, or `""` if algorithm name is `None`

196 3. instance name, or `""` if instance name is `None`

197 4. objective name, `""` objective name is `None`

198 5. encoding name, or `""` encoding name is `None`

199 6. number of runs, or `0` if no number of runs is specified or `1` if

200 the data concerns exactly one run

201 7. the random seed, or `-1` if no random seed is specified

202 8. the string time unit, or `""` if no time unit is given

203 9. the scaling name of the objective function, or `""` if no scaling

204 name is given

205

206 If the tuples are longer, then all values following after this must be

207 integers or floats.

208

209 >>> EvaluationDataElement()._tuple()

210 ('EvaluationDataElement',)

211

212 :returns: a tuple with all the data of this class, where `None` values

213 are masked out

214 """

215 return (self.__class__.__name__, )

216

217 def __hash__(self) -> int:

218 """

219 Compute the hash code of this object.

220

221 :returns: the hash code

222 """

223 return hash(self._tuple())

224

225 def __eq__(self, other) -> bool:

226 """

227 Compare for `==` with another object based on the `_tuple()` value.

228

229 :param other: the other object to compare to, must be an instance of

230 :class:`EvaluationDataElement`

231 :retval `True`: if the `other` object's `_tuple()` representation is

232 `==` with this object's `_tuple()` representation

233 :retval `False`: otherwise

234 :raises NotImplementedError: if the other object is not an instance of

235 :class:`EvaluationDataElement` and therefore cannot be compared.

236

237 >>> PerRunData("a", "i", "f", "e", 234) == PerRunData(

238 ... "a", "i", "f", "e", 234)

239 True

240 >>> PerRunData("a", "i", "f", "e", 234) == PerRunData(

241 ... "a", "j", "f", "e", 234)

242 False

243 >>> try:

244 ... PerRunData("a", "i", "f", "e", 234) == 3

245 ... except NotImplementedError as ni:

246 ... print(ni)

247 Cannot compare PerRunData(algorithm='a', instance='i', \

248objective='f', encoding='e', rand_seed=234) with 3 for ==.

249 """

250 if isinstance(other, EvaluationDataElement):

251 return self._tuple() == other._tuple()

252 raise NotImplementedError(

253 f"Cannot compare {self} with {other} for ==.")

254

255 def __ne__(self, other) -> bool:

256 """

257 Compare for `!=` with another object based on the `_tuple()` value.

258

259 :param other: the other object to compare to, must be an instance of

260 :class:`EvaluationDataElement`

261 :retval `True`: if the `other` object's `_tuple()` representation is

262 `!=` with this object's `_tuple()` representation

263 :retval `False`: otherwise

264 :raises NotImplementedError: if the other object is not an instance of

265 :class:`EvaluationDataElement` and therefore cannot be compared.

266

267 >>> PerRunData("a", "i", "f", "e", 234) != PerRunData(

268 ... "a", "i", "f", "e", 234)

269 False

270 >>> PerRunData("a", "i", "f", "e", 234) != PerRunData(

271 ... "a", "j", "f", "e", 234)

272 True

273 >>> try:

274 ... PerRunData("a", "i", "f", "e", 234) != 3

275 ... except NotImplementedError as ni:

276 ... print(ni)

277 Cannot compare PerRunData(algorithm='a', instance='i', \

278objective='f', encoding='e', rand_seed=234) with 3 for !=.

279 """

280 if isinstance(other, EvaluationDataElement):

281 return self._tuple() != other._tuple()

282 raise NotImplementedError(

283 f"Cannot compare {self} with {other} for !=.")

284

285 def __lt__(self, other) -> bool:

286 """

287 Compare for `<` with another object based on the `_tuple()` value.

288

289 :param other: the other object to compare to, must be an instance of

290 :class:`EvaluationDataElement`

291 :retval `True`: if the `other` object's `_tuple()` representation is

292 `<` with this object's `_tuple()` representation

293 :retval `False`: otherwise

294 :raises NotImplementedError: if the other object is not an instance of

295 :class:`EvaluationDataElement` and therefore cannot be compared.

296

297 >>> PerRunData("a", "i", "f", "e", 234) < PerRunData(

298 ... "a", "i", "f", "e", 234)

299 False

300 >>> PerRunData("a", "i", "f", "e", 234) < PerRunData(

301 ... "a", "j", "f", "e", 234)

302 True

303 >>> PerRunData("a", "j", "f", "e", 234) < PerRunData(

304 ... "a", "i", "f", "e", 234)

305 False

306 >>> try:

307 ... PerRunData("a", "i", "f", "e", 234) < 3

308 ... except NotImplementedError as ni:

309 ... print(ni)

310 Cannot compare PerRunData(algorithm='a', instance='i', \

311objective='f', encoding='e', rand_seed=234) with 3 for <.

312 """

313 if isinstance(other, EvaluationDataElement):

314 return self._tuple() < other._tuple()

315 raise NotImplementedError(

316 f"Cannot compare {self} with {other} for <.")

317

318 def __le__(self, other) -> bool:

319 """

320 Compare for `<=` with another object based on the `_tuple()` value.

321

322 :param other: the other object to compare to, must be an instance of

323 :class:`EvaluationDataElement`

324 :retval `True`: if the `other` object's `_tuple()` representation is

325 `<=` with this object's `_tuple()` representation

326 :retval `False`: otherwise

327 :raises NotImplementedError: if the other object is not an instance of

328 :class:`EvaluationDataElement` and therefore cannot be compared.

329

330 >>> PerRunData("a", "i", "f", "e", 234) <= PerRunData(

331 ... "a", "i", "f", "e", 234)

332 True

333 >>> PerRunData("a", "i", "f", "e", 234) <= PerRunData(

334 ... "a", "j", "f", "e", 234)

335 True

336 >>> PerRunData("a", "j", "f", "e", 234) < PerRunData(

337 ... "a", "i", "f", "e", 234)

338 False

339 >>> try:

340 ... PerRunData("a", "i", "f", "e", 234) <= 3

341 ... except NotImplementedError as ni:

342 ... print(ni)

343 Cannot compare PerRunData(algorithm='a', instance='i', \

344objective='f', encoding='e', rand_seed=234) with 3 for <=.

345 """

346 if isinstance(other, EvaluationDataElement):

347 return self._tuple() <= other._tuple()

348 raise NotImplementedError(

349 f"Cannot compare {self} with {other} for <=.")

350

351 def __gt__(self, other) -> bool:

352 """

353 Compare for `>` with another object based on the `_tuple()` value.

354

355 :param other: the other object to compare to, must be an instance of

356 :class:`EvaluationDataElement`

357 :retval `True`: if the `other` object's `_tuple()` representation is

358 `>` with this object's `_tuple()` representation

359 :retval `False`: otherwise

360 :raises NotImplementedError: if the other object is not an instance of

361 :class:`EvaluationDataElement` and therefore cannot be compared.

362

363 >>> PerRunData("a", "i", "f", "e", 234) > PerRunData(

364 ... "a", "i", "f", "e", 234)

365 False

366 >>> PerRunData("a", "i", "f", "e", 234) > PerRunData(

367 ... "a", "j", "f", "e", 234)

368 False

369 >>> PerRunData("a", "j", "f", "e", 234) > PerRunData(

370 ... "a", "i", "f", "e", 234)

371 True

372 >>> try:

373 ... PerRunData("a", "i", "f", "e", 234) > 3

374 ... except NotImplementedError as ni:

375 ... print(ni)

376 Cannot compare PerRunData(algorithm='a', instance='i', \

377objective='f', encoding='e', rand_seed=234) with 3 for >.

378 """

379 if isinstance(other, EvaluationDataElement):

380 return self._tuple() > other._tuple()

381 raise NotImplementedError(

382 f"Cannot compare {self} with {other} for >.")

383

384 def __ge__(self, other) -> bool:

385 """

386 Compare for `>=` with another object based on the `_tuple()` value.

387

388 :param other: the other object to compare to, must be an instance of

389 :class:`EvaluationDataElement`

390 :retval `True`: if the `other` object's `_tuple()` representation is

391 `>=` with this object's `_tuple()` representation

392 :retval `False`: otherwise

393 :raises NotImplementedError: if the other object is not an instance of

394 :class:`EvaluationDataElement` and therefore cannot be compared.

395

396 >>> PerRunData("a", "i", "f", "e", 234) >= PerRunData(

397 ... "a", "i", "f", "e", 234)

398 True

399 >>> PerRunData("a", "i", "f", "e", 234) >= PerRunData(

400 ... "a", "j", "f", "e", 234)

401 False

402 >>> PerRunData("a", "j", "f", "e", 234) >= PerRunData(

403 ... "a", "i", "f", "e", 234)

404 True

405 >>> try:

406 ... PerRunData("a", "i", "f", "e", 234) >= 3

407 ... except NotImplementedError as ni:

408 ... print(ni)

409 Cannot compare PerRunData(algorithm='a', instance='i', \

410objective='f', encoding='e', rand_seed=234) with 3 for >=.

411 """

412 if isinstance(other, EvaluationDataElement):

413 return self._tuple() >= other._tuple()

414 raise NotImplementedError(

415 f"Cannot compare {self} with {other} for >=.")

416

417

418@dataclass(frozen=True, init=False, order=False, eq=False)

419class PerRunData(EvaluationDataElement):

420 """

421 An immutable record of information over a single run.

422

423 >>> p = PerRunData("a", "i", "f", None, 234)

424 >>> p.instance

425 'i'

426 >>> p.algorithm

427 'a'

428 >>> p.objective

429 'f'

430 >>> print(p.encoding)

431 None

432 >>> p.rand_seed

433 234

434 >>> p = PerRunData("a", "i", "f", "e", 234)

435 >>> p.instance

436 'i'

437 >>> p.algorithm

438 'a'

439 >>> p.objective

440 'f'

441 >>> p.encoding

442 'e'

443 >>> p.rand_seed

444 234

445 >>> try:

446 ... PerRunData(3, "i", "f", "e", 234)

447 ... except TypeError as te:

448 ... print(te)

449 algorithm name should be an instance of str but is int, namely 3.

450 >>> try:

451 ... PerRunData("@1 2", "i", "f", "e", 234)

452 ... except ValueError as ve:

453 ... print(ve)

454 Invalid algorithm name '@1 2'.

455 >>> try:

456 ... PerRunData("x", 3.2, "f", "e", 234)

457 ... except TypeError as te:

458 ... print(te)

459 instance name should be an instance of str but is float, namely 3.2.

460 >>> try:

461 ... PerRunData("x", "sdf i", "f", "e", 234)

462 ... except ValueError as ve:

463 ... print(ve)

464 Invalid instance name 'sdf i'.

465 >>> try:

466 ... PerRunData("a", "i", True, "e", 234)

467 ... except TypeError as te:

468 ... print(te)

469 objective name should be an instance of str but is bool, namely True.

470 >>> try:

471 ... PerRunData("x", "i", "d-f", "e", 234)

472 ... except ValueError as ve:

473 ... print(ve)

474 Invalid objective name 'd-f'.

475 >>> try:

476 ... PerRunData("x", "i", "f", 54.2, 234)

477 ... except TypeError as te:

478 ... print(te)

479 encoding name should be an instance of any in {None, str} but is float, \

480namely 54.2.

481 >>> try:

482 ... PerRunData("y", "i", "f", "x x", 234)

483 ... except ValueError as ve:

484 ... print(ve)

485 Invalid encoding name 'x x'.

486 >>> try:

487 ... PerRunData("x", "i", "f", "e", 3.3)

488 ... except TypeError as te:

489 ... print(te)

490 rand_seed should be an instance of int but is float, namely 3.3.

491 >>> try:

492 ... PerRunData("x", "i", "f", "e", -234)

493 ... except ValueError as ve:

494 ... print(ve)

495 rand_seed=-234 is invalid, must be in 0..18446744073709551615.

496 """

497

498 #: The algorithm that was applied.

499 algorithm: str

500 #: The problem instance that was solved.

501 instance: str

502 #: the name of the objective function

503 objective: str

504 #: the encoding, if any, or `None` if no encoding was used

505 encoding: str | None

506 #: The seed of the random number generator.

507 rand_seed: int

508

509 def __init__(self, algorithm: str, instance: str, objective: str,

510 encoding: str | None, rand_seed: int):

511 """

512 Create a per-run data record.

513

514 :param algorithm: the algorithm name

515 :param instance: the instance name

516 :param objective: the name of the objective function

517 :param encoding: the name of the encoding that was used, if any, or

518 `None` if no encoding was used

519 :param rand_seed: the random seed

520 """

521 _set_name(self, algorithm, "algorithm")

522 _set_name(self, instance, "instance")

523 _set_name(self, objective, "objective")

524 _set_name(self, encoding, "encoding", True, False)

525 object.__setattr__(self, "rand_seed", rand_seed_check(rand_seed))

526

527 def _tuple(self) -> tuple[Any, ...]:

528 """

529 Get the tuple representation of this object used in comparisons.

530

531 :return: the comparison-relevant data of this object in a tuple

532

533 >>> PerRunData("a", "i", "f", "e", 234)._tuple()

534 ('PerRunData', 'a', 'i', 'f', 'e', 1, 234)

535 >>> PerRunData("a", "i", "f", None, 234)._tuple()

536 ('PerRunData', 'a', 'i', 'f', '', 1, 234)

537 """

538 return (self.__class__.__name__, self.algorithm, self.instance,

539 self.objective,

540 "" if self.encoding is None else self.encoding, 1,

541 self.rand_seed)

542

543 def path_to_file(self, base_dir: str) -> Path:

544 """

545 Get the path that would correspond to the log file of this end result.

546

547 Obtain a path that would correspond to the log file of this end

548 result, resolved from a base directory `base_dir`.

549

550 :param base_dir: the base directory

551 :returns: the path to a file corresponding to the end result record

552 """

553 return Path(base_dir).resolve_inside(

554 self.algorithm).resolve_inside(self.instance).resolve_inside(

555 sanitize_names([self.algorithm, self.instance,

556 hex(self.rand_seed)]) + FILE_SUFFIX)

557

558

559@dataclass(frozen=True, init=False, order=False, eq=False)

560class MultiRunData(EvaluationDataElement):

561 """

562 A class that represents statistics over a set of runs.

563

564 If one algorithm*instance is used, then `algorithm` and `instance` are

565 defined. Otherwise, only the parameter which is the same over all recorded

566 runs is defined.

567

568 >>> p = MultiRunData("a", "i", "f", None, 3)

569 >>> p.instance

570 'i'

571 >>> p.algorithm

572 'a'

573 >>> p.objective

574 'f'

575 >>> print(p.encoding)

576 None

577 >>> p.n

578 3

579 >>> p = MultiRunData(None, None, None, "x", 3)

580 >>> print(p.instance)

581 None

582 >>> print(p.algorithm)

583 None

584 >>> print(p.objective)

585 None

586 >>> p.encoding

587 'x'

588 >>> p.n

589 3

590 >>> try:

591 ... MultiRunData(1, "i", "f", "e", 234)

592 ... except TypeError as te:

593 ... print(te)

594 algorithm name should be an instance of any in {None, str} but is int, \

595namely 1.

596 >>> try:

597 ... MultiRunData("x x", "i", "f", "e", 234)

598 ... except ValueError as ve:

599 ... print(ve)

600 Invalid algorithm name 'x x'.

601 >>> try:

602 ... MultiRunData("a", 5.5, "f", "e", 234)

603 ... except TypeError as te:

604 ... print(te)

605 instance name should be an instance of any in {None, str} but is float, \

606namely 5.5.

607 >>> try:

608 ... MultiRunData("x", "a-i", "f", "e", 234)

609 ... except ValueError as ve:

610 ... print(ve)

611 Invalid instance name 'a-i'.

612 >>> try:

613 ... MultiRunData("a", "i", True, "e", 234)

614 ... except TypeError as te:

615 ... print(te)

616 objective name should be an instance of any in {None, str} but is bool, \

617namely True.

618 >>> try:

619 ... MultiRunData("xx", "i", "d'@f", "e", 234)

620 ... except ValueError as ve:

621 ... print(ve)

622 Invalid objective name "d'@f".

623 >>> try:

624 ... MultiRunData("yy", "i", "f", -9.4, 234)

625 ... except TypeError as te:

626 ... print(te)

627 encoding name should be an instance of any in {None, str} but is float, \

628namely -9.4.

629 >>> try:

630 ... MultiRunData("xx", "i", "f", "e-{a", 234)

631 ... except ValueError as ve:

632 ... print(ve)

633 Invalid encoding name 'e-{a'.

634 >>> try:

635 ... MultiRunData("x", "i", "f", "e", -1.234)

636 ... except TypeError as te:

637 ... print(te)

638 n should be an instance of int but is float, namely -1.234.

639 >>> try:

640 ... MultiRunData("xx", "i", "f", "e", 1_000_000_000_000_000_000_000)

641 ... except ValueError as ve:

642 ... print(ve)

643 n=1000000000000000000000 is invalid, must be in 1..1000000000000000.

644 """

645

646 #: The algorithm that was applied, if the same over all runs.

647 algorithm: str | None

648 #: The problem instance that was solved, if the same over all runs.

649 instance: str | None

650 #: the name of the objective function, if the same over all runs

651 objective: str | None

652 #: the encoding, if any, or `None` if no encoding was used or if it was

653 #: not the same over all runs

654 encoding: str | None

655 #: The number of runs over which the statistic information is computed.

656 n: int

657

658 def __init__(self, algorithm: str | None, instance: str | None,

659 objective: str | None, encoding: str | None, n: int):

660 """

661 Create the dataset of an experiment-setup combination.

662

663 :param algorithm: the algorithm name, if all runs are with the same

664 algorithm, `None` otherwise

665 :param instance: the instance name, if all runs are on the same

666 instance, `None` otherwise

667 :param objective: the objective name, if all runs are on the same

668 objective function, `None` otherwise

669 :param encoding: the encoding name, if all runs are on the same

670 encoding and an encoding was actually used, `None` otherwise

671 :param n: the total number of runs

672 """

673 _set_name(self, algorithm, "algorithm", True, False)

674 _set_name(self, instance, "instance", True, False)

675 _set_name(self, objective, "objective", True, False)

676 _set_name(self, encoding, "encoding", True, False)

677 object.__setattr__(self, "n", check_int_range(

678 n, "n", 1, 1_000_000_000_000_000))

679

680 def _tuple(self) -> tuple[Any, ...]:

681 """

682 Get the tuple representation of this object used in comparisons.

683

684 :return: the comparison-relevant data of this object in a tuple

685

686 >>> MultiRunData("a", "i", "f", None, 3)._tuple()

687 ('MultiRunData', 'a', 'i', 'f', '', 3, -1)

688 >>> MultiRunData(None, "i", "f", "e", 31)._tuple()

689 ('MultiRunData', '', 'i', 'f', 'e', 31, -1)

690 >>> MultiRunData("x", None, "fy", "e1", 131)._tuple()

691 ('MultiRunData', 'x', '', 'fy', 'e1', 131, -1)

692 >>> MultiRunData("yx", "z", None, "xe1", 2131)._tuple()

693 ('MultiRunData', 'yx', 'z', '', 'xe1', 2131, -1)

694 """

695 return (self.__class__.__name__,

696 "" if self.algorithm is None else self.algorithm,

697 "" if self.instance is None else self.instance,

698 "" if self.objective is None else self.objective,

699 "" if self.encoding is None else self.encoding,

700 self.n, -1)

701

702

703@dataclass(frozen=True, init=False, order=False, eq=False)

704class MultiRun2DData(MultiRunData):

705 """

706 A multi-run data based on one time and one objective dimension.

707

708 >>> p = MultiRun2DData("a", "i", "f", None, 3,

709 ... TIME_UNIT_FES, F_NAME_SCALED)

710 >>> p.instance

711 'i'

712 >>> p.algorithm

713 'a'

714 >>> p.objective

715 'f'

716 >>> print(p.encoding)

717 None

718 >>> p.n

719 3

720 >>> print(p.time_unit)

721 FEs

722 >>> print(p.f_name)

723 scaledF

724 >>> try:

725 ... MultiRun2DData("a", "i", "f", None, 3,

726 ... 3, F_NAME_SCALED)

727 ... except TypeError as te:

728 ... print(te)

729 time_unit should be an instance of str but is int, namely 3.

730 >>> try:

731 ... MultiRun2DData("a", "i", "f", None, 3,

732 ... "sdfjsdf", F_NAME_SCALED)

733 ... except ValueError as ve:

734 ... print(ve)

735 Invalid time unit 'sdfjsdf', only 'FEs' and 'ms' are permitted.

736 >>> try:

737 ... MultiRun2DData("a", "i", "f", None, 3,

738 ... TIME_UNIT_FES, True)

739 ... except TypeError as te:

740 ... print(te)

741 f_name should be an instance of str but is bool, namely True.

742 >>> try:

743 ... MultiRun2DData("a", "i", "f", None, 3,

744 ... TIME_UNIT_FES, "blablue")

745 ... except ValueError as ve:

746 ... print(ve)

747 Invalid f name 'blablue', only 'plainF', 'scaledF', and 'normalizedF' \

748are permitted.

749 """

750

751 #: The unit of the time axis.

752 time_unit: str

753 #: the name of the objective value axis.

754 f_name: str

755

756 def __init__(self, algorithm: str | None, instance: str | None,

757 objective: str | None, encoding: str | None, n: int,

758 time_unit: str, f_name: str):

759 """

760 Create multi-run data based on one time and one objective dimension.

761

762 :param algorithm: the algorithm name, if all runs are with the same

763 algorithm

764 :param instance: the instance name, if all runs are on the same

765 instance

766 :param objective: the objective name, if all runs are on the same

767 objective function, `None` otherwise

768 :param encoding: the encoding name, if all runs are on the same

769 encoding and an encoding was actually used, `None` otherwise

770 :param n: the total number of runs

771 :param time_unit: the time unit

772 :param f_name: the objective dimension name

773 """

774 super().__init__(algorithm, instance, objective, encoding, n)

775 object.__setattr__(self, "time_unit", check_time_unit(time_unit))

776 object.__setattr__(self, "f_name", check_f_name(f_name))

777

778 def _tuple(self) -> tuple[Any, ...]:

779 """

780 Get the tuple representation of this object used in comparisons.

781

782 :return: the comparison-relevant data of this object in a tuple

783

784 >>> MultiRun2DData("a", "i", "f", None, 3,

785 ... TIME_UNIT_FES, F_NAME_SCALED)._tuple()

786 ('MultiRun2DData', 'a', 'i', 'f', '', 3, -1, 'FEs', 'scaledF')

787 >>> MultiRun2DData(None, "ix", None, "x", 43,

788 ... TIME_UNIT_MILLIS, F_NAME_RAW)._tuple()

789 ('MultiRun2DData', '', 'ix', '', 'x', 43, -1, 'ms', 'plainF')

790 >>> MultiRun2DData("xa", None, None, None, 143,

791 ... TIME_UNIT_MILLIS, F_NAME_NORMALIZED)._tuple()

792 ('MultiRun2DData', 'xa', '', '', '', 143, -1, 'ms', 'normalizedF')

793 """

794 return (self.__class__.__name__,

795 "" if self.algorithm is None else self.algorithm,

796 "" if self.instance is None else self.instance,

797 "" if self.objective is None else self.objective,

798 "" if self.encoding is None else self.encoding,

799 self.n, -1, self.time_unit, self.f_name)

800

801

802def get_instance(obj: PerRunData | MultiRunData) -> str | None:

803 """

804 Get the instance of a given object.

805

806 :param obj: the object

807 :return: the instance string, or `None` if no instance is specified

808

809 >>> p1 = MultiRunData("a", "i1", None, "x", 3)

810 >>> get_instance(p1)

811 'i1'

812 >>> p2 = PerRunData("a", "i2", "f", "x", 31)

813 >>> get_instance(p2)

814 'i2'

815 """

816 return obj.instance

817

818

819def get_algorithm(obj: PerRunData | MultiRunData) -> str | None:

820 """

821 Get the algorithm of a given object.

822

823 :param obj: the object

824 :return: the algorithm string, or `None` if no algorithm is specified

825

826 >>> p1 = MultiRunData("a1", "i1", "f", "y", 3)

827 >>> get_algorithm(p1)

828 'a1'

829 >>> p2 = PerRunData("a2", "i2", "y", None, 31)

830 >>> get_algorithm(p2)

831 'a2'

832 """

833 return obj.algorithm

834

835

836def sort_key(obj: PerRunData | MultiRunData) -> tuple[Any, ...]:

837 """

838 Get the default sort key for the given object.

839

840 The sort key is a tuple with well-defined field elements that should

841 allow for a default and consistent sorting over many different elements of

842 the experiment evaluation data API. Sorting should work also for lists

843 containing elements of different classes.

844

845 :param obj: the object

846 :return: the sort key

847

848 >>> p1 = MultiRunData("a1", "i1", "f", None, 3)

849 >>> p2 = PerRunData("a2", "i2", "f", None, 31)

850 >>> sort_key(p1) < sort_key(p2)

851 True

852 >>> sort_key(p1) >= sort_key(p2)

853 False

854 >>> p3 = MultiRun2DData("a", "i", "f", None, 3,

855 ... TIME_UNIT_FES, F_NAME_SCALED)

856 >>> sort_key(p3) < sort_key(p1)

857 True

858 >>> sort_key(p3) >= sort_key(p1)

859 False

860 """

861 # noinspection PyProtectedMember

862 return obj._tuple()

863

864

865def motipy_footer_bottom_comments(

866 _: Any, additional: str | None = None) -> Iterable[str]:

867 """

868 Print the standard csv footer for moptipy.

869

870 :param _: the setup object, ignored

871 :param dest: the destination callable

872 :param additional: any additional output string

873 :returns: the iterable with the footer comments

874

875 >>> for s in motipy_footer_bottom_comments(None, "bla"):

876 ... print(s[:49])

877 This data has been generated with moptipy version

878 bla

879 You can find moptipy at https://thomasweise.githu

880

881 >>> for s in motipy_footer_bottom_comments(None, None):

882 ... print(s[:49])

883 This data has been generated with moptipy version

884 You can find moptipy at https://thomasweise.githu

885 """

886 yield ("This data has been generated with moptipy version "

887 f"{moptipy_version}.")

888 if (additional is not None) and (str.__len__(additional) > 0):

889 yield additional

890 yield "You can find moptipy at https://thomasweise.github.io/mopitpy."

891

892

893#: a description of the algorithm field

894DESC_ALGORITHM: Final[str] = "the name of the algorithm setup that was used."

895#: a description of the instance field

896DESC_INSTANCE: Final[str] = ("the name of the problem instance to which the "

897 "algorithm was applied.")

898#: a description of the objective function field

899DESC_OBJECTIVE_FUNCTION: Final[str] = \

900 ("the name of the objective function (often also called fitness function "

901 "or cost function) that was used to rate the solution quality.")

902#: a description of the encoding field

903DESC_ENCODING: Final[str] = \

904 ("the name of the encoding, often also called genotype-phenotype mapping"

905 ", used. In some problems, the search space on which the algorithm "

906 "works is different from the space of possible solutions. For example, "

907 "when solving a scheduling problem, maybe our optimization algorithm "

908 "navigates in the space of permutations, but the solutions are Gantt "

909 "charts. The encoding is the function that translates the points in "

910 "the search space (e.g., permutations) to the points in the solution "

911 "space (e.g., Gantt charts). Nothing if no encoding was used.")

Coverage for moptipy / evaluation / base.py: 99%

127 statements