Coverage for moptipy/utils/sys

1"""A tool for writing a section with system information into log files."""

2import contextlib

3import importlib.metadata as ilm

4import os

5import platform

6import re

7import socket

8import sys

9from contextlib import suppress

10from datetime import UTC, datetime

11from typing import Final, Iterable

13import psutil # type: ignore

14from pycommons.io.csv import CSV_SEPARATOR, SCOPE_SEPARATOR

15from pycommons.io.path import Path

16from pycommons.processes.caller import is_build

18import moptipy.version as ver

19from moptipy.api import logging

20from moptipy.utils.logger import (

21 KEY_VALUE_SEPARATOR,

22 InMemoryLogger,

23 KeyValueLogSection,

24 Logger,

25)

28def __cpu_affinity(proc: psutil.Process | None = None) -> str | None:

29 """

30 Get the CPU affinity.

32 :param proc: the process handle

33 :return: the CPU affinity string.

34 """

35 if proc is None:

36 proc = psutil.Process()

37 if proc is None:

38 return None

39 cpua = proc.cpu_affinity()

40 if cpua is not None:

41 cpua = CSV_SEPARATOR.join(map(str, cpua))

42 if len(cpua) > 0:

43 return cpua

44 return None

47#: the dependencies

48__DEPENDENCIES: set[str] | None = {

49 "cmaes", "contourpy", "cycler", "fonttools", "intel-cmplr-lib-rt",

50 "joblib", "kiwisolver", "llvmlite", "matplotlib", "moptipy", "numba",

51 "numpy", "packaging", "pdfo", "pillow", "psutil", "pycommons",

52 "pyparsing", "python-dateutil", "scikit-learn", "scipy", "setuptools",

53 "six", "threadpoolctl"}

56def add_dependency(dependency: str,

57 ignore_if_make_build: bool = False) -> None:

58 """

59 Add a dependency so that its version can be stored in log files.

61 Warning: You must add all dependencies *before* the first log file is

62 written. As soon as the :func:`log_sys_info` is invoked for the first

63 time, adding new dependencies will cause an error. And writing a log

64 file via the :mod:`~moptipy.api.experiment` API or to a file specified in

65 the :mod:`~moptipy.api.execution` API will invoke this function.

67 :param dependency: the basic name of the library, exactly as you would

68 `import` it in a Python module. For example, to include the version of

69 `numpy` in the log files, you would do `add_dependency("numpy")` (of

70 course, the version of `numpy` is already automatically included

71 anyway).

72 :param ignore_if_make_build: should this dependency be ignored if this

73 method is invoked during a `make` build? This makes sense if the

74 dependency itself is a package which is uninstalled and then

75 re-installed during a `make` build process. In such a situation, the

76 dependency version may be unavailable and cause an exception.

77 :raises TypeError: if `dependency` is not a string

78 :raises ValueError: if `dependency` is an invalid string or the log

79 information has already been accessed before and modifying it now is

80 not permissible.

81 """

82 if (str.__len__(dependency) <= 0) or (dependency != str.strip(dependency))\

83 or (" " in dependency):

84 raise ValueError(f"Invalid dependency string {dependency!r}.")

85 if __DEPENDENCIES is None:

86 raise ValueError(

87 f"Too late. Cannot add dependency {dependency!r} anymore.")

88 if ignore_if_make_build and is_build():

89 return

90 __DEPENDENCIES.add(dependency)

93# noinspection PyBroadException

94def __make_sys_info() -> str:

95 """

96 Build the system info string.

98 This method is only used once and then deleted.

100 :returns: the system info string.

101 """

102 global __DEPENDENCIES # noqa: PLW0603 # pylint: disable=W0603

103 if __DEPENDENCIES is None:

104 raise ValueError("Cannot re-create log info.")

105 dep: Final[set[str]] = __DEPENDENCIES

106 __DEPENDENCIES = None # noqa: PLW0603 # pylint: disable=W0603

107

108 def __v(sec: KeyValueLogSection, key: str, value) -> None:

109 """

110 Create a key-value pair if value is not empty.

111

112 :param sec: the section to write to.

113 :param key: the key

114 :param value: an arbitrary value, maybe consisting of multiple lines.

115 """

116 if value is None:

117 return

118 value = str.strip(" ".join([str.strip(ts) for ts in

119 str.strip(str(value)).split("\n")]))

120 if str.__len__(value) <= 0:

121 return

122 sec.key_value(key, value)

123

124 def __get_processor_name() -> str | None:

125 """

126 Get the processor name.

127

128 :returns: a string if there is any processor information

129 """

130 with contextlib.suppress(Exception):

131 if platform.system() == "Windows":

132 return platform.processor()

133 if platform.system() == "Linux":

134 with Path("/proc/cpuinfo").open_for_read() as rd:

135 for line in rd:

136 if "model name" in line:

137 return re.sub(

138 pattern=".*model name.*:",

139 repl="", string=line, count=1).strip()

140 return None

141

142 def __get_mem_size_sysconf() -> int | None:

143 """

144 Get the memory size information from sysconf.

145

146 :returns: an integer with the memory size if available

147 """

148 with contextlib.suppress(Exception):

149 k1 = "SC_PAGESIZE"

150 if k1 not in os.sysconf_names:

151 k1 = "SC_PAGE_SIZE"

152 if k1 not in os.sysconf_names:

153 return None

154 v1 = os.sysconf(k1)

155 if not isinstance(v1, int):

156 return None

157 k2 = "SC_PHYS_PAGES"

158 if k2 not in os.sysconf_names:

159 k2 = "_SC_PHYS_PAGES"

160 if k2 not in os.sysconf_names:

161 return None

162 v2 = os.sysconf(k2)

163 if not isinstance(v1, int):

164 return None

165

166 if (v1 > 0) and (v2 > 0):

167 return v1 * v2

168 return None

169

170 def __get_mem_size_meminfo() -> int | None:

171 """

172 Get the memory size information from meminfo.

173

174 :returns: an integer with the memory size if available

175 """

176 with contextlib.suppress(Exception):

177 with Path("/proc/meminfo").open_for_read() as rd:

178 meminfo = {i.split()[0].rstrip(":"): int(i.split()[1])

179 for i in rd}

180 mem_kib = meminfo["MemTotal"] # e.g. 3921852

181 mem_kib = int(mem_kib)

182 if mem_kib > 0:

183 return 1024 * mem_kib

184 return None

185

186 def __get_mem_size() -> int | None:

187 """

188 Get the memory size information from any available source.

189

190 :returns: an integer with the memory size if available

191 """

192 vs = __get_mem_size_sysconf()

193 if vs is None:

194 vs = __get_mem_size_meminfo()

195 if vs is None:

196 return psutil.virtual_memory().total

197 return vs

198

199 # log all information in memory to convert it to one constant string.

200 with InMemoryLogger() as imr:

201 with imr.key_values(logging.SECTION_SYS_INFO) as kv:

202 with kv.scope(logging.SCOPE_SESSION) as k:

203 __v(k, logging.KEY_SESSION_START,

204 datetime.now(tz=UTC))

205 __v(k, logging.KEY_NODE_NAME, platform.node())

206 proc = psutil.Process()

207 __v(k, logging.KEY_PROCESS_ID, hex(proc.pid))

208 cpua = __cpu_affinity(proc)

209 if cpua is not None:

210 __v(k, logging.KEY_CPU_AFFINITY, cpua)

211 del proc, cpua

212

213 # get the command line and working directory of the process

214 with contextlib.suppress(Exception):

215 proc = psutil.Process(os.getpid())

216 cmd = proc.cmdline()

217 if isinstance(cmd, Iterable):

218 __v(k, logging.KEY_COMMAND_LINE,

219 " ".join(map(repr, proc.cmdline())))

220 cwd = proc.cwd()

221 if isinstance(cwd, str):

222 __v(k, logging.KEY_WORKING_DIRECTORY,

223 repr(proc.cwd()))

224

225 # see https://stackoverflow.com/questions/166506/.

226 s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)

227 try:

228 # doesn't even have to be reachable

229 s.connect(("10.255.255.255", 1))

230 ip = s.getsockname()[0]

231 except Exception: # noqa: BLE001

232 ip = "127.0.0.1"

233 finally:

234 s.close()

235 __v(k, logging.KEY_NODE_IP, ip)

236

237 with kv.scope(logging.SCOPE_VERSIONS) as k:

238 for package in sorted(dep):

239 if package == "moptipy":

240 __v(k, "moptipy", ver.__version__)

241 else:

242 use_v: str | None = None

243 with suppress(BaseException):

244 use_v = str.strip(ilm.version(package))

245 if (use_v is None) or (str.__len__(use_v) <= 0):

246 use_v = "notFound"

247 __v(k, package.replace("-", ""), use_v)

248

249 with kv.scope(logging.SCOPE_HARDWARE) as k:

250 __v(k, logging.KEY_HW_MACHINE, platform.machine())

251 __v(k, logging.KEY_HW_N_PHYSICAL_CPUS,

252 psutil.cpu_count(logical=False))

253 __v(k, logging.KEY_HW_N_LOGICAL_CPUS,

254 psutil.cpu_count(logical=True))

255

256 # store the CPU speed information

257 cpuf: dict[tuple[int, int], int] = {}

258 total: int = 0

259 for cf in psutil.cpu_freq(True):

260 t = (int(cf.min), int(cf.max))

261 cpuf[t] = cpuf.get(t, 0) + 1

262 total += 1

263 memlst: list[tuple[int, ...]]

264 if total > 1:

265 memlst = [(key[0], key[1], value) for

266 key, value in cpuf.items()]

267 memlst.sort()

268 else:

269 memlst = list(cpuf)

270

271 def __make_mhz_str(tpl: tuple[int, ...]) -> str:

272 """Convert a MHz tuple to a string."""

273 base: str = f"({tpl[0]}MHz..{tpl[1]}MHz)" \

274 if tpl[1] > tpl[0] else f"{tpl[0]}MHz"

275 return base if (len(tpl) < 3) or (tpl[2] <= 1) \

276 else f"{base}*{tpl[2]}"

277 __v(k, logging.KEY_HW_CPU_MHZ,

278 "+".join([__make_mhz_str(t) for t in memlst]))

279

280 __v(k, logging.KEY_HW_BYTE_ORDER, sys.byteorder)

281 __v(k, logging.KEY_HW_CPU_NAME, __get_processor_name())

282 __v(k, logging.KEY_HW_MEM_SIZE, __get_mem_size())

283

284 with kv.scope(logging.SCOPE_PYTHON) as k:

285 __v(k, logging.KEY_PYTHON_VERSION, sys.version)

286 __v(k, logging.KEY_PYTHON_IMPLEMENTATION,

287 platform.python_implementation())

288

289 with kv.scope(logging.SCOPE_OS) as k:

290 __v(k, logging.KEY_OS_NAME, platform.system())

291 __v(k, logging.KEY_OS_RELEASE, platform.release())

292 __v(k, logging.KEY_OS_VERSION, platform.version())

293

294 lst = imr.get_log()

295

296 if len(lst) < 3:

297 raise ValueError("sys info turned out to be empty?")

298 __DEPENDENCIES = None

299 return "\n".join(lst[1:(len(lst) - 1)])

300

301

302def get_sys_info() -> str:

303 r"""

304 Get the system information as string.

305

306 :returns: the system information as string

307

308 >>> raw_infos = get_sys_info()

309 >>> raw_infos is get_sys_info() # caching!

310 True

311 >>> for k in raw_infos.split("\n"):

312 ... print(k[:k.find(": ")])

313 session.start

314 session.node

315 session.processId

316 session.cpuAffinity

317 session.commandLine

318 session.workingDirectory

319 session.ipAddress

320 version.cmaes

321 version.contourpy

322 version.cycler

323 version.fonttools

324 version.intelcmplrlibrt

325 version.joblib

326 version.kiwisolver

327 version.llvmlite

328 version.matplotlib

329 version.moptipy

330 version.numba

331 version.numpy

332 version.packaging

333 version.pdfo

334 version.pillow

335 version.psutil

336 version.pycommons

337 version.pyparsing

338 version.pythondateutil

339 version.scikitlearn

340 version.scipy

341 version.setuptools

342 version.six

343 version.threadpoolctl

344 hardware.machine

345 hardware.nPhysicalCpus

346 hardware.nLogicalCpus

347 hardware.cpuMhz

348 hardware.byteOrder

349 hardware.cpu

350 hardware.memSize

351 python.version

352 python.implementation

353 os.name

354 os.release

355 os.version

356 """

357 the_object: Final[object] = get_sys_info

358 the_attr: Final[str] = "__the_sysinfo"

359 if hasattr(the_object, the_attr):

360 return getattr(the_object, the_attr)

361 sys_info: Final[str] = __make_sys_info()

362 setattr(the_object, the_attr, sys_info)

363 return sys_info

364

365

366def update_sys_info_cpu_affinity() -> None:

367 """Update the CPU affinity of the system information."""

368 sys_info_str = get_sys_info()

369 start = (f"\n{logging.SCOPE_SESSION}{SCOPE_SEPARATOR}"

370 f"{logging.KEY_CPU_AFFINITY}{KEY_VALUE_SEPARATOR}")

371 start_i = sys_info_str.find(start)

372 if start_i < 0:

373 return # no affinity, don't need to update

374 start_i += len(start)

375 end_i = sys_info_str.find("\n", start_i)

376 if end_i <= start_i:

377 raise ValueError(f"Empty {logging.KEY_CPU_AFFINITY}?")

378 affinity = __cpu_affinity()

379 if affinity is None:

380 raise ValueError(

381 f"first affinity query is {sys_info_str[start_i:end_i]},"

382 f" but second one is None?")

383 sys_info_str = f"{sys_info_str[:start_i]}{affinity}{sys_info_str[end_i:]}"

384

385 the_object: Final[object] = get_sys_info

386 the_attr: Final[str] = "__the_sysinfo"

387 setattr(the_object, the_attr, sys_info_str)

388

389

390def log_sys_info(logger: Logger) -> None:

391 r"""

392 Write the system information section to a logger.

393

394 The concept of this method is that we only construct the whole system

395 configuration exactly once in memory and then always directly flush it

396 as a string to the logger. This is much more efficient than querying it

397 every single time.

398

399 :param logger: the logger

400

401 >>> from moptipy.utils.logger import InMemoryLogger

402 >>> with InMemoryLogger() as l:

403 ... log_sys_info(l)

404 ... log = l.get_log()

405 >>> print(log[0])

406 BEGIN_SYS_INFO

407 >>> print(log[-1])

408 END_SYS_INFO

409 >>> log[1:-1] == get_sys_info().split('\n')

410 True

411 """

412 with logger.text(logging.SECTION_SYS_INFO) as txt:

413 txt.write(get_sys_info())

Coverage for moptipy / utils / sys_info.py: 79%

194 statements