Coverage for pycommons / dev / building / make_documentation.py: 97%
184 statements
« prev ^ index » next coverage.py v7.13.0, created at 2025-12-11 03:04 +0000
1"""Create the documentation."""
3from argparse import ArgumentParser
4from os import remove as osremove
5from typing import Callable, Final, cast
7import minify_html
9from pycommons.dev.building.build_info import (
10 BuildInfo,
11 parse_project_arguments,
12)
13from pycommons.dev.doc.doc_info import (
14 DocInfo,
15 extract_md_infos,
16 load_doc_info_from_setup_cfg,
17)
18from pycommons.dev.url_replacer import make_url_replacer
19from pycommons.io.arguments import pycommons_argparser
20from pycommons.io.console import logger
21from pycommons.io.path import Path, delete_path
22from pycommons.processes.shell import STREAM_CAPTURE, STREAM_FORWARD, Command
23from pycommons.types import type_error
26def __get_source(
27 source: Path, __dst: set[Path] | None = None) \
28 -> Callable[[Path], bool]:
29 """
30 Get the existing files in a directory.
32 :param source: the directory
33 :param __dst: the destination
34 :returns: the set of files
36 >>> from pycommons.io.temp import temp_dir
37 >>> with temp_dir() as td:
38 ... f1 = td.resolve_inside("a")
39 ... f1.ensure_file_exists() # gives False
40 ... f2 = td.resolve_inside("b")
41 ... f2.ensure_file_exists() # gives False
42 ... d = td.resolve_inside("x")
43 ... d.ensure_dir_exists() # gives False
44 ... f3 = d.resolve_inside("a")
45 ... f3.ensure_file_exists()
46 ... g = __get_source(td)
47 ... g(f1) # is True
48 ... g(f2) # is True
49 ... g(f3) # is True
50 ... g(d) # is True
51 ... g("bla") # is not True
52 ... f4 = d.resolve_inside("x")
53 ... f4.ensure_file_exists() # gives False
54 ... g(f4) # is also not True, because generated later
55 False
56 False
57 False
58 True
59 True
60 True
61 True
62 False
63 False
64 False
65 """
66 if __dst is None:
67 __dst = {source}
68 for k in source.list_dir():
69 __dst.add(k)
70 if k.is_dir():
71 __get_source(k, __dst)
72 return __dst.__contains__
75def __keep_only_source(
76 source: Path, keep: Callable[[Path], bool],
77 __collect: Callable[[Path], None] | None = None) -> None:
78 """
79 Keep only the source items, delete the rest.
81 :param source: the source path
82 :param keep: the set of files and directories to keep
83 :param __collect: the collector
85 >>> from pycommons.io.temp import temp_dir
86 >>> with temp_dir() as td:
87 ... f1 = td.resolve_inside("a")
88 ... f1.ensure_file_exists() # gives False
89 ... f2 = td.resolve_inside("b")
90 ... f2.ensure_file_exists() # gives False
91 ... d = td.resolve_inside("x")
92 ... d.ensure_dir_exists() # gives False
93 ... f3 = d.resolve_inside("a")
94 ... f3.ensure_file_exists()
95 ... g = __get_source(td)
96 ... f4 = d.resolve_inside("x")
97 ... f4.ensure_file_exists() # gives False
98 ... e = td.resolve_inside("y")
99 ... e.ensure_dir_exists() # gives False
100 ... f5 = e.resolve_inside("a")
101 ... f5.ensure_file_exists()
102 ... f3.is_file() # gives True - should be preserved
103 ... f4.is_file() # gives True - will be deleted
104 ... f5.is_file() # gives True - will be deleted
105 ... d.is_dir() # True - will be preserved
106 ... e.is_dir() # True - will be deleted
107 ... __keep_only_source(td, g)
108 ... f3.is_file() # gives True - was preserved
109 ... f4.is_file() # gives False
110 ... f5.is_file() # gives False
111 ... d.is_dir() # True - was preserved
112 ... e.is_dir() # False
113 False
114 False
115 False
116 False
117 False
118 True
119 True
120 True
121 True
122 True
123 True
124 False
125 False
126 True
127 False
128 """
129 lst: list[Path] | None = None
130 if __collect is None:
131 lst = []
132 __collect = lst.append
133 for f in source.list_dir():
134 if not keep(f):
135 __collect(f)
136 if f.is_dir():
137 __keep_only_source(f, keep, __collect)
138 if lst:
139 lst.sort(key=cast("Callable[[Path], int]", str.__len__), reverse=True)
140 for k in lst:
141 delete_path(k)
144def __pygmentize(source: Path, info: BuildInfo,
145 dest: Path | None = None) -> None:
146 """
147 Pygmentize a source file to a destination.
149 :param source: the source file
150 :param info: the information
151 :param dest: the destination folder
153 >>> root = Path(__file__).up(4)
154 >>> bf = BuildInfo(root, "pycommons",
155 ... examples_dir=root.resolve_inside("examples"),
156 ... tests_dir=root.resolve_inside("tests"),
157 ... doc_source_dir=root.resolve_inside("docs/source"),
158 ... doc_dest_dir=root.resolve_inside("docs/build"))
160 >>> from contextlib import redirect_stdout
161 >>> from pycommons.io.temp import temp_dir
162 >>> with temp_dir() as td:
163 ... with redirect_stdout(None):
164 ... __pygmentize(root.resolve_inside("README.md"), bf, td)
165 ... readme = td.resolve_inside("README_md.html").is_file()
166 ... __pygmentize(root.resolve_inside("setup.py"), bf, td)
167 ... setuppy = td.resolve_inside("setup_py.html").is_file()
168 ... __pygmentize(root.resolve_inside("setup.cfg"), bf, td)
169 ... setup_cfg = td.resolve_inside("setup_cfg.html").is_file()
170 ... __pygmentize(root.resolve_inside("make.sh"), bf, td)
171 ... makefile = td.resolve_inside("make_sh.html").is_file()
172 ... __pygmentize(root.resolve_inside("LICENSE"), bf, td)
173 ... xlicense = td.resolve_inside("LICENSE.html").is_file()
174 ... __pygmentize(root.resolve_inside("requirements.txt"), bf, td)
175 ... req = td.resolve_inside("requirements_txt.html").is_file()
176 ... try:
177 ... __pygmentize(root.resolve_inside("LICENSE"), bf, td)
178 ... except ValueError as ve:
179 ... ver = str(ve)
180 >>> readme
181 True
182 >>> setuppy
183 True
184 >>> setup_cfg
185 True
186 >>> makefile
187 True
188 >>> xlicense
189 True
190 >>> req
191 True
192 >>> "already exists" in ver
193 True
194 """
195 logger(f"Trying to pygmentize {source!r} to {dest!r}.")
196 name: Final[str] = source.basename()
197 language: str = "text"
198 if name.endswith(".py"):
199 language = "python3"
200 elif name.endswith((".cfg", ".toml")):
201 language = "INI"
202 elif name.lower() == "makefile":
203 language = "make"
204 elif name.endswith(".sh"):
205 language = "bash"
206 if dest is None:
207 dest = info.doc_dest_dir
208 dest_file: Final[Path] = dest.resolve_inside(
209 f"{name.replace('.', '_')}.html")
210 if dest_file.exists():
211 raise ValueError(f"File {dest_file!r} already exists, "
212 f"cannot pygmentize {source!r}.")
213 info.command(("pygmentize", "-f", "html", "-l", language, "-O", "full",
214 "-O", "style=default", "-o", dest_file, source)).execute()
215 logger(f"Done pygmentizing {source!r} to {dest_file!r}.")
218#: the default files to pygmentize
219__PYGMENTIZE_DEFAULT: Final[tuple[str, ...]] = (
220 "conftest.py", "LICENSE", "make.sh", "Makefile", "pyproject.toml",
221 "requirements.txt", "requirements-dev.txt", "setup.cfg", "setup.py",
222)
224#: the possible styles
225__STYLES: Final[tuple[str, ...]] = ("bizstyle.css", )
227#: the html header
228__HTML_HEADER: Final[str] = "<!DOCTYPE html><html><title>"
229#: the top of the html body if styles are present
230__HTML_BODY_STYLE_1: Final[str] =\
231 ('</title><link href={STYLE} rel="stylesheet">'
232 '<body style="background-image:none"><div class="document">'
233 '<div class="documentwrapper"><div class="bodywrapper">'
234 '<div class="body" role="main"><section>')
235#: the bottom of the html body if styles are present
236__HTML_BODY_STYLE_2: Final[str] = \
237 "</section></div></div></div></div></body></html>"
238#: the top of the html body if not styles are present
239__HTML_BODY_NO_STYLE_1: Final[str] = "</title><body><section>"
240#: the bottom of the html body if styles are present
241__HTML_BODY_NO_STYLE_2: Final[str] = "</section></body></html>"
244def __render_markdown(markdown: Path, info: BuildInfo, dest: Path | None,
245 css: str | None,
246 url_fixer: Callable[[str], str]) -> None:
247 """
248 Render a markdown file.
250 :param markdown: the markdown file
251 :param dest: the destination path
252 :param info: the build info
253 :param css: the relative path to the style sheet
254 :param url_fixer: the URL fixer
256 >>> root = Path(__file__).up(4)
257 >>> bf = BuildInfo(root, "pycommons",
258 ... examples_dir=root.resolve_inside("examples"),
259 ... tests_dir=root.resolve_inside("tests"),
260 ... doc_source_dir=root.resolve_inside("docs/source"),
261 ... doc_dest_dir=root.resolve_inside("docs/build"))
263 >>> from io import StringIO
264 >>> from contextlib import redirect_stdout
265 >>> from pycommons.io.temp import temp_dir
266 >>> with temp_dir() as td:
267 ... with redirect_stdout(None):
268 ... __render_markdown(root.resolve_inside("README.md"), bf,
269 ... td, None, lambda s: s)
270 ... readme = td.resolve_inside("README_md.html").is_file()
271 ... try:
272 ... __render_markdown(root.resolve_inside("README.md"), bf,
273 ... td, None, lambda s: s)
274 ... except ValueError as ve:
275 ... vstr = str(ve)
276 ... __render_markdown(root.resolve_inside("CONTRIBUTING.md"), bf,
277 ... td, "dummy.css", lambda s: s)
278 ... cb = td.resolve_inside("CONTRIBUTING_md.html").is_file()
279 >>> readme
280 True
281 >>> vstr.endswith("already exists.")
282 True
283 >>> cb
284 True
285 """
286 logger(f"Trying to render {markdown!r}.")
288 # find the destination file
289 basename: Final[str] = markdown.basename()
290 if dest is None:
291 dest = info.doc_dest_dir
292 dest_path: Final[Path] = dest.resolve_inside(
293 f"{basename.replace('.', '_')}.html")
294 if dest_path.exists():
295 raise ValueError(f"Destination path {dest_path!r} already exists.")
297 # get the title
298 title, _ = extract_md_infos(markdown)
299 title = str.strip(title).replace("`", "").replace("*", "")
301 # set up the body
302 body_1: str = __HTML_BODY_NO_STYLE_1
303 body_2: str = __HTML_BODY_NO_STYLE_2
304 if css is not None:
305 body_1 = __HTML_BODY_STYLE_1.replace("{STYLE}", css)
306 body_2 = __HTML_BODY_STYLE_2
308 text: Final[str] = url_fixer(str.strip(Command((
309 "markdown_py", "-o", "html", markdown),
310 stderr=STREAM_FORWARD, stdout=STREAM_CAPTURE, timeout=info.timeout,
311 working_dir=info.base_dir).execute()[0]))
312 dest_path.write_all_str(f"{__HTML_HEADER}{title}{body_1}{text}{body_2}")
313 logger(f"Finished rendering {markdown!r} to {dest_path!r}.")
316def __minify(file: Path) -> None:
317 """
318 Minify the given HTML file.
320 :param file: the file
322 >>> root = Path(__file__).up(4)
323 >>> bf = BuildInfo(root, "pycommons",
324 ... examples_dir=root.resolve_inside("examples"),
325 ... tests_dir=root.resolve_inside("tests"),
326 ... doc_source_dir=root.resolve_inside("docs/source"),
327 ... doc_dest_dir=root.resolve_inside("docs/build"))
329 >>> from io import StringIO
330 >>> from contextlib import redirect_stdout
331 >>> from pycommons.io.temp import temp_dir
332 >>> with temp_dir() as td:
333 ... with redirect_stdout(None):
334 ... __render_markdown(root.resolve_inside("README.md"), bf,
335 ... td, None, lambda s: s)
336 ... readme = td.resolve_inside("README_md.html")
337 ... long = str.__len__(readme.read_all_str())
338 ... __minify(readme)
339 ... short = str.__len__(readme.read_all_str())
340 >>> 0 < short < long
341 True
342 """
343 logger(f"Minifying HTML in {file!r}.")
344 text: str = str.strip(file.read_all_str())
345 text = str.strip(minify_html.minify( # pylint: disable=E1101
346 text,
347 allow_noncompliant_unquoted_attribute_values=False,
348 allow_optimal_entities=True,
349 allow_removing_spaces_between_attributes=True,
350 keep_closing_tags=False,
351 keep_comments=False,
352 keep_html_and_head_opening_tags=False,
353 keep_input_type_text_attr=False,
354 keep_ssi_comments=False,
355 minify_css=True,
356 minify_doctype=False,
357 minify_js=True,
358 preserve_brace_template_syntax=False,
359 preserve_chevron_percent_template_syntax=False,
360 remove_bangs=True,
361 remove_processing_instructions=True))
362 if "<pre" not in text:
363 text = " ".join(map(str.strip, text.splitlines()))
364 file.write_all_str(str.strip(text))
367def __minify_all(dest: Path, skip: Callable[[str], bool]) -> None:
368 """
369 Minify all files in the given destination folder.
371 :param dest: the destination
372 :param skip: the files to skip
374 >>> root = Path(__file__).up(4)
375 >>> bf = BuildInfo(root, "pycommons",
376 ... examples_dir=root.resolve_inside("examples"),
377 ... tests_dir=root.resolve_inside("tests"),
378 ... doc_source_dir=root.resolve_inside("docs/source"),
379 ... doc_dest_dir=root.resolve_inside("docs/build"))
381 >>> from io import StringIO
382 >>> from contextlib import redirect_stdout
383 >>> from pycommons.io.temp import temp_dir
384 >>> with temp_dir() as td:
385 ... with redirect_stdout(None):
386 ... __render_markdown(root.resolve_inside("README.md"), bf,
387 ... td, None, lambda s: s)
388 ... readme = td.resolve_inside("README_md.html")
389 ... __render_markdown(root.resolve_inside("CONTRIBUTING.md"), bf,
390 ... td, None, lambda s: s)
391 ... cb = td.resolve_inside("CONTRIBUTING_md.html")
392 ... longr = str.__len__(readme.read_all_str())
393 ... longc = str.__len__(cb.read_all_str())
394 ... __minify_all(td, lambda x: False)
395 ... shortr = str.__len__(readme.read_all_str())
396 ... shortc = str.__len__(cb.read_all_str())
397 >>> 0 < shortr < longr
398 True
399 >>> 0 < shortc < longc
400 True
401 """
402 if skip(dest):
403 return
404 if dest.is_file():
405 if dest.endswith(".html"):
406 __minify(dest)
407 elif dest.is_dir():
408 for f in dest.list_dir():
409 __minify_all(f, skip)
412def __put_nojekyll(dest: Path) -> None:
413 """
414 Put a `.nojekyll` file into each directory.
416 :param dest: the destination path.
418 >>> from pycommons.io.temp import temp_dir, temp_file
419 >>> with temp_dir() as td:
420 ... x = td.resolve_inside("x")
421 ... x.ensure_dir_exists()
422 ... y = x.resolve_inside("y")
423 ... y.ensure_dir_exists()
424 ... __put_nojekyll(td)
425 ... td.resolve_inside(".nojekyll").is_file()
426 ... x.resolve_inside(".nojekyll").is_file()
427 ... y.resolve_inside(".nojekyll").is_file()
428 True
429 True
430 True
432 >>> with temp_file() as tf:
433 ... __put_nojekyll(tf) # nothing happens
434 """
435 if not dest.is_dir():
436 return
437 dest.resolve_inside(".nojekyll").ensure_file_exists()
438 for f in dest.list_dir(files=False):
439 __put_nojekyll(f)
442def make_documentation(info: BuildInfo) -> None:
443 """
444 Make the documentation of the project.
446 :param info: the build information
448 >>> root = Path(__file__).up(4)
449 >>> bf = BuildInfo(root, "pycommons",
450 ... examples_dir=root.resolve_inside("examples"),
451 ... tests_dir=root.resolve_inside("tests"),
452 ... doc_source_dir=root.resolve_inside("docs/source"),
453 ... doc_dest_dir=root.resolve_inside("docs/build"))
454 >>> from io import StringIO
455 >>> from contextlib import redirect_stdout
456 >>> with redirect_stdout(None):
457 ... make_documentation(bf)
459 >>> try:
460 ... make_documentation(1)
461 ... except TypeError as te:
462 ... print(str(te)[-13:])
463 nt, namely 1.
464 """
465 if not isinstance(info, BuildInfo):
466 raise type_error(info, "info", BuildInfo)
468 source: Final[Path | None] = info.doc_source_dir
469 dest: Final[Path | None] = info.doc_dest_dir
470 if (source is None) or (dest is None):
471 raise ValueError("Need both documentation source and "
472 f"destination, but got {info}.")
473 logger(f"Building documentation with setup {info}.")
475 logger(f"First clearing {dest!r}.")
476 if dest.exists():
477 if not dest.is_dir():
478 raise ValueError(f"{dest!r} exists but is no directory?")
479 delete_path(dest)
480 dest.ensure_dir_exists()
482 logger("Collecting all documentation source files.")
483 retain: Final[Callable[[Path], bool]] = __get_source(source)
484 try:
485 logger("Building the documentation files via Sphinx.")
486 info.command(("sphinx-apidoc", "-M", "--ext-autodoc",
487 "-o", source, info.sources_dir)).execute()
488 info.command(("sphinx-build", "-W", "-a", "-E", "-b", "html",
489 source, dest)).execute()
490 logger("Finished the Sphinx executions.")
491 finally:
492 logger("Clearing all auto-generated files.")
493 __keep_only_source(source, retain)
495 logger("Now building the additional files.")
497 if info.examples_dir is not None:
498 examples_dest: Final[Path] = dest.resolve_inside("examples")
499 examples_dest.ensure_dir_exists()
500 logger(f"Now pygmentizing example files to {examples_dest!r}.")
501 for f in info.examples_dir.list_dir(directories=False):
502 if f.endswith(".py"):
503 __pygmentize(f, info, examples_dest)
505 logger("Now pygmentizing default files.")
506 for fn in __PYGMENTIZE_DEFAULT:
507 f = info.base_dir.resolve_inside(fn)
508 if f.is_file():
509 __pygmentize(f, info)
511 # now printing coverage information
512 coverage_file: Final[Path] = info.base_dir.resolve_inside(".coverage")
513 coverage_dest: Path | None = None
514 if coverage_file.is_file():
515 logger(f"Generating coverage from file {coverage_file!r}.")
516 coverage_dest = dest.resolve_inside("tc")
517 coverage_dest.ensure_dir_exists()
518 delete_path(coverage_dest)
519 try:
520 info.command(("coverage", "html", "-d", coverage_dest,
521 f"--data-file={coverage_file}",
522 f"--include={info.package_name}/*")).execute()
523 except ValueError as ve:
524 if coverage_dest.is_dir():
525 raise
526 logger(f"No coverage to report: {ve}.")
527 else:
528 info.command((
529 "coverage-badge", "-o", coverage_dest.resolve_inside(
530 "badge.svg"))).execute()
531 gitignore: Path = coverage_dest.resolve_inside(".gitignore")
532 if gitignore.is_file(): # remove gitignore
533 logger(f"Found .gitignore file {gitignore!r} - deleting it.")
534 osremove(gitignore)
535 else:
536 logger("No coverage data found.")
538 # find potential style sheet
539 static: Final[Path] = dest.resolve_inside("_static")
540 css: str | None = None
541 if static.is_dir():
542 for sst in __STYLES:
543 css_path: Path = static.resolve_inside(sst)
544 if css_path.is_file():
545 css = css_path.relative_to(dest)
546 break
547 if css is None:
548 logger("Found no static css style.")
549 else:
550 logger(f"Using style sheet {css!r}.")
552 setup_cfg: Final[Path] = info.base_dir.resolve_inside("setup.cfg")
553 url_fixer: Callable[[str], str] = str.strip
554 if setup_cfg.is_file():
555 logger("Loading documentation information.")
556 doc_info: Final[DocInfo] = load_doc_info_from_setup_cfg(setup_cfg)
557 url_fixer = make_url_replacer(
558 {doc_info.doc_url: "./"}, for_markdown=False)
560 logger("Now rendering all markdown files in the root directory.")
561 for f in info.base_dir.list_dir(directories=False):
562 if f.is_file() and f.endswith(".md") and (
563 not f.basename().startswith("README")):
564 __render_markdown(f, info, dest, css, url_fixer)
566 logger("Now minifying all generated html files.")
567 __minify_all(dest, {coverage_dest}.__contains__)
569 logger("Now putting a .nojekyll file into each directory.")
570 __put_nojekyll(dest)
572 logger(f"Finished building documentation with setup {info}.")
575# Run documentation generation process if executed as script
576if __name__ == "__main__":
577 parser: Final[ArgumentParser] = pycommons_argparser(
578 __file__,
579 "Build the Documentation",
580 "This utility uses sphinx to build the documentation.")
581 make_documentation(parse_project_arguments(parser))