Coverage for pycommons / dev / building / make_documentation.py: 98%
186 statements
« prev ^ index » next coverage.py v7.13.4, created at 2026-02-24 03:11 +0000
1"""Create the documentation."""
3from argparse import ArgumentParser
4from typing import Callable, Final, cast
6import minify_html
8from pycommons.dev.building.build_info import (
9 BuildInfo,
10 parse_project_arguments,
11)
12from pycommons.dev.doc.doc_info import (
13 DocInfo,
14 extract_md_infos,
15 load_doc_info_from_setup_cfg,
16)
17from pycommons.dev.url_replacer import make_url_replacer
18from pycommons.io.arguments import pycommons_argparser
19from pycommons.io.console import logger
20from pycommons.io.path import UTF8, Path, delete_path
21from pycommons.processes.shell import STREAM_CAPTURE, STREAM_FORWARD, Command
22from pycommons.types import type_error
25def __get_source(
26 source: Path, __dst: set[Path] | None = None) \
27 -> Callable[[Path], bool]:
28 """
29 Get the existing files in a directory.
31 :param source: the directory
32 :param __dst: the destination
33 :returns: the set of files
35 >>> from pycommons.io.temp import temp_dir
36 >>> with temp_dir() as td:
37 ... f1 = td.resolve_inside("a")
38 ... f1.ensure_file_exists() # gives False
39 ... f2 = td.resolve_inside("b")
40 ... f2.ensure_file_exists() # gives False
41 ... d = td.resolve_inside("x")
42 ... d.ensure_dir_exists() # gives False
43 ... f3 = d.resolve_inside("a")
44 ... f3.ensure_file_exists()
45 ... g = __get_source(td)
46 ... g(f1) # is True
47 ... g(f2) # is True
48 ... g(f3) # is True
49 ... g(d) # is True
50 ... g("bla") # is not True
51 ... f4 = d.resolve_inside("x")
52 ... f4.ensure_file_exists() # gives False
53 ... g(f4) # is also not True, because generated later
54 False
55 False
56 False
57 True
58 True
59 True
60 True
61 False
62 False
63 False
64 """
65 if __dst is None:
66 __dst = {source}
67 for k in source.list_dir():
68 __dst.add(k)
69 if k.is_dir():
70 __get_source(k, __dst)
71 return __dst.__contains__
74def __keep_only_source(
75 source: Path, keep: Callable[[Path], bool],
76 __collect: Callable[[Path], None] | None = None) -> None:
77 """
78 Keep only the source items, delete the rest.
80 :param source: the source path
81 :param keep: the set of files and directories to keep
82 :param __collect: the collector
84 >>> from pycommons.io.temp import temp_dir
85 >>> with temp_dir() as td:
86 ... f1 = td.resolve_inside("a")
87 ... f1.ensure_file_exists() # gives False
88 ... f2 = td.resolve_inside("b")
89 ... f2.ensure_file_exists() # gives False
90 ... d = td.resolve_inside("x")
91 ... d.ensure_dir_exists() # gives False
92 ... f3 = d.resolve_inside("a")
93 ... f3.ensure_file_exists()
94 ... g = __get_source(td)
95 ... f4 = d.resolve_inside("x")
96 ... f4.ensure_file_exists() # gives False
97 ... e = td.resolve_inside("y")
98 ... e.ensure_dir_exists() # gives False
99 ... f5 = e.resolve_inside("a")
100 ... f5.ensure_file_exists()
101 ... f3.is_file() # gives True - should be preserved
102 ... f4.is_file() # gives True - will be deleted
103 ... f5.is_file() # gives True - will be deleted
104 ... d.is_dir() # True - will be preserved
105 ... e.is_dir() # True - will be deleted
106 ... __keep_only_source(td, g)
107 ... f3.is_file() # gives True - was preserved
108 ... f4.is_file() # gives False
109 ... f5.is_file() # gives False
110 ... d.is_dir() # True - was preserved
111 ... e.is_dir() # False
112 False
113 False
114 False
115 False
116 False
117 True
118 True
119 True
120 True
121 True
122 True
123 False
124 False
125 True
126 False
127 """
128 lst: list[Path] | None = None
129 if __collect is None:
130 lst = []
131 __collect = lst.append
132 for f in source.list_dir():
133 if not keep(f):
134 __collect(f)
135 if f.is_dir():
136 __keep_only_source(f, keep, __collect)
137 if lst:
138 lst.sort(key=cast("Callable[[Path], int]", str.__len__), reverse=True)
139 for k in lst:
140 delete_path(k)
143def __pygmentize(source: Path, info: BuildInfo,
144 dest: Path | None = None) -> None:
145 """
146 Pygmentize a source file to a destination.
148 :param source: the source file
149 :param info: the information
150 :param dest: the destination folder
152 >>> root = Path(__file__).up(4)
153 >>> bf = BuildInfo(root, "pycommons",
154 ... examples_dir=root.resolve_inside("examples"),
155 ... tests_dir=root.resolve_inside("tests"),
156 ... doc_source_dir=root.resolve_inside("docs/source"),
157 ... doc_dest_dir=root.resolve_inside("docs/build"))
159 >>> from contextlib import redirect_stdout
160 >>> from pycommons.io.temp import temp_dir
161 >>> with temp_dir() as td:
162 ... with redirect_stdout(None):
163 ... __pygmentize(root.resolve_inside("README.md"), bf, td)
164 ... readme = td.resolve_inside("README_md.html").is_file()
165 ... __pygmentize(root.resolve_inside("setup.py"), bf, td)
166 ... setuppy = td.resolve_inside("setup_py.html").is_file()
167 ... __pygmentize(root.resolve_inside("setup.cfg"), bf, td)
168 ... setup_cfg = td.resolve_inside("setup_cfg.html").is_file()
169 ... __pygmentize(root.resolve_inside("make.sh"), bf, td)
170 ... makefile = td.resolve_inside("make_sh.html").is_file()
171 ... __pygmentize(root.resolve_inside("LICENSE"), bf, td)
172 ... xlicense = td.resolve_inside("LICENSE.html").is_file()
173 ... __pygmentize(root.resolve_inside("requirements.txt"), bf, td)
174 ... req = td.resolve_inside("requirements_txt.html").is_file()
175 ... try:
176 ... __pygmentize(root.resolve_inside("LICENSE"), bf, td)
177 ... except ValueError as ve:
178 ... ver = str(ve)
179 >>> readme
180 True
181 >>> setuppy
182 True
183 >>> setup_cfg
184 True
185 >>> makefile
186 True
187 >>> xlicense
188 True
189 >>> req
190 True
191 >>> "already exists" in ver
192 True
193 """
194 logger(f"Trying to pygmentize {source!r} to {dest!r}.")
195 name: Final[str] = source.basename()
196 language: str = "text"
197 if name.endswith(".py"):
198 language = "python3"
199 elif name.endswith((".cfg", ".toml")):
200 language = "INI"
201 elif name.lower() == "makefile":
202 language = "make"
203 elif name.endswith(".sh"):
204 language = "bash"
205 if dest is None:
206 dest = info.doc_dest_dir
207 dest_file: Final[Path] = dest.resolve_inside(
208 f"{name.replace('.', '_')}.html")
209 if dest_file.exists():
210 raise ValueError(f"File {dest_file!r} already exists, "
211 f"cannot pygmentize {source!r}.")
212 info.command(("pygmentize", "-f", "html", "-l", language, "-O", "full",
213 "-O", "style=default", "-o", dest_file, source)).execute()
214 logger(f"Done pygmentizing {source!r} to {dest_file!r}.")
217#: the default files to pygmentize
218__PYGMENTIZE_DEFAULT: Final[tuple[str, ...]] = (
219 "conftest.py", "LICENSE", "make.sh", "Makefile", "pyproject.toml",
220 "requirements.txt", "requirements-dev.txt", "setup.cfg", "setup.py",
221)
223#: the possible styles
224__STYLES: Final[tuple[str, ...]] = ("bizstyle.css", )
226#: additions that should be included in the styles
227__STYLE_ADDITIONS: Final[tuple[tuple[str, str], ...]] = (
228 ("bizstyle.css",
229 ("\n.sphinxsidebarwrapper span.pre{white-space:wrap;-epub-hyphens:auto;"
230 "-webkit-hyphens:auto;-moz-hyphens:auto;hyphens: auto;}\n")), )
232#: the html header
233__HTML_HEADER: Final[str] = "<!DOCTYPE html><html><title>"
234#: the top of the html body if styles are present
235__HTML_BODY_STYLE_1: Final[str] =\
236 ('</title><link href={STYLE} rel="stylesheet">'
237 '<body style="background-image:none"><div class="document">'
238 '<div class="documentwrapper"><div class="bodywrapper">'
239 '<div class="body" role="main"><section>')
240#: the bottom of the html body if styles are present
241__HTML_BODY_STYLE_2: Final[str] = \
242 "</section></div></div></div></div></body></html>"
243#: the top of the html body if not styles are present
244__HTML_BODY_NO_STYLE_1: Final[str] = "</title><body><section>"
245#: the bottom of the html body if styles are present
246__HTML_BODY_NO_STYLE_2: Final[str] = "</section></body></html>"
249def __render_markdown(markdown: Path, info: BuildInfo, dest: Path | None,
250 css: str | None,
251 url_fixer: Callable[[str], str]) -> None:
252 """
253 Render a markdown file.
255 :param markdown: the markdown file
256 :param dest: the destination path
257 :param info: the build info
258 :param css: the relative path to the style sheet
259 :param url_fixer: the URL fixer
261 >>> root = Path(__file__).up(4)
262 >>> bf = BuildInfo(root, "pycommons",
263 ... examples_dir=root.resolve_inside("examples"),
264 ... tests_dir=root.resolve_inside("tests"),
265 ... doc_source_dir=root.resolve_inside("docs/source"),
266 ... doc_dest_dir=root.resolve_inside("docs/build"))
268 >>> from io import StringIO
269 >>> from contextlib import redirect_stdout
270 >>> from pycommons.io.temp import temp_dir
271 >>> with temp_dir() as td:
272 ... with redirect_stdout(None):
273 ... __render_markdown(root.resolve_inside("README.md"), bf,
274 ... td, None, lambda s: s)
275 ... readme = td.resolve_inside("README_md.html").is_file()
276 ... try:
277 ... __render_markdown(root.resolve_inside("README.md"), bf,
278 ... td, None, lambda s: s)
279 ... except ValueError as ve:
280 ... vstr = str(ve)
281 ... __render_markdown(root.resolve_inside("CONTRIBUTING.md"), bf,
282 ... td, "dummy.css", lambda s: s)
283 ... cb = td.resolve_inside("CONTRIBUTING_md.html").is_file()
284 >>> readme
285 True
286 >>> vstr.endswith("already exists.")
287 True
288 >>> cb
289 True
290 """
291 logger(f"Trying to render {markdown!r}.")
293 # find the destination file
294 basename: Final[str] = markdown.basename()
295 if dest is None:
296 dest = info.doc_dest_dir
297 dest_path: Final[Path] = dest.resolve_inside(
298 f"{basename.replace('.', '_')}.html")
299 if dest_path.exists():
300 raise ValueError(f"Destination path {dest_path!r} already exists.")
302 # get the title
303 title, _ = extract_md_infos(markdown)
304 title = str.strip(title).replace("`", "").replace("*", "")
306 # set up the body
307 body_1: str = __HTML_BODY_NO_STYLE_1
308 body_2: str = __HTML_BODY_NO_STYLE_2
309 if css is not None:
310 body_1 = __HTML_BODY_STYLE_1.replace("{STYLE}", css)
311 body_2 = __HTML_BODY_STYLE_2
313 text: Final[str] = url_fixer(str.strip(Command((
314 "markdown_py", "-o", "html", markdown),
315 stderr=STREAM_FORWARD, stdout=STREAM_CAPTURE, timeout=info.timeout,
316 working_dir=info.base_dir).execute()[0]))
317 dest_path.write_all_str(f"{__HTML_HEADER}{title}{body_1}{text}{body_2}")
318 logger(f"Finished rendering {markdown!r} to {dest_path!r}.")
321def __minify(file: Path) -> None:
322 """
323 Minify the given HTML file.
325 :param file: the file
327 >>> root = Path(__file__).up(4)
328 >>> bf = BuildInfo(root, "pycommons",
329 ... examples_dir=root.resolve_inside("examples"),
330 ... tests_dir=root.resolve_inside("tests"),
331 ... doc_source_dir=root.resolve_inside("docs/source"),
332 ... doc_dest_dir=root.resolve_inside("docs/build"))
334 >>> from io import StringIO
335 >>> from contextlib import redirect_stdout
336 >>> from pycommons.io.temp import temp_dir
337 >>> with temp_dir() as td:
338 ... with redirect_stdout(None):
339 ... __render_markdown(root.resolve_inside("README.md"), bf,
340 ... td, None, lambda s: s)
341 ... readme = td.resolve_inside("README_md.html")
342 ... long = str.__len__(readme.read_all_str())
343 ... __minify(readme)
344 ... short = str.__len__(readme.read_all_str())
345 >>> 0 < short < long
346 True
347 """
348 logger(f"Minifying HTML in {file!r}.")
349 text: str = str.strip(file.read_all_str())
350 text = str.strip(minify_html.minify( # pylint: disable=E1101
351 text,
352 allow_noncompliant_unquoted_attribute_values=False,
353 allow_optimal_entities=True,
354 allow_removing_spaces_between_attributes=True,
355 keep_closing_tags=False,
356 keep_comments=False,
357 keep_html_and_head_opening_tags=False,
358 keep_input_type_text_attr=False,
359 keep_ssi_comments=False,
360 minify_css=True,
361 minify_doctype=False,
362 minify_js=True,
363 preserve_brace_template_syntax=False,
364 preserve_chevron_percent_template_syntax=False,
365 remove_bangs=True,
366 remove_processing_instructions=True))
367 if "<pre" not in text:
368 text = " ".join(map(str.strip, text.splitlines()))
369 file.write_all_str(str.strip(text))
372def __minify_all(dest: Path, skip: Callable[[str], bool]) -> None:
373 """
374 Minify all files in the given destination folder.
376 :param dest: the destination
377 :param skip: the files to skip
379 >>> root = Path(__file__).up(4)
380 >>> bf = BuildInfo(root, "pycommons",
381 ... examples_dir=root.resolve_inside("examples"),
382 ... tests_dir=root.resolve_inside("tests"),
383 ... doc_source_dir=root.resolve_inside("docs/source"),
384 ... doc_dest_dir=root.resolve_inside("docs/build"))
386 >>> from io import StringIO
387 >>> from contextlib import redirect_stdout
388 >>> from pycommons.io.temp import temp_dir
389 >>> with temp_dir() as td:
390 ... with redirect_stdout(None):
391 ... __render_markdown(root.resolve_inside("README.md"), bf,
392 ... td, None, lambda s: s)
393 ... readme = td.resolve_inside("README_md.html")
394 ... __render_markdown(root.resolve_inside("CONTRIBUTING.md"), bf,
395 ... td, None, lambda s: s)
396 ... cb = td.resolve_inside("CONTRIBUTING_md.html")
397 ... longr = str.__len__(readme.read_all_str())
398 ... longc = str.__len__(cb.read_all_str())
399 ... __minify_all(td, lambda x: False)
400 ... shortr = str.__len__(readme.read_all_str())
401 ... shortc = str.__len__(cb.read_all_str())
402 >>> 0 < shortr < longr
403 True
404 >>> 0 < shortc < longc
405 True
406 """
407 if skip(dest):
408 return
409 if dest.is_file():
410 if dest.endswith(".html"):
411 __minify(dest)
412 elif dest.is_dir():
413 for f in dest.list_dir():
414 __minify_all(f, skip)
417def __put_nojekyll(dest: Path) -> None:
418 """
419 Put a `.nojekyll` file into each directory.
421 :param dest: the destination path.
423 >>> from pycommons.io.temp import temp_dir, temp_file
424 >>> with temp_dir() as td:
425 ... x = td.resolve_inside("x")
426 ... x.ensure_dir_exists()
427 ... y = x.resolve_inside("y")
428 ... y.ensure_dir_exists()
429 ... __put_nojekyll(td)
430 ... td.resolve_inside(".nojekyll").is_file()
431 ... x.resolve_inside(".nojekyll").is_file()
432 ... y.resolve_inside(".nojekyll").is_file()
433 True
434 True
435 True
437 >>> with temp_file() as tf:
438 ... __put_nojekyll(tf) # nothing happens
439 """
440 if not dest.is_dir():
441 return
442 dest.resolve_inside(".nojekyll").ensure_file_exists()
443 for f in dest.list_dir(files=False):
444 __put_nojekyll(f)
447def make_documentation(info: BuildInfo) -> None:
448 """
449 Make the documentation of the project.
451 :param info: the build information
453 >>> root = Path(__file__).up(4)
454 >>> bf = BuildInfo(root, "pycommons",
455 ... examples_dir=root.resolve_inside("examples"),
456 ... tests_dir=root.resolve_inside("tests"),
457 ... doc_source_dir=root.resolve_inside("docs/source"),
458 ... doc_dest_dir=root.resolve_inside("docs/build"))
459 >>> from io import StringIO
460 >>> from contextlib import redirect_stdout
461 >>> with redirect_stdout(None):
462 ... make_documentation(bf)
464 >>> try:
465 ... make_documentation(1)
466 ... except TypeError as te:
467 ... print(str(te)[-13:])
468 nt, namely 1.
469 """
470 if not isinstance(info, BuildInfo):
471 raise type_error(info, "info", BuildInfo)
473 source: Final[Path | None] = info.doc_source_dir
474 dest: Final[Path | None] = info.doc_dest_dir
475 if (source is None) or (dest is None):
476 raise ValueError("Need both documentation source and "
477 f"destination, but got {info}.")
478 logger(f"Building documentation with setup {info}.")
480 logger(f"First clearing {dest!r}.")
481 if dest.exists():
482 if not dest.is_dir():
483 raise ValueError(f"{dest!r} exists but is no directory?")
484 delete_path(dest)
485 dest.ensure_dir_exists()
487 logger("Collecting all documentation source files.")
488 retain: Final[Callable[[Path], bool]] = __get_source(source)
489 try:
490 logger("Building the documentation files via Sphinx.")
491 info.command(("sphinx-apidoc", "-M", "--ext-autodoc",
492 "-o", source, info.sources_dir)).execute()
493 info.command(("sphinx-build", "-W", "-a", "-E", "-b", "html",
494 source, dest)).execute()
495 logger("Finished the Sphinx executions.")
496 finally:
497 logger("Clearing all auto-generated files.")
498 __keep_only_source(source, retain)
500 logger("Now building the additional files.")
502 if info.examples_dir is not None:
503 examples_dest: Final[Path] = dest.resolve_inside("examples")
504 examples_dest.ensure_dir_exists()
505 logger(f"Now pygmentizing example files to {examples_dest!r}.")
506 for f in info.examples_dir.list_dir(directories=False):
507 if f.endswith(".py"):
508 __pygmentize(f, info, examples_dest)
510 logger("Now pygmentizing default files.")
511 for fn in __PYGMENTIZE_DEFAULT:
512 f = info.base_dir.resolve_inside(fn)
513 if f.is_file():
514 __pygmentize(f, info)
516 # now printing coverage information
517 coverage_file: Final[Path] = info.base_dir.resolve_inside(".coverage")
518 coverage_dest: Path | None = None
519 if coverage_file.is_file():
520 logger(f"Generating coverage from file {coverage_file!r}.")
521 coverage_dest = dest.resolve_inside("tc")
522 coverage_dest.ensure_dir_exists()
523 delete_path(coverage_dest)
524 try:
525 info.command(("coverage", "html", "-d", coverage_dest,
526 f"--data-file={coverage_file}",
527 f"--include={info.package_name}/*")).execute()
528 except ValueError as ve:
529 if coverage_dest.is_dir():
530 raise
531 logger(f"No coverage to report: {ve}.")
532 else:
533 gitignore: Path = coverage_dest.resolve_inside(".gitignore")
534 if gitignore.is_file(): # remove gitignore
535 logger(f"Found .gitignore file {gitignore!r} - deleting it.")
536 delete_path(gitignore)
537 else:
538 logger("No coverage data found.")
540 # find potential style sheet
541 static: Final[Path] = dest.resolve_inside("_static")
542 css: str | None = None
543 if static.is_dir():
544 for sst in __STYLES:
545 css_path: Path = static.resolve_inside(sst)
546 if css_path.is_file():
547 css = css_path.relative_to(dest)
548 # To some styles, we need to add minor modifications.
549 for add_style, add_text in __STYLE_ADDITIONS:
550 if add_style == sst:
551 logger(f"Improving style {css_path!r}.")
552 with open(css_path, "a", encoding=UTF8) as stream:
553 stream.write(add_text)
554 break
555 break
556 if css is None:
557 logger("Found no static css style.")
558 else:
559 logger(f"Using style sheet {css!r}.")
561 setup_cfg: Final[Path] = info.base_dir.resolve_inside("setup.cfg")
562 url_fixer: Callable[[str], str] = str.strip
563 if setup_cfg.is_file():
564 logger("Loading documentation information.")
565 doc_info: Final[DocInfo] = load_doc_info_from_setup_cfg(setup_cfg)
566 url_fixer = make_url_replacer(
567 {doc_info.doc_url: "./"}, for_markdown=False)
569 logger("Now rendering all markdown files in the root directory.")
570 for f in info.base_dir.list_dir(directories=False):
571 if f.is_file() and f.endswith(".md") and (
572 not f.basename().startswith("README")):
573 __render_markdown(f, info, dest, css, url_fixer)
575 logger("Now minifying all generated html files.")
576 __minify_all(dest, {coverage_dest}.__contains__)
578 logger("Now putting a .nojekyll file into each directory.")
579 __put_nojekyll(dest)
581 logger(f"Finished building documentation with setup {info}.")
584# Run documentation generation process if executed as script.
585# This part cannot appear in unit test coverage, but we do test it.
586if __name__ == "__main__": # pragma: no cover
587 parser: Final[ArgumentParser] = pycommons_argparser(
588 __file__,
589 "Build the Documentation",
590 "This utility uses sphinx to build the documentation.")
591 make_documentation(parse_project_arguments(parser))