Coverage for pycommons / dev / building / make_documentation.py: 97%
191 statements
« prev ^ index » next coverage.py v7.13.0, created at 2025-12-15 10:00 +0000
1"""Create the documentation."""
3from argparse import ArgumentParser
4from os import remove as osremove
5from typing import Callable, Final, cast
7import minify_html
9from pycommons.dev.building.build_info import (
10 BuildInfo,
11 parse_project_arguments,
12)
13from pycommons.dev.doc.doc_info import (
14 DocInfo,
15 extract_md_infos,
16 load_doc_info_from_setup_cfg,
17)
18from pycommons.dev.url_replacer import make_url_replacer
19from pycommons.io.arguments import pycommons_argparser
20from pycommons.io.console import logger
21from pycommons.io.path import UTF8, Path, delete_path
22from pycommons.processes.shell import STREAM_CAPTURE, STREAM_FORWARD, Command
23from pycommons.types import type_error
26def __get_source(
27 source: Path, __dst: set[Path] | None = None) \
28 -> Callable[[Path], bool]:
29 """
30 Get the existing files in a directory.
32 :param source: the directory
33 :param __dst: the destination
34 :returns: the set of files
36 >>> from pycommons.io.temp import temp_dir
37 >>> with temp_dir() as td:
38 ... f1 = td.resolve_inside("a")
39 ... f1.ensure_file_exists() # gives False
40 ... f2 = td.resolve_inside("b")
41 ... f2.ensure_file_exists() # gives False
42 ... d = td.resolve_inside("x")
43 ... d.ensure_dir_exists() # gives False
44 ... f3 = d.resolve_inside("a")
45 ... f3.ensure_file_exists()
46 ... g = __get_source(td)
47 ... g(f1) # is True
48 ... g(f2) # is True
49 ... g(f3) # is True
50 ... g(d) # is True
51 ... g("bla") # is not True
52 ... f4 = d.resolve_inside("x")
53 ... f4.ensure_file_exists() # gives False
54 ... g(f4) # is also not True, because generated later
55 False
56 False
57 False
58 True
59 True
60 True
61 True
62 False
63 False
64 False
65 """
66 if __dst is None:
67 __dst = {source}
68 for k in source.list_dir():
69 __dst.add(k)
70 if k.is_dir():
71 __get_source(k, __dst)
72 return __dst.__contains__
75def __keep_only_source(
76 source: Path, keep: Callable[[Path], bool],
77 __collect: Callable[[Path], None] | None = None) -> None:
78 """
79 Keep only the source items, delete the rest.
81 :param source: the source path
82 :param keep: the set of files and directories to keep
83 :param __collect: the collector
85 >>> from pycommons.io.temp import temp_dir
86 >>> with temp_dir() as td:
87 ... f1 = td.resolve_inside("a")
88 ... f1.ensure_file_exists() # gives False
89 ... f2 = td.resolve_inside("b")
90 ... f2.ensure_file_exists() # gives False
91 ... d = td.resolve_inside("x")
92 ... d.ensure_dir_exists() # gives False
93 ... f3 = d.resolve_inside("a")
94 ... f3.ensure_file_exists()
95 ... g = __get_source(td)
96 ... f4 = d.resolve_inside("x")
97 ... f4.ensure_file_exists() # gives False
98 ... e = td.resolve_inside("y")
99 ... e.ensure_dir_exists() # gives False
100 ... f5 = e.resolve_inside("a")
101 ... f5.ensure_file_exists()
102 ... f3.is_file() # gives True - should be preserved
103 ... f4.is_file() # gives True - will be deleted
104 ... f5.is_file() # gives True - will be deleted
105 ... d.is_dir() # True - will be preserved
106 ... e.is_dir() # True - will be deleted
107 ... __keep_only_source(td, g)
108 ... f3.is_file() # gives True - was preserved
109 ... f4.is_file() # gives False
110 ... f5.is_file() # gives False
111 ... d.is_dir() # True - was preserved
112 ... e.is_dir() # False
113 False
114 False
115 False
116 False
117 False
118 True
119 True
120 True
121 True
122 True
123 True
124 False
125 False
126 True
127 False
128 """
129 lst: list[Path] | None = None
130 if __collect is None:
131 lst = []
132 __collect = lst.append
133 for f in source.list_dir():
134 if not keep(f):
135 __collect(f)
136 if f.is_dir():
137 __keep_only_source(f, keep, __collect)
138 if lst:
139 lst.sort(key=cast("Callable[[Path], int]", str.__len__), reverse=True)
140 for k in lst:
141 delete_path(k)
144def __pygmentize(source: Path, info: BuildInfo,
145 dest: Path | None = None) -> None:
146 """
147 Pygmentize a source file to a destination.
149 :param source: the source file
150 :param info: the information
151 :param dest: the destination folder
153 >>> root = Path(__file__).up(4)
154 >>> bf = BuildInfo(root, "pycommons",
155 ... examples_dir=root.resolve_inside("examples"),
156 ... tests_dir=root.resolve_inside("tests"),
157 ... doc_source_dir=root.resolve_inside("docs/source"),
158 ... doc_dest_dir=root.resolve_inside("docs/build"))
160 >>> from contextlib import redirect_stdout
161 >>> from pycommons.io.temp import temp_dir
162 >>> with temp_dir() as td:
163 ... with redirect_stdout(None):
164 ... __pygmentize(root.resolve_inside("README.md"), bf, td)
165 ... readme = td.resolve_inside("README_md.html").is_file()
166 ... __pygmentize(root.resolve_inside("setup.py"), bf, td)
167 ... setuppy = td.resolve_inside("setup_py.html").is_file()
168 ... __pygmentize(root.resolve_inside("setup.cfg"), bf, td)
169 ... setup_cfg = td.resolve_inside("setup_cfg.html").is_file()
170 ... __pygmentize(root.resolve_inside("make.sh"), bf, td)
171 ... makefile = td.resolve_inside("make_sh.html").is_file()
172 ... __pygmentize(root.resolve_inside("LICENSE"), bf, td)
173 ... xlicense = td.resolve_inside("LICENSE.html").is_file()
174 ... __pygmentize(root.resolve_inside("requirements.txt"), bf, td)
175 ... req = td.resolve_inside("requirements_txt.html").is_file()
176 ... try:
177 ... __pygmentize(root.resolve_inside("LICENSE"), bf, td)
178 ... except ValueError as ve:
179 ... ver = str(ve)
180 >>> readme
181 True
182 >>> setuppy
183 True
184 >>> setup_cfg
185 True
186 >>> makefile
187 True
188 >>> xlicense
189 True
190 >>> req
191 True
192 >>> "already exists" in ver
193 True
194 """
195 logger(f"Trying to pygmentize {source!r} to {dest!r}.")
196 name: Final[str] = source.basename()
197 language: str = "text"
198 if name.endswith(".py"):
199 language = "python3"
200 elif name.endswith((".cfg", ".toml")):
201 language = "INI"
202 elif name.lower() == "makefile":
203 language = "make"
204 elif name.endswith(".sh"):
205 language = "bash"
206 if dest is None:
207 dest = info.doc_dest_dir
208 dest_file: Final[Path] = dest.resolve_inside(
209 f"{name.replace('.', '_')}.html")
210 if dest_file.exists():
211 raise ValueError(f"File {dest_file!r} already exists, "
212 f"cannot pygmentize {source!r}.")
213 info.command(("pygmentize", "-f", "html", "-l", language, "-O", "full",
214 "-O", "style=default", "-o", dest_file, source)).execute()
215 logger(f"Done pygmentizing {source!r} to {dest_file!r}.")
218#: the default files to pygmentize
219__PYGMENTIZE_DEFAULT: Final[tuple[str, ...]] = (
220 "conftest.py", "LICENSE", "make.sh", "Makefile", "pyproject.toml",
221 "requirements.txt", "requirements-dev.txt", "setup.cfg", "setup.py",
222)
224#: the possible styles
225__STYLES: Final[tuple[str, ...]] = ("bizstyle.css", )
227#: additions that should be included in the styles
228__STYLE_ADDITIONS: Final[tuple[tuple[str, str], ...]] = (
229 ("bizstyle.css",
230 "\n.sphinxsidebarwrapper span.pre{white-space:wrap;-epub-hyphens:auto;"
231 "-webkit-hyphens:auto;-moz-hyphens:auto;hyphens: auto;}\n"), )
233#: the html header
234__HTML_HEADER: Final[str] = "<!DOCTYPE html><html><title>"
235#: the top of the html body if styles are present
236__HTML_BODY_STYLE_1: Final[str] =\
237 ('</title><link href={STYLE} rel="stylesheet">'
238 '<body style="background-image:none"><div class="document">'
239 '<div class="documentwrapper"><div class="bodywrapper">'
240 '<div class="body" role="main"><section>')
241#: the bottom of the html body if styles are present
242__HTML_BODY_STYLE_2: Final[str] = \
243 "</section></div></div></div></div></body></html>"
244#: the top of the html body if not styles are present
245__HTML_BODY_NO_STYLE_1: Final[str] = "</title><body><section>"
246#: the bottom of the html body if styles are present
247__HTML_BODY_NO_STYLE_2: Final[str] = "</section></body></html>"
250def __render_markdown(markdown: Path, info: BuildInfo, dest: Path | None,
251 css: str | None,
252 url_fixer: Callable[[str], str]) -> None:
253 """
254 Render a markdown file.
256 :param markdown: the markdown file
257 :param dest: the destination path
258 :param info: the build info
259 :param css: the relative path to the style sheet
260 :param url_fixer: the URL fixer
262 >>> root = Path(__file__).up(4)
263 >>> bf = BuildInfo(root, "pycommons",
264 ... examples_dir=root.resolve_inside("examples"),
265 ... tests_dir=root.resolve_inside("tests"),
266 ... doc_source_dir=root.resolve_inside("docs/source"),
267 ... doc_dest_dir=root.resolve_inside("docs/build"))
269 >>> from io import StringIO
270 >>> from contextlib import redirect_stdout
271 >>> from pycommons.io.temp import temp_dir
272 >>> with temp_dir() as td:
273 ... with redirect_stdout(None):
274 ... __render_markdown(root.resolve_inside("README.md"), bf,
275 ... td, None, lambda s: s)
276 ... readme = td.resolve_inside("README_md.html").is_file()
277 ... try:
278 ... __render_markdown(root.resolve_inside("README.md"), bf,
279 ... td, None, lambda s: s)
280 ... except ValueError as ve:
281 ... vstr = str(ve)
282 ... __render_markdown(root.resolve_inside("CONTRIBUTING.md"), bf,
283 ... td, "dummy.css", lambda s: s)
284 ... cb = td.resolve_inside("CONTRIBUTING_md.html").is_file()
285 >>> readme
286 True
287 >>> vstr.endswith("already exists.")
288 True
289 >>> cb
290 True
291 """
292 logger(f"Trying to render {markdown!r}.")
294 # find the destination file
295 basename: Final[str] = markdown.basename()
296 if dest is None:
297 dest = info.doc_dest_dir
298 dest_path: Final[Path] = dest.resolve_inside(
299 f"{basename.replace('.', '_')}.html")
300 if dest_path.exists():
301 raise ValueError(f"Destination path {dest_path!r} already exists.")
303 # get the title
304 title, _ = extract_md_infos(markdown)
305 title = str.strip(title).replace("`", "").replace("*", "")
307 # set up the body
308 body_1: str = __HTML_BODY_NO_STYLE_1
309 body_2: str = __HTML_BODY_NO_STYLE_2
310 if css is not None:
311 body_1 = __HTML_BODY_STYLE_1.replace("{STYLE}", css)
312 body_2 = __HTML_BODY_STYLE_2
314 text: Final[str] = url_fixer(str.strip(Command((
315 "markdown_py", "-o", "html", markdown),
316 stderr=STREAM_FORWARD, stdout=STREAM_CAPTURE, timeout=info.timeout,
317 working_dir=info.base_dir).execute()[0]))
318 dest_path.write_all_str(f"{__HTML_HEADER}{title}{body_1}{text}{body_2}")
319 logger(f"Finished rendering {markdown!r} to {dest_path!r}.")
322def __minify(file: Path) -> None:
323 """
324 Minify the given HTML file.
326 :param file: the file
328 >>> root = Path(__file__).up(4)
329 >>> bf = BuildInfo(root, "pycommons",
330 ... examples_dir=root.resolve_inside("examples"),
331 ... tests_dir=root.resolve_inside("tests"),
332 ... doc_source_dir=root.resolve_inside("docs/source"),
333 ... doc_dest_dir=root.resolve_inside("docs/build"))
335 >>> from io import StringIO
336 >>> from contextlib import redirect_stdout
337 >>> from pycommons.io.temp import temp_dir
338 >>> with temp_dir() as td:
339 ... with redirect_stdout(None):
340 ... __render_markdown(root.resolve_inside("README.md"), bf,
341 ... td, None, lambda s: s)
342 ... readme = td.resolve_inside("README_md.html")
343 ... long = str.__len__(readme.read_all_str())
344 ... __minify(readme)
345 ... short = str.__len__(readme.read_all_str())
346 >>> 0 < short < long
347 True
348 """
349 logger(f"Minifying HTML in {file!r}.")
350 text: str = str.strip(file.read_all_str())
351 text = str.strip(minify_html.minify( # pylint: disable=E1101
352 text,
353 allow_noncompliant_unquoted_attribute_values=False,
354 allow_optimal_entities=True,
355 allow_removing_spaces_between_attributes=True,
356 keep_closing_tags=False,
357 keep_comments=False,
358 keep_html_and_head_opening_tags=False,
359 keep_input_type_text_attr=False,
360 keep_ssi_comments=False,
361 minify_css=True,
362 minify_doctype=False,
363 minify_js=True,
364 preserve_brace_template_syntax=False,
365 preserve_chevron_percent_template_syntax=False,
366 remove_bangs=True,
367 remove_processing_instructions=True))
368 if "<pre" not in text:
369 text = " ".join(map(str.strip, text.splitlines()))
370 file.write_all_str(str.strip(text))
373def __minify_all(dest: Path, skip: Callable[[str], bool]) -> None:
374 """
375 Minify all files in the given destination folder.
377 :param dest: the destination
378 :param skip: the files to skip
380 >>> root = Path(__file__).up(4)
381 >>> bf = BuildInfo(root, "pycommons",
382 ... examples_dir=root.resolve_inside("examples"),
383 ... tests_dir=root.resolve_inside("tests"),
384 ... doc_source_dir=root.resolve_inside("docs/source"),
385 ... doc_dest_dir=root.resolve_inside("docs/build"))
387 >>> from io import StringIO
388 >>> from contextlib import redirect_stdout
389 >>> from pycommons.io.temp import temp_dir
390 >>> with temp_dir() as td:
391 ... with redirect_stdout(None):
392 ... __render_markdown(root.resolve_inside("README.md"), bf,
393 ... td, None, lambda s: s)
394 ... readme = td.resolve_inside("README_md.html")
395 ... __render_markdown(root.resolve_inside("CONTRIBUTING.md"), bf,
396 ... td, None, lambda s: s)
397 ... cb = td.resolve_inside("CONTRIBUTING_md.html")
398 ... longr = str.__len__(readme.read_all_str())
399 ... longc = str.__len__(cb.read_all_str())
400 ... __minify_all(td, lambda x: False)
401 ... shortr = str.__len__(readme.read_all_str())
402 ... shortc = str.__len__(cb.read_all_str())
403 >>> 0 < shortr < longr
404 True
405 >>> 0 < shortc < longc
406 True
407 """
408 if skip(dest):
409 return
410 if dest.is_file():
411 if dest.endswith(".html"):
412 __minify(dest)
413 elif dest.is_dir():
414 for f in dest.list_dir():
415 __minify_all(f, skip)
418def __put_nojekyll(dest: Path) -> None:
419 """
420 Put a `.nojekyll` file into each directory.
422 :param dest: the destination path.
424 >>> from pycommons.io.temp import temp_dir, temp_file
425 >>> with temp_dir() as td:
426 ... x = td.resolve_inside("x")
427 ... x.ensure_dir_exists()
428 ... y = x.resolve_inside("y")
429 ... y.ensure_dir_exists()
430 ... __put_nojekyll(td)
431 ... td.resolve_inside(".nojekyll").is_file()
432 ... x.resolve_inside(".nojekyll").is_file()
433 ... y.resolve_inside(".nojekyll").is_file()
434 True
435 True
436 True
438 >>> with temp_file() as tf:
439 ... __put_nojekyll(tf) # nothing happens
440 """
441 if not dest.is_dir():
442 return
443 dest.resolve_inside(".nojekyll").ensure_file_exists()
444 for f in dest.list_dir(files=False):
445 __put_nojekyll(f)
448def make_documentation(info: BuildInfo) -> None:
449 """
450 Make the documentation of the project.
452 :param info: the build information
454 >>> root = Path(__file__).up(4)
455 >>> bf = BuildInfo(root, "pycommons",
456 ... examples_dir=root.resolve_inside("examples"),
457 ... tests_dir=root.resolve_inside("tests"),
458 ... doc_source_dir=root.resolve_inside("docs/source"),
459 ... doc_dest_dir=root.resolve_inside("docs/build"))
460 >>> from io import StringIO
461 >>> from contextlib import redirect_stdout
462 >>> with redirect_stdout(None):
463 ... make_documentation(bf)
465 >>> try:
466 ... make_documentation(1)
467 ... except TypeError as te:
468 ... print(str(te)[-13:])
469 nt, namely 1.
470 """
471 if not isinstance(info, BuildInfo):
472 raise type_error(info, "info", BuildInfo)
474 source: Final[Path | None] = info.doc_source_dir
475 dest: Final[Path | None] = info.doc_dest_dir
476 if (source is None) or (dest is None):
477 raise ValueError("Need both documentation source and "
478 f"destination, but got {info}.")
479 logger(f"Building documentation with setup {info}.")
481 logger(f"First clearing {dest!r}.")
482 if dest.exists():
483 if not dest.is_dir():
484 raise ValueError(f"{dest!r} exists but is no directory?")
485 delete_path(dest)
486 dest.ensure_dir_exists()
488 logger("Collecting all documentation source files.")
489 retain: Final[Callable[[Path], bool]] = __get_source(source)
490 try:
491 logger("Building the documentation files via Sphinx.")
492 info.command(("sphinx-apidoc", "-M", "--ext-autodoc",
493 "-o", source, info.sources_dir)).execute()
494 info.command(("sphinx-build", "-W", "-a", "-E", "-b", "html",
495 source, dest)).execute()
496 logger("Finished the Sphinx executions.")
497 finally:
498 logger("Clearing all auto-generated files.")
499 __keep_only_source(source, retain)
501 logger("Now building the additional files.")
503 if info.examples_dir is not None:
504 examples_dest: Final[Path] = dest.resolve_inside("examples")
505 examples_dest.ensure_dir_exists()
506 logger(f"Now pygmentizing example files to {examples_dest!r}.")
507 for f in info.examples_dir.list_dir(directories=False):
508 if f.endswith(".py"):
509 __pygmentize(f, info, examples_dest)
511 logger("Now pygmentizing default files.")
512 for fn in __PYGMENTIZE_DEFAULT:
513 f = info.base_dir.resolve_inside(fn)
514 if f.is_file():
515 __pygmentize(f, info)
517 # now printing coverage information
518 coverage_file: Final[Path] = info.base_dir.resolve_inside(".coverage")
519 coverage_dest: Path | None = None
520 if coverage_file.is_file():
521 logger(f"Generating coverage from file {coverage_file!r}.")
522 coverage_dest = dest.resolve_inside("tc")
523 coverage_dest.ensure_dir_exists()
524 delete_path(coverage_dest)
525 try:
526 info.command(("coverage", "html", "-d", coverage_dest,
527 f"--data-file={coverage_file}",
528 f"--include={info.package_name}/*")).execute()
529 except ValueError as ve:
530 if coverage_dest.is_dir():
531 raise
532 logger(f"No coverage to report: {ve}.")
533 else:
534 info.command((
535 "coverage-badge", "-o", coverage_dest.resolve_inside(
536 "badge.svg"))).execute()
537 gitignore: Path = coverage_dest.resolve_inside(".gitignore")
538 if gitignore.is_file(): # remove gitignore
539 logger(f"Found .gitignore file {gitignore!r} - deleting it.")
540 osremove(gitignore)
541 else:
542 logger("No coverage data found.")
544 # find potential style sheet
545 static: Final[Path] = dest.resolve_inside("_static")
546 css: str | None = None
547 if static.is_dir():
548 for sst in __STYLES:
549 css_path: Path = static.resolve_inside(sst)
550 if css_path.is_file():
551 css = css_path.relative_to(dest)
552 # To some styles, we need to add minor modifications.
553 for add_style, add_text in __STYLE_ADDITIONS:
554 if add_style == sst:
555 logger(f"Improving style {css_path!r}.")
556 with open(css_path, "a", encoding=UTF8) as stream:
557 stream.write(add_text)
558 break
559 break
560 if css is None:
561 logger("Found no static css style.")
562 else:
563 logger(f"Using style sheet {css!r}.")
565 setup_cfg: Final[Path] = info.base_dir.resolve_inside("setup.cfg")
566 url_fixer: Callable[[str], str] = str.strip
567 if setup_cfg.is_file():
568 logger("Loading documentation information.")
569 doc_info: Final[DocInfo] = load_doc_info_from_setup_cfg(setup_cfg)
570 url_fixer = make_url_replacer(
571 {doc_info.doc_url: "./"}, for_markdown=False)
573 logger("Now rendering all markdown files in the root directory.")
574 for f in info.base_dir.list_dir(directories=False):
575 if f.is_file() and f.endswith(".md") and (
576 not f.basename().startswith("README")):
577 __render_markdown(f, info, dest, css, url_fixer)
579 logger("Now minifying all generated html files.")
580 __minify_all(dest, {coverage_dest}.__contains__)
582 logger("Now putting a .nojekyll file into each directory.")
583 __put_nojekyll(dest)
585 logger(f"Finished building documentation with setup {info}.")
588# Run documentation generation process if executed as script
589if __name__ == "__main__":
590 parser: Final[ArgumentParser] = pycommons_argparser(
591 __file__,
592 "Build the Documentation",
593 "This utility uses sphinx to build the documentation.")
594 make_documentation(parse_project_arguments(parser))