, # content is the FOLLOWING sibling
) # 'sectN' -> a module (e.g.
) HEADER_CLASSES = ("title", "sect1", "sect2", "sect3", "sect4", "sect5") # anchor -> header element, built once per parsed book (keyed by id(soup)) _INDEX_CACHE = {} def _header_classes(tag): return set(tag.get("class", []) or []) def build_index(soup): idx = {} selector = ", ".join(f"{h}.{c}" for h in ("h1", "h2", "h3", "h4", "h5") for c in HEADER_CLASSES) for header in soup.select(selector): a = header.find(lambda t: t.name == "a" and (t.get("id") or t.get("name"))) if a is None: continue anchor = a.get("id") or a.get("name") if anchor and anchor not in idx: idx[anchor] = header # THE ID IS NOT ALWAYS INSIDE THE HEADER. # # A page whose anchor sits on the section DIV rather than on an inside # its title header is invisible to the loop above -- it is not "missing # from the book", it is missing from the index, and the two look identical # from outside: `packagemanager stack network` said NOT IN BOOK for # dhcpcd, whose page is right there in basicnet. # # Only ids that name a section carrying a Package Information block are # added, and only when nothing already claims the anchor, so this can add # pages and can never move an existing one. for div in soup.select("div.sect1[id], div.sect2[id]"): anchor = div.get("id") if not anchor or anchor in idx: continue if div.find("div", class_="package", recursive=False) is None: continue # the DIV is what gets indexed, not its header: for this shape the # header sits INSIDE the section, so the "titlepage then sibling # sect" walk that resolves the common shape finds div.package next # and gives up. resolve_package takes a section div directly. if div.find(lambda t: t.name in H_TAGS and (_header_classes(t) & set(HEADER_CLASSES))) is not None: idx[anchor] = div return idx def get_index(soup): key = id(soup) if key not in _INDEX_CACHE: _INDEX_CACHE[key] = build_index(soup) return _INDEX_CACHE[key] def resolve_package(soup, anchor): """Resolve an anchor to a package/module. Returns {anchor, title, content, is_module} or None. Handles both real BLFS 13.0 shapes: * package:
.. then a sibling
holding package/installation/... * module: .. inside a
that itself holds package/installation/... """ header = get_index(soup).get(anchor) if header is None: return None # A SECTION DIV THAT CARRIES THE ID IS ALREADY THE CONTENT. build_index # adds these (see there); they are whole pages whose anchor is an # attribute of the section rather than an in its title. if getattr(header, "name", None) == "div": h = _content_title_header(header) title = " ".join(h.get_text().split()) if h is not None else anchor if header.find("div", class_="package", recursive=False) is None: return None if not extract_link(header).strip(): return None return {"anchor": anchor, "title": title, "content": header, "is_module": "sect1" not in _header_classes(header)} classes = _header_classes(header) title = " ".join(header.get_text().split()) if "title" in classes: tp = header.find_parent("div", class_="titlepage") content = None if tp is not None: # a real package's content is the IMMEDIATE next sibling sect; # a Part/Chapter title is followed by intro text or a child # titlepage, so this correctly rejects those. sib = tp.find_next_sibling(True) if (sib is not None and getattr(sib, "name", None) == "div" and (_header_classes(sib) & set(CONTENT_CLASSES))): content = sib is_module = False elif header.name in H_TAGS and ("sect1" in classes): # chunked-book shape (stitched dev/svn book): the anchor lives in an # that is a SIBLING immediately before the #
under an
-- the list vanished and `blfs deps libheif` answered "libheif". Losing a dependency silently is the worst way to be wrong: the build gets further before failing, and fails somewhere else. """ nodes = [n for n in pd.select(f"p.{kind} a.xref") if not _in_parenthetical(n)] if nodes: return nodes for h in pd.find_all(["h3", "h4", "h5", "h6"]): if " ".join(h.get_text().split()).strip().lower() != kind: continue out = [] for sib in h.next_siblings: name = getattr(sib, "name", None) if name is None: continue if name in ("h3", "h4", "h5", "h6"): break if name == "p": # a classed paragraph belongs to the kind its class names classes = sib.get("class") or [] if any(c in DEP_KINDS and c != kind for c in classes): break out.extend(n for n in sib.select("a.xref") if not _in_parenthetical(n)) if out: return out return [] def _xref_titles(sec, kind): """The dependencies of one kind (required / recommended / optional). The book usually marks the list with a class --
-- and this selected on that alone. But the class is not guaranteed: on libheif the Recommended list is a plain
under an
, so the whole list vanished and `blfs deps libheif` answered "libheif". Silently losing dependencies is the worst way to be wrong: the build gets further before it fails, and fails somewhere else. So: the class when it is there, and otherwise the heading that says what the list is. """ pd = package_div(sec) if not pd: return [] def _titles_from(nodes): out = [] for x in nodes: title = " ".join((x.get("title") or "").split()) if not title or re.match(r"(Chapter|Part)\b", title): continue out.append(title.replace(" ", "-")) return out return _titles_from(_dep_xrefs(pd, kind)) def extract_installed_content(sec): content = {} for item in sec.select("div.content div.seglistitem"): for seg in item.find_all("div", class_="seg"): title = seg.find("strong", class_="segtitle") body = seg.find("span", class_="segbody") if title and body: key = re.sub(r"[^a-z0-9]+", "_", title.get_text().lower()).strip("_") if not key: continue val = " ".join(body.get_text().split()).replace(" and ", " ") content[key] = val return content def fix_heredocs(text): """BLFS often renders a heredoc terminator with a trailing shell operator, e.g. `EOF &&`. Bash requires the closing delimiter to sit alone on its line, so as-written the heredoc never closes and swallows the rest of the script. Move any trailing operator up onto the opener line, which is the equivalent, valid form (`cat << "EOF" && ... EOF`).""" lines = text.split("\n") opener = re.compile(r'<<-?\s*(["\']?)([A-Za-z_][A-Za-z0-9_]+)\1') i = 0 while i < len(lines): m = opener.search(lines[i]) if not m: i += 1 continue delim = re.escape(m.group(2)) term_op = re.compile(r'^(\s*)' + delim + r'\s+(&&|\|\||;|&)\s*$') term_plain = re.compile(r'^\s*' + delim + r'\s*$') j = i + 1 while j < len(lines): tm = term_op.match(lines[j]) if tm: lines[i] = lines[i].rstrip() + " " + tm.group(2) lines[j] = tm.group(1) + m.group(2) break if term_plain.match(lines[j]): break j += 1 i = j + 1 return "\n".join(lines) def _pre_command_text(pre): """Full runnable text of a command
.  BLFS puts only the heredoc opener
    (cat > f << "EOF") in  and the body + closing EOF in
    sibling nodes of the same ; so we take the whole  text, not just
    the kbd, or the heredoc would lose its body and delimiter.

    BLFS user-substitution placeholders render as  with
    literal angle brackets (e.g. ), which bash reads as a redirection and
    rejects.  Rewrite them to a safe, obvious __PREFIX__ token instead."""
    if pre.find("kbd", class_="command") is None:
        return ""
    if pre.find(class_="replaceable") is None:
        return pre.get_text().strip()
    clone = _need_bs4()(str(pre), "html.parser")
    for rep in clone.select(".replaceable"):
        txt = rep.get_text().strip()
        if txt.startswith("<") and txt.endswith(">"):
            txt = txt[1:-1]
        token = "__" + re.sub(r"[^A-Za-z0-9_]", "_", txt).strip("_") + "__"
        rep.replace_with(token)
    return clone.get_text().strip()


def extract_commands(sec, selector):
    elements = sec.select(selector + " > *")
    if not elements:
        return ""
    out = []
    _emit_command_elements(elements, out, nested=False)
    return "\n".join(out)


# A SUB-SECTION IS PART OF THE SECTION.
#
# Shadow's configure phase came out as prose and a bare `:` -- no
# /etc/pam.d/login, passwd, chage or the ten admin tools -- so `passwd` said
# `pam_start() failed, error 26` and login `PAM Failure, aborting`.  Every one
# of those blocks is on the page, one level down:
#     div.configuration > div.sect3 > div.sect4 > pre.root
# and this walk read the children of sect3 and dropped any div that was not a
# note.  linux-pam's four system-* files sit directly in its sect3, which is
# why those four existed and shadow's sixteen did not.
#
# So a nested div.sect4/sect5 is walked with the same rules, and its title is
# kept as a comment.  The TOP level is unchanged byte for byte: its own
# titlepage divs were never emitted and still are not.
_NESTED_SECT_CLASSES = {"sect4", "sect5"}


def _emit_command_elements(elements, out, nested):
    for el in elements:
        name = getattr(el, "name", None)
        if name == "h2":
            out.append("\n\n\n#### " + "".join(l.strip() for l in el.get_text().splitlines()) + "\n")
        elif name == "h3":
            out.append("\n\n### " + "".join(l.strip() for l in el.get_text().splitlines()))
        elif name == "h4":
            out.append("\n\n## " + "".join(l.strip() for l in el.get_text().splitlines()))
        elif name == "p":
            out.append("\n\n# " + " ".join(l.strip() for l in el.get_text().splitlines()).strip() + "\n")
        elif name == "pre":
            text = _pre_command_text(el)
            if text:
                # THE GUARD BELONGS HERE TOO.  extract_commands_phased checks
                # each  for a conditional sentence; this extractor (used
                # for the CONFIGURATION section) did not -- so cyrus-sasl ran
                #     make install-saslauthd
                # which the book introduces with "If you want the saslauthd
                # daemon to start at system startup ... install the
                # saslauthd.service unit included in blfs-systemd-units".
                # A conditional block is conditional wherever it appears.
                _g = _conditional_guard(el)
                if _g:
                    out.append(SKIP_MARK + _g)
                    out.append(_commented(text))
                else:
                    text, _dropped = _strip_test_lines(text)
                    if _dropped:
                        out.append(SKIP_MARK + "the test suite, chained into "
                                   "this block -- `--test` runs it")
                        out.append(_commented("\n".join(_dropped)))
                    if text.strip():
                        out.append(text)
        elif name == "div":
            classes = list(el.get("class", []))
            if "admon" in classes:
                classes.remove("admon")
                out.append("\n#### " + "".join(c.upper() for c in classes) + " ####")
                for adm in el:
                    if getattr(adm, "name", None) == "p":
                        lines = []
                        for item in adm.contents:
                            if isinstance(item, str):
                                line = re.sub(r"\s+", " ", item.strip())
                                if line:
                                    lines.append(line)
                            elif item.name:
                                child = re.sub(r"\s+", " ", item.get_text().strip())
                                if child:
                                    lines.append(child)
                        out.append("## " + " ".join(lines).replace(" , ", ", "))
                        out.append("")
            elif _NESTED_SECT_CLASSES & set(classes):
                _emit_command_elements(el.find_all(True, recursive=False),
                                       out, nested=True)
            elif nested and "titlepage" in classes:
                title = " ".join(el.get_text().split())
                if title:
                    out.append("\n\n## " + title)


# The book's conditionals live in PROSE: "If you wish to build and install
# the manual pages ... run:" precedes a command block that hard-requires
# doxygen.  The extractor took every block unconditionally, so a fresh system
# died in libxml2's documentation build for want of a tool the book calls
# optional.  A block whose introducing paragraph is conditional is emitted
# COMMENTED, with the book's own sentence above it -- visible, one uncomment
# away, never a surprise requirement.
# ANY sentence that opens with a condition, not a list of verbs.
#
# The verb list caught libuv's "If you installed the optional sphinx module,
# create the man page:" and missed the very next block, "If you BUILT the man
# page, install it as the root user:" -- so the build of the page was skipped
# and its install was not, and libuv died on a file that was never made.
# Enumerating verbs is guessing at English; the pattern is the conditional
# opening.  Skipping is safe either way: the block is emitted commented with
# the book's own sentence above it, one uncomment away.
_COND_GUARD_RE = re.compile(r"^(if\b|optionally\b|should you\b)", re.I)

# A block the book offers as a TEST is not part of installing.
#
# "To test the results, issue: gs -q -dBATCH .../tiger.eps" sits in the build
# section and runs a program that is not installed yet.  It was harmless while
# a failing command was ignored; the moment the dispatcher started honouring
# set -e it would fail every such package.  Tests belong to `test`
# (PM_TEST=1), not to a build.
# ...AND THE BOOK DOES NOT ALWAYS START THE SENTENCE WITH "To test".
# numpy's is
#     "The installation can be tested with the following commands:"
# which this matched none of, so a venv, `pip3 install hypothesis attrs`
# and `numpy.test()` were generated into the INSTALL phase.  numpy itself
# installed (1320 files) and then the package failed on
#     ModuleNotFoundError: No module named 'pytest'
# -- a test dependency, in an install.  So these are searched for anywhere
# in the introducing sentence, not just at its start.
_TEST_BLOCK_RE = re.compile(
    r"^(to test\b|to check\b|the test suite\b|tests? (can|may) be run\b)"
    r"|\bcan be tested\b|\bto test the (results|installation|build)\b"
    r"|\bto run the test"
    # LVM2 13.0: "If you wish to run the tests, install a minimal set of
    # ... first" -- an INSTALL command that is part of the test suite, and
    # the only thing marking it as such is the sentence above it.
    r"|\bif you (wish|want) to run the test"
    r"|\bbefore running the test"
    r"|\bfor the test suite\b", re.I)

# THE COMMANDS SAY WHAT THEY ARE, AND PROSE IS A SECOND-HAND ACCOUNT.
#
# 1.14.211 widened the sentence match so numpy's "The installation can be
# tested with the following commands" would be recognised -- and the
# regenerated script STILL had the venv, `pip3 install hypothesis attrs`
# and `numpy.test()` in its install phase, because the book being read on
# that machine is 13.0 and words or nests it differently again.
#
# Chasing the prose is a losing game: `numpy.test()` IS a test suite
# whatever sentence introduces it.  These patterns match the block's own
# commands, so a rewording upstream cannot bring the bug back.
_TEST_CONTENT_RE = re.compile(
    r"""(?mx)
      ^\s*(\w+=\S*\s+)*(make|ninja)\s+(-\w+\s+)*(check|test)\b
                                                    # make check / ninja test
    | ^\s*(\w+=\S*\s+)*(make|ninja)\s+(-\w+\s+)*(check|test)[_-]\w+
                                                    # LC_ALL=... make check_local
                                                    # (the old pattern needed a
                                                    #  word boundary after
                                                    #  "check", which
                                                    #  "check_local" has not --
                                                    #  and no env assignment in
                                                    #  front, which LVM2's page
                                                    #  has)
    | ^\s*meson\s+test\b
    | ^\s*ctest\b
    | ^\s*py\.test\b | ^\s*pytest\b
    | ^\s*python3?\s+-m\s+(pytest|unittest)\b
    | \.test\(\)                                    # numpy.test()
    | ^\s*python3?\s+-m\s+venv\b                    # a throwaway test venv
    | ^\s*source\s+\S*testenv/bin/activate
    """)


# A COMMAND THAT ONLY ROOT CAN RUN IS NOT AN INSTALL STEP.
#
# LVM2's page (13.0) puts three blocks between the build and the install:
# a partial install for the test suite, then
#     mount -o remount,dev /tmp
# and then `make check_local`.  The middle one is not about installing
# anything -- it prepares /tmp so the tests may create device nodes -- and
# it ran in the install phase, as the package user:
#     mount: /tmp: must be superuser to use mount.
#     !! phase 'install' FAILED (exit 32)
# A package user cannot mount, load a module or make a device node, and no
# BLFS page needs it to: those commands belong to the system, not to the
# package.  (`make check_local` was missed too -- the test pattern required
# a word boundary after "check", and "check_local" has none.)
_ROOT_ONLY_RE = re.compile(
    r"""(?mx)
      ^\s*mount\b | ^\s*umount\b
    | ^\s*modprobe\b | ^\s*insmod\b
    | ^\s*mknod\b
    | ^\s*swapon\b | ^\s*swapoff\b
    """)


def _root_only_reason(pre):
    """Why this block cannot run as the package user, or None."""
    m = _ROOT_ONLY_RE.search(pre.get_text())
    if not m:
        return None
    hit = " ".join(m.group(0).split())
    return ("it needs root and the system, not this package (`%s`) -- run it "
            "yourself if you want it" % hit)


# A TEST COMMAND CHAINED INTO THE BUILD BLOCK.
#
# Every Perl module in BLFS uses "the standard build and installation
# instructions", which are ONE block:
#     perl Makefile.PL &&
#     make &&
#     make test
# So the test suite is not a block to skip -- it is the last line of the
# block that builds the package, joined with &&.  XML::Parser's tests fail
# on this machine (62 files, 60 of them cannot even load the module) and
# took the whole gparted chain down in the BUILD phase; Class::Inspector's
# passed, which is why nobody noticed the tests were running at all.
#
# Tests run here only when they are asked for (PKGUSR_TESTS=1, which is
# `--test`), and then the runner runs them ITSELF, in the test phase.  So
# the line comes out of the build, with the book's own text left in a
# comment, and the trailing && of the line before it goes too.
_TEST_LINE_RE = re.compile(
    r"""^\s*(\w+=\S*\s+)*(make|ninja|meson|ctest|py\.test|pytest)\b[^&|;]*?
         \b(check|test)([_-]\w+)?\b[^&|;]*$""", re.X)


def _strip_test_lines(text):
    """Drop `make test` / `make check` lines from a block that also builds.

    Returns (text, dropped-lines).  A block that is ONLY tests comes back
    empty and its caller keeps treating it as a test block."""
    lines = text.splitlines()
    keep, dropped = [], []
    for ln in lines:
        if _TEST_LINE_RE.match(ln.strip()) and not re.search(r"\binstall\b", ln):
            dropped.append(ln.strip())
            continue
        keep.append(ln)
    if not dropped:
        return text, []
    # the line before a dropped last line ends in `&&` (or `;`) that now
    # chains to nothing -- and `make &&` on its own is a syntax error
    while keep and not keep[-1].strip():
        keep.pop()
    if keep:
        keep[-1] = re.sub(r"\s*(&&|\|\||;)\s*$", "", keep[-1])
    return "\n".join(keep), dropped


def _test_content_reason(pre):
    """Why this block is a test suite, judged by its commands, or None."""
    txt = pre.get_text()
    m = _TEST_CONTENT_RE.search(txt)
    if not m:
        return None
    hit = " ".join(m.group(0).split())
    return "it IS the test suite -- these commands run it (`%s`)" % hit


def _conditional_guard(pre):
    """The conditional sentence introducing this , or None if the block
    must run.

    A condition is only worth skipping when it depends on something we do NOT
    guarantee.  These tools install required AND recommended dependencies by
    default -- that is the contract -- so a block conditioned on those is
    satisfied by construction and must RUN.  Ghostscript proved the cost of
    not knowing that:
        If you have installed the recommended dependencies on your system,
        remove the copies of freetype, lcms2, libjpeg, libpng, and openjpeg:
            rm -rf freetype lcms2mt jpeg libpng openjpeg
    Commented out, ghostscript kept its bundled copies and configure died:
        Mixing local libtiff with shared libjpeg not supported
    "optional" wins over both: an optional tool is exactly what we do not
    install."""
    # content first: it does not depend on the book's wording or nesting
    _by_content = _root_only_reason(pre) or _test_content_reason(pre)
    if _by_content:
        return _by_content
    prev = pre.find_previous(["p", "pre", "h3", "h4"])
    if prev is None or prev.name != "p":
        return None
    txt = " ".join(prev.get_text().split())
    if _TEST_BLOCK_RE.search(txt):
        return "the book says: " + txt  # a test, never part of installing
    if not _COND_GUARD_RE.match(txt):
        return None
    low = txt.lower()
    # "If sudo or su is invoked for switching to the root user, ensure ..."
    # is about HOW to become root, not whether to run the command -- and the
    # command it introduces was `./x.py install`, the whole point of the
    # package.  A sentence about privilege mechanics guards nothing.
    if re.match(r"^if\s+(sudo|su)\b", low):
        return None
    if "optional" not in low and ("recommended" in low or "required" in low):
        return None                     # our contract satisfies it
    return "the book says: " + txt


def _commented(text):
    return "\n".join("#" + (" " + l if l and not l.startswith("#") else l)
                     for l in text.splitlines())


def extract_commands_phased(sec):
    """Split the installation section into (build, install).  BLFS marks the
    privileged install step as  (make install, install -v ...);
    everything else (configure, make, ...) is the build.  In the pkgusr model the
    'root' step runs as the package user and is where permission problems appear,
    so isolating it lets you fix perms and re-run just the install phase without
    recompiling."""
    inst = sec.find("div", class_="installation")
    if inst is None:
        return "", ""
    build, install = [], []

    # Walk the whole subtree in document order, not just the direct children.
    # Packages that build several components -- gst-plugins-rs builds
    # libgstdav1d and libgstgtk4 -- put each one in its own sub-section, so
    # every command block sits one level down.  Looking only at direct
    # children found nothing at all and produced an empty, silently useless
    # script.
    seen = set()
    for el in inst.find_all(["pre", "h3", "h4", "div"]):
        if id(el) in seen:
            continue
        classes = el.get("class", []) or []
        name = el.name

        if name == "div":
            # a note or warning: keep it as a comment, and don't descend into
            # it again through its own children
            if "admon" in classes or {"note", "warning", "important"} & set(classes):
                for d in el.find_all(True):
                    seen.add(id(d))
                build.append("## NOTE: " + " ".join(el.get_text().split()))
            continue

        if name == "pre":
            # configuration blocks are a separate phase, handled elsewhere
            if el.find_parent("div", class_="configuration"):
                continue
            text = _pre_command_text(el)
            if not text:
                continue
            # ORDER ACROSS THE STREAMS, not just within them.
            #
            # A page can go user -> root -> user, and GLib does:
            #   build glib with introspection disabled           (user)
            #   ninja -C gi-build install                        (root)
            #   meson configure -D introspection=enabled; ninja  (user)
            # Sorting blocks into "build" and "install" kept each stream's
            # order but lost the INTERLEAVING, so the reconfigure ran in the
            # build phase, before gobject-introspection was installed:
            #   ERROR: Program 'g-ir-scanner' not found
            # Once a root block has appeared, everything that follows belongs
            # after it -- so later user blocks join the install stream, in
            # the book's own order.  Both streams run as the same package
            # user here (a package user installs its own files), so nothing
            # is lost by moving a command across.
            # A BLOCK THAT IS COMMENTED OUT DOES NOT MOVE THE STREAM.
            #
            # "everything after a root block belongs after it" is right for a
            # root step that RUNS.  gegl's first root block is conditional --
            # "If you are installing over a previous version ... remove it" --
            # so it is emitted commented, and nothing has happened; but it
            # flipped the latch anyway, and the whole build (mkdir build,
            # meson setup, ninja) landed in install_pkg while build_pkg came
            # out as `:`.  The build then ran inside the install phase, and
            # when meson failed the following `ninja install` still ran:
            #     ninja: error: loading 'build.ninja': No such file
            guard = _conditional_guard(el)
            is_root = "root" in classes and not guard
            stream = install if (is_root or install) else build
            if guard:
                stream.append(SKIP_MARK + guard)
                stream.append(_commented(text))
                continue
            text, _dropped = _strip_test_lines(text)
            if _dropped:
                stream.append(SKIP_MARK + "the test suite, chained into this "
                              "block with && -- run it with `--test`, which "
                              "runs it in the test phase")
                stream.append(_commented("\n".join(_dropped)))
            if not text.strip():
                continue
            stream.append(text)
        else:                                            # h3 / h4
            title = " ".join(el.get_text().split())
            if title:
                build.append("\n### " + title)

    return "\n".join(build).strip(), "\n".join(install).strip()


def extract_raw_commands(sec):
    cmds = []
    for pre in sec.select("pre"):
        if pre.find_parent("div", class_="configuration"):
            continue
        text = _pre_command_text(pre)
        if text:
            cmds.append(text)
    return "\n".join(cmds)


def subsection_titled(sec, *prefixes):
    """Find a sub-section whose title starts with any prefix."""
    for sub in sec.select("div.sect1, div.sect2, div.sect3"):
        h = sub.find(lambda t: t.name in H_TAGS
                     and (_header_classes(t) & set(HEADER_CLASSES)))
        if h is not None:
            title = " ".join(h.get_text().split())
            if any(title.startswith(p) for p in prefixes):
                return sub
    return None


def extract_between_package_and_installation(sec):
    pkg = sec.find("div", class_="package")
    inst = sec.find("div", class_="installation")
    if not (pkg and inst):
        return []
    names = []
    for sib in pkg.next_siblings:
        if sib == inst:
            break
        if getattr(sib, "name", None):
            names.extend(sib.get("class", []))
    return names


# --------------------------------------------------------------------------- #
# dependencies
# --------------------------------------------------------------------------- #

def anchor_from_href(href):
    """A cross-ref href -> (anchor, external_url).  In the nochunks book local
    refs are '#anchor'; https refs are out-of-book."""
    if not href:
        return None, None
    if href.startswith(("http://", "https://")):
        return None, href
    if "#" in href:
        return href.split("#", 1)[1], None
    if href.endswith(".html"):
        return os.path.basename(href)[:-5], None
    return href, None


def section_deps(sec, kinds=("required", "recommended")):
    """Return list of dicts: {title, anchor, external, kind}."""
    pd = package_div(sec)
    if not pd:
        return []
    deps = []
    for kind in kinds:
        for x in _dep_xrefs(pd, kind):
            title = strip_paren(" ".join(x.get("title", "").split()))
            # xrefs to whole chapters/parts aren't packages -- skip them
            if not title or re.match(r"(Chapter|Part)\b", title):
                continue
            anchor, external = anchor_from_href(x.get("href"))
            deps.append({"title": title.replace(" ", "-"), "anchor": anchor,
                         "external": external, "kind": kind})
    return deps


def selected_kinds(args):
    """Which dependency classes a deps/order run should follow."""
    kinds = ["required"]
    if not getattr(args, "no_required", False):
        pass
    if not getattr(args, "no_recommended", False):
        kinds.append("recommended")
    if getattr(args, "optional", False):
        kinds.append("optional")
    return kinds


def deps_of(data, kinds):
    """Dependency dicts of a cached package, filtered to the chosen kinds."""
    return [d for d in data.get("deps", []) if d["kind"] in kinds]


_EXTRA_DEPS_PATH = os.environ.get(
    "BLFS_PACKAGE_FIXES", "/etc/pkgusr/stacks/package-fixes.conf")


def _extra_deps_for(anchor):
    """Dependencies the BOOK does not record but the build needs.

    libass hard-requires harfbuzz in its configure and its page lists only
    FreeType, FriBidi, Fontconfig and NASM -- so no ordering could place it
    (libass built at 143, harfbuzz at 199) and no entry order could either,
    because both sit inside one plan.  glslang for mesa was the same class,
    patched by hand into the stack.

    package-fixes.conf can add the edge:
        [libass]
        needs = harfbuzz
    They are treated as REQUIRED, because that is what they are.
    """
    if not hasattr(_extra_deps_for, "_map"):
        m = {}
        try:
            import configparser
            cp = configparser.ConfigParser()
            cp.read(_EXTRA_DEPS_PATH)
            for sect in cp.sections():
                v = cp[sect].get("needs", "")
                if v.strip():
                    m[sect.lower()] = [x.strip() for x in
                                       re.split(r"[,\s]+", v) if x.strip()]
        except Exception:
            m = {}
        _extra_deps_for._map = m
    # the same shape as a dep parsed from the book, or consumers KeyError
    return [{"anchor": n, "kind": "required", "name_version": n,
             "external": False, "title": n, "optional": False}
            for n in _extra_deps_for._map.get(anchor.lower(), [])]


def walk_deps(book, anchor, kinds=("required", "recommended"), ignore=()):
    """Depth-first walk over cached package data.  Returns (order, edges,
    missing, via).  `order` is install order (deps first); `edges` maps anchor
    -> child dep dicts; `via` maps each dependency anchor -> (parent_anchor,
    kind) recording who first pulled it in and how.

    AN AVOIDED PACKAGE IS NOT WALKED INTO.  Ignoring happened afterwards, on
    the finished list, so `--ignore libreoffice` dropped LibreOffice and kept
    everything only LibreOffice wanted: PostgreSQL, unixODBC, CLucene,
    Redland, OpenJDK, apache-ant, Cups...  Planning harfbuzz with libreoffice
    ignored came to 463 packages, and the plan stopped on a JT_JAVA guard for
    a JDK nobody asked for.  sway.stack says what was meant -- "everything
    they alone pull in goes with them" -- and it has an avoid list 80 names
    long because that is what it took to say it by hand.

    Not walking into it is the whole fix: anything ANOTHER package needs is
    still reached through that package, so shared dependencies survive.
    """
    order, edges, missing, via, visited = [], {}, [], {}, set()
    ignored_required = []
    ignore = {x.lower() for x in (ignore or ())}

    def visit(a, parent="", kind=""):
        if a in visited:
            return
        if a.lower() in ignore and a != anchor:
            # AN IGNORED PACKAGE THAT SOMETHING REQUIRES.
            #
            # Not walking into an avoided branch is right (1.14.24).  But
            # webkitgtk REQUIRES Ruby, the stack's avoid list contains ruby,
            # and the plan therefore left out a hard requirement of a package
            # the user had asked for -- silently, until cmake said "Ruby 2.5
            # or higher is required" after everything else had built.
            # Skipping is still correct; saying nothing is not.
            if kind == "required" and parent:
                ignored_required.append((parent, a))
            return
        visited.add(a)
        data = book.get(a)
        if data is None:
            missing.append(a)
            return
        deps = deps_of(data, kinds) + _extra_deps_for(a)
        edges[a] = deps
        for d in deps:
            if d["external"]:
                missing.append(d["external"])
                via.setdefault(d["external"], (a, d["kind"]))
            elif d["anchor"]:
                via.setdefault(d["anchor"], (a, d["kind"]))
                visit(d["anchor"], a, d["kind"])
        order.append(a)

    visit(anchor)
    order = _dependencies_come_first(order, edges)
    walk_deps.ignored_required = ignored_required
    return order, edges, missing, via


# (dependant, dependency) pairs whose recommended edge was dropped to break a
# cycle -- the dependant is built BEFORE something it recommends.
_CYCLE_DROPS = []


def _dependencies_come_first(order, edges):
    """A real topological sort: every dependency before its dependant.

    This was three times a REPAIR PASS -- walk the list, notice a dependency
    that landed too late, move it earlier -- and three times something moved
    it back (1.12.33, 1.12.52, 1.12.68: texlive, then GLib, then libxcb).  A
    pass that repairs an order cannot guarantee one; it can only fix the
    cases it happens to look at, and the next pass undoes them.

    So: Kahn's algorithm over the dependency graph.  A node is emitted when
    everything it depends on has been emitted, and nothing else can move it.
    Ties keep the walk's original order, so the result stays stable and
    readable.

    Cycles are real in BLFS (GLib recommends shared-mime-info, which requires
    GLib).  They are broken by strength: RECOMMENDED edges inside a cycle are
    dropped first, because a recommendation is a preference and a requirement
    is a fact.  A cycle made only of required edges is the book's own knot --
    those nodes are emitted in their original order, and no reordering
    pretends to have solved it.
    """
    del _CYCLE_DROPS[:]
    nodes = list(order)
    present = set(nodes)
    deps = {a: [(d.get("anchor"), d.get("kind"))
                for d in edges.get(a, [])
                if d.get("anchor") in present and d.get("anchor") != a]
            for a in nodes}

    result, remaining = [], set(nodes)
    while remaining:
        ready = [a for a in nodes
                 if a in remaining
                 and not any(dep in remaining for dep, _k in deps[a])]
        if ready:
            for a in ready:
                result.append(a)
                remaining.discard(a)
            continue
        # Nothing is ready: something is in a cycle.  BREAK ONLY INSIDE IT.
        #
        # "Fewest blocking dependencies" is not the same as "in the cycle":
        # it picked ffmpeg, whose recommended edge to libass was then
        # dropped, and ffmpeg was ordered at 179 with libass at 208 --
        #     ERROR: libass >= 0.11.0 not found using pkg-config
        # The cycle was elsewhere (libass -> harfbuzz -> freetype2 ->
        # libass, after 1.13.23 made libass REQUIRE harfbuzz).
        #
        # So find the strongly-connected components of what is left, and
        # drop recommended edges only where BOTH ends are in the same one.
        # An edge into or out of a cycle is not what is holding it.
        _idx, _low, _on, _stack, _sccs, _counter = {}, {}, set(), [], [], [0]

        def _strong(v):
            _idx[v] = _low[v] = _counter[0]
            _counter[0] += 1
            _stack.append(v)
            _on.add(v)
            for _w, _k in deps.get(v, []):
                if _w not in remaining:
                    continue
                if _w not in _idx:
                    _strong(_w)
                    _low[v] = min(_low[v], _low[_w])
                elif _w in _on:
                    _low[v] = min(_low[v], _idx[_w])
            if _low[v] == _idx[v]:
                comp = []
                while True:
                    _w = _stack.pop()
                    _on.discard(_w)
                    comp.append(_w)
                    if _w == v:
                        break
                if len(comp) > 1:
                    _sccs.append(set(comp))

        import sys as _sys
        _lim = _sys.getrecursionlimit()
        _sys.setrecursionlimit(max(_lim, len(nodes) * 4 + 100))
        try:
            for v in nodes:
                if v in remaining and v not in _idx:
                    _strong(v)
        finally:
            _sys.setrecursionlimit(_lim)

        broke = False
        for comp in _sccs:
            for a in comp:
                kept = [(dep, k) for dep, k in deps[a]
                        if not (dep in comp and k != "required")]
                if len(kept) != len(deps[a]):
                    # SAY WHAT WAS GIVEN UP.
                    #
                    # Breaking the cycle is right, and silent.  vala
                    # recommends graphviz; graphviz -> gegl -> babl -> librsvg
                    # -> vala closes the loop, so vala's edge was dropped and
                    # vala built FIRST -- without valadoc, and with configure
                    # stopping on "Package 'libgvc' not found".  The order was
                    # the best available; not mentioning it was not.
                    for _dep, _k in deps[a]:
                        if _dep in comp and _k != "required":
                            _CYCLE_DROPS.append((a, _dep))
                    deps[a] = kept
                    broke = True
        if broke:
            continue
        # A REQUIRED-ONLY CYCLE IS THE BOOK'S KNOT -- but only the knot.
        #
        # This emitted EVERY remaining node in its original order, edges and
        # all, so one unbreakable cycle threw away the ordering of everything
        # after it: ffmpeg came out at 192 with libass, which it needs, at
        # 205.  Release just the smallest strongly-connected component and
        # carry on sorting the rest.
        _knot = min(_sccs, key=len) if _sccs else set()
        if not _knot:
            _knot = {next(a for a in nodes if a in remaining)}
        for a in nodes:
            if a in _knot:
                result.append(a)
                remaining.discard(a)
        for a in list(remaining):
            deps[a] = [(dep, k) for dep, k in deps[a] if dep not in _knot]
        continue
    return result

def _active_command_text(*chunks):
    """The command text with commented-out lines removed -- what will really
    run."""
    out = []
    for c in chunks:
        for line in (c or "").splitlines():
            t = line.strip()
            if t.startswith("#"):
                continue
            out.append(line)
    return "\n".join(out)


def _page_env_definitions(sec, names):
    """Variable definitions the PAGE itself makes, for variables its build
    commands use.

    TeX Live's build uses $TEXLIVE_PREFIX, and the same page sets it -- in
    the block that writes /etc/profile.d/texlive.sh, further down:
        TEXLIVE_PREFIX=/opt/texlive/2025
    The extractor takes install commands and drops that block, so the script
    used a variable the page defines a few paragraphs later.  A definition on
    the page is the book's answer; asking the user to supply it again is
    asking them to repeat the book.

    Only simple, self-contained assignments (no command substitution beyond
    what the page itself spells out), and only for variables actually used.
    """
    if not names:
        return {}
    # FOLLOW THE CHAIN.  The book defines
    #     export XORG_PREFIX="/usr"
    #     export XORG_CONFIG="--prefix=$XORG_PREFIX --sysconfdir=/etc ..."
    # and rejecting a value because it names another variable ("the same
    # question moved") threw away an answer the same page gives.  A value
    # that depends on a variable we can ALSO find here is answered; only a
    # chain that runs out is not.
    names = set(names)
    found = {}
    # The definition may be on ANOTHER page: TeX Live's build uses
    # $TEXLIVE_PREFIX and the value is set in "Setting the PATH for TeX Live"
    # (anchor tex-path), which is its own section.  The book is one document;
    # search it whole, nearest section first.
    scopes = [sec]
    root = sec
    while getattr(root, "parent", None) is not None:
        root = root.parent
    if root is not sec:
        scopes.append(root)
    for pre in [q for sc in scopes for q in sc.find_all("pre")]:
        text = pre.get_text()
        for n in names:
            if n in found:
                continue
            m = re.search(r"^\s*(?:export\s+)?%s=([^\n]+(?:\\\n[^\n]+)*)$"
                          % re.escape(n), text, re.M)
            if m:
                # a value may continue over several lines with backslashes;
                # a fragment ending in "\" is not a value, it is half of one
                val = " ".join(x.strip() for x in
                               m.group(1).replace("\\\n", " ").splitlines())
                val = val.strip().strip("&").strip()
                # a definition that needs another unknown variable is not an
                # answer, it is the same question moved
                # "" and friends are placeholders in prose, not values
                if "<" in val or ">" in val:
                    continue
                found[n] = val
    # pull in what those values reference, up to a few links
    for _ in range(4):
        need = set()
        for v in found.values():
            need |= set(re.findall(r"\$\{?([A-Z][A-Z0-9_]{2,})", v))
        need -= set(found)
        if not need:
            break
        names |= need
        for pre in [q for sc in scopes for q in sc.find_all("pre")]:
            text = pre.get_text()
            for n in list(need):
                m = re.search(r"^\s*(?:export\s+)?%s=([^\n]+(?:\\\n[^\n]+)*)$"
                              % re.escape(n), text, re.M)
                if m and n not in found:
                    v = " ".join(x.strip() for x in
                                 m.group(1).replace("\\\n", " ").splitlines())
                    found[n] = v.strip().strip("&").strip()
    # drop anything still referencing something we could not find: an
    # unanswered chain is not an answer
    for n in list(found):
        unresolved = set(re.findall(r"\$\{?([A-Z][A-Z0-9_]{2,})", found[n])) - set(found)
        if unresolved or found[n].endswith("\\") or '"' in found[n].rstrip('"'):
            del found[n]
    return found


def _required_env_guard(cmds, known):
    """Variables the book expects the USER to set, checked before use.

    TeX Live's page says to `export TEXLIVE_PREFIX=...` first; the commands
    then use it.  Unset, the shell expands it to nothing and configure gets
        error: expected an absolute directory name for --datarootdir:
    followed by a tar reading "--strip-components=1" as a filename.  An empty
    expansion produces nonsense far from its cause, so the script now refuses
    at the top of the phase and names the variable.

    Only variables the script does not set itself and that are not part of
    its own vocabulary."""
    # SINGLE-QUOTED TEXT IS NOT THE SHELL'S.
    #
    # $PACKAGE_PREFIX_DIR and $SAVE_PACKAGE_PREFIX_DIR are CMake variables,
    # written by a sed expression into a .cmake file:
    #     sed -e '/PACKAGE_INIT/i set(SAVE_PACKAGE_PREFIX_DIR "${PACKAGE_PREFIX_DIR}"...
    # The shell never expands them -- they are inside single quotes -- so
    # asking the user to set them was asking about another language's
    # variables.  Strip single-quoted spans before looking for names.
    # A COMMENTED LINE IS NOT A COMMAND.
    #
    # OpenJDK's page uses $JT_JAVA only in its test block, which the generated
    # script comments out -- and it EXPORTS the variable there itself:
    #     # export JT_JAVA=$(echo $PWD/build/*/jdk) &&
    #     # jtreg/bin/jtreg -jdk:$JT_JAVA ...
    # Scanning the commented text anyway produced a guard for a variable
    # nothing live would ever read, and the whole plan refused to start until
    # the user invented a value for it.  Read only what will run.
    _live = "\n".join(l for l in (cmds or "").splitlines()
                      if not l.strip().startswith("#"))
    _scan = re.sub(r"'[^']*'", "''", _live)
    names = set(re.findall(r"\$\{([A-Z][A-Z0-9_]{2,})\}", _scan))
    names |= set(re.findall(r"\$([A-Z][A-Z0-9_]{2,})\b", _scan))
    names -= set(known or ())
    # ...and a variable the commands SET is not one to ask the user for,
    # whether the assignment is live or not: the page is telling us its value.
    names -= set(re.findall(r"(?:^|\s|;)(?:export\s+)?([A-Z][A-Z0-9_]{2,})=",
                            cmds or "", re.M))
    # The shell's own variables are not the user's to set, and asking for
    # them is worse than useless: a list containing EUID teaches the user to
    # ignore the list.
    names -= {"PATH", "HOME", "PWD", "OLDPWD", "SHELL", "USER", "LOGNAME",
              "TERM", "MAKEFLAGS", "MAKELEVEL", "CFLAGS", "CXXFLAGS",
              "LDFLAGS", "CPPFLAGS", "PKG_CONFIG_PATH", "LD_LIBRARY_PATH",
              "BUILD_ROOT", "PM_TEST", "DESTDIR", "SHLVL", "IFS", "EUID",
              "UID", "PPID", "RANDOM", "SECONDS", "LINENO", "HOSTNAME",
              "BASH", "BASH_SOURCE", "BASH_VERSION", "FUNCNAME", "REPLY",
              "PIPESTATUS", "OSTYPE", "MACHTYPE", "LANG", "LC_ALL", "TMPDIR"}
    if not names:
        return cmds
    lines = []
    for n in sorted(names):
        lines.append('\t[ -n "${%s:-}" ] || { echo "this package needs %s to be set '
                     '(the book says to export it before building)" >&2; exit 1; }'
                     % (n, n))
    return "\n".join(lines) + "\n" + (cmds or "")


# Subprojects the book has NO PAGE for, per package.
#
# --wrap-mode=nofallback exists to stop a package installing its own copy of
# something another package owns (cairo once vendored freetype).  That is only
# the right answer when the book HAS the dependency.  gegl needs libnsgif and
# poly2tri-c; neither has a BLFS page, so no amount of planning can supply
# them and the flag simply makes gegl unbuildable -- a package the book builds
# with a plain `meson setup`.
#
# `--force-fallback-for` overrides the wrap mode for exactly the named
# subprojects, so the protection stays for everything else.  Add an entry only
# after checking `blfs search ` finds nothing: if the book gains a page,
# the entry must go, and the failure box (1.14.39) covers anything not listed.
_FALLBACK_ALLOWED = {
    "gegl": ("libnsgif", "poly2tri-c"),
}


def _allow_missing_subprojects(cmds, anchor):
    allowed = _FALLBACK_ALLOWED.get((anchor or "").lower())
    if not allowed:
        return cmds
    out = []
    for line in (cmds or "").splitlines():
        if (re.search(r"\bmeson\s+setup\b", line)
                and "force-fallback-for" not in line
                and not line.strip().startswith("#")):
            line = re.sub(r"(\bmeson\s+setup\b)",
                          r"\1 --force-fallback-for=" + ",".join(allowed),
                          line, count=1)
        out.append(line)
    return "\n".join(out)


def _no_vendored_subprojects(cmds):
    """A package must not quietly build a dependency it could not find.

    cairo's meson fell back to its bundled subprojects and INSTALLED its own
    freetype and fontconfig:
        /usr/lib/libfreetype.so.6.20.1   p_cairo   (and no libfreetype.so)
    There was no p_freetype2 at all -- never built, because the planner then
    saw a libfreetype on disk and considered it done.  Every later
    "cannot find -lfreetype" came from that one silent fallback: a vendored
    copy is owned by the wrong package, carries no development symlink, and
    hides the missing dependency instead of reporting it.

    --wrap-mode=nofallback makes meson FAIL instead, naming the dependency,
    which is what the planner and the user need to see.  It is added only
    where it is not already specified."""
    out = []
    for line in (cmds or "").splitlines():
        if (re.search(r"\bmeson\s+setup\b", line)
                and "wrap-mode" not in line and "wrap_mode" not in line
                and not line.strip().startswith("#")):
            line = re.sub(r"(\bmeson\s+setup\b)", r"\1 --wrap-mode=nofallback", line, count=1)
        out.append(line)
    return "\n".join(out)


# The tools the package-user system replaces with wrappers.  Kept here
# because the generator has to know them: see _use_wrapped_tools.
_WRAPPED_TOOLS = ("ldconfig", "install", "chown", "chgrp", "chmod", "mkdir", "cp")


def _use_wrapped_tools(cmds):
    r"""Call wrapped tools by NAME, so the wrappers are actually used.

    The wrappers live in a directory on the package user's PATH, which means
    an ABSOLUTE path walks straight past them.  The book's Xorg loop says
        as_root /sbin/ldconfig
    so the package user ran the real ldconfig and hit
        Renaming of /etc/ld.so.cache~ to /etc/ld.so.cache failed
    with the wrapper sitting unused.  This is not specific to ldconfig: any
    of the wrapped tools written as /sbin/x or /usr/bin/x bypasses the whole
    mechanism.
    """
    out = []
    for line in (cmds or "").splitlines():
        if not line.strip().startswith("#"):
            for tool in _WRAPPED_TOOLS:
                line = re.sub(r"(?= 0 and not out[_j].strip():
            _j -= 1
        if _j >= 0 and re.match(r"^\s*popd\s*(&&)?\s*$", out[_j]):
            _k = _j
            while _k >= 0 and not re.match(r"^\s*(mkdir\s+\S+\s*&&\s*)?pushd\b",
                                          out[_k]):
                _k -= 1
            if _k >= 0:
                # `mkdir tools` and `pushd tools` are often separate lines;
                # the directory is made only for the suite, so it goes too
                _m = _k - 1
                while _m >= 0 and not out[_m].strip():
                    _m -= 1
                if _m >= 0 and re.match(r"^\s*mkdir\s+\S+\s*(&&)?\s*$", out[_m]):
                    _k = _m
                _span = "\n".join(out[_k:_j + 1])
                if _TEST_SETUP_RE.search(_span):
                    for _n in range(_k, _j + 1):
                        if out[_n].strip() and not out[_n].strip().startswith("#"):
                            out[_n] = "# " + out[_n]
                    out.insert(_k, "## SKIPPED -- builds the test suite that the"
                                   " commented run below uses")
        # A MATCH ON A CONTINUATION LINE BELONGS TO THE COMMAND ABOVE IT.
        #
        # ffmpeg's sample fetch is
        #     rsync -vrltLW --delete ... \
        #           rsync://fate-suite.ffmpeg.org/... fate-suite/
        # and only the SECOND line matches, so the first stayed active and
        # ran rsync with no arguments to speak of.
        while out and re.search(r"\\\s*$", out[-1]) and not out[-1].lstrip().startswith("#"):
            out[-1] = "# " + out[-1]
        out.append("## SKIPPED -- runs the test suite; this script has a test "
                   "phase for that (PM_TEST=1)")
        out.append("# " + line)
        # whatever reads the log it would have written goes too: rust pipes
        # the run into rustc-testlog and greps it on the next lines
        _logs = set(m for m in _TEST_LOG_RE.findall(line) for m in m if m)
        i += 1
        while i < len(lines) and _logs and any(l in lines[i] for l in _logs):
            out.append("# " + lines[i])
            i += 1
        # a pipeline continues over lines: comment the rest of it, or the
        # `awk` after a commented `grep ... |` runs on its own
        while (out and re.search(r"(\||&&|\\)\s*$", out[-1])
               and i < len(lines) and lines[i].strip()):
            out.append("# " + lines[i])
            i += 1
        # and the cd back out
        if i < len(lines) and re.match(r"^\s*cd\s+\.\.?/?\s*&*\s*$", lines[i]):
            out.append("# " + lines[i])
            i += 1
    return "\n".join(out)


_TYPESET_RE = re.compile(
    r"\b(texi2pdf|texi2dvi|dvips|dvipdf|pdflatex|dblatex|latex)\b")


def _comment_typeset_blocks(cmds):
    r"""A loop that typesets the manuals needs TeX; skip the whole block.

    ffmpeg's page ends its build with
        pushd doc &&
        for DOCNAME in `basename -s .html *.html`
        do
            texi2pdf -b $DOCNAME.texi &&
            texi2dvi -b $DOCNAME.texi &&
            dvips    -o $DOCNAME.ps $DOCNAME.dvi
        done &&
    which fails on a system that deliberately has no TeX.  Commenting only
    the texi2* lines would leave an empty loop body, so the block goes as a
    unit: from the `pushd`/`for` that opens it to the `done`/`popd` that
    closes it.
    """
    lines = (cmds or "").splitlines()
    out, i = [], 0
    while i < len(lines):
        line = lines[i]
        if line.strip().startswith("#") or not re.match(r"^\s*(pushd|for)\b", line):
            out.append(line)
            i += 1
            continue
        # find the end of this block, and whether it typesets anything
        j, depth, typesets = i, 0, False
        while j < len(lines):
            if re.match(r"^\s*(for|while|until)\b", lines[j]):
                depth += 1
            if re.match(r"^\s*done\b", lines[j]):
                depth -= 1
            if _TYPESET_RE.search(lines[j]):
                typesets = True
            if depth <= 0 and j > i and re.match(r"^\s*(done|popd)\b", lines[j]):
                break
            j += 1
        if not typesets or j >= len(lines):
            out.append(line)
            i += 1
            continue
        # swallow a trailing popd too, and any && that chains on
        while j + 1 < len(lines) and re.match(r"^\s*popd\b", lines[j + 1]):
            j += 1
        out.append("## SKIPPED -- typesets the manuals (texi2pdf/dvips need TeX); "
                   "the package itself does not need them")
        for k in range(i, j + 1):
            out.append("# " + lines[k])
        i = j + 1
    return "\n".join(out)


def _comment_doc_format_builds(cmds):
    r"""Comment `make ... pdf/ps/dvi` -- the book's OPTIONAL manuals.

    libassuan's page ends with
        make -C doc pdf ps
    which typesets the manual and needs a TeX installation.  --disable-doc
    does not help (libassuan has no such switch; autotools accepts unknown
    --disable-* silently), because nothing is misconfigured: the command
    explicitly asks for PDF.

    A library is not installed by typesetting its manual, so these lines are
    emitted commented, with the book's own command one uncomment away.  info
    and html builds are left alone -- they need only texinfo, which LFS has.
    """
    out = []
    for line in (cmds or "").splitlines():
        if (not line.strip().startswith("#") and not _is_definition(line)
                and re.search(r"\bmake\b[^#\n]*\s(pdf|ps|dvi)\b", line)):
            out.append("## SKIPPED -- typesets the manual (needs TeX); the "
                       "package itself does not need it")
            out.append("# " + line)
        else:
            out.append(line)
    return "\n".join(out)


_PROFILE_RUN_RE = re.compile(
    r"^\s*(source|\.)\s+/etc/profile(\.d/\S+)?\s*$|^\s*/etc/profile\.d/\S+\.sh\s*$")


# `make install-` in a CONFIGURATION section is the SysV books' idiom
# for installing a boot script, and the target lives in the blfs-bootscripts
# package -- not in the package being built.  Run as-is it says:
#     make: *** No rule to make target 'install-bluetooth'.  Stop.
# after everything else has installed correctly, which turns a finished build
# into a failure.  The systemd books have no such sections, so this only
# appears once a SysV book is in use.
_BOOTSCRIPT_RE = re.compile(r"^\s*make\s+(?:-C\s+\S+\s+)?install-([A-Za-z0-9_.+-]+)\s*$")
_AUTOMAKE_INSTALL_TARGETS = {
    "strip", "exec", "data", "html", "pdf", "ps", "dvi", "info", "man",
    "exec-am", "data-am", "dist", "recursive",
}


# "You can test your installation ..." -- prose that introduces an EXAMPLE,
# not a configuration step.  enchant's page ends with a spell-check demo that
# needs an aspell dictionary; run as a configure phase it failed with
#     No dictionary available for 'en_GB'
# after the package had installed perfectly.  A demonstration is not part of
# installing something.
_CONFIG_TEST_INTRO_RE = re.compile(
    r"you (can |may )?(now )?(test|verify|try)\b|to test (your|the) install"
    r"|test your installation", re.I)


# A SITE CONFIGURATION FILE IS NOT AN INSTALL STEP.
#
# The dhcpcd page ends with an example interface file:
#     cat > /etc/sysconfig/ifconfig.eth0 << "EOF"
#     ONBOOT="yes"
#     IFACE="eth0"
#     SERVICE="dhcpcd"
#     ...
# Run as a configure phase by p_dhcpcd it died with
#     /etc/sysconfig/ifconfig.eth0: Permission denied
# -- correctly: /etc/sysconfig is drwxrwxr-t root:install and that file is
# root's.  Granting the package user a collector group for /etc/sysconfig
# would "fix" it by letting a package write the machine's network
# configuration, which is worse than the failure.
#
# And the content is wrong anyway: `eth0` is the book's placeholder, and this
# machine's interfaces are `enp5s0` and `wlp3s0`.  A file naming an interface
# that does not exist does nothing, or shadows the one you meant.
_SITE_CONFIG_RE = re.compile(
    r"/etc/sysconfig/(ifconfig\.|network-devices|wpa_supplicant)"
    r"|/etc/wpa_supplicant"
    r"|/etc/sysconfig/console|/etc/sysconfig/clock", re.I)


def _comment_site_config_writes(cmds, name=""):
    """Comment out writes to files that hold THIS machine's settings."""
    out, skipping, heredoc = [], False, None
    for line in (cmds or "").splitlines():
        st = line.strip()
        # PROSE IS NOT CODE: a comment mentioning the path is not a write.
        if st.startswith("#") and not heredoc:
            out.append(line)
            continue
        if heredoc is not None:
            out.append("# " + line if skipping else line)
            if st == heredoc or st == heredoc.strip('"\''):
                heredoc, skipping = None, False
            continue
        if _SITE_CONFIG_RE.search(st) and re.search(r">|tee |install |cp |mv ", st):
            out.append("## SKIPPED -- this writes a file that belongs to the"
                       " MACHINE, not to the package:")
            out.append("##   %s" % st[:120])
            out.append("## The interface name in the book is a placeholder, and"
                       " /etc/sysconfig is root's.")
            out.append("## Write yours by hand; what is missing is reported by:")
            out.append("##   packagemanager stack network --only net-wifi --run")
            out.append("# " + line)
            m = re.search(r"<<-?\s*([\"']?)([A-Za-z_][A-Za-z0-9_]*)\1", st)
            if m:
                heredoc, skipping = m.group(2), True
            continue
        out.append(line)
    return "\n".join(out)


def _comment_config_tests(cmds):
    out, skipping = [], False
    for line in (cmds or "").splitlines():
        st = line.strip()
        if st.startswith("#"):
            if _CONFIG_TEST_INTRO_RE.search(st):
                if not skipping:
                    out.append("## SKIPPED -- the book offers this as a way to"
                               " TEST the installation,")
                    out.append("## not as a step of it.  Run it by hand if you"
                               " want to check the package.")
                skipping = True
            elif skipping and st not in ("#", "##"):
                skipping = False            # a new paragraph ends the example
            out.append(line)
            continue
        if skipping and st:
            out.append("# " + line)
        else:
            out.append(line)
    return "\n".join(out)


def _comment_bootscript_installs(cmds, anchor=""):
    out = []
    for line in (cmds or "").splitlines():
        m = _BOOTSCRIPT_RE.match(line)
        # install-strip, install-exec, install-data and friends are REAL
        # automake targets in the package itself -- only a name that is not
        # one of those is a boot script.
        if m and m.group(1) in _AUTOMAKE_INSTALL_TARGETS:
            m = None
        if m and not line.strip().startswith("#"):
            out.append("## SKIPPED -- installs the '%s' boot script, whose "
                       "target lives in blfs-bootscripts," % m.group(1))
            out.append("## not in this package.  Install it from there:")
            out.append("##   packagemanager install blfs-bootscripts --run")
            out.append("##   cd  && %s"
                       % line.strip())
            out.append("# " + line.strip())
            continue
        out.append(line)
    return "\n".join(out)


def _comment_profile_sourcing(cmds):
    r"""Writing a login profile is not running it.

    Rust's configuration writes /etc/profile.d/rustc.sh and the page then
    sources it.  Those snippets call pathprepend/pathappend -- helpers from
    the BLFS bash startup files, absent from a build shell -- so a package
    that had built and installed 2390 files failed at:
        /etc/profile.d/rustc.sh: line 3: pathprepend: command not found

    The file is written; it takes effect at the next login, which is what it
    is for.  Sourcing it here tests the login shell, not the package.
    """
    out = []
    for line in (cmds or "").splitlines():
        if not line.strip().startswith("#") and _PROFILE_RUN_RE.match(line):
            out.append("## SKIPPED -- sourcing a login profile; the file is "
                       "written and applies at next login")
            out.append("# " + line)
        else:
            out.append(line)
    return "\n".join(out)


_LIST_LOOP_RE = re.compile(
    r"^\s*for\s+\w+\s+in\s+\$\(.*?(\.\./[\w.-]+|\b[\w-]+\.md5)\b.*?\)")


def _guard_list_loops(cmds):
    r"""A loop over a list file must fail when the file is missing.

    Xorg's build is
        for package in $(grep -v '^#' ../lib-7.md5 | awk '{print $2}')
    and when ../lib-7.md5 is not there, grep prints an error, the loop runs
    ZERO times, and the `for` exits 0 -- so a package that built nothing was
    recorded as installed.  Twice, before anyone noticed.

    A guard in front of the loop turns that into a failure with a name.
    """
    out = []
    for line in (cmds or "").splitlines():
        m = (None if line.strip().startswith("#")
             else _LIST_LOOP_RE.match(line))
        if m:
            f = m.group(1)
            out.append('[ -s "%s" ] || { echo "the list this build loops over '
                       'is missing or empty: %s" >&2; exit 1; }' % (f, f))
        out.append(line)
    return "\n".join(out)


def _comment_orphan_doc_installs(cmds, context=""):
    r"""`make install-man` when nothing active ran `make man`.

    Git's page generates the man pages and html docs under "If you have
    installed asciidoc-10.2.1 and xmlto-0.0.29 you can create ..." -- which
    the conditional guard skips -- and then installs them unconditionally:
        make install-man
        make htmldir=/usr/share/doc/git-2.53.0 install-html
    So the install tried to build them itself and died on
        /bin/sh: line 1: asciidoc: command not found
    after everything else had installed.

    Fourth in this family (libuv's man page, the orphan log reader, brotli's
    wheel): whenever the guard comments a block, something may still consume
    what that block would have produced.
    """
    lines = (cmds or "").splitlines()
    active = "\n".join(l for l in lines if not l.strip().startswith("#"))
    if context:
        active += "\n" + "\n".join(l for l in context.splitlines()
                                   if not l.strip().startswith("#"))
    # A BARE `make` BUILDS THE PACKAGE -- BUT NOT ITS MANUALS.
    #
    # Whois builds with `make` and installs with `make prefix=/usr
    # install-whois`; there is no `make whois` target to look for, so all
    # three of its install lines were commented and it installed nothing.
    # But git ALSO has a bare `make`, and git's manuals still need `make man`
    # -- which is exactly the conditional block this filter was written for.
    # So a bare make covers ordinary targets and never the documentation ones.
    _bare_make = bool(re.search(r"^\s*make\s*(?:\S+=\S+\s*)*$", active, re.M))
    _DOC_TARGETS = {"man", "html", "info", "pdf", "ps", "dvi", "doc", "docs",
                    "manual", "manuals"}
    # `make htmldir=/usr/... install-html`: the book puts variable assignments
    # between make and the target, so allow them.
    pat = re.compile(r"\bmake\s+(?:\S+=\S+\s+)*(?:-C\s+\S+\s+)?install-(\w+)")
    out, i = [], 0
    while i < len(lines):
        line = lines[i]
        m = (None if line.strip().startswith("#") or _is_definition(line)
             else pat.search(line))
        if not m:
            out.append(line)
            i += 1
            continue
        what = m.group(1)
        if (_bare_make and what not in _DOC_TARGETS) \
                or re.search(r"\bmake\s+(?:\S+=\S+\s+)*(?:-C\s+\S+\s+)?%s\b"
                             % re.escape(what), active):
            out.append(line)          # something really does build it
            i += 1
            continue
        out.append("## SKIPPED -- installs '%s', which nothing here builds "
                   "(the book generates it in an optional step)" % what)
        out.append("# " + line)
        # The rest of ITS chain goes too: git follows install-html with
        # `mkdir -vp .../html && mv ...`, moving files that only install-html
        # would have put there.
        i += 1
        while (re.search(r"&&\s*$", out[-1])
               and i < len(lines) and lines[i].strip()):
            out.append("# " + lines[i])
            i += 1
    return "\n".join(out)

def _comment_orphan_tree_shuffles(cmds):
    r"""Rearranging a directory that only a SKIPPED command fills.

    After git's optional "untar the html docs" block is commented, the page
    still reorganises what it would have extracted:
        mkdir -vp /usr/share/doc/git-2.53.0/man-pages/{html,text} &&
        mv        /usr/share/doc/git-2.53.0/{git*.adoc,man-pages/text} &&
    and the mv finds nothing:
        mv: cannot stat '/usr/share/doc/git-2.53.0/git*.adoc'

    A `mv` (or find/sed) whose source directory appears ONLY in commented
    commands is moving something nobody produced.  `mkdir` does not count as
    filling a directory -- it makes it empty.
    """
    lines = (cmds or "").splitlines()
    commented, active = [], []
    for l in lines:
        (commented if l.strip().startswith("#") else active).append(l)
    ctext, atext = "\n".join(commented), "\n".join(active)

    def _fills(d):
        """Does an ACTIVE command put anything INTO d (not just create it)?"""
        for l in active:
            if d not in l:
                continue
            if re.match(r"^\s*mkdir\b", l):
                continue                       # creating it is not filling it
            # mv/find/sed REARRANGE a directory; they do not fill it.  Only
            # something that brings files in counts.
            if re.match(r"^\s*(mv|find|sed)\b", l):
                continue
            if re.search(r"\b(tar|cp|install|make|ninja)\b", l):
                return True
        return False

    out, i = [], 0
    while i < len(lines):
        line = lines[i]
        if (line.strip().startswith("#") or _is_definition(line)
                or not re.match(r"^\s*(mv|find|sed)\b", line)):
            out.append(line)
            i += 1
            continue
        dirs = [d.rstrip("/") for d in
                re.findall(r"(/[A-Za-z0-9._/+-]{6,})", line)]
        orphan = False
        for d in dirs:
            base = d.rstrip("/").rsplit("/", 1)[0] if "{" not in d else d
            for probe in (d, base):
                if probe and probe in ctext and not _fills(probe):
                    orphan = True
                    break
            if orphan:
                break
        if not orphan:
            out.append(line)
            i += 1
            continue
        out.append("## SKIPPED -- rearranges files that only a skipped step "
                   "would have put there")
        out.append("# " + line)
        i += 1
        # and the rest of its chain, plus the mkdir that led into it
        while out and re.search(r"&&\s*$", out[-1]) and i < len(lines) and lines[i].strip():
            out.append("# " + lines[i])
            i += 1
        for j in range(len(out) - 1, -1, -1):
            if out[j].startswith("#"):
                continue
            if re.match(r"^\s*mkdir\b", out[j]) and re.search(r"&&\s*$", out[j]):
                out[j] = "# " + out[j]
            break
    return "\n".join(out)


def _repair_dangling_operators(cmds):
    r"""Commenting a command must not leave the previous one dangling.

    libpwquality's install ended:
        make install &&
        # pip3 install ... --find-links dist pwquality     <- commented (1.13.3)
    and bash refused the whole file:
        syntax error near unexpected token `}'

    Every filter that comments a line can do this, so the repair belongs at
    the end, once: an ACTIVE line ending in && || or | with no active line
    after it loses the operator.
    """
    lines = (cmds or "").splitlines()
    for i, line in enumerate(lines):
        if line.strip().startswith("#") or not re.search(r"(&&|\|\||\|)\s*$", line):
            continue
        nxt = next((l for l in lines[i + 1:]
                    if l.strip() and not l.strip().startswith("#")), None)
        if nxt is None:
            lines[i] = re.sub(r"\s*(&&|\|\||\|)\s*$", "", line)
    return "\n".join(lines)


def _comment_orphan_pip_installs(cmds, context=""):
    r"""A pip install whose wheel nothing built.

    brotli's page builds optional Python bindings in two steps:
        pip3 wheel -w dist --no-build-isolation --no-deps --no-cache-dir $PWD
        pip3 install --no-index --find-links dist Brotli
    The first is conditional ("if you want the bindings") and is commented
    out; the second was not, so pip looked in a directory nobody had made:
        WARNING: Location 'dist' is ignored: ... non-existing path
        ERROR: No matching distribution found for Brotli
    -- after the C library had installed perfectly.

    An install that reads from a --find-links directory no ACTIVE command
    fills is commented with its producer.

    CONTEXT MATTERS.  The producer and the consumer are usually on one page
    but NOT in one block: for a Python module the wheel is built in the build
    phase and installed in the install phase, and judging the install block
    alone found no producer and commented the only line that installs
    anything.  Every python module page came out as "build a wheel, install
    nothing" -- pathspec's manifest then held one path, /var/mail/p_pathspec,
    created by useradd, which was enough to call it installed, and Hatchling
    failed on `No module named 'pathspec'` twenty-two packages later.
    """
    lines = (cmds or "").splitlines()
    active = "\n".join(l for l in lines if not l.strip().startswith("#"))
    if context:
        active += "\n" + "\n".join(l for l in context.splitlines()
                                   if not l.strip().startswith("#"))
    out = []
    for line in lines:
        m = (None if line.strip().startswith("#") or _is_definition(line)
             else re.search(r"--find-links[= ]+(\S+)", line))
        if m:
            d = m.group(1)
            builds = re.search(r"(pip3?\s+wheel[^\n]*-w[= ]+%s\b"
                               r"|mkdir[^\n]*\b%s\b)" % (re.escape(d), re.escape(d)),
                               active)
            if not builds:
                out.append("## SKIPPED -- installs from '%s', which nothing here "
                           "builds (the wheel step is optional)" % d)
                out.append("# " + line)
                continue
        out.append(line)
    return "\n".join(out)


def _comment_orphan_log_readers(cmds):
    """Comment lines that read a log no ACTIVE command writes.

    Xorg Libraries ends with
        grep -A9 summary *make_check.log
    which inspects the test logs -- and the line that WRITES them
    (make check ... | tee ../$package-make_check.log) is part of the optional
    test step, correctly commented out.  Under set -e the orphan grep then
    failed the whole page:
        grep: *make_check.log: No such file or directory

    Same shape as the libuv man page (1.12.19), one layer earlier: a command
    whose input nothing produces is not part of installing.  Only .log files
    are treated this way -- a build artifact with a real name might be
    produced by make itself, and guessing there would do harm."""
    lines = (cmds or "").splitlines()
    active = [l for l in lines if not l.strip().startswith("#")]
    writes_log = any(re.search(r"(tee|>|>>)\s*\S*\.log", l) for l in active)
    if writes_log:
        return cmds
    out = []
    for line in lines:
        if (not line.strip().startswith("#")
                and re.search(r"\S*\.log\b", line)
                and re.match(r"\s*(grep|cat|less|tail|head|awk|sed)\b", line)):
            out.append("## SKIPPED -- reads a log that nothing here writes "
                       "(it belongs to the optional test step)")
            out.append("# " + line)
        else:
            out.append(line)
    return "\n".join(out)


def _comment_own_program_demos(cmds, installed):
    """Comment out BUILD lines that run a program this package installs.

    The book demonstrates the result -- "You can now test the rendering ... To
    do this, run the following command: gs -q -dBATCH .../tiger.eps" -- and
    that command lands in the build, where the program does not exist yet:
        install_gs: line 118: gs: command not found
    It was harmless while failures were ignored; with the dispatcher honouring
    set -e it fails the package.  Chasing the wording ("To do this...") is the
    mistake this file keeps making; the structural fact is that a package
    cannot run its own not-yet-installed program as part of building itself.
    """
    progs = set()
    for chunk in (installed or {}).values():
        for tok in re.split(r"[,\s]+", str(chunk)):
            tok = tok.strip()
            if tok and re.fullmatch(r"[A-Za-z0-9._+-]+", tok):
                progs.add(tok)
    if not progs:
        return cmds
    out = []
    for line in (cmds or "").splitlines():
        stripped = line.strip()
        first = stripped.split()[0] if stripped else ""
        if (first in progs and not stripped.startswith("#")
                and not re.match(r"^(make|ninja|meson|cmake)\b", stripped)):
            out.append("## SKIPPED -- runs %s, which this package has not "
                       "installed yet (the book is demonstrating the result)"
                       % first)
            out.append("# " + line)
        else:
            out.append(line)
    return "\n".join(out)


def _needed_additional_links(data, emitted_lines):
    """Of the page's additional downloads, the ones an ACTIVE command uses.

    A file nothing runs is not fetched (optional doc tarballs, test suites --
    their commands are emitted commented), and a file an active command needs
    IS fetched whatever the book called it.  Patches are always kept: the
    command that applies one is `patch -Np1 -i ../x.patch`, which does name
    it, but a patch missed for any reason is a silent mis-build."""
    links = data.get("additional_links", []) or []
    if not links:
        return []
    # the PHASED text only: data["install_cmds"] is the raw page, where the
    # conditional blocks are still live and would vote for their own
    # downloads
    active = _active_command_text(data.get("build_cmds", ""),
                                  data.get("install_root_cmds", ""))
    keep = []
    for href in links:
        base = href.rstrip("/").split("/")[-1]
        if href.endswith(".patch") or (base and base in active):
            keep.append(href)
    return keep


def bash_str(v):
    """A book string as a bash literal that the shell will NOT interpret.

    These values are PROSE lifted out of the book -- "Installed Directories:
    $TEXLIVE_PREFIX/bin, $TEXLIVE_PREFIX/lib" -- and they were emitted inside
    double quotes, so the shell expanded them.  pm-install sources the script
    to read its metadata, and under set -u that killed the install before it
    started:
        install_texlive-20250308-source: line 25: TEXLIVE_PREFIX: unbound variable
    A dollar sign in documentation is not a variable.  Single quotes, with
    the only escape single quoting needs."""
    return "'" + str(v).replace("'", "'\\''") + "'"


def bash_array(items):
    return "(" + " ".join(bash_str(i) for i in items) + ")"


def extract_info(sec, is_module):
    pd = package_div(sec)
    if not pd:
        return ""
    start = pd.find(class_="sect3" if is_module else "sect2")
    if not start:
        return ""
    stop = "h4" if is_module else "h3"
    text = ""
    for el in start.next_siblings:
        if getattr(el, "name", None) == stop and "Package Information" in el.get_text():
            break
        if getattr(el, "name", None) == "p":
            text += "\\n\\n" + " ".join(l.strip() for l in el.get_text().strip().splitlines())
    return text.replace('"', '\\"').lstrip("\\n")


def extract_package_data(sec, is_module, group, anchor):
    """Pull everything a script / dependency walk needs out of the soup, once.
    The result is plain JSON-serialisable data (the cache stores this)."""
    title = full_title(sec, is_module)

    install_cmds = extract_commands(sec, "div.installation")
    if not install_cmds.strip():
        sub = subsection_titled(sec, "Installation", "Install")
        if sub is not None:
            install_cmds = (extract_commands(sub, "div.installation")
                            or extract_raw_commands(sub))
    if is_module and not install_cmds.strip():
        install_cmds = extract_raw_commands(sec)
    install_cmds = fix_heredocs(install_cmds)

    # split the installation section into build (configure/make) vs install
    # (the privileged  step) for the phased script
    build_cmds, install_root = extract_commands_phased(sec)
    if not build_cmds.strip() and not install_root.strip():
        sub = subsection_titled(sec, "Installation", "Install")
        if sub is not None:
            build_cmds, install_root = extract_commands_phased(sub)
    build_cmds = fix_heredocs(build_cmds or "")
    install_root = fix_heredocs(install_root or "")

    config_cmds = extract_commands(sec, "div.configuration > div.sect3")
    if not config_cmds.strip():
        sub = subsection_titled(sec, "Configuring", "Configuration")
        if sub is not None:
            config_cmds = (extract_commands(sub, "div.configuration > div.sect3")
                           or extract_commands(sub, "div.configuration"))
    config_cmds = fix_heredocs(config_cmds or "")

    between = extract_between_package_and_installation(sec)
    between_cmds = [extract_commands(sec, "div." + cls) for cls in between]

    link = extract_link(sec)
    return {
        "anchor": anchor,
        "title": title,
        "name": short_name(anchor),
        "name_version": sanitize(title),
        "is_module": is_module,
        "group": group,
        "link": link,
        "pkg": os.path.basename(link),
        "md5": extract_md5(sec),
        "additional_links": extract_additional_downloads(sec),
        "info": extract_info(sec, is_module),
        "required": _xref_titles(sec, "required"),
        "recommended": _xref_titles(sec, "recommended"),
        "optional": _xref_titles(sec, "optional"),
        "installed_content": extract_installed_content(sec),
        # variables the PAGE defines for its own commands (see
        # _page_env_definitions): collected here, where the section is in
        # hand, and used by render_script
        "page_env": _page_env_definitions(
            sec, set(re.findall(r"\$\{?([A-Z][A-Z0-9_]{2,})", build_cmds or ""))),
        "install_cmds": install_cmds,
        "build_cmds": build_cmds,
        "install_root_cmds": install_root,
        "config_cmds": config_cmds,
        "between_cmds": between_cmds,
        "deps": section_deps(sec, DEP_KINDS),
    }


def _drop_orphaned_variant_install(cmds):
    """An install step whose build variant was commented out.

    WebKitGTK's page gives two builds, each with its own `ninja install`.  One
    build is skipped, but its install line stayed live, so the install phase
    ran `ninja install` twice.  The second is a no-op rather than a second
    build -- but it reads exactly like one, and someone watching a package
    that takes hours has no way to tell.  An identical command repeated after
    a skipped block belongs to that block.
    """
    # ONLY THE PATTERN "X ;  ; X".
    #
    # A first attempt commented any repeat after a skipped block, and dropped
    # GLib's LAST `ninja install` -- the one that installs the reconfigured
    # build with introspection enabled, several live commands later.  A repeat
    # is only redundant when nothing happened in between: the command
    # immediately before the skipped block and the one immediately after it
    # must be the same.
    lines = (cmds or "").split("\n")
    out, last_active, pending_skip = [], None, False
    for line in lines:
        st = line.strip()
        if st.startswith("## OPTIONAL, skipped"):
            pending_skip = True
            out.append(line)
            continue
        if not st or st.startswith("#"):
            out.append(line)
            continue
        if pending_skip and last_active is not None and st == last_active:
            out.append("## SKIPPED -- the same command again, belonging to the "
                       "build variant commented out above.")
            out.append("# " + st)
            pending_skip = False
            continue
        last_active = st
        pending_skip = False
        out.append(line)
    return "\n".join(out)


# ONE MARKER, ONE CONSTANT.  `_enable_one_alternative` looked for
# "## OPTIONAL, skipped" and 1.14.212 renamed the marker to "## SKIPPED --"
# -- so the picker silently stopped firing, and webkitgtk's script came out
# with BOTH build blocks commented: a build phase of `:` and an install
# phase that runs `ninja install` against a directory with no build.ninja.
# Two strings for one fact, drifting apart, exactly as the rule says.
SKIP_MARK = "## SKIPPED -- "


def _is_skip_line(line):
    """A skipped-block header, under either spelling this tool has used."""
    st = line.strip()
    return st.startswith(SKIP_MARK.strip()) or st.startswith("## OPTIONAL, skipped")


def _enable_one_alternative(cmds):
    """A page that offers a CHOICE must still build something.

    WebKitGTK's page gives two complete builds -- "If you want to install the
    GTK-3 version, run the following commands" and the same for GTK-4 -- and
    each is introduced by a conditional, so `_conditional_guard` commented
    BOTH.  The build phase came out as `:` and the install phase then ran
    `ninja install` against a directory with no build.ninja in it.

    When nothing active is left but a skipped block exists, the first block is
    restored: a package that builds one of two ways is not a package that
    builds no ways.  The others stay commented, with a line saying so, because
    which one is wanted is the reader's decision, not ours.
    """
    lines = (cmds or "").split("\n")
    if any(l.strip() and not l.strip().startswith("#") and l.strip() != ":"
           for l in lines):
        return cmds
    starts = [i for i, l in enumerate(lines) if _is_skip_line(l)]
    if not starts:
        return cmds
    i = starts[0]
    end = starts[1] if len(starts) > 1 else len(lines)
    out = lines[:i]
    out.append("## Every build block on this page is conditional, so none "
               "would have run.")
    _why = lines[i]
    for _sep in ("the book says:", SKIP_MARK.strip()):
        if _sep in _why:
            _why = _why.split(_sep, 1)[-1]
    out.append("## ENABLED HERE: %s" % _why.strip()[:110])
    out.append("## If you want one of the page's other variants instead, "
               "comment these")
    out.append("## lines and uncomment that block.")
    for l in lines[i + 1:end]:
        st = l.strip()
        if st.startswith("##"):
            out.append(l)
        elif st.startswith("#"):
            out.append(l.replace("#", "", 1).lstrip() if st != "#" else "")
        else:
            out.append(l)
    out.extend(lines[end:])
    return "\n".join(out)


def _fn_body(cmds, empty=": # (nothing to do in this phase)"):
    """A command block as a function body.  NOT indented: the book is full of
    heredocs, and an indented terminator is no terminator.  A body of comments
    alone is a syntax error to bash, so it gets a command."""
    if not cmds or not cmds.strip():
        return empty
    body = cmds.strip("\n")
    if not any(ln.strip() and not ln.lstrip().startswith("#")
               for ln in body.split("\n")):
        body += "\n:"
    return body


def render_script(data, source_url, book_version=""):
    """Turn cached package data into an install_ script: the
    book's variables at the top, its commands in build_pkg / install_pkg /
    configure_pkg, and one line at the end that hands over to the shared
    runner (lfs-phases), which does the rest -- fetch, unpack, run each phase
    as its own process, resume in the build dir, the dispatcher.  A failed
    install can be fixed and re-run WITHOUT recompiling:
        bash install_ install"""
    name_version = data["name_version"]
    pkg = data["pkg"]
    build_cmds = data.get("build_cmds") or data["install_cmds"]
    install_root = data.get("install_root_cmds") or ""
    config_cmds = data["config_cmds"]
    # let packagemanager's --reinstall actually force pip to reinstall: when
    # PM_REINSTALL=1 is set, pip install picks up --force-reinstall; otherwise the
    # expansion is empty and nothing changes.
    def _pip_reinstall(cmds):
        return re.sub(r'\bpip3?\s+install\b',
                      lambda m: m.group(0) + ' ${PM_REINSTALL:+--force-reinstall}',
                      cmds or "")
    build_cmds = _pip_reinstall(build_cmds)
    build_cmds = _comment_own_program_demos(build_cmds,
                                            data.get("installed_content"))
    build_cmds = _unwrap_interactive_subshell(build_cmds)
    install_root = _unwrap_interactive_subshell(install_root)
    build_cmds = _use_wrapped_tools(build_cmds)
    install_root = _use_wrapped_tools(install_root)
    build_cmds = _define_as_root(build_cmds)
    install_root = _define_as_root(install_root)
    build_cmds = _comment_typeset_doc_builds(build_cmds)
    install_root = _comment_typeset_doc_builds(install_root)
    config_cmds = _comment_profile_sourcing(data.get("config_cmds") or "")
    config_cmds = _comment_bootscript_installs(config_cmds, data.get("name"))
    config_cmds = _comment_config_tests(config_cmds)
    config_cmds = _comment_site_config_writes(config_cmds, data.get("name"))
    data["config_cmds"] = config_cmds
    build_cmds = _comment_profile_sourcing(build_cmds)
    install_root = _comment_profile_sourcing(install_root)
    # each block is judged with the other in view: they run in one directory,
    # one after the other, and a wheel built by one is installed by the next
    build_cmds = _comment_orphan_pip_installs(build_cmds, install_root)
    install_root = _comment_orphan_pip_installs(install_root, build_cmds)
    # LAST: whatever the filters above commented, the command before it must
    # not be left ending in `&&`.
    build_cmds = _comment_orphan_doc_installs(build_cmds, install_root)
    install_root = _comment_orphan_doc_installs(install_root, build_cmds)
    build_cmds = _comment_orphan_tree_shuffles(build_cmds)
    install_root = _comment_orphan_tree_shuffles(install_root)
    build_cmds = _guard_list_loops(build_cmds)
    install_root = _guard_list_loops(install_root)
    build_cmds = _comment_test_runs(build_cmds)
    install_root = _comment_test_runs(install_root)
    build_cmds = _comment_typeset_blocks(build_cmds)
    install_root = _comment_typeset_blocks(install_root)
    build_cmds = _comment_doc_format_builds(build_cmds)
    install_root = _comment_doc_format_builds(install_root)
    build_cmds = _comment_orphan_log_readers(build_cmds)
    install_root = _comment_orphan_log_readers(install_root)
    build_cmds = _no_vendored_subprojects(build_cmds)
    install_root = _no_vendored_subprojects(install_root)
    build_cmds = _allow_missing_subprojects(build_cmds, data.get("name"))
    install_root = _allow_missing_subprojects(install_root, data.get("name"))
    # LAST, once, after EVERY filter that comments a line: a command left
    # ending in `&&` with nothing active after it is a syntax error.  It ran
    # before five of them, and the old scaffolding line that followed each
    # phase body absorbed the dangling operator -- except in install_pkg,
    # where cachecontrol and 145 others failed `bash -n`.
    # ...and if every way of building it was skipped, enable one
    build_cmds = _enable_one_alternative(build_cmds)
    install_root = _drop_orphaned_variant_install(install_root)
    build_cmds = _repair_dangling_operators(build_cmds)
    install_root = _repair_dangling_operators(install_root)
    config_cmds = _repair_dangling_operators(config_cmds)
    data["config_cmds"] = config_cmds
    _known = {"pkg", "link", "name", "name_version", "url", "md5_sum", "info",
              "git_ref", "installed_program", "BUILD_ROOT"}
    # Everything the SCRIPT itself sets, anywhere: plain assignments, exports,
    # `for NAME in ...`, `read NAME`, and assignments inside heredocs.  These
    # were only scanned in the build text, so a loop variable in the install
    # section (DOCNAME, PACKAGE_PREFIX_DIR, JT_JAVA...) looked like something
    # the user had to provide.
    _alltext = (build_cmds or "") + "\n" + (install_root or "") + "\n" \
             + (data.get("install_cmds") or "")
    _setvars = set(re.findall(r"^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)=",
                              _alltext, re.M))
    _setvars |= set(re.findall(r"\bfor\s+([A-Za-z_][A-Za-z0-9_]*)\s+in\b", _alltext))
    _setvars |= set(re.findall(r"\bread\s+(?:-\S+\s+)*([A-Za-z_][A-Za-z0-9_]*)", _alltext))
    _setvars |= set(re.findall(r"^\s*([A-Za-z_][A-Za-z0-9_]*)=", _alltext, re.M))
    # what the page itself defines comes first; only what remains is asked of
    # the user
    _used = set(re.findall(r"\$\{([A-Z][A-Z0-9_]{2,})\}", build_cmds or "")) \
          | set(re.findall(r"\$([A-Z][A-Z0-9_]{2,})\b", build_cmds or ""))
    _defs = {k: v for k, v in (data.get("page_env") or {}).items()
             if k in (_used - (_known | _setvars))}
    if _defs:
        # dependencies first: XORG_PREFIX before XORG_CONFIG, or the second
        # expands to a flag with an empty --prefix
        _ordered, _left = [], dict(_defs)
        for _ in range(len(_defs) + 1):
            for k in sorted(_left):
                refs = set(re.findall(r"\$\{?([A-Z][A-Z0-9_]{2,})", _left[k]))
                if not (refs & set(_left)):
                    _ordered.append((k, _left.pop(k)))
                    break
            else:
                break
        _ordered += sorted(_left.items())
        _pre = "\n".join('\t%s="%s"   # set by the book'
                          % (k, v.strip('"\''))
                          for k, v in _ordered)
        build_cmds = _pre + "\n" + (build_cmds or "")
        _setvars |= set(_defs)
    build_cmds = _required_env_guard(build_cmds, _known | _setvars)
    install_root = _pip_reinstall(install_root)
    installed = data["installed_content"]
    unpacks = not re.search(rf"^tar .* {re.escape(pkg)} .*", data["install_cmds"])

    L = []
    w = L.append
    w("#!/bin/bash")
    w("############################################")
    w(f"### {name_version}")
    # 'origin=' is the shared contract: both generators stamp format + origin,
    # so no tool has to recognise the other's dialect.
    w(f"### generated by blfs {BLFS_VERSION} -- script format v{SCRIPT_VERSION}"
      f" origin=blfs")
    w("### phases: all (default) | unpack | build | test | install | configure | update")
    w("###   update = install without reconfiguring (keeps your config)")
    w("### re-run just 'install' after fixing permissions, without recompiling.\n")
    w(f'url={bash_str(source_url)}')
    w(f'name={bash_str(data["name"])}')
    w(f'name_version={bash_str(name_version)}')
    w(f'link={bash_str(data["link"])}')
    w(f'pkg={bash_str(pkg)}')
    w(f'md5_sum={bash_str(data["md5"])}')
    w(f"additional_links={bash_array(_needed_additional_links(data, L))}")
    w(f'info={bash_str(data["info"])}')
    w(f"required={bash_array(data['required'])}")
    w(f"recommended={bash_array(data['recommended'])}")
    w(f"optional={bash_array(data['optional'])}")
    w("installed_content=(" + " ".join(bash_str(k.rstrip(":")) for k in installed) + ")")
    for k, v in installed.items():
        w(f'{k}={bash_str(v)}')
    if data["is_module"] and data["group"]:
        w(f"install_groups={bash_array([data['group']])}")
    if not unpacks:
        w('pkg_unpack="no"     # the page\'s own commands unpack the tarball')
    # the mirror of last resort keeps most BLFS sources by filename -- the
    # BOOK's release, not a literal that goes stale with the next one
    if book_version and book_version != "unknown":
        w(f'pkgusr_book_mirror={bash_str(f"{SITE}/blfs/downloads/{book_version}")}')
    w("")

    # The commands, one function per phase.  NOT indented: the book is full of
    # heredocs, and an indented terminator is no terminator.
    between = [c for c in (data["between_cmds"] or []) if c.strip()]
    if between:
        w("prepare_pkg() {")
        w(_fn_body("\n\n".join(between)))
        w("}\n")
    w("build_pkg() {")
    w(_fn_body(build_cmds, ": # (no separate build step)"))
    w("}\n")
    w("install_pkg() {")
    w(_fn_body(install_root,
               ": # (this package installs during the build phase; nothing separate)"))
    w("}\n")
    if config_cmds.strip():
        w("configure_pkg() {")
        w('echo "## Configuration"')
        w(_fn_body(config_cmds))
        w("}\n")
    # no test_pkg: the runner autodetects `meson test` / `make check`; define
    # one here to override.
    w('. "${PKGUSR_LIB:-lfs-phases}" && pkgusr_run "$@"')

    safe = re.sub(r"[^A-Za-z0-9._-]+", "-", name_version).strip("-")
    body = "\n".join(L) + "\n"
    # embed a checksum of the pristine script so packagemanager can tell later
    # whether the user edited it (edit-detection for migrate).  The checksum is
    # computed over the body WITHOUT this line, so verification just strips the
    # 'pm_md5' line and recomputes.
    digest = hashlib.md5(body.encode()).hexdigest()
    stamp = (f"### generated by blfs {BLFS_VERSION} -- script format "
             f"v{SCRIPT_VERSION} origin=blfs")
    body = body.replace(stamp, stamp + f"\n### pm_md5={digest}", 1)
    return f"install_{safe}", body


def build_script(sec, is_module, group, anchor, source_url):
    """Convenience: extract + render straight from the soup (used at cache build
    time and by the whole-book test harness)."""
    return render_script(
        extract_package_data(sec, is_module, group, anchor), source_url)


# --------------------------------------------------------------------------- #
# subcommands
# --------------------------------------------------------------------------- #

def _find_packagemanager():
    here = os.path.dirname(os.path.abspath(__file__))
    local = os.path.join(here, "packagemanager")
    if os.path.isfile(local):
        ex = bool(os.stat(local).st_mode & 0o111)
        return [local] if ex else [sys.executable or "python3", local]
    import shutil as _sh
    return ["packagemanager"] if _sh.which("packagemanager") else None


def cmd_install(args):
    """Delegate to packagemanager, which owns the package users.

    Kept here so `blfs install ` works the way `lfs install ` does --
    one obvious verb per tool -- but there is exactly ONE install path: the one
    that creates the package user, resolves dependencies and tracks files."""
    pm = _find_packagemanager()
    if not pm:
        sys.stderr.write("packagemanager is not installed next to blfs; "
                         "install the tools first:\n"
                         "    lfs build-system install-tools --run\n")
        sys.exit(2)
    verb = "update" if args.update else "install"
    argv = pm + [verb] + list(args.packages) + list(args.rest or [])
    os.execvp(argv[0], argv)


def cmd_fetch(args):
    """Download (or refresh) a book into the cache."""
    sel = args.selector or DEFAULT_SELECTOR
    # SAY IT WHEN THERE IS NO SUCH BOOK.
    #
    # `blfs fetch stable-sysv` printed the routine "no book matched
    # 'stable-sysv'; using the cached ...-systemd" note and then "cached." --
    # so a request that fetched NOTHING reported success, and the user
    # reasonably believed they now had the sysv book.  There is no STABLE
    # 13.0 sysv book to have.  (The development book, `svn`, IS the SysV
    # edition -- r13.1-245 on 2026-09-23 -- so `blfs fetch svn` is the way
    # to a current SysV book.)
    avail = load_available()
    if sel not in avail and not sel.startswith(("http://", "https://")) \
            and not os.path.isfile(sel):
        try:
            avail = discover_all() or avail
        except Exception:
            pass
    if sel not in avail and not sel.startswith(("http://", "https://")) \
            and not os.path.isfile(sel):
        sys.stderr.write("no book is published as '%s'.\n" % sel)
        _sysv = sorted((k for k, v in avail.items()
                        if (v.get("init") or "") == "sysv"),
                       key=str)
        if _sysv:
            sys.stderr.write("  The editions that exist:  blfs books\n")
            sys.stderr.write("  Newest SysV edition:      %s\n" % _sysv[-1])
        sys.stderr.write("  The development book is SysV and current:"
                         "  blfs fetch svn\n")
        sys.stderr.write("  Nothing was fetched.\n")
        sys.exit(1)
    path = ensure_book(sel)
    print("cached: %s" % path)


def cmd_path(args):
    """Print the local book file a selector reads -- for other tools.

    packagemanager asks this once per run for every book it needs (the
    default, a stack's `from-book svn`).  Resolving `svn` through
    resolve_book asks the website every time, and a planner that passes
    `--book svn` to each of its hundred blfs calls would ask a hundred
    times.  A cached book is answered from disk; only a book that was never
    fetched goes to the network (or fails, with --offline)."""
    sel = args.selector or getattr(args, "book", None) or default_selector()
    if os.path.isfile(sel):
        print(os.path.abspath(sel))
        return
    p = _cached_for_selector(sel)
    if not (p and os.path.isfile(p)):
        if getattr(args, "offline", False):
            sys.stderr.write("no cached book for '%s' (blfs fetch %s)\n"
                             % (sel, sel))
            sys.exit(1)
        p = ensure_book(sel, quiet=True)
    print(p)


def cmd_import(args):
    """Add a local book file to the cache -- no network involved."""
    src = os.path.expanduser(args.file)
    if not os.path.isfile(src):
        sys.stderr.write("no such file: %s\n" % src)
        sys.exit(1)
    dst = os.path.join(store_dir(), "books")
    os.makedirs(dst, exist_ok=True)
    import shutil as _sh
    out = os.path.join(dst, os.path.basename(src))
    _sh.copy(src, out)
    print("imported: %s" % out)
    print("use it with:  blfs --book-file %s " % out)


def cmd_sources(args):
    """The download URLs for a package, exactly as the book gives them."""
    book = get_book(args)
    anchor = anchor_from_input(args.package)
    anchor = resolve_anchor(book, anchor)
    data = book.get(anchor)
    if data is None:
        sys.stderr.write("no package '%s' in the book (try: blfs search %s)\n"
                         % (args.package, args.package))
        sys.exit(1)
    urls = []
    if data.get("link"):
        urls.append(data["link"])
    urls += data.get("additional_links", []) or []
    if not urls:
        print("(the book lists no downloads for %s)" % args.package)
        return
    for u in urls:
        print(u)


# What makes a difference an INIT difference rather than version churn.
_INIT_WORDS_RE = re.compile(
    r"\b(systemd|systemctl|sysv|bootscript|init\.d|rc\.d|tmpfiles|sysusers|"
    r"unit[-_ ]?dir|--(en|dis)able-systemd|service)\b", re.I)


def cmd_diff_init(args):
    """Show where a SysV book and a systemd book disagree.

    The generated scripts come from ONE book.  Using the systemd book on a
    SysV system is workable -- a missing systemctl becomes a visible skip --
    right up to a page whose CONFIGURE line differs, and then it is not:

        BlueZ: configure: error: systemd system unit directory is required

    because the sysv page passes --disable-systemd and the systemd page has
    no reason to.  Per-package machine.conf entries paper over that one page
    at a time, and only after each has failed.  This says which pages differ
    BEFORE any of them are built -- and for one package, exactly how.
    """
    import difflib
    other = args.other
    if not os.path.isfile(other):
        url, local, _is_svn, _v = resolve_book(other)
        if not os.path.isfile(local):
            ensure_book(other)
        other = local
    here = get_book(args)
    # the same loader the rest of the tool uses: cached if the cache is
    # current, indexed once otherwise
    there = get_book(argparse.Namespace(book_file=other, book=None))
    # NAME BOTH SIDES, AND REFUSE TO COMPARE ONE WITH ITSELF.
    #
    # `blfs diff-init 12.4` reported "0 of 947 shared pages differ", which
    # was true and useless: the default selector (stable-sysv) has no book,
    # so it fell back to the newest cached one -- 12.4, the very book being
    # compared against.  A comparison that silently compares a thing with
    # itself is worse than an error.
    print("comparing:\n  A  %s\n  B  %s\n" % (here.path, there.path))
    if os.path.realpath(here.path) == os.path.realpath(there.path):
        sys.stderr.write("those are the same file -- nothing to compare.\n")
        sys.stderr.write("  Name the book to compare AGAINST, and make sure a\n")
        sys.stderr.write("  different one is in use here:\n")
        sys.stderr.write("    blfs --book-file  diff-init \n")
        sys.exit(1)
    names = args.packages or sorted(set(here.packages) & set(there.packages))
    # ONLY THE PAGES THIS MACHINE WILL EVER BUILD.
    #
    # 47 of 929 shared pages differ in their init handling, and most of them
    # are for software this system does not have and never will (GNOME, samba,
    # qemu, postgresql...).  The useful question is not "what differs" but
    # "what differs AND is on my machine", and the answer is on disk: a
    # package with an account has been built or is about to be.
    if getattr(args, "installed", False):
        root = os.environ.get("LFS_PKGUSR_ROOT", "/usr/src/pkgusr")
        pfx = os.environ.get("LFS_PKGUSR_PREFIX", "p_")
        try:
            have = {d[len(pfx):] for d in os.listdir(root) if d.startswith(pfx)}
        except OSError:
            have = set()
        if not have:
            sys.stderr.write("no package users under %s -- --installed has "
                             "nothing to filter by.\n" % root)
            sys.exit(1)
        names = [n for n in names if n.lower() in {h.lower() for h in have}]
    shown = 0
    for anchor in names:
        a = here.get(resolve_anchor(here, anchor_from_input(anchor)))
        try:
            b = there.get(resolve_anchor(there, anchor_from_input(anchor)))
        except SystemExit:
            b = None
        if not a or not b:
            continue
        ka = "\n".join([a.get("build_cmds") or "", a.get("install_root_cmds") or ""])
        kb = "\n".join([b.get("build_cmds") or "", b.get("install_root_cmds") or ""])
        if ka.strip() == kb.strip():
            continue
        # ACROSS VERSIONS, ALMOST EVERYTHING DIFFERS.
        #
        # There is no stable 13.0 sysv book, so a stable-vs-stable comparison
        # is 12.4-sysv against 13.0-systemd -- where most
        # pages differ for version reasons that have nothing to do with init.
        # Default to the differences that MENTION init, which is what the
        # question is; --all for the rest.
        if not getattr(args, "all", False):
            # A COMMENTED LINE IS NOT A COMMAND -- HERE TOO (1.14.23 again).
            #
            # Of eight pages this reported for a real system, four were
            # nothing: gcc, libxml2 and rust differed only in NOTE prose that
            # happens to say "systemd journal", "systemctl stop httpd.service"
            # or a book URL containing "13.0-systemd".  Judge the difference
            # on lines that RUN.
            _live = lambda t: [l for l in t.splitlines()
                               if l.strip() and not l.strip().startswith("#")]
            _delta = "\n".join(l for l in difflib.ndiff(_live(kb), _live(ka))
                               if l[:1] in "+-")
            if not _INIT_WORDS_RE.search(_delta):
                continue
        shown += 1
        if args.packages:
            print("=== %s ===" % anchor)
            for line in difflib.unified_diff(
                    kb.splitlines(), ka.splitlines(),
                    fromfile="%s (%s)" % (anchor, os.path.basename(other)),
                    tofile="%s (%s)" % (anchor, os.path.basename(here.path)),
                    lineterm=""):
                print(line)
            print()
        else:
            print("  %-28s commands differ" % anchor)
    if not args.packages:
        _what = "differ" if getattr(args, "all", False) \
            else "differ in their init handling"
        print("\n%d of %d shared pages %s." % (shown, len(names), _what))
        if not getattr(args, "all", False):
            print("(--all compares everything, including version churn)")
        print("For one of them:  blfs diff-init %s " % args.other)
    return 0


def cmd_books(args):
    if getattr(args, "rebuild_cache", False):
        book = get_book(args, rebuild=True)
        print(f"Re-indexed {book.path}: {len(book.packages)} packages cached.")
        return
    cfg = load_config()
    default = cfg.get("default", DEFAULT_SELECTOR)
    print(f"Store:   {store_dir()}")
    print(f"Default: {default}\n")

    print("Cached books:")
    found = False
    for f in sorted(os.listdir(books_dir())):
        if f.endswith(".meta") or f.endswith(".part"):
            continue
        found = True
        meta = _load_meta(os.path.join(books_dir(), f))
        # NO .meta IS NOT "?".  A book imported by hand, or a dated svn
        # download, has no record -- but its NAME says what it is, and the
        # file's date says when it came.
        _fv, _fi = parse_book_filename(f)
        _tm = re.search(r"html-(\d{4}-\d{2}-\d{2})\.tar\.", f)
        if _tm:
            _gs, _gv = "(svn source)", _tm.group(1)
        elif _fv != "unknown":
            _gs, _gv = _fv + ("-systemd" if _fi == "systemd" else ""), _fv
        else:
            _gs, _gv = "?", "?"
        sel = meta.get("selector", _gs)
        ver = meta.get("version", _gv)
        try:
            _mt = meta.get("fetched") or os.path.getmtime(os.path.join(books_dir(), f))
        except OSError:
            _mt = 0
        when = time.strftime("%Y-%m-%d %H:%M", time.localtime(_mt))
        mark = " (default)" if sel == default else ""
        print(f"  {sel:<18} version {ver:<8} fetched {when}{mark}  [{f}]")
    if not found:
        print("  (none yet)")
    else:
        print("  (which of these anything reads, and removing the rest:"
              " packagemanager books)")

    # --brief is the wrapper's contract: the Default line and the cached
    # list, NOTHING live.  packagemanager reads the book name through this,
    # and the auto-discovery below is for a person at a terminal -- routed
    # through the wrapper it made every `packagemanager update` probe the
    # site, twice, and print the probes' notes.
    if getattr(args, "brief", False):
        return

    def _print_available(books, note):
        print(note)
        for sel, rec in sorted(books.items()):
            mark = " (default)" if sel == default else ""
            print(f"  {sel:<22} version {rec['version']:<8} [{rec['init']}]{mark}")
        print(f"\nPick one with:  blfs set-default ")

    # Discovery is automatic: with a connection, the list is live; without
    # one, the cached list from the last discovery is shown and SAYS it is
    # cached.  --discover is kept as the explicit refresh (it reports the
    # failure loudly instead of falling back).
    if args.discover:
        try:
            _print_available(discover_all(),
                             "\nAvailable books (from the downloads index):")
        except Exception as e:
            print(f"  could not read {downloads_index_url()} ({e})")
            return
    else:
        try:
            _print_available(discover_all(),
                             "\nAvailable books (from the downloads index):")
        except Exception:
            avail = load_available()
            if avail:
                _print_available(avail, "\nAvailable books (offline -- cached "
                                        "from the last discovery):")
            else:
                print("\n(offline, and no cached discovery yet -- re-run with "
                      "a connection to list what the site offers)")


def cmd_set_default(args):
    cfg = load_config()
    cfg["default"] = args.selector
    save_config(cfg)
    print(f"Default book set to: {args.selector}")
    # A default that is not cached is a decision waiting to be silently
    # overridden: every lookup would fall back to the closest cached book.
    # That happened -- set-default 13.0-systemd, and update read 12.4 with
    # the note swallowed.  So the book is cached NOW, while the person who
    # made the choice is looking.
    try:
        ensure_book(args.selector)
    except Exception as e:
        warn(f"note: could not cache '{args.selector}' ({e}).")
        warn("      Until it is cached, every lookup falls back to the "
             "closest cached book, and says so.")
        warn(f"      Fetch it when there is a connection:  blfs fetch "
             f"{args.selector}")


def cmd_search(args):
    book = get_book(args)
    needle = args.name.lower().replace(" ", "-")
    hits = []
    for anchor, data in sorted(book.packages.items()):
        if needle in (anchor + " " + data["title"]).lower():
            hits.append((anchor, data["title"], data["is_module"]))
    if not hits:
        print(f"No package matching '{args.name}'.")
        return
    for anchor, title, is_module in hits:
        kind = "module " if is_module else "package"
        print(f"  [{kind}] {anchor:<28} {title}")
    print(f"\n{len(hits)} result(s).")


def cmd_script(args):
    # `blfs script ` has always BEEN fetch; `blfs script fetch `
    # is accepted so the verb reads the same across lfs, blfs and
    # packagemanager.
    _t = list(args.target)
    if _t and _t[0] == "fetch" and len(_t) > 1:
        _t = _t[1:]
    # SEVERAL PACKAGES, ONE BOOK.  The parser has always taken a list and this
    # used _t[0] and dropped the rest, so packagemanager regenerating a plan
    # ran one `blfs script` per package -- 151 subprocesses, each loading and
    # indexing the book again.  Loading it once and writing them all is the
    # same work minus 150 book loads.
    book = get_book(args)
    ver = _book_version_of(book)
    os.makedirs(args.outdir, exist_ok=True)
    written = []
    for target in _t:
        anchor = anchor_from_input(target)
        anchor = resolve_anchor(book, anchor)
        data = book.get(anchor)
        if data is None:
            print(f"ERROR: could not find anchor '{anchor}' in the book.")
            print("       run:  blfs debug " + target)
            if len(_t) == 1:
                sys.exit(1)
            continue
        src = target if target.startswith("http") else f"#{anchor}"
        filename, script = render_script(data, src, ver)
        path = os.path.join(args.outdir, filename)
        with open(path, "w") as f:
            f.write(script)
        print(f"Created {path}")
        try:
            subprocess.run(["bash", "-n", path], check=True)
            print("Bash syntax OK.")
        except subprocess.CalledProcessError:
            print("!!! bash syntax error -- review the file")
        written.append(path)
    if not written:
        sys.exit(1)
    return written[0] if len(written) == 1 else written


def cmd_deps(args):
    book = get_book(args)
    anchor = anchor_from_input(args.name)
    anchor = resolve_anchor(book, anchor)
    if book.get(anchor) is None:
        print(f"ERROR: '{anchor}' not found.  try: blfs debug {args.name}")
        sys.exit(1)
    kinds = selected_kinds(args)

    if args.tree:
        printed = set()

        def show(a, prefix="", last=True):
            data = book.get(a)
            title = data["title"] if data else a + " (missing)"
            connector = "└─ " if last else "├─ "
            print(prefix + (connector if prefix else "") + title)
            if a in printed or data is None:
                if a in printed and data is not None:
                    print(prefix + ("   " if last else "│  ") + "   … (already shown)")
                return
            printed.add(a)
            deps = deps_of(data, kinds)
            child_prefix = prefix + ("   " if last else "│  ")
            for i, d in enumerate(deps):
                is_last = i == len(deps) - 1
                if d["anchor"]:
                    show(d["anchor"], child_prefix, is_last)
                else:
                    label = d["external"] or d["title"]
                    conn = "└─ " if is_last else "├─ "
                    print(child_prefix + conn + label + f"  [{d['kind']}, external]")
        show(anchor)
        return

    order, edges, missing, via = walk_deps(book, anchor, kinds)
    print(f"Install order (dependencies first) [{'+'.join(kinds)}]:")
    for a in order:
        print(f"  {book.get(a)['title']}")
    if missing:
        print("\nOut-of-book / missing:")
        for m in sorted(set(missing)):
            print(f"  {m}")


def cmd_versions(args):
    """Machine-readable: every package as 'anchorname_version', in ONE call
    (so callers don't spawn a process per package)."""
    book = get_book(args)
    for anchor in sorted(book.packages):
        data = book.get(anchor)
        if data:
            print(f"{anchor}\t{data['name_version']}")


def _reverse_graph(book, kinds):
    """anchor -> set of anchors that directly depend on it (of the given kinds)."""
    rev = {}
    for anchor, data in book.packages.items():
        for d in deps_of(data, kinds):
            if d.get("anchor"):
                rev.setdefault(d["anchor"], set()).add(anchor)
    return rev


def cmd_rdeps(args):
    """Reverse dependencies: packages that (recursively) depend ON the target --
    i.e. what may need rebuilding when the target is updated."""
    book = get_book(args)
    target = anchor_from_input(args.name)
    if book.get(target) is None:
        print(f"ERROR: '{target}' not found.  try: blfs debug {args.name}")
        sys.exit(1)
    rev = _reverse_graph(book, selected_kinds(args))
    order, seen, queue = [], set(), [target]
    while queue:
        cur = queue.pop(0)
        for dependent in sorted(rev.get(cur, ())):
            if dependent not in seen:
                seen.add(dependent)
                order.append(dependent)
                queue.append(dependent)
    seen_anchors = {a for a in order}
    if getattr(args, "anchors", False):
        for a in order:
            data = book.get(a)
            print(f"{a}\t{data['name_version'] if data else ''}")
        return
    print(f"{len(order)} package(s) (recursively) depend on {target}:")
    for a in order:
        data = book.get(a)
        print(f"  {a:<28} {data['name_version'] if data else ''}")


def cmd_order(args):
    book = get_book(args)
    anchor = anchor_from_input(args.name)
    anchor = resolve_anchor(book, anchor)
    if book.get(anchor) is None:
        print(f"ERROR: '{anchor}' not found.  try: blfs debug {args.name}")
        sys.exit(1)
    kinds = selected_kinds(args)
    _ign = [x for x in (getattr(args, "ignore", "") or "").split(",") if x]
    _ign = [resolve_anchor(book, anchor_from_input(x)) for x in _ign]

    order, edges, missing, via = walk_deps(book, anchor, kinds, ignore=_ign)

    # machine-readable mode: one TSV line per package in install order,
    # "anchorname_versionkindparent_anchor" (root has empty
    # kind/parent), then "#missingnamekindparent".
    seen_anchors = {a for a in order}
    if getattr(args, "anchors", False):
        for a in order:
            parent, kind = ("", "") if a == anchor else via.get(a, ("", ""))
            print(f"{a}\t{book.get(a)['name_version']}\t{kind}\t{parent}")
        for m in sorted(set(missing)):
            parent, kind = via.get(m, ("", ""))
            print(f"#missing\t{m}\t{kind}\t{parent}")
        # a dependant built before something it recommends, because a cycle
        # left no order that satisfies both -- the caller decides what to say
        for _a, _d in _CYCLE_DROPS:
            if _a in seen_anchors and _d in seen_anchors:
                print(f"#cycle\t{_a}\t{_d}")
        for _parent, _ign in getattr(walk_deps, "ignored_required", ()):
            if _parent in seen_anchors:
                print(f"#ignored-required\t{_parent}\t{_ign}")
        return

    os.makedirs(args.outdir, exist_ok=True)
    order_file = os.path.join(args.outdir, "install_order.txt")

    lines = []
    for a in order:
        data = book.get(a)
        lines.append(data["name_version"])
        if args.scripts:
            fn, script = render_script(data, f"#{a}", _book_version_of(book))
            with open(os.path.join(args.outdir, fn), "w") as f:
                f.write(script)

    with open(order_file, "w") as f:
        f.write("\n".join(lines) + "\n")

    print(f"Install order ({len(order)} packages) [{'+'.join(kinds)}] "
          f"written to {order_file}")
    for ln in lines:
        print(f"  {ln}")
    if args.scripts:
        print(f"\nGenerated {len(order)} install scripts in {args.outdir}")
    if missing:
        print("\nOut-of-book / missing (handle manually):")
        for m in sorted(set(missing)):
            print(f"  {m}")


def cmd_debug(args):
    book = get_book(args)
    anchor = anchor_from_input(args.target)
    print(f"input          : {args.target}")
    print(f"anchor         : {anchor}")
    # WHICH BOOK AM I EVEN READING.  The first question when a page that
    # plainly exists upstream is "not in the book" is whether this is the
    # book you think it is, and how much of it parsed.
    try:
        _st = os.stat(book.path)
        print("book           : %s" % book.path)
        print("                 %.1f MB, modified %s"
              % (_st.st_size / 1048576.0,
                 time.strftime("%Y-%m-%d %H:%M", time.localtime(_st.st_mtime))))
    except OSError:
        print("book           : %s (cannot stat it)" % book.path)
    print("pages parsed   : %d" % len(book.packages))

    anchor = resolve_anchor(book, anchor)
    data = book.get(anchor)
    if data is None:
        # NAME THE TEST THAT FAILED.  "not an installable package (no package
        # block + download link)" covered three different causes with three
        # different fixes, and told you which of them applied: none.
        soup = book.soup
        in_index = anchor in get_index(soup)
        node = soup.find(attrs={"id": anchor}) or soup.find("a", attrs={"name": anchor})
        print("in page index  : %s" % ("yes" if in_index else "no"))
        if node is None:
            print("resolve        : NOT FOUND -- no element in this book carries "
                  "that id or name")
            near = sorted({(e.get("id") or e.get("name"))
                           for e in soup.find_all(attrs={"id": True})
                           if anchor in (e.get("id") or "")}
                          | {a.get("name") for a in soup.find_all("a", attrs={"name": True})
                             if anchor in (a.get("name") or "")})
            near = [n for n in near if n and n != anchor][:8]
            if near:
                print("similar ids    : %s" % ", ".join(near))
                print("hint           : the page is there under another anchor; "
                      "use one of those")
            elif re.search(re.escape(anchor), soup.get_text(" ", strip=True)[:4000000],
                           re.IGNORECASE):
                print("hint           : the NAME appears in the text but no anchor "
                      "matches -- likely a different book edition")
            else:
                print("hint           : this book really does not have it.  A tar or "
                      "git entry with build=/install= is the way in.")
        else:
            print("resolve        : found <%s id=%s>, and it is not accepted as a "
                  "package.  Which test fails:" % (node.name, node.get("id")))
            content = node if node.name == "div" else node.find_parent(
                "div", class_=["sect1", "sect2", "sect3", "sect4", "sect5"])
            if content is None:
                print("  content      : no enclosing section div")
            else:
                pkg_div = content.find("div", class_="package", recursive=False)
                print("  package block: %s" % ("present" if pkg_div is not None
                                               else "MISSING (a direct child "
                                                    " is required)"))
                link = extract_link(content).strip() if pkg_div is not None else ""
                if pkg_div is not None:
                    print("  download link: %s" % (link or "MISSING (no upstream tarball url)"))
                sa = sect_anchor(content)
                print("  titles it    : %s" % ("yes" if (sa == anchor
                                               or content.get("id") == anchor)
                                               else "no -- %r titles that section, so "
                                                    "this is a subsection anchor" % sa))
        print("hint           : blfs search " + anchor)
        return

    print(f"title          : {data['title']}")
    print(f"is_module      : {data['is_module']}  group={data['group']}")
    print(f"link           : {data['link']}")
    print(f"md5            : {data['md5']}")
    print(f"required       : {[d['title'] for d in data['deps'] if d['kind']=='required']}")
    print(f"recommended    : {[d['title'] for d in data['deps'] if d['kind']=='recommended']}")
    print(f"optional       : {[d['title'] for d in data['deps'] if d['kind']=='optional']}")

    if args.raw:
        pkg = resolve_package(book.soup, anchor)
        if pkg is not None:
            print("\n--- content section (first ~1600 chars) ---")
            print(str(pkg["content"])[:1600])


# --------------------------------------------------------------------------- #
# cli
# --------------------------------------------------------------------------- #

BLFS_HELP_EPILOG = """\
commands, by what you are doing:

  find packages
    search               find a package in the book
    versions []           every package and its version
    deps / rdeps          what it needs / what needs it
    order                 the full build order, dependencies first
    sources               its download URLs, as the book gives them

  build and install (packagemanager owns the package users)
    install  [...]        build as its package user; hands over
    update  [...]         rebuild from the current book; hands over
    script                just write the install script

  books
    books                      list the books available
    fetch []         download one into the cache
    import               add a local book file (no network)
    set-default      the book used when none is named
    debug                 why a package resolves the way it does

Dry run is the default everywhere; --run applies.
"""


def main():
    # Pre-parse the book selectors from anywhere in argv (before OR after the
    # subcommand), so both `blfs --book-file X script vim` and
    # `blfs script --book-file X vim` work.
    pre = argparse.ArgumentParser(add_help=False)
    pre.add_argument("--book")
    pre.add_argument("--book-file")
    book_args, remaining = pre.parse_known_args()

    ap = argparse.ArgumentParser(prog="blfs", description="BLFS book tool")
    ap.add_argument("-V", "--version", action="version",
                    version=f"blfs {BLFS_VERSION} (build {_build_id()}, "
                            f"cache v{CACHE_VERSION}, "
                            f"script v{SCRIPT_VERSION})")
    ap.add_argument("--book", help="book selector or url for this run (overrides default)")
    ap.add_argument("--book-file", help="use a local nochunks html file (offline/testing)")
    ap.epilog = _colour_epilog(BLFS_HELP_EPILOG)
    ap.formatter_class = argparse.RawDescriptionHelpFormatter
    sub = ap.add_subparsers(dest="command", required=True, metavar="")

    p = sub.add_parser("versions", help="dump every anchor + version (machine-readable)")
    p.set_defaults(func=cmd_versions)

    p = sub.add_parser("rdeps", help="reverse deps: what depends on a package "
                       "(needs rebuilding when it changes)")
    p.add_argument("name")
    p.add_argument("--anchors", action="store_true", help="machine-readable TSV")
    p.add_argument("--no-recommended", action="store_true")
    p.add_argument("--optional", action="store_true")
    p.set_defaults(func=cmd_rdeps)

    p = sub.add_parser("install", help="build and install (hands over to "
                       "packagemanager, which owns the package users)")
    p.add_argument("packages", nargs="+")
    p.add_argument("rest", nargs=argparse.REMAINDER,
                   help="passed through, e.g. --recursive --run --yes")
    p.set_defaults(func=cmd_install, update=False)

    p = sub.add_parser("update", help="rebuild from the current book "
                       "(hands over to packagemanager)")
    p.add_argument("packages", nargs="+")
    p.add_argument("rest", nargs=argparse.REMAINDER)
    p.set_defaults(func=cmd_install, update=True)

    p = sub.add_parser("fetch", help="download a book into the cache")
    p.add_argument("selector", nargs="?", help="e.g. stable-systemd, 12.4, svn")
    p.set_defaults(func=cmd_fetch)

    p = sub.add_parser("path", help="print the local file a book selector "
                       "reads (cached: no network)")
    p.add_argument("selector", nargs="?", help="e.g. svn; default: the default")
    p.add_argument("--offline", action="store_true",
                   help="fail instead of fetching a book that is not cached")
    p.set_defaults(func=cmd_path)

    p = sub.add_parser("import", help="add a local book file (no network)")
    p.add_argument("file")
    p.add_argument("--selector", help="name to file it under")
    p.set_defaults(func=cmd_import)

    p = sub.add_parser("sources", help="a package's download URLs")
    p.add_argument("package")
    p.set_defaults(func=cmd_sources)

    p = sub.add_parser("diff-init",
                       help="which pages differ between this book and another "
                            "(e.g. the sysv book vs the systemd one)")
    p.add_argument("other", help="selector (stable-sysv) or a book file")
    p.add_argument("packages", nargs="*",
                   help="show the command diff for these; omit to list every "
                        "page that differs")
    p.add_argument("--all", action="store_true",
                   help="every difference, not only the init-related ones "
                        "(across two book VERSIONS most pages differ anyway)")
    p.add_argument("--installed", action="store_true",
                   help="only pages this machine has a package user for -- "
                        "what differs AND concerns you")
    p.set_defaults(func=cmd_diff_init)

    p = sub.add_parser("books", help="list/inspect books")
    p.add_argument("--discover", action="store_true", help="probe the live site")
    p.add_argument("--brief", action="store_true",
                   help="default + cached list only, no network (for scripts)")
    p.add_argument("--rebuild-cache", action="store_true",
                   help="re-index the current book (after upgrading blfs)")
    p.set_defaults(func=cmd_books)

    p = sub.add_parser("set-default", help="set the default book")
    p.add_argument("selector")
    p.set_defaults(func=cmd_set_default)

    p = sub.add_parser("search", help="find packages / modules")
    p.add_argument("name")
    p.set_defaults(func=cmd_search)

    p = sub.add_parser("script", help="write an install_ script")
    p.add_argument("target", nargs="+",
                   help="package name, or a BLFS url (optionally #fragment); "
                        "'fetch ' is accepted too, matching the other "
                        "tools' script commands")
    p.add_argument("-o", "--outdir", default=".")
    p.set_defaults(func=cmd_script)

    p = sub.add_parser("deps", help="show dependencies")
    p.add_argument("name")
    p.add_argument("--tree", action="store_true")
    p.add_argument("--no-recommended", action="store_true",
                   help="follow required deps only (skip recommended)")
    p.add_argument("--optional", action="store_true",
                   help="also follow optional dependencies")
    p.set_defaults(func=cmd_deps)

    p = sub.add_parser("order", help="recursive install order (+ optional scripts)")
    p.add_argument("name")
    p.add_argument("-o", "--outdir", default=".")
    p.add_argument("--scripts", action="store_true", help="also emit every install script")
    p.add_argument("--no-recommended", action="store_true",
                   help="follow required deps only (skip recommended)")
    p.add_argument("--optional", action="store_true",
                   help="also follow optional dependencies")
    p.add_argument("--anchors", action="store_true",
                   help="machine-readable: print 'anchorname_version' in order")
    p.add_argument("--ignore", default="",
                   help="comma-separated packages not to build -- their own "
                        "dependencies are not walked into either")
    p.set_defaults(func=cmd_order)

    p = sub.add_parser("debug", help="explain how an anchor resolves")
    p.add_argument("target")
    p.add_argument("--raw", action="store_true", help="also dump raw HTML around the anchor")
    p.set_defaults(func=cmd_debug)

    args = ap.parse_args(remaining)
    # book selectors come from the pre-parser (either position wins over none)
    args.book = book_args.book
    args.book_file = book_args.book_file
    args.func(args)


if __name__ == "__main__":
    try:
        main()
    except BrokenPipeError:
        # output was piped into something like `head` that closed early
        try:
            sys.stdout.close()
        except Exception:
            pass
        os._exit(0)
    except KeyboardInterrupt:
        sys.exit(130)
    except RuntimeError as e:
        # An expected problem (no book cached, no network, a bad selector) is
        # not a bug -- print it plainly instead of a stack trace.
        sys.stderr.write("error: %s\n" % e)
        sys.exit(1)