Packagemanager-LFS-PackageUser-System git · main
Package-user based package manager for LFS/BLFS
Shell 63.7% Markdown 35.8%git clone https://git.christianimmanuel.de/linux-from-scratch/Packagemanager-LFS-PackageUser-System.gitwget https://git.christianimmanuel.de/linux-from-scratch/Packagemanager-LFS-PackageUser-System/archive/Packagemanager-LFS-PackageUser-System.tar.gzlfs build packageuser system and stuff
Makefile | 84 + README.md | 251 +- blfs | 348 ++- last_build_step.sh | 251 ++ lfs | 5966 ++++++++++++++++++++++++++++++++++++++++++++++++ lfs-completion.bash | 134 ++ lfs-helper | 4250 ++++++++++++++++++++++++++++++++++ packagemanager | 1295 ++++++++++- packagemanager_install | 312 ++- test_lfs_crosschain.sh | 4242 ++++++++++++++++++++++++++++++++++ test_packagemanager.sh | 228 ++ 11 files changed, 17200 insertions(+), 161 deletions(-) diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..52e27a3 --- /dev/null +++ b/Makefile @@ -0,0 +1,84 @@ +# lfs-pkgusr -- install the package-user toolchain. +# +# make install -> /usr/local +# make install PREFIX=/usr -> /usr +# make uninstall +# make check -> run the regression tests +# +# DESTDIR is honoured, so this works for staged installs and for installing +# into a tree that is not the running system: +# make install DESTDIR=/mnt/lfs PREFIX=/usr + +PREFIX ?= /usr/local +DESTDIR ?= +BINDIR := $(DESTDIR)$(PREFIX)/bin +SHAREDIR := $(DESTDIR)$(PREFIX)/share/lfs-pkgusr +COMPDIR := $(DESTDIR)$(PREFIX)/share/bash-completion/completions +SYSCONFDIR ?= $(DESTDIR)/etc + +# The four tools, plus the bash engine packagemanager calls. +TOOLS := lfs lfs-helper packagemanager packagemanager_install blfs +COMPLETION := lfs-completion.bash +LAST_STEP := last_build_step.sh +TESTS := test_lfs_crosschain.sh + +INSTALL ?= install + +.PHONY: all install uninstall check help + +all: help + +help: + @echo "make install [PREFIX=/usr] [DESTDIR=...] install the tools" + @echo "make uninstall [PREFIX=/usr] remove them" + @echo "make check run the tests" + @echo "" + @echo "installs into: $(PREFIX)/bin" + +install: + @echo "==> tools -> $(BINDIR)" + $(INSTALL) -d $(BINDIR) + @for t in $(TOOLS); do \ + if [ -f "$$t" ]; then \ + $(INSTALL) -m 755 "$$t" "$(BINDIR)/$$t" && echo " $$t"; \ + else \ + echo " !! missing: $$t" >&2; exit 1; \ + fi; \ + done + @echo "==> bash completion -> $(COMPDIR)" + $(INSTALL) -d $(COMPDIR) + $(INSTALL) -m 644 $(COMPLETION) $(COMPDIR)/lfs + @for t in blfs packagemanager lfs-helper; do \ + ln -sf lfs "$(COMPDIR)/$$t"; \ + done + @echo "==> last build step -> $(SYSCONFDIR)/lfs" + $(INSTALL) -d $(SYSCONFDIR)/lfs + @if [ -f "$(SYSCONFDIR)/lfs/$(LAST_STEP)" ]; then \ + echo " kept your existing $(SYSCONFDIR)/lfs/$(LAST_STEP)"; \ + $(INSTALL) -m 644 $(LAST_STEP) "$(SYSCONFDIR)/lfs/$(LAST_STEP).new"; \ + echo " new version alongside it as $(LAST_STEP).new"; \ + else \ + $(INSTALL) -m 755 $(LAST_STEP) "$(SYSCONFDIR)/lfs/$(LAST_STEP)"; \ + fi + @echo "==> tests -> $(SHAREDIR)" + $(INSTALL) -d $(SHAREDIR) + $(INSTALL) -m 644 $(TESTS) $(SHAREDIR)/ 2>/dev/null || true + @echo "" + @echo "Installed. Check with:" + @echo " $(PREFIX)/bin/lfs --version" + @echo " $(PREFIX)/bin/lfs --help" + +uninstall: + @for t in $(TOOLS); do rm -fv "$(BINDIR)/$$t"; done + @for t in lfs blfs packagemanager lfs-helper; do \ + rm -fv "$(COMPDIR)/$$t"; \ + done + rm -rfv $(SHAREDIR) + @echo "" + @echo "Left alone (they are yours):" + @echo " $(SYSCONFDIR)/lfs/$(LAST_STEP)" + @echo " /usr/share/lfs books, settings, snapshots" + @echo " /etc/pkgusr packagemanager settings" + +check: + @bash $(TESTS) ./lfs diff --git a/README.md b/README.md index e6e65c5..0ffd5e7 100644 --- a/README.md +++ b/README.md @@ -1,116 +1,161 @@ -# pkgusr package manager for BLFS / LFS - -> **🎲 100% vibecode** — this whole toolchain was built by prompting an AI, not -> hand-written line by line. It's not a polished distro tool. That said, I -> actually run it on my own BLFS/LFS system, so it's vibecode I trust enough to -> point at my own machine — still, read the code and test on something -> disposable before you point it at yours. - -A small toolchain for building and maintaining a BLFS/LFS system with the -**package-user** scheme: every package is built and owned by its own user, so -you always know which package installed which file, and removing a package is -just removing its files. - -It works **offline** from a downloaded copy of the BLFS book — no network needed -to resolve dependencies, versions, or build commands. - -## The package-user scheme - -Instead of installing everything as `root`, each package gets its own -unprivileged user that owns that package's files. Because ownership maps 1:1 to a -package, you can list exactly what a package installed (via `find` / the -`pkg.lst` manifest), spot stray files, and uninstall by removing one user's -files — no separate package database to drift out of sync. Shared write access to -install directories is granted through an `install` group, and packages that drop -files into another package's tree are tracked with **collector groups**. - -This is the classic LFS "More Control and Package Management" hint; if the idea -is new to you, read it first: -<https://www.linuxfromscratch.org/hints/downloads/files/more_control_and_pkg_man.txt> - -## The three programs - -- **`blfs`** — parses the BLFS book (one cached HTML copy) and answers questions - about it: versions, dependencies, build order, and it generates phased install - scripts. Machine-readable output; safe to script against. -- **`packagemanager`** — the thing you use day to day: install, update, verify, - and manage package-users, their scripts, and their groups. -- **`packagemanager_install`** — the privileged engine `packagemanager` calls to - run an install script *as the package user*. You normally don't call it - directly. - -Run any command with `-h`/`--help` for its options — this README stays short on -purpose and does **not** document every flag. - -## What makes it different - -- **One user per package.** Files are owned by the package's user; `verify` - tells you what's installed and *why* (see validation below). -- **Phased install scripts.** Generated scripts split into - `unpack | build | test | install | configure`. You can re-run a single phase — - e.g. `packagemanager script install <pkg>` reinstalls without recompiling. -- **Real install validation.** A package counts as *installed* only when it can - be validated: the script's `installed_program/library/directory` exist, a - versioned directory matches, or a per-package `validate` command reports the - right version (`verify --set-validate-command <pkg>`). Otherwise it's - *unvalidated* — looks installed but couldn't be confirmed. -- **Dry-run by default.** `install`/`update` show a plan first; add `--run` to - execute. Plans are cached, so `--run` reuses/resumes them. `--select` lets you - pick a subset (`1,3,5-8`, `all,!4`, `!4,!7`). -- **Safe updates.** `update` shows *required/recommended by which package*, - can rebuild reverse-dependencies (`-R`), and `--clean` removes old-version - files an upgrade left behind while keeping configs and the source tree - (install → clean → reapply install, so nothing current is lost). -- **Edit-tracked scripts.** Generated scripts carry a checksum, so - `script-status` shows whether you edited a script or the book moved ahead of - it. `migrate-scripts` upgrades unedited scripts automatically and opens - `vimdiff` (new on the left, yours on the right) only for the ones you changed. -- **Custom & git packages.** `template` scaffolds an install script for anything - not in the book, including git repos; `install`/`update --local <script>` build - from it. - -## Typical flow +# lfs-pkgusr + +Build and maintain an LFS/BLFS system where **every package is owned by its own +user**. + +> **100% vibecode, but tested.** Prompted into existence rather than hand +> written. 387 regression tests pass. It builds my own system — read it before +> you point it at yours. + +--- + +## What it does + +Each package gets an unprivileged user that owns the files it installs. + +- You always know which package installed which file. +- Removing a package is removing that user's files. +- A broken build cannot overwrite another package's files. + +Directories shared by several packages are handled by **collector groups** +(`<prefix>_<owner>`), so a package may install into another's directory without +owning it. + +Works **offline** from a downloaded copy of the books. + +--- + +## The four tools + +| Tool | Runs | Does | +|---|---|---| +| `lfs` | host | Drives the whole build, chapters 1-10 | +| `lfs-helper` | inside the chroot | Builds each package as its user (bash, no Python needed) | +| `packagemanager` | built system | Installs and updates packages | +| `blfs` | built system | Reads the BLFS book, generates install scripts | + +--- + +## Install ```sh -packagemanager config --collector-prefix nimgnu --user-prefix u --main-user <you> -packagemanager install <pkg> # dry-run plan (deps first) -packagemanager install <pkg> --run # build + install -packagemanager update # what's outdated -packagemanager update <pkg> --run # upgrade (add --clean to drop old files) -packagemanager verify # installed / unvalidated / not installed -packagemanager errors # failures from the last run + log paths +make install # to /usr/local +make install PREFIX=/usr +make uninstall ``` -## Configuration +Installs the four tools, the completion script, and `last_build_step.sh`. -`packagemanager config` shows and sets a few things once, up front: +**Requires:** Python 3.9+, bash, coreutils, tar. Book parsing needs +`beautifulsoup4` and `requests`. + +--- + +## Build a system + +```sh +lfs build-system session # partition, mount point, prefix, locale +lfs run # everything up to the chroot, then enter it +lfs-helper build-all # inside: build chapters 7-9 +``` + +`lfs run` continues from wherever it stopped. Run it again after a reboot, a +failure, or a cancel. + +| Command | Does | +|---|---| +| `lfs build-system next` | What is done, what comes next | +| `lfs build-system list` | Every step and its state | +| `lfs build-system snapshot save <name> --run` | Save the build | +| `lfs build-system snapshot restore <name> --run` | Put it back | +| `lfs build-system restart --run` | Delete it all and start over | + +--- + +## Use the built system + +```sh +packagemanager install <pkg> --recursive --run +packagemanager update <pkg> --run +packagemanager which-package /usr/bin/foo +blfs order <pkg> # build order, dependencies first +blfs sources <pkg> # download URLs +``` -- **`--collector-prefix`** — the prefix for *collector groups*. When package A - needs to install files into package B's directory, those files are made - group-owned by a collector group named `<prefix>_<owner>` (e.g. - `nimgnu_python-modules`), and A is added to it. This is how cross-package - installs stay tracked under the package-user scheme instead of silently - becoming root-owned. Pick a short, distinctive prefix — it namespaces every - collector group on your system. -- **`--user-prefix`** — the prefix for *shared users* you create to **run** - software (as opposed to the package-users that **build** it). A shared user is - named `<prefix>_<name>` (e.g. `u_media`) and can be granted supplementary - groups. Keeps run-time accounts clearly separate from package accounts. -- **`--main-user`** — your normal login account (used as a sensible default in a - few places). -- **`--editor`** / **`--difftool`** — the editor for `verify --set-validate-command` - and the diff tool for `migrate-scripts` (default `vimdiff`). +### Application users -Set them once, e.g.: +Separate from package users. An account a program **runs as**, so a browser +cannot read your files: ```sh -packagemanager config --collector-prefix nimgnu --user-prefix u --main-user <you> +packagemanager user create firefox --shared --launcher ``` -Storage locations are shown by `packagemanager paths`. +--- + +## Repair + +Everything is dry-run by default. `--run` applies. + +| Command | Fixes | +|---|---| +| `lfs-helper check-toolchain` | Can the compiler build anything? | +| `lfs-helper fix-ownership --run` | Give each package its files | +| `lfs-helper fix-orphans --run` | Files whose owner does not exist | +| `lfs-helper which-package <path>` | Which package installed this | +| `lfs-helper manifests` | What each package installed | +| `lfs-helper sort-users --run` | Sort passwd and group by id | + +--- + +## Conventions + +| Range | Used for | +|---|---| +| 9998 | `lfs` build user | +| 9999 | `install` group | +| 10000+ | Package users, in build order | +| 90000+ | Collector groups | + +Install directories are group-writable during the build and become sticky +(`o+t`) at the end, so packages can no longer overwrite each other. + +--- + +## Checking your version + +The tools print a fingerprint of their own contents: + +```sh +lfs --version # lfs 1.7.0 (build d64e55c) +lfs-helper --version # inside the chroot +``` + +Same build id means same code. The chroot copy refreshes automatically when you +enter it. + +--- + +## Tests + +```sh +bash test_lfs_crosschain.sh ./lfs +``` + +387 tests. Each encodes a bug that actually happened, with a comment explaining +what broke. + +--- + +## Caveats + +- The kernel and the bootloader's kernel line are yours to finish. +- `lfs-helper` and `packagemanager_install` duplicate some logic. Unifying them + is worthwhile but not done. +- Tested on x86_64 only. + +--- -## Notes +## Licence -- Most operations need root (they create users, groups, and write system files). -- Set `BLFS_BOOK_FILE` to your book HTML, or point the store with `BLFS_STORE`. -- Not for reproducing distro packaging — this is for hand-built BLFS/LFS systems. +GPLv2-or-later. diff --git a/blfs b/blfs old mode 100755 new mode 100644 index 7a76549..99ffc4a --- a/blfs +++ b/blfs @@ -45,6 +45,13 @@ import hashlib import json import os import re +try: + # arrow keys, backspace and history inside input() prompts. Importing it + # is enough -- Python wires it into input() automatically. Optional: a + # Python built without readline still runs, just without line editing. + import readline # noqa: F401 +except ImportError: + pass import shutil import subprocess import sys @@ -53,8 +60,41 @@ import time import warnings from urllib.parse import urlsplit, unquote -import requests -from bs4 import BeautifulSoup +# Both of these are optional at IMPORT time. Base LFS ships neither, and a +# cached book can be parsed and turned into install scripts without ever +# touching the network -- so failing at startup would make the tool useless in +# exactly the situation it is most needed: +# ModuleNotFoundError: No module named 'requests' +try: + import requests +except ImportError: + requests = None + +try: + from bs4 import BeautifulSoup +except ImportError: + BeautifulSoup = None + + +def _need_requests(): + if requests is None: + sys.stderr.write( + "this command needs the 'requests' module, which base LFS does not\n" + "ship. Install it as a package user:\n" + " packagemanager pip install requests\n" + "Commands that only read an already-downloaded book work without it.\n") + sys.exit(2) + return requests + + +def _need_bs4(): + if BeautifulSoup is None: + sys.stderr.write( + "this command needs 'beautifulsoup4' to parse the book. Install it\n" + "as a package user:\n" + " packagemanager pip install beautifulsoup4\n") + sys.exit(2) + return BeautifulSoup # The nochunks book is XHTML; silence the (correct but noisy) heads-up. try: @@ -74,14 +114,63 @@ VALID_INITS = ("systemd", "sysv") # Bump whenever the extracted-data shape or extraction logic changes, so stale # on-disk caches are rebuilt automatically. -CACHE_VERSION = 3 +CACHE_VERSION = 4 # tool + generated-script format versions. Bump SCRIPT_VERSION when the shape # of generated install scripts changes (phases, dispatcher, ...), and # CACHE_VERSION when the parsed/cached data structure changes. -BLFS_VERSION = "1.0.0" +BLFS_VERSION = "1.7.0" + + +def _build_id(): + """A short fingerprint of this file's own contents. + + A hand-maintained version number goes stale the moment someone forgets to + bump it -- and the whole reason for printing a version is to answer "am I + running the code that was just fixed?". This cannot go stale: it changes + whenever the file does. + """ + import hashlib + try: + with open(os.path.abspath(__file__), "rb") as fh: + return hashlib.md5(fh.read()).hexdigest()[:7] + except OSError: + return "unknown" + SCRIPT_VERSION = 4 +def _tty_out(): + return sys.stdout.isatty() and os.environ.get("NO_COLOR") is None + + +_C_OFF = "\033[0m" +_C_HEAD = "\033[1m" +_C_GROUP = "\033[1;36m" +_C_CMD = "\033[0;32m" + + +def _colour_epilog(text): + """Colour the group headings in the help -- same scheme as `lfs`.""" + if not _tty_out(): + return text + out = [] + for line in text.split("\n"): + if line and not line.startswith(" ") and line.endswith(":"): + out.append("%s%s%s" % (_C_HEAD, line, _C_OFF)) + elif line.startswith(" ") and not line.startswith(" ") and line.strip(): + out.append("%s%s%s" % (_C_GROUP, line, _C_OFF)) + elif line.startswith(" ") and line.strip(): + m = re.match(r"^(\s+)(\S+(?: \S+)*?)(\s{2,}.*)$", line.rstrip()) + if m: + out.append("%s%s%s%s%s" % (m.group(1), _C_CMD, m.group(2), + _C_OFF, m.group(3))) + else: + out.append(line) + else: + out.append(line) + return "\n".join(out) + + def store_dir(): for candidate in (os.environ.get("BLFS_STORE"), "/usr/share/blfs", @@ -183,7 +272,7 @@ KNOWN_NAMED_BOOKS = ["svn", "systemd"] def _find_nochunks_at(base): """Return (filename, url) for a nochunks book under `base`, or None.""" try: - resp = requests.get(base, timeout=60) + resp = _need_requests().get(base, timeout=60) resp.raise_for_status() except Exception: return None @@ -199,7 +288,7 @@ def _find_svn_archive(base): (blfs-book-svn-html-YYYY-MM-DD.tar.xz); the date changes over time, so read the directory live and pick the newest. Returns (filename, url) or None.""" try: - resp = requests.get(base, timeout=60) + resp = _need_requests().get(base, timeout=60) resp.raise_for_status() except Exception: return None @@ -239,7 +328,7 @@ def discover_dir(selector): f"{SITE}/blfs/view/{selector}/"): url = base + fn try: - if requests.head(url, timeout=30, + if _need_requests().head(url, timeout=30, allow_redirects=True).status_code == 200: version, init2 = parse_book_filename(fn) return {"selector": selector, "url": url, "filename": fn, @@ -247,7 +336,54 @@ def discover_dir(selector): "init": init2, "archive": False} except Exception: continue - raise RuntimeError(f"no book for '{selector}'") + # Nothing on the site matched -- but a book may already be cached here. + # A book imported by hand (or copied in by `lfs build-system install-tools`) + # has no metadata recording which selector it is, so the lookup failed and + # every package came back "not in the BLFS book" while the book sat right + # there. Fall back to what is actually on disk. + cached = _cached_book_files() + if cached: + pick = _best_cached_for(selector, cached) + sys.stderr.write( + "note: no book matched '%s'; using the cached %s\n" + % (selector, os.path.basename(pick))) + fn = os.path.basename(pick) + version, init2 = parse_book_filename(fn) + return {"selector": selector, "url": None, "filename": fn, + "version": version if version != "unknown" else selector, + "init": init2, "archive": False} + raise RuntimeError( + "no book for '%s', and none is cached.\n" + " Download one: blfs fetch\n" + " or import a local copy: blfs import <file>" % selector) + + +def _cached_book_files(): + d = os.path.join(store_dir(), "books") + try: + return sorted(os.path.join(d, f) for f in os.listdir(d) + if f.endswith((".html", ".html.gz"))) + except OSError: + return [] + + +def _best_cached_for(selector, cached): + """Pick the cached book that best matches what was asked for. + + 'stable-systemd' should prefer a systemd book over a SysV one, and a + numbered release over the development snapshot.""" + want_systemd = "systemd" in selector + def score(p): + n = os.path.basename(p).lower() + s = 0 + if ("systemd" in n) == want_systemd: + s += 4 + if "svn" not in n: + s += 2 # a release beats the dev snapshot + if selector.lower() in n: + s += 8 # an exact selector match wins + return s + return sorted(cached, key=lambda p: (-score(p), p))[0] def book_record_from_selector(selector): @@ -278,7 +414,7 @@ def discover_all(): dirs = set(KNOWN_NAMED_BOOKS) for index in (downloads_index_url(), view_index_url()): try: - resp = requests.get(index, timeout=60) + resp = _need_requests().get(index, timeout=60) resp.raise_for_status() dirs.update(_extract_subdirs(resp.text)) except Exception: @@ -369,7 +505,15 @@ def resolve_book(selector): save_available(avail) is_svn = "svn" in selector.lower() or "svn" in str(version).lower() - filename = os.path.basename(urlsplit(url).path) + # A book that is already cached has no URL -- it was imported, or copied in + # with the tools. urlsplit(None) yields bytes, which then fails deep in + # os.path.join with "Can't mix strings and bytes"; take the filename the + # record carries instead. + if url is None: + filename = rec["filename"] if isinstance(rec, dict) and rec.get("filename") \ + else os.path.basename(_best_cached_for(selector, _cached_book_files())) + else: + filename = os.path.basename(urlsplit(url).path) return url, os.path.join(books_dir(), filename), is_svn, version @@ -506,7 +650,7 @@ def ensure_book(selector, quiet=False): if not have_tar: if not quiet: print(f"Fetching dev book ({selector}) archive: {url}") - resp = requests.get(url, timeout=300, stream=True) + resp = _need_requests().get(url, timeout=300, stream=True) resp.raise_for_status() tmp = local + ".part" with open(tmp, "wb") as f: @@ -542,7 +686,7 @@ def ensure_book(selector, quiet=False): if not quiet: print(f"Fetching book ({selector}): {url}") - resp = requests.get(url, headers=headers, timeout=180, stream=True) + resp = _need_requests().get(url, headers=headers, timeout=180, stream=True) if resp.status_code == 304: meta = _load_meta(local) meta["fetched"] = time.time() @@ -572,10 +716,10 @@ def load_soup(book_path): html = f.read() for parser in ("lxml", "html.parser"): try: - return BeautifulSoup(html, parser) + return _need_bs4()(html, parser) except Exception: continue - return BeautifulSoup(html, "html.parser") + return _need_bs4()(html, "html.parser") def get_book_soup(args): @@ -1009,7 +1153,7 @@ def _pre_command_text(pre): return "" if pre.find(class_="replaceable") is None: return pre.get_text().strip() - clone = BeautifulSoup(str(pre), "html.parser") + clone = _need_bs4()(str(pre), "html.parser") for rep in clone.select(".replaceable"): txt = rep.get_text().strip() if txt.startswith("<") and txt.endswith(">"): @@ -1071,21 +1215,42 @@ def extract_commands_phased(sec): if inst is None: return "", "" build, install = [], [] - for el in inst.children: - name = getattr(el, "name", None) - if name is None: + + # Walk the whole subtree in document order, not just the direct children. + # Packages that build several components -- gst-plugins-rs builds + # libgstdav1d and libgstgtk4 -- put each one in its own sub-section, so + # every command block sits one level down. Looking only at direct + # children found nothing at all and produced an empty, silently useless + # script. + seen = set() + for el in inst.find_all(["pre", "h3", "h4", "div"]): + if id(el) in seen: + continue + classes = el.get("class", []) or [] + name = el.name + + if name == "div": + # a note or warning: keep it as a comment, and don't descend into + # it again through its own children + if "admon" in classes or {"note", "warning", "important"} & set(classes): + for d in el.find_all(True): + seen.add(id(d)) + build.append("## NOTE: " + " ".join(el.get_text().split())) continue + if name == "pre": + # configuration blocks are a separate phase, handled elsewhere + if el.find_parent("div", class_="configuration"): + continue text = _pre_command_text(el) if not text: continue - (install if "root" in el.get("class", []) else build).append(text) - elif name in ("h3", "h4"): - build.append("\n### " + " ".join(el.get_text().split())) - elif name == "p": - build.append("\n# " + " ".join(el.get_text().split()) + "\n") - elif name == "div" and "admon" in el.get("class", []): - build.append("## NOTE: " + " ".join(el.get_text().split())) + (install if "root" in classes else build).append(text) + else: # h3 / h4 + title = " ".join(el.get_text().split()) + if title: + build.append("\n### " + title) + return "\n".join(build).strip(), "\n".join(install).strip() @@ -1522,6 +1687,75 @@ def build_script(sec, is_module, group, anchor, source_url): # subcommands # --------------------------------------------------------------------------- # +def _find_packagemanager(): + here = os.path.dirname(os.path.abspath(__file__)) + local = os.path.join(here, "packagemanager") + if os.path.isfile(local): + ex = bool(os.stat(local).st_mode & 0o111) + return [local] if ex else [sys.executable or "python3", local] + import shutil as _sh + return ["packagemanager"] if _sh.which("packagemanager") else None + + +def cmd_install(args): + """Delegate to packagemanager, which owns the package users. + + Kept here so `blfs install <pkg>` works the way `lfs install <pkg>` does -- + one obvious verb per tool -- but there is exactly ONE install path: the one + that creates the package user, resolves dependencies and tracks files.""" + pm = _find_packagemanager() + if not pm: + sys.stderr.write("packagemanager is not installed next to blfs; " + "install the tools first:\n" + " lfs build-system install-tools --run\n") + sys.exit(2) + verb = "update" if args.update else "install" + argv = pm + [verb] + list(args.packages) + list(args.rest or []) + os.execvp(argv[0], argv) + + +def cmd_fetch(args): + """Download (or refresh) a book into the cache.""" + sel = args.selector or DEFAULT_SELECTOR + ensure_book(sel) + print("cached.") + + +def cmd_import(args): + """Add a local book file to the cache -- no network involved.""" + src = os.path.expanduser(args.file) + if not os.path.isfile(src): + sys.stderr.write("no such file: %s\n" % src) + sys.exit(1) + dst = os.path.join(store_dir(), "books") + os.makedirs(dst, exist_ok=True) + import shutil as _sh + out = os.path.join(dst, os.path.basename(src)) + _sh.copy(src, out) + print("imported: %s" % out) + print("use it with: blfs --book-file %s <command>" % out) + + +def cmd_sources(args): + """The download URLs for a package, exactly as the book gives them.""" + book = get_book(args) + anchor = anchor_from_input(args.package) + data = book.get(anchor) + if data is None: + sys.stderr.write("no package '%s' in the book (try: blfs search %s)\n" + % (args.package, args.package)) + sys.exit(1) + urls = [] + if data.get("link"): + urls.append(data["link"]) + urls += data.get("additional_links", []) or [] + if not urls: + print("(the book lists no downloads for %s)" % args.package) + return + for u in urls: + print(u) + + def cmd_books(args): if getattr(args, "rebuild_cache", False): book = get_book(args, rebuild=True) @@ -1797,6 +2031,32 @@ def cmd_debug(args): # cli # --------------------------------------------------------------------------- # +BLFS_HELP_EPILOG = """\ +commands, by what you are doing: + + find packages + search <text> find a package in the book + versions [<pkg>] every package and its version + deps / rdeps <pkg> what it needs / what needs it + order <pkg> the full build order, dependencies first + sources <pkg> its download URLs, as the book gives them + + build and install (packagemanager owns the package users) + install <pkg> [...] build as its package user; hands over + update <pkg> [...] rebuild from the current book; hands over + script <pkg> just write the install script + + books + books list the books available + fetch [<selector>] download one into the cache + import <file> add a local book file (no network) + set-default <selector> the book used when none is named + debug <pkg> why a package resolves the way it does + +Dry run is the default everywhere; --run applies. +""" + + def main(): # Pre-parse the book selectors from anywhere in argv (before OR after the # subcommand), so both `blfs --book-file X script vim` and @@ -1808,11 +2068,14 @@ def main(): ap = argparse.ArgumentParser(prog="blfs", description="BLFS book tool") ap.add_argument("--version", action="version", - version=f"blfs {BLFS_VERSION} (cache v{CACHE_VERSION}, " + version=f"blfs {BLFS_VERSION} (build {_build_id()}, " + f"cache v{CACHE_VERSION}, " f"script v{SCRIPT_VERSION})") ap.add_argument("--book", help="book selector or url for this run (overrides default)") ap.add_argument("--book-file", help="use a local nochunks html file (offline/testing)") - sub = ap.add_subparsers(dest="command", required=True) + ap.epilog = _colour_epilog(BLFS_HELP_EPILOG) + ap.formatter_class = argparse.RawDescriptionHelpFormatter + sub = ap.add_subparsers(dest="command", required=True, metavar="<command>") p = sub.add_parser("versions", help="dump every anchor + version (machine-readable)") p.set_defaults(func=cmd_versions) @@ -1825,6 +2088,32 @@ def main(): p.add_argument("--optional", action="store_true") p.set_defaults(func=cmd_rdeps) + p = sub.add_parser("install", help="build and install (hands over to " + "packagemanager, which owns the package users)") + p.add_argument("packages", nargs="+") + p.add_argument("rest", nargs=argparse.REMAINDER, + help="passed through, e.g. --recursive --run --yes") + p.set_defaults(func=cmd_install, update=False) + + p = sub.add_parser("update", help="rebuild from the current book " + "(hands over to packagemanager)") + p.add_argument("packages", nargs="+") + p.add_argument("rest", nargs=argparse.REMAINDER) + p.set_defaults(func=cmd_install, update=True) + + p = sub.add_parser("fetch", help="download a book into the cache") + p.add_argument("selector", nargs="?", help="e.g. stable-systemd, 12.4, svn") + p.set_defaults(func=cmd_fetch) + + p = sub.add_parser("import", help="add a local book file (no network)") + p.add_argument("file") + p.add_argument("--selector", help="name to file it under") + p.set_defaults(func=cmd_import) + + p = sub.add_parser("sources", help="a package's download URLs") + p.add_argument("package") + p.set_defaults(func=cmd_sources) + p = sub.add_parser("books", help="list/inspect books") p.add_argument("--discover", action="store_true", help="probe the live site") p.add_argument("--rebuild-cache", action="store_true", @@ -1889,3 +2178,8 @@ if __name__ == "__main__": os._exit(0) except KeyboardInterrupt: sys.exit(130) + except RuntimeError as e: + # An expected problem (no book cached, no network, a bad selector) is + # not a bug -- print it plainly instead of a stack trace. + sys.stderr.write("error: %s\n" % e) + sys.exit(1) diff --git a/last_build_step.sh b/last_build_step.sh new file mode 100644 index 0000000..c546513 --- /dev/null +++ b/last_build_step.sh @@ -0,0 +1,251 @@ +#!/bin/bash +# last_build_step.sh -- the final step of an LFS build. +# +# EDIT THIS FILE. It is yours: whatever a freshly built system should have +# that the book does not provide goes here, and it runs as the last step of +# `lfs-helper build-all`. +# +# * SOURCES below is read by `lfs build-system get-sources`, so anything you +# add is downloaded along with the book's own packages. +# * The rest runs as root inside the chroot, after every book package. +# * It is copied into the chroot by `lfs build-system gen-chroot-scripts`, +# so edit it HERE (on the host) and regenerate -- not in the chroot. +# +# It ships with wget because LFS has no download tool at all: without one a +# freshly booted system cannot fetch even the sources for its next package. + +# --- extra sources -------------------------------------------------------- # +# Downloaded into $LFS/sources by `lfs build-system get-sources`. +SOURCES=( + # wget, so the new system can fetch anything at all + "https://ftpmirror.gnu.org/wget/wget-1.25.0.tar.gz" + + # What `lfs` and `blfs` need to parse the books. WHEELS, not source + # tarballs: a modern sdist needs its build backend (beautifulsoup4 wants + # hatchling, requests wants setuptools) and with no package index reachable + # pip cannot fetch one -- + # BackendUnavailable: Cannot import 'hatchling.build' + # A wheel is already built, so pip just unpacks it. Both are pure Python, + # so the "any" wheel works on every architecture. + "https://files.pythonhosted.org/packages/py3/r/requests/requests-2.32.3-py3-none-any.whl" + "https://files.pythonhosted.org/packages/py3/b/beautifulsoup4/beautifulsoup4-4.12.3-py3-none-any.whl" + # urllib3, charset-normalizer, idna and certifi are what requests imports + "https://files.pythonhosted.org/packages/py3/u/urllib3/urllib3-2.2.3-py3-none-any.whl" + "https://files.pythonhosted.org/packages/py3/c/charset_normalizer/charset_normalizer-3.4.0-py3-none-any.whl" + "https://files.pythonhosted.org/packages/py3/i/idna/idna-3.10-py3-none-any.whl" + "https://files.pythonhosted.org/packages/py3/c/certifi/certifi-2024.8.30-py3-none-any.whl" + "https://files.pythonhosted.org/packages/py3/s/soupsieve/soupsieve-2.6-py3-none-any.whl" +) + +# --- the work ------------------------------------------------------------- # +set -e + +# Build one of the SOURCES tarballs as its own package user. +# build_as_package_user <user> <tarball-glob> <configure options...> +build_as_package_user() { + local user="$1" glob="$2"; shift 2 + local src="/sources" + + local tarball="" + local c + for c in $src/$glob; do [ -e "$c" ] && { tarball="$c"; break; }; done + if [ -z "$tarball" ]; then + echo "!! no $glob in $src -- add it to SOURCES and run:" >&2 + echo " lfs build-system get-sources --run" >&2 + return 1 + fi + + lfs-helper add-user "$user" >/dev/null 2>&1 || true + + local dir + dir="$(tar tf "$tarball" | head -n1 | cut -d/ -f1)" + rm -rf "${src:?}/${dir:?}" + tar --no-same-owner -xf "$tarball" -C "$src" + chown -R "$user:$user" "$src/$dir" + + echo "# building $user from $(basename "$tarball") as package user '$user'" + su - "$user" -c "cd '$src/$dir' && ./configure $* && make && make install" + + # record what was installed, the same way lfs-helper does + printf '%s\n# installed %s by last_build_step.sh\n' \ + "$dir" "$(date '+%Y-%m-%d %H:%M:%S')" > "/usr/src/$user/VERSION" \ + 2>/dev/null || true + chown "$user:$user" "/usr/src/$user/VERSION" 2>/dev/null || true +} + +# Install a Python module as ITS OWN package user, the same as any other +# package. requests and beautifulsoup4 are what `lfs` and `blfs` need to parse +# the books, so without them the tools cannot run on the new system. +# Let a package user install into site-packages. +# +# site-packages belongs to the python package and is shared through a collector +# group. Ask lfs-helper for access, exactly as a failed build would: it joins +# the collector group the directory already carries, or asks which group should +# own it. It must NOT be handed to the install group -- that means "anyone may +# install here", and site-packages is python's tree, not shared infrastructure. +_grant_site_packages() { + local user="$1" sp + sp="$(python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])' \ + 2>/dev/null)" + if [ -z "$sp" ] || [ ! -d "$sp" ]; then + echo "note: cannot find site-packages -- skipping $user" >&2 + return 1 + fi + lfs-helper grant-dir "$sp" "$user" --run +} + +install_python_module() { + local user="$1" module="$2" glob="$3" + local src="/sources" wheel="" c + for c in $src/$glob; do [ -e "$c" ] && { wheel="$c"; break; }; done + if [ -z "$wheel" ]; then + echo "note: no $glob in $src -- skipping $module" >&2 + echo " add it to SOURCES and run: lfs build-system get-sources --run" >&2 + return 0 + fi + command -v python3 >/dev/null 2>&1 || { + echo "note: no python3 yet -- skipping $module" >&2; return 0; } + + lfs-helper add-user "$user" >/dev/null 2>&1 || true + chown "$user:$user" "$wheel" 2>/dev/null || true + _grant_site_packages "$user" || return 0 + + echo "# installing $module from $(basename "$wheel") as package user '$user'" + # A wheel needs no build backend and no network: pip unpacks it. + # PIP_USER=0 / PYTHONNOUSERSITE keep it out of the package user's ~/.local, + # where nothing else would ever find it. + su - "$user" -c "PIP_USER=0 PYTHONNOUSERSITE=1 \ + python3 -m pip install --no-index --no-deps '$wheel'" \ + || echo "!! $module did not install -- do it later with: + packagemanager pip install $module" >&2 +} + +# wget: the one thing a new system cannot bootstrap without. +# Deliberately minimal -- reinstall it properly once the system boots: +# packagemanager install wget --recursive --run +build_as_package_user wget "wget-*.tar.*" \ + --prefix=/usr --sysconfdir=/etc --with-ssl=openssl + +# The two Python modules the tools themselves need. Each gets its own package +# user, exactly like every other package -- there is no "last-step" package and +# nothing here is owned by one. +# Dependencies first -- --no-deps means pip will not pull them in itself. +install_python_module urllib3 urllib3 "urllib3-*.whl" +install_python_module charset-normalizer charset-normalizer "charset_normalizer-*.whl" +install_python_module idna idna "idna-*.whl" +install_python_module certifi certifi "certifi-*.whl" +install_python_module soupsieve soupsieve "soupsieve-*.whl" +install_python_module requests requests "requests-*.whl" +install_python_module beautifulsoup4 beautifulsoup4 "beautifulsoup4-*.whl" + +# --- wget without certificates -------------------------------------------- # +# A freshly built LFS has no CA certificates -- they come from BLFS' make-ca -- +# so every HTTPS fetch fails with "cannot verify ... certificate". That is a +# chicken-and-egg problem: you need wget to fetch make-ca. +# +# Turn verification off in /etc/wgetrc rather than with a wrapper script in +# $PATH. wget reads /etc/wgetrc no matter how it is invoked or what PATH the +# caller has, so this works for builds run through `su -` with the package +# user's own environment -- where a $PATH wrapper is simply never seen. +# +# This is a deliberate, TEMPORARY weakening: downloads are unverified until +# make-ca is installed. The marker below is how the tools find and remove it. +disable_wget_verification() { + local rc=/etc/wgetrc + grep -q 'lfs-temporary-no-verify' "$rc" 2>/dev/null && return 0 + cat >> "$rc" <<'WRC' + +# --- lfs-temporary-no-verify --------------------------------------------- +# This system has no CA certificates yet, so wget cannot verify anything. +# Downloads are UNVERIFIED until you install make-ca: +# packagemanager install make-ca --recursive --run +# The tools remove these two lines automatically once certificates exist. +check_certificate = off +# --- end lfs-temporary-no-verify ----------------------------------------- +WRC + echo "# certificate checking disabled in $rc until make-ca is installed" + echo "# (downloads are UNVERIFIED until then -- the tools re-enable it" + echo "# automatically as soon as certificates exist)" +} + +disable_wget_verification + +# --- add your own steps below --------------------------------------------- # +# e.g. a package the book does not carry, a config file, a user account. + +# --- accounts you can actually log in with -------------------------------- # +# A freshly built system has root with NO password. Depending on the login +# manager that is either "anyone can log in as root" or "nobody can log in at +# all" -- and it is discovered at the worst moment, after a reboot, with no +# way in. Ask now, while there is still a working shell here. +set_up_login_accounts() { + # No terminal (a scripted or resumed run): say what is missing rather than + # blocking, and leave the system unbootable-but-known instead of hanging. + if [ ! -t 0 ]; then + echo + echo "!! Not running interactively, so no passwords were set." + echo " Before rebooting, set at least the root password:" + echo " chroot /mnt/lfs /usr/bin/passwd root" + return 0 + fi + + echo + echo "--- accounts ------------------------------------------------" + + # root + if grep -qE '^root:[^:]*:' /etc/shadow 2>/dev/null && + ! grep -qE '^root:(\*|!|)?:' /etc/shadow 2>/dev/null; then + echo "root already has a password." + else + echo "root has NO password yet. Without one you may not be able to" + echo "log in after rebooting." + passwd root || echo "!! setting the root password failed -- do it before rebooting" >&2 + fi + + # a login account for a person: the name is already configured, so this + # only has to create it and set a password + local main + main="$(sed -n 's/^main_user=\(.*\)$/\1/p' /etc/pkgusr/packagemanager.conf \ + 2>/dev/null | head -n1)" + if [ -z "$main" ]; then + echo + echo "No login account is configured. You can add one now, or later" + echo "with: useradd -m -G users <name> && passwd <name>" + printf "Create a login account now? [name, or blank to skip]: " + read -e -r main + fi + [ -n "$main" ] || { echo; return 0; } + + if id "$main" >/dev/null 2>&1; then + echo "Account '$main' already exists." + else + echo "Creating login account '$main' ..." + useradd -m -k /etc/skel -G users -s /bin/bash "$main" \ + || { echo "!! could not create $main" >&2; return 0; } + fi + if grep -qE "^$main:(\*|!|)?:" /etc/shadow 2>/dev/null; then + echo "Set a password for '$main':" + passwd "$main" || echo "!! setting the password for $main failed" >&2 + fi + + # so the account can reach the application users created later + echo + echo " '$main' can be given access to application users with:" + echo " packagemanager user create <app> --shared --launcher" +} + +set_up_login_accounts + +echo +echo "==============================================================" +echo " The base system is built." +echo +echo " wget was installed with a minimal configure line so the system" +echo " can fetch things at all. Reinstall it properly -- with its" +echo " dependencies, from the BLFS book -- once you have booted:" +echo +echo " packagemanager install wget --recursive --run" +echo +echo " Anything else you want on a fresh system belongs in" +echo " last_build_step.sh on the host." +echo "==============================================================" diff --git a/lfs b/lfs new file mode 100644 index 0000000..2b33f81 --- /dev/null +++ b/lfs @@ -0,0 +1,5966 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# lfs -- Linux From Scratch book tool (follow one cached book) +# +# The LFS counterpart to the `blfs` tool. You cache one or more LFS "nochunks" +# HTML books, pick a default (main) book, and query it: +# * list chapter-8 build packages and their versions, +# * dump the shell commands for any section, and +# * print the package source URLs (from the book's wget-list). +# +# Building (chroot / cross-compile / package-user aware install + update) comes +# in later steps on top of this. +# +# Usage: +# lfs fetch VERSION download + cache a book (+ wget-list) +# lfs import FILE --version V cache a LOCAL book file (offline) +# lfs books [--remote] list cached books + the default +# lfs set-default VERSION choose the main book +# lfs packages chapter-8 packages (name version id) +# lfs sections [BOOK] list section ids + titles +# lfs commands SECTION-ID [BOOK] a section's shell commands +# lfs sources package source URLs (wget-list) +# +# Global: --book VERSION (use a cached version) --book-file FILE (local html) +# Store: $LFS_STORE (default /usr/share/lfs, then ~/.cache/lfs). +# +# GPLv2-or-later. Part of the pkgusr toolchain. + +import argparse +import json +import os +import re +import shlex +try: + # arrow keys, backspace and history inside input() prompts. Importing it + # is enough -- Python wires it into input() automatically. Optional: a + # Python built without readline still runs, just without line editing. + import readline # noqa: F401 +except ImportError: + pass +import subprocess +import sys +import time + +try: + from bs4 import BeautifulSoup + try: + from bs4 import XMLParsedAsHTMLWarning + import warnings + warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning) + except Exception: + pass +except ImportError: + sys.stderr.write( + "error: this needs beautifulsoup4 to parse the book.\n" + " Install it as a package user (not with a bare pip install, which\n" + " would leave the files owned by root):\n" + " packagemanager pip install beautifulsoup4\n") + sys.exit(2) + +LFS_VERSION = "1.7.0" + + +def _build_id(): + """A short fingerprint of this file's own contents. + + A hand-maintained version number goes stale the moment someone forgets to + bump it -- and the whole reason for printing a version is to answer "am I + running the code that was just fixed?". This cannot go stale: it changes + whenever the file does. + """ + import hashlib + try: + with open(os.path.abspath(__file__), "rb") as fh: + return hashlib.md5(fh.read()).hexdigest()[:7] + except OSError: + return "unknown" + +LFS_SITE = os.environ.get("LFS_SITE", "https://linuxfromscratch.org") + + +# --------------------------------------------------------------------------- # +# book store: cache books + a default selection, just like blfs +# --------------------------------------------------------------------------- # +# ---- output helpers ------------------------------------------------------- # + +def _tty(): + return sys.stderr.isatty() and os.environ.get("NO_COLOR") is None + + +_C_WARN = "\033[0;33m" # orange: something did NOT happen +_C_ERR = "\033[0;31m" +_C_OK = "\033[0;32m" +_C_OFF = "\033[0m" + + +def warn(msg): + """A dry run, a skip, a refusal -- anything the user might otherwise read + as 'done'. Coloured, because the whole point is that nothing happened.""" + if _tty(): + sys.stderr.write("%s%s%s\n" % (_C_WARN, msg, _C_OFF)) + else: + sys.stderr.write("%s\n" % msg) + + +def dry_run_note(what="apply"): + warn("\n(dry run -- nothing was changed. Re-run with --run to %s.)" % what) + + +def next_step(*cmds, label="next run:"): + """Print a follow-up command on its OWN line, unindented and unprefixed, + so it can be copied straight out of the terminal.""" + print("# " + label) + for c in cmds: + print(c) + + + +def _colour_epilog(text): + """Colour the group headings in the help. + + The command list is what people scan to find a command, and an unbroken + wall of text is hard to search. Headings are the lines that are not + indented; commands are indented. Colour is dropped when the output is not + a terminal, so piping help into a file or a pager stays clean.""" + if not _tty_out(): + return text + out = [] + for line in text.split("\n"): + if line and not line.startswith(" ") and line.endswith(":"): + out.append("%s%s%s" % (_C_HEAD, line, _C_OFF)) # "commands, by..." + elif line.startswith(" ") and not line.startswith(" ") and line.strip(): + out.append("%s%s%s" % (_C_GROUP, line, _C_OFF)) # a group heading + elif line.startswith(" ") and line.strip(): + # the command itself, up to the first run of two spaces + stripped = line.rstrip() + m = re.match(r"^(\s+)(\S+(?: \S+)*?)(\s{2,}.*)$", stripped) + if m: + out.append("%s%s%s%s%s" % (m.group(1), _C_CMD, m.group(2), + _C_OFF, m.group(3))) + else: + out.append(line) + else: + out.append(line) + return "\n".join(out) + + +def _tty_out(): + return sys.stdout.isatty() and os.environ.get("NO_COLOR") is None + + +_C_HEAD = "\033[1m" # bold +_C_GROUP = "\033[1;36m" # bold cyan -- the group you are scanning for +_C_CMD = "\033[0;32m" # green -- the thing you actually type + + +def _wrap(text, width): + out, line = [], "" + for w in text.split(): + if len(line) + len(w) + 1 > width: + out.append(line); line = w + else: + line = (line + " " + w).strip() + if line: + out.append(line) + return out + + +# The directories handed to the lfs user at book 4.3 and handed BACK to root +# at 7.2. One list, used by both -- they were written separately and drifted: +# 4.3 included `lib` and 7.2 did not, so /lib stayed owned by the lfs user for +# the rest of the build while /bin and /sbin (never handed over) stayed root. +# /sources is deliberately absent: the book leaves it to the lfs user. +LFS_HANDOVER_DIRS = ("usr", "lib", "lib64", "var", "etc", "tools", "bin", "sbin") + + +def store_dir(): + """One shared location so root and the lfs user see the SAME books. + Override with $LFS_STORE.""" + return os.environ.get("LFS_STORE") or "/usr/share/lfs" + + +def _ensure_store(): + """Create the shared store (needs root/owner) and keep it world-readable so + every user can READ the cached books.""" + d = store_dir() + os.makedirs(os.path.join(d, "books"), exist_ok=True) + for p in (d, os.path.join(d, "books")): + try: + os.chmod(p, 0o755) + except OSError: + pass + return d + + +def _make_world_readable(path): + try: + os.chmod(path, 0o644) + except OSError: + pass + + +def books_dir(): + return os.path.join(store_dir(), "books") + + +def config_path(): + return os.path.join(store_dir(), "config.json") + + +_RENAMED_KEYS = { + # old name -> new. "collector_groups" read like "the groups themselves"; + # it is the file the group decisions are imported FROM. + "collector_groups": "collector_import_file", +} + + +def load_config(): + try: + with open(config_path()) as f: + cfg = json.load(f) + except Exception: + return {} + # migrate renamed keys in memory, so an existing config keeps working + for old_k, new_k in _RENAMED_KEYS.items(): + if old_k in cfg and not cfg.get(new_k): + cfg[new_k] = cfg.pop(old_k) + return cfg + + +def save_config(cfg): + _ensure_store() + with open(config_path(), "w") as f: + json.dump(cfg, f, indent=2) + _make_world_readable(config_path()) + + +def default_version(): + """The book used for installing packages on a running system.""" + return load_config().get("default") + + +def build_version(): + """The book used to BUILD the system (chapters 5-9). + + Deliberately separate from `default`: you might build the system from 12.4 + but then install packages on it from a newer book, or pin the build to the + exact version you started with while the default moves on. Falls back to + `default` when unset, so nothing changes for anyone who doesn't care.""" + cfg = load_config() + return cfg.get("build_book") or cfg.get("default") + + +def book_path(ver): + return os.path.join(books_dir(), f"LFS-BOOK-{ver}-NOCHUNKS.html") + + +def wget_list_path(ver): + return os.path.join(books_dir(), f"wget-list-{ver}.txt") + + +def cached_versions(): + out = [] + try: + for f in sorted(os.listdir(books_dir())): + m = re.match(r"LFS-BOOK-(.+)-NOCHUNKS\.html$", f) + if m: + out.append(m.group(1)) + except OSError: + pass + return out + + +def _http_get(url, binary=False): + import urllib.request + timeout = float(os.environ.get("LFS_HTTP_TIMEOUT", "30")) + req = urllib.request.Request(url, headers={"User-Agent": "lfs-tool"}) + with urllib.request.urlopen(req, timeout=timeout) as r: + data = r.read() + return data if binary else data.decode("utf-8", "replace") + + +def _download_to(url, dest): + _ensure_store() + data = _http_get(url, binary=True) + with open(dest, "wb") as f: + f.write(data) + _make_world_readable(dest) + return len(data) + + +DEFAULT_MIRROR = "https://lfs.gnlug.org/pub/lfs/lfs-packages/{book}" + + +def mirror_base(ver): + """Base URL of a full package mirror, tried BEFORE the wget-list URL. + Upstream links rot (ncurses' snapshot vanishes from invisible-mirror), while + the LFS package mirrors keep the exact versions a book release needs.""" + cfg = load_config() + if cfg.get("mirror") == "off": + return None + tmpl = cfg.get("mirror") or DEFAULT_MIRROR + return tmpl.replace("{book}", ver) + + +def remote_versions(): + """All book versions listed on the LFS downloads index (10.0, 10.0-systemd, + 11.0-rc2, ...).""" + html = _http_get(f"{LFS_SITE}/lfs/downloads/") + vers = re.findall(r'href="([^"/]+)/"', html) + return [v for v in vers if re.match(r"\d", v)] + + +def _find_nochunks_link(page_html): + m = re.search(r'href="([^"]*NOCHUNKS\.html)"', page_html) + return m.group(1) if m else None + + +def md5sums_path(ver): + return os.path.join(books_dir(), f"md5sums-{ver}.txt") + + +def _fetch_version(ver, set_default=False, quiet=False): + """Download a book's NOCHUNKS html + plain wget-list + md5sums + the + lfs-bootscripts tarball into the store.""" + base = f"{LFS_SITE}/lfs/downloads/{ver}/" + page = _http_get(base) + link = _find_nochunks_link(page) + if not link: + raise RuntimeError(f"no NOCHUNKS.html at {base}") + book_url = link if link.startswith("http") else base + link.lstrip("/") + n = _download_to(book_url, book_path(ver)) + if not quiet: + print(f" book: {n} bytes -> {book_path(ver)}") + # always the plain wget-list (not -sysv / .old) + for name, dest, why in ( + ("wget-list", wget_list_path(ver), "wget-list"), + ("md5sums", md5sums_path(ver), "md5sums")): + try: + b = _download_to(base + name, dest) + if not quiet: + print(f" {why}: {b} bytes -> {dest}") + except Exception as e: + sys.stderr.write(f" ({why} not fetched: {e})\n") + # the lfs-bootscripts tarball (filename carries a date; read it off the page) + m = re.search(r'href="(lfs-bootscripts-[^"]+\.tar\.[a-z0-9]+)"', page) + if m: + fn = m.group(1) + try: + b = _download_to(base + fn, os.path.join(books_dir(), fn)) + if not quiet: + print(f" bootscripts: {b} bytes -> {os.path.join(books_dir(), fn)}") + except Exception as e: + sys.stderr.write(f" (bootscripts not fetched: {e})\n") + if set_default or not default_version(): + cfg = load_config() + cfg["default"] = ver + save_config(cfg) + if not quiet: + print(f" default book set to {ver}") + + +def resolve_book(args): + """Return (label, html_path) for the book this run should use: + --book-file wins; else --book <version>; else the cached default. A named + version that isn't cached is fetched automatically.""" + if getattr(args, "book_file", None): + return (os.path.basename(args.book_file), args.book_file) + # `--book` always wins; otherwise which book depends on what is being + # done: building the system, or installing onto a running one. + ver = (getattr(args, "book", None) + or (build_version() if getattr(args, "_use_build_book", False) + else default_version())) + if not ver: + sys.stderr.write( + "no book selected. See what's available and pick one:\n" + " lfs books # list cached + all site versions\n" + " lfs fetch 12.4 # download + cache (becomes default)\n" + "or import a local file:\n" + " lfs import LFS-BOOK-12.4-NOCHUNKS.html --version 12.4\n") + sys.exit(2) + p = book_path(ver) + if not os.path.isfile(p): + sys.stderr.write(f"book {ver} not cached -- fetching from {LFS_SITE}...\n") + try: + _fetch_version(ver, quiet=True) + except Exception as e: + sys.stderr.write(f"could not fetch {ver}: {e}\n" + f" see available versions: lfs books\n") + sys.exit(2) + return (ver, p) + + +def resolve_book_arg(val): + """An argument that is either a local file path or a cached version.""" + if os.path.isfile(val): + return val + if os.path.isfile(book_path(val)): + return book_path(val) + sys.stderr.write(f"not a file or cached version: {val}\n") + sys.exit(2) + + +# --------------------------------------------------------------------------- # +# book loading + section walking +# --------------------------------------------------------------------------- # +def load_soup(path): + with open(path, "rb") as f: + html = f.read() + for parser in ("lxml", "html.parser"): + try: + return BeautifulSoup(html, parser) + except Exception: + continue + return BeautifulSoup(html, "html.parser") + + +def _is_admonition(tag): + """A note/tip/caution/warning box -- its commands are illustrative.""" + cls = getattr(tag, "get", lambda _k: None)("class") or [] + return bool({"admon", "note", "tip", "caution", "warning", + "important"} & set(cls)) + + +def _heading_anchor_id(tag): + """The section id of a heading (hN) is the id of the <a id> it contains.""" + a = tag.find("a", id=True) + return a["id"] if a else None + + +_HEADING_RE = re.compile(r"^h[1-6]$") + + +def parse_sections(soup): + """Linear walk over the flat/stitched book: a heading (h1-h6) carrying an + <a id> starts a section; every <pre class="userinput"> until the next such + heading is one of its command blocks. Returns an ordered list of dicts: + {id, title, level, commands:[str], heading:<tag>}.""" + sections, cur = [], None + for tag in soup.find_all([_HEADING_RE, "pre"]): + if tag.name != "pre": + aid = _heading_anchor_id(tag) + if aid: + cur = {"id": aid, + "title": re.sub(r"\s+", " ", tag.get_text(" ", strip=True)), + "level": int(tag.name[1]), "commands": [], "heading": tag} + sections.append(cur) + continue + if cur is None: + continue + classes = tag.get("class") or [] + if "userinput" in classes: + # Skip commands inside admonitions (note/tip/caution/warning). + # Those are conditional examples, not part of the build -- GMP's + # note shows "ABI=32 ./configure ..." for 32-bit hosts, and running + # that literally passes an ellipsis to configure: + # Invalid configuration '...': machine '...-unknown' not recognized + if any(_is_admonition(p) for p in tag.parents): + continue + # the whole <pre class="userinput"> is the command block as the reader + # would copy it; taking per-<kbd> text splits inline tokens wrongly. + text = tag.get_text("", strip=False).replace("\u00a0", " ").rstrip() + if text.strip(): + cur["commands"].append(text) + return sections + + +def iter_sections(soup): + """Yield (id, title, section_dict) in book order.""" + for sec in parse_sections(soup): + yield sec["id"], sec["title"], sec + + +def section_commands(sec): + return sec["commands"] + + +# --------------------------------------------------------------------------- # +# commands: subcommands +# --------------------------------------------------------------------------- # +def _explain_fetch_failure(e): + """Turn a network failure into something worth reading. + + Inside a fresh chroot the usual cause is that no CA certificates exist yet + -- they come from BLFS' make-ca, not from LFS -- and the raw exception + ("CERTIFICATE_VERIFY_FAILED ... unable to get local issuer certificate") + reads like something is broken when nothing is.""" + msg = str(e) + if "CERTIFICATE_VERIFY_FAILED" in msg or "SSLCertVerification" in msg: + return ("(cannot verify HTTPS certificates: this system has no CA\n" + " certificates yet -- they come from BLFS' make-ca, not from\n" + " LFS. The cached book still works: lfs books --local)") + if "Name or service not known" in msg or "Temporary failure in name" in msg: + return ("(no DNS: nothing can be resolved from here.\n" + " In the chroot: lfs build-system chroot resolv\n" + " Or work offline: lfs books --local)") + if "Network is unreachable" in msg or "Connection refused" in msg: + return ("(no network from here -- work offline with: lfs books --local)") + return "(could not reach %s: %s -- use --local to skip)" % (LFS_SITE, e) + + +def cmd_books(args): + cfg = load_config() + default = cfg.get("default") + cached = set(cached_versions()) + print(f"Cached books (in {books_dir()}):") + if not cached: + print(" (none)") + for v in sorted(cached): + mark = " *" if v == default else " " + wl = " [wget-list]" if os.path.isfile(wget_list_path(v)) else "" + print(f" {mark} {v}{wl}") + if default: + print(f"default: {default}") + + if args.local: + return + try: + vers = remote_versions() + except Exception as e: + print("\n" + _explain_fetch_failure(e)) + return + sysv = [v for v in vers if "-systemd" not in v] + systemd = [v for v in vers if "-systemd" in v] + print(f"\nAvailable on the site ({len(vers)}): * = cached") + print(" SysV:") + for v in sysv: + print(f" {'*' if v in cached else ' '} {v}") + print(" systemd:") + for v in systemd: + print(f" {'*' if v in cached else ' '} {v}") + print("\nFetch: lfs fetch <version> One-off: lfs --book <version> ... " + "(auto-fetches)") + + +# --------------------------------------------------------------------------- # +# snapshots: save the build so a bad step can be undone +# --------------------------------------------------------------------------- # +# Building LFS is a long sequence of irreversible steps. One bad package, one +# mistaken chown, and the choice is "start over" or "hope". A snapshot makes +# it "go back to before that". +# +# What is saved: the whole $LFS tree EXCEPT /sources (large, re-downloadable, +# and unchanged by a build) and the virtual filesystems (/dev /proc /sys /run, +# which are the host's and must never be captured or restored). Ownership, +# permissions, symlinks, hardlinks and sparse files are preserved exactly -- +# the package-user model IS ownership, so a backup that loses it is worthless. + +def snapshots_dir(): + cfg = load_config() + d = cfg.get("snapshot_dir") or os.path.join(store_dir(), "snapshots") + os.makedirs(d, exist_ok=True) + return d + + +_SNAP_EXCLUDE = ["./dev/*", "./proc/*", "./sys/*", "./run/*", "./sources/*", + "./tmp/*", "./var/tmp/*"] + + +def _snapshot_path(name): + return os.path.join(snapshots_dir(), "%s.tar" % name) + + +def _list_snapshots(): + out = [] + for f in sorted(os.listdir(snapshots_dir())): + if not f.endswith((".tar", ".tar.zst", ".tar.gz")): + continue + p = os.path.join(snapshots_dir(), f) + meta = p.rsplit(".tar", 1)[0] + ".json" + info = {} + if os.path.isfile(meta): + try: + info = json.load(open(meta)) + except ValueError: + pass + out.append((f.rsplit(".tar", 1)[0], p, os.path.getsize(p), info)) + return out + + +def _tar_compressor(): + """zstd if available (fast, and these trees are big), else gzip.""" + import shutil as _sh + if _sh.which("zstd"): + return "--zstd", ".tar.zst" + return "--gzip", ".tar.gz" + + +def cmd_snapshot(args): + action = args.action or "list" + if action == "list": + snaps = _list_snapshots() + if not snaps: + print("No snapshots yet. Take one before a risky step:") + print(" %slfs snapshot save before-gcc --run" % sudo_prefix()) + return + print("Snapshots in %s:\n" % snapshots_dir()) + for name, path, size, info in snaps: + print(" %-24s %8s %s" % (name, _human_size(size), + info.get("when", ""))) + if info.get("note"): + print(" %-24s %s" % ("", info["note"])) + if info.get("progress"): + print(" %-24s %s" % ("", info["progress"])) + print("\nRestore one with: %slfs snapshot restore <name> --run" + % sudo_prefix()) + return + + lfs = require_mounted_lfs() + + if action == "save": + name = args.name or time.strftime("%Y%m%d-%H%M%S") + comp, ext = _tar_compressor() + dest = os.path.join(snapshots_dir(), name + ext) + if os.path.exists(dest) and not args.force: + sys.stderr.write("a snapshot named '%s' already exists " + "(use --force to replace it)\n" % name) + sys.exit(1) + excl = " ".join("--exclude='%s'" % e for e in _SNAP_EXCLUDE) + print("Save %s -> %s" % (lfs, dest)) + print(" excluded: /sources (re-downloadable) and the virtual " + "filesystems") + if not args.run: + dry_run_note("save it") + return + # --numeric-owner: inside the chroot the uids have no names yet, and + # names would be resolved against the HOST's /etc/passwd -- restoring + # that would silently reassign every package user's files. + # --one-file-system: do not descend into anything mounted inside the + # tree. Excluding "./sys/*" skipped the CONTENTS but still archived + # the ./sys directory itself, and reading a live kernel filesystem's + # metadata gave + # tar: ./sys: file changed as we read it + # and exit 1. The mount points are still recorded as empty + # directories, which is exactly what a restore needs. + cmd = ("tar -C '%s' %s --numeric-owner --acls --xattrs --sparse " + "--one-file-system %s -cf '%s' ." % (lfs, comp, excl, dest)) + rc = _run_bash(cmd) + if rc == 1: + # tar's exit 1 is "some files differ from what was archived" -- on + # a live tree that is a warning, not a failure. Verify the archive + # rather than throwing away a snapshot that is probably fine. + print("# tar reported changes during the read -- checking the " + "archive ...") + if _run_bash("tar -tf '%s' >/dev/null 2>&1" % dest) == 0: + print("# the archive is readable and complete.") + rc = 0 + else: + sys.stderr.write("the archive is not readable -- discarding\n") + try: + os.remove(dest) + except OSError: + pass + if rc != 0: + sys.stderr.write("snapshot failed (tar exited %d)\n" % rc) + sys.exit(rc) + prog = _load_crosschain_progress(lfs) or {} + done = len(prog.get("done", [])) + meta = { + "when": time.strftime("%Y-%m-%d %H:%M:%S"), + "lfs": lfs, + "book": build_version(), + "note": args.note or "", + "progress": "chapters 5-6: %d/%d steps" + % (done, len(CROSSCHAIN_STEPS)) if prog else "", + } + with open(os.path.join(snapshots_dir(), name + ".json"), "w") as f: + json.dump(meta, f, indent=2) + print("\nsaved %s (%s)" % (name, _human_size(os.path.getsize(dest)))) + return + + if action == "restore": + if not args.name: + sys.stderr.write("which snapshot? see: lfs snapshot list\n") + sys.exit(2) + match = [s for s in _list_snapshots() if s[0] == args.name] + if not match: + sys.stderr.write("no snapshot named '%s'\n see: lfs snapshot " + "list\n" % args.name) + sys.exit(1) + name, path, size, info = match[0] + + warn("!! This REPLACES the current build at %s" % lfs) + print() + print(" restore: %s (%s, %s)" % (name, _human_size(size), + info.get("when", ""))) + if info.get("progress"): + print(" %s" % info["progress"]) + print(" onto: %s" % lfs) + print() + print("Everything built since that snapshot is lost.") + print("/sources is left alone (it was never captured).") + if not args.run: + dry_run_note("restore it") + return + + # Restoring into a tree with the virtual filesystems mounted would + # write through them onto the HOST. Refuse rather than risk it. + mounted = [d for d in ("dev", "proc", "sys", "run") + if _is_mounted(os.path.join(lfs, d))] + if mounted: + sys.stderr.write( + "\n! these are still mounted in the tree: %s\n" + " Restoring now could write through them onto this system.\n" + " Unmount first:\n" + " %slfs build-system chroot unmount --run\n" + % (", ".join(mounted), sudo_prefix())) + sys.exit(2) + + if not args.yes: + try: + ans = input("Type the snapshot name to confirm: ").strip() + except (EOFError, KeyboardInterrupt): + print() + return + if ans != name: + print("Not confirmed -- nothing was changed.") + return + + # Remove the current contents (except /sources) and unpack. Deleting + # first matters: tar alone would leave files the snapshot does not + # contain, giving a tree that is neither the old state nor the new one. + keep = "sources" + print("\nclearing the tree (keeping /%s) ..." % keep) + rc = _run_bash( + "shopt -s dotglob; for e in '%s'/*; do " + " b=\"$(basename \"$e\")\"; " + " case \"$b\" in %s|dev|proc|sys|run) continue ;; esac; " + " rm -rf \"$e\"; done" % (lfs, keep)) + print("unpacking %s ..." % name) + rc = _run_bash("tar -C '%s' --numeric-owner --acls --xattrs " + "-xf '%s'" % (lfs, path)) + if rc != 0: + sys.stderr.write("restore failed (tar exited %d)\n" % rc) + sys.exit(rc) + print("\nrestored %s" % name) + print() + next_step("%slfs build-system run" % sudo_prefix(), + label="continue with:") + return + + if action == "remove": + if not args.name: + sys.stderr.write("which snapshot? see: lfs snapshot list\n") + sys.exit(2) + match = [s for s in _list_snapshots() if s[0] == args.name] + if not match: + sys.stderr.write("no snapshot named '%s'\n" % args.name) + sys.exit(1) + name, path, size, _i = match[0] + print("remove %s (%s)" % (name, _human_size(size))) + if not args.run: + dry_run_note("remove it") + return + os.remove(path) + meta = path.rsplit(".tar", 1)[0] + ".json" + if os.path.isfile(meta): + os.remove(meta) + print("removed %s" % name) + + +def cmd_reset(args): + """Delete the tools' configuration so the next run starts fresh. + + Deliberately narrow: this removes what `lfs` and `packagemanager` remember + -- settings, the step list, generated scripts -- and NOTHING else. It does + not touch the LFS tree, the downloaded sources, the cached books, or any + package user. Wiping a half-built system is never what "start over with + the settings" should mean.""" + store = store_dir() + targets = [ + (os.path.join(store, "config.json"), "settings (book, mount, device, prefix)"), + (os.path.join(store, "session.json"), "the saved session"), + ("/etc/pkgusr/packagemanager.conf", "packagemanager's settings"), + ] + lfs_mount = load_config().get("lfs_mount") or os.environ.get("LFS") or "" + if lfs_mount: + targets.append((os.path.join(lfs_mount, ".lfs-pkgusr"), + "the in-chroot step list and generated scripts")) + + present = [(p, d) for p, d in targets if os.path.exists(p)] + if not present: + print("Nothing to clear -- no configuration found.") + return + + warn("!! This will DELETE the configuration listed below.") + print() + for p, d in present: + kind = "dir " if os.path.isdir(p) else "file" + print(" %s %s" % (kind, p)) + print(" %s" % d) + print() + print("Kept, untouched:") + print(" the LFS tree and everything built in it") + print(" downloaded sources and cached books") + print(" every package user and its files") + + if not args.run: + dry_run_note("delete them") + return + + if not args.yes: + warn("\nThis cannot be undone.") + try: + ans = input("Type 'reset' to confirm: ").strip() + except (EOFError, KeyboardInterrupt): + print() + return + if ans != "reset": + print("Not confirmed -- nothing was deleted.") + return + + import shutil as _sh + n = 0 + for p, _d in present: + try: + if os.path.isdir(p): + _sh.rmtree(p) + else: + os.remove(p) + n += 1 + except OSError as e: + sys.stderr.write("could not remove %s: %s\n" % (p, e)) + print("\ncleared %d item(s)." % n) + print() + next_step("%slfs build-system run" % sudo_prefix(), label="start again with:") + + +def cmd_bs_set_default(args): + """Choose the book the SYSTEM is built from. + + Separate from `lfs set-default`, which chooses the book packages are + installed from on a running system. One command that quietly meant two + different things was the confusing part.""" + ver = args.version + if not os.path.isfile(book_path(ver)): + known = [] + try: + known = list(remote_versions()) + except Exception: + known = [] + if known and ver not in known: + sys.stderr.write("no such book: %s\n see them all: lfs books\n" % ver) + sys.exit(1) + sys.stderr.write("note: %s isn't downloaded yet -- fetching it now\n" % ver) + try: + _fetch_version(ver, set_default=False) + except Exception as e: + sys.stderr.write("fetch failed: %s\n" % e) + sys.exit(1) + cfg = load_config() + cfg["build_book"] = ver + save_config(cfg) + print(f"book for BUILDING the system set to {ver}") + if cfg.get("default") and cfg["default"] != ver: + print(f" (packages are installed from {cfg['default']} -- " + f"lfs set-default changes that)") + + +def cmd_set_default(args): + """Make a book the default -- but only if it actually exists. + + This used to warn and save anyway, so a typo became the configured book and + every later command failed with a 404 far from where the mistake was made + ("book ls not cached -- fetching ... 404"). A setting that cannot work + should not be accepted.""" + ver = args.version + if not os.path.isfile(book_path(ver)): + # not cached: is it even a real version? + known = [] + try: + known = list(remote_versions()) + except Exception: + known = [] # offline: fall back to the cache + if known and ver not in known: + sys.stderr.write("no such book: %s\n" % ver) + near = [k for k in known if ver in k or k.startswith(ver)][:5] + if near: + sys.stderr.write("did you mean: %s\n" % ", ".join(near)) + sys.stderr.write(" see them all: lfs books\n") + sys.exit(1) + if not known: + cached = sorted(cached_versions()) + if ver not in cached: + sys.stderr.write( + "no such book: %s (and %s could not be reached to check)\n" + % (ver, LFS_SITE)) + if cached: + sys.stderr.write(" downloaded already: %s\n" + % ", ".join(cached)) + sys.exit(1) + sys.stderr.write("note: %s isn't downloaded yet -- fetching it now\n" + % ver) + try: + _fetch_version(ver, set_default=False) + except Exception as e: + sys.stderr.write("fetch failed: %s\n" % e) + sys.exit(1) + cfg = load_config() + cfg["default"] = ver + save_config(cfg) + print(f"book for installing packages set to {ver}") + print(" (the book the SYSTEM is built from is 'build_book':") + print(" lfs build-system set-default <version>)") + + +def cmd_fetch(args): + try: + print(f"fetching {args.version} from {LFS_SITE} ...") + _fetch_version(args.version, set_default=args.set_default) + except Exception as e: + sys.stderr.write(f"fetch failed: {e}\n see: lfs books\n") + sys.exit(1) + + +def cmd_import(args): + import shutil + ver = args.version + try: + _ensure_store() + dest = book_path(ver) + shutil.copy(args.file, dest) + _make_world_readable(dest) + except PermissionError: + sys.stderr.write(f"cannot write to {books_dir()} -- run this as root " + f"(the store is shared for all users).\n") + sys.exit(1) + print(f"imported book -> {dest}") + if args.wget_list: + shutil.copy(args.wget_list, wget_list_path(ver)) + _make_world_readable(wget_list_path(ver)) + print(f"imported wget-list -> {wget_list_path(ver)}") + if args.set_default or not default_version(): + cfg = load_config() + cfg["default"] = ver + save_config(cfg) + print(f"book for installing packages set to {ver}") + print(" (the book the SYSTEM is built from is 'build_book':") + print(" lfs build-system set-default <version>)") + + +def cmd_sources(args): + """Print the package source URLs (from the cached wget-list).""" + ver = args.book or default_version() + if not ver: + sys.stderr.write("no book selected (see: lfs books)\n") + sys.exit(2) + wl = wget_list_path(ver) + if not os.path.isfile(wl): + sys.stderr.write(f"no wget-list cached for {ver}. Re-fetch: lfs fetch {ver}\n") + sys.exit(1) + with open(wl) as f: + sys.stdout.write(f.read()) + + +def cmd_sections(args): + label, path = resolve_book(args) if not args.book_pos else \ + (args.book_pos, resolve_book_arg(args.book_pos)) + soup = load_soup(path) + n = 0 + for sid, title, _sec in iter_sections(soup): + print(f"{sid}\t{title}") + n += 1 + sys.stderr.write(f"{n} section(s) [{label}].\n") + + +def cmd_commands(args): + label, path = (args.book_pos, resolve_book_arg(args.book_pos)) \ + if args.book_pos else resolve_book(args) + soup = load_soup(path) + for sid, title, sec in iter_sections(soup): + if sid == args.section or title == args.section: + sys.stderr.write(f"# {sid} -- {title} [{label}]\n") + for b in section_commands(sec): + print(b) + print() + return + sys.stderr.write(f"section not found: {args.section}\n") + sys.exit(1) + + +# a chapter-8 package section: id ch-system-<name>, a versioned title, and it +# actually carries build commands (skips the intro / package-management pages). +_PKG_ID_RE = re.compile(r"^ch-system-(.+)$") +_PKG_TITLE_RE = re.compile(r"^\d+\.\d+\.\s+(.*\S-\S.*)$") + + +def package_name_version(sid, title): + m = _PKG_ID_RE.match(sid) + if not m: + return None + t = _PKG_TITLE_RE.match(title.strip()) + if not t: + return None + return m.group(1), t.group(1).strip() # (name, "Glibc-2.42") + + +def cmd_packages(args): + label, path = resolve_book(args) + soup = load_soup(path) + n = 0 + for sid, title, sec in iter_sections(soup): + nv = package_name_version(sid, title) + if not nv or not sec["commands"]: + continue + name, version = nv + print(f"{name}\t{version}\t{sid}") + n += 1 + sys.stderr.write(f"{n} chapter-8 package(s) [{label}].\n") + + +# --------------------------------------------------------------------------- # +# --------------------------------------------------------------------------- # +# build-system: create an LFS system, step by step, straight from the book. +# (install / update commands live alongside this later.) +# --------------------------------------------------------------------------- # +def _section_script(args, section_id, join="\n\n"): + """(label, title, script) for one section of the main book, or (label,None,None).""" + label, path = resolve_book(args) + soup = load_soup(path) + for sid, title, sec in iter_sections(soup): + if sid == section_id: + return label, title, join.join(sec["commands"]) + return label, None, None + + +def _run_bash(script, cwd=None, strict=False): + import subprocess + flags = ["-e"] if strict else [] + # never inherit the caller's cwd blindly -- if it's inside a dir a build + # step deleted, EVERY subprocess we spawn afterwards fails with a cryptic + # "getcwd: cannot access parent directories" error. Default to a dir that + # always exists. + return subprocess.run(["bash", *flags, "-c", script], + cwd=(cwd or "/")).returncode + + +def _lfs_env_prefix(lfs): + """The exact variable set the book's ~/.bashrc (4.4) exports, embedded + directly rather than relied on via sourcing (see run_as_lfs). Missing PATH + here would be a real bug: gcc-pass1 needs binutils-pass1's tools on PATH, + and glibc/libstdc++ need the cross compiler -- not just LFS/LFS_TGT.""" + if not lfs: + return "" + cfg = load_config() + tgt = cfg.get("lfs_tgt") or "$(uname -m)-lfs-linux-gnu" + mkf = cfg.get("makeflags") or "-j$(nproc)" + return ( + f'export LFS="{lfs}"\n' + f'export LC_ALL=POSIX\n' + f'LFS_TGT={tgt}\n' + f'export LFS_TGT\n' + f'PATH=/usr/bin\n' + f'if [ ! -L /bin ]; then PATH=/bin:$PATH; fi\n' + f'PATH="$LFS/tools/bin:$PATH"\n' + f'export PATH\n' + f'CONFIG_SITE="$LFS/usr/share/config.site"\n' + f'export CONFIG_SITE\n' + f'export MAKEFLAGS="{mkf}"\n' + ) + + +def run_as_lfs(script, lfs=None): + """Run a script as the lfs user via `su - lfs`. IMPORTANT: `su -` fully + resets the environment (that's what login simulation does), so anything the + root process exported -- $LFS included -- is gone in the lfs shell. We embed + the book's whole env block explicitly at the top of the script instead of + relying on ~/.bashrc being sourced (it usually isn't, in this non-interactive + invocation).""" + import subprocess + lfs = lfs or _lfs_dir() + prefix = _lfs_env_prefix(lfs) + return subprocess.run(["su", "-", "lfs", "-c", "bash -s"], cwd="/", + input=prefix + script, text=True).returncode + + +def _lfs_dir(): + """The LFS mount point: $LFS if exported, else the saved session config.""" + return os.environ.get("LFS") or load_config().get("lfs_mount") + + +def require_lfs_for_root(): + """Ensure an LFS mount is known (book rule: root needs $LFS after 2.4). Falls + back to the saved session config and exports it so child shells inherit it.""" + d = _lfs_dir() + if not d: + sys.stderr.write( + "! no LFS mount set (neither $LFS nor a saved session).\n" + " Configure it once: lfs build-system session\n" + " or for this shell: export LFS=/mnt/lfs\n") + sys.exit(2) + os.environ["LFS"] = d + return d + + +def require_mounted_lfs(): + """Resolve $LFS AND verify the configured partition is actually mounted + there. Building against an unmounted $LFS silently operates on a plain + empty directory instead of the real filesystem (confusing 'No such file or + directory' errors deep inside a build), so this is a hard stop with a hint + to the exact command that fixes it -- we already know the device and mount + point from the saved session config. If no device was ever recorded (e.g. + a bind-mount / manual setup), we can't verify a mount, so we don't second- + guess it -- same behavior as require_lfs_for_root().""" + lfs = require_lfs_for_root() + device = load_config().get("lfs_device") + if device and not _is_mounted(lfs): + sys.stderr.write( + f"! {lfs} is not mounted -- the build session isn't active.\n" + f" {device} needs to be mounted there first. Start/rejoin the " + f"session (mounts it, sets $LFS, chowns the tree):\n" + f" lfs build-system session --run\n") + sys.exit(2) + return lfs + + +_SESSION_KEYS = [ + ("lfs_mount", "LFS mount point", "/mnt/lfs"), + ("lfs_device", "root partition device (e.g. /dev/sda2)", ""), + ("lfs_fstype", "root filesystem type", "ext4"), + ("lfs_swap", "swap device (optional, blank to skip)", ""), + ("lfs_home_device", "separate /home device (optional)", ""), + ("collector_prefix", "prefix for collector groups (packages that install " + "into another package's directory join <prefix>_<owner>)", + "sysgroup"), + # Asked here with the rest of the session settings: a groups export from a + # previous build saves answering the same sharing questions again, and it + # has to be known BEFORE the chroot scripts are generated (that is when the + # file is copied into the tree). Blank means "start fresh". + ("collector_import_file", "collector-group export to import from a previous " + "build (blank for none)", ""), + # Asked here rather than left to a default nobody sees. These end up in + # the generated chroot scripts, and a wrong locale or triplet only shows + # up much later -- a mismatched $LFS_TGT means the compiler looks for its + # headers under a target the headers were never installed under. + ("lfs_tgt", "cross-compile target triplet ($LFS_TGT)", + "%s-lfs-linux-gnu" % os.uname().machine), + ("makeflags", "parallel jobs for builds", "-j%s" % (os.cpu_count() or 1)), + ("hostname", "hostname for the built system", "lfs"), + ("locale", "system locale (see 'locale -a')", "en_US.UTF-8"), + ("paper_size", "paper size for groff: A4 or letter", "A4"), +] + + +def _is_mounted(path): + try: + real = os.path.realpath(path) + with open("/proc/mounts") as f: + return any(ln.split()[1] == real for ln in f if len(ln.split()) > 1) + except OSError: + return False + + +def _prompt_session_config(cfg, force=False): + """Ask for each session setting (keeping saved answers as defaults) + save.""" + if not sys.stdin.isatty(): + return cfg + for k, desc, default in _SESSION_KEYS: + cur = cfg.get(k, default) + if k == "collector_import_file" and not cur: + # offer the one install-tools wrote last time, if it is still there + guess = os.path.join(store_dir(), "collector-groups.export") + if os.path.isfile(guess): + cur = guess + if cfg.get(k) and not force: + continue + val = input(f" {desc} [{cur}]: ").strip() + val = val if val else cur + if k == "collector_import_file" and val: + # Accepting a path that is not there means the groups silently + # never get imported and every sharing question comes back. + p = os.path.expanduser(val) + if not os.path.isfile(p): + sys.stderr.write(" no such file: %s -- leaving it unset\n" % val) + val = "" + else: + val = os.path.abspath(p) + cfg[k] = val + save_config(cfg) + print(f" saved session config -> {config_path()}") + return cfg + + +def _heredoc_writes(script): + """The `cat > FILE << \"TAG\" ... TAG` blocks in a script (the real file + creations), skipping example commands like a bare `make -j32`.""" + out = [] + for m in re.finditer(r'(cat >>? \S+ << *"?(\w+)"?\n.*?\n\2)\b', script, re.S): + out.append(m.group(1)) + return out + + +def _emit_or_run(args, name, script, needs_lfs=False): + """Print a step's commands; run them only with --run. Enforces $LFS (book + rule for root after 2.4) and warns if not root.""" + if script is None: + sys.stderr.write(f"{name}: section not found in this book.\n") + sys.exit(1) + print(f"# === {name} ===") + print(script) + if not args.run: + if needs_lfs and not _lfs_dir(): + sys.stderr.write("\n! no LFS mount set. Configure it: " + "lfs build-system session\n") + dry_run_note("execute") + return + if needs_lfs: + require_mounted_lfs() + if os.geteuid() != 0: + sys.stderr.write("\n! not root -- these steps write under $LFS / add " + "users and will likely fail.\n") + rc = _run_bash(script) + if rc != 0: + sys.stderr.write(f"\n{name}: exited {rc}\n") + sys.exit(rc) + print(f"\n# {name}: done") + + +def cmd_bs_version_check(args): + """Chapter 2.2 -- run the book's version-check.sh on the HOST (read-only).""" + label, _title, script = _section_script(args, "version-check") + if not script: + sys.stderr.write("version-check section not found in this book.\n") + sys.exit(1) + if args.show: + print(script) + return + import tempfile, shutil + d = tempfile.mkdtemp(prefix="lfs-vc-") + try: + sys.stderr.write(f"# host version check [{label}]\n") + rc = _run_bash(script, cwd=d) + finally: + shutil.rmtree(d, ignore_errors=True) + sys.exit(rc) + + +def cmd_bs_layout(args): + _emit_or_run(args, "4.2 Creating a Limited Directory Layout", + _section_script(args, "ch-tools-creatingminlayout")[2], + needs_lfs=True) + if args.run: + # Book 4.3's chown belongs HERE, not only in `session`: layout runs as + # root and creates these directories as root. session chowns the tree + # too, but it runs BEFORE layout, when there is nothing to chown yet -- + # so without this the tree stays root-owned and chapter 5 dies with + # permission errors it blames on chapter 7 having already happened. + _chown_tree_to_lfs(require_mounted_lfs()) + + +def _chown_tree_to_lfs(lfs, quiet=False): + """Hand the build tree to the lfs user (book 4.3). A no-op if there is no + lfs user yet -- add-user calls this again once there is.""" + try: + import pwd + pwd.getpwnam("lfs") + except (KeyError, ImportError): + return False + # the handover set, plus /sources which the lfs user keeps + dirs = [d for d in LFS_HANDOVER_DIRS + ("sources",) + if os.path.exists(os.path.join(lfs, d))] + if not dirs: + return False + _run_bash('chown -R lfs "%s"/{%s} 2>/dev/null || true' + % (lfs, ",".join(dirs))) + if not quiet: + print("# gave %s to the lfs user (book 4.3): %s" + % (lfs, ", ".join(dirs))) + return True + + +def _ensure_lfs_home(run): + """Make sure the lfs user actually has a home dir (the book's `useradd -m` + should create /home/lfs, but if it didn't the env files would land in / ).""" + import subprocess + try: + import pwd + pw = pwd.getpwnam("lfs") + except (KeyError, ImportError): + return + home = pw.pw_dir if pw.pw_dir and pw.pw_dir != "/" else "/home/lfs" + if os.path.isdir(home) and pw.pw_dir == home: + return + print(f"# lfs has no proper home -- creating {home}") + if not run: + return + subprocess.run(["mkdir", "-pv", home], cwd="/") + subprocess.run(["usermod", "-d", home, "lfs"], cwd="/") + subprocess.run(["chown", "lfs:lfs", home], cwd="/") + subprocess.run(["chmod", "755", home], cwd="/") + + +def _setup_lfs_env(args, run): + """Chapter 4.4 -- WRITE the lfs user's ~/.bash_profile and ~/.bashrc AS the + lfs user (into ~lfs). This only SETS UP the environment; it does not start + it (the book's `source ~/.bash_profile` is intentionally left out). Only the + file-creation heredocs are used, so the book's inline examples (make -jN, + source ...) are skipped, and LFS= is pinned to the current $LFS.""" + _label, _title, script = _section_script(args, "ch-preps-settingenviron") + if not script: + sys.stderr.write("4.4 environment section not found.\n") + return 1 + blocks = _heredoc_writes(script) + if not blocks: + sys.stderr.write("4.4: no ~/.bash_profile / ~/.bashrc blocks found.\n") + return 1 + combined = "\n".join(blocks) + lfs = os.environ.get("LFS") + if lfs: # pin LFS= to the real path + combined = re.sub(r"^LFS=\S+", f"LFS={lfs}", combined, flags=re.M) + print("# === 4.4 Setting Up the Environment (writes ~lfs config; does NOT " + "start it) ===") + print(combined) + if not run: + dry_run_note("write them") + return 0 + require_mounted_lfs() + rc = run_as_lfs(combined) + print(f"\n# 4.4 environment: {'written' if rc == 0 else 'exited ' + str(rc)}") + return rc + + +def cmd_bs_add_user(args): + """4.3 create the lfs user (as root), ensure its home exists, then 4.4 write + its environment (as lfs). 4.3 + 4.4 belong together.""" + script = _section_script(args, "ch-preps-addinguser")[2] or "" + # Drop the book's INTERACTIVE lines: + # * `su - lfs` -- the tool switches to the lfs user itself for 4.4. + # * `passwd lfs` -- prompts for a password and would hang a non-tty --run. + # The lfs account is a build-only account; the book only sets a password + # so you can `su` to it, which the tool does as root (no password needed). + dropped = [] + kept = [] + for ln in script.splitlines(): + if ln.strip() in ("su - lfs", "su lfs", "passwd lfs"): + dropped.append(ln.strip()) + continue + kept.append(ln) + script = "\n".join(kept) + if dropped and not args.run: + print(f"# note: skipping interactive book command(s): " + f"{', '.join(dropped)}") + _emit_or_run(args, "4.3 Adding the LFS User", script, needs_lfs=True) + print() + _ensure_lfs_home(args.run) + if args.run: + # the user now exists, so the tree can finally be handed over -- this + # is a no-op when layout already did it + _chown_tree_to_lfs(require_mounted_lfs()) + if args.run: + _verify_lfs_ownership() + print() + rc = _setup_lfs_env(args, args.run) + if rc: + sys.exit(rc) + + +def _verify_lfs_ownership(): + """The book's 4.3 chown is what makes $LFS writable by the lfs user. If it + silently didn't apply, every later build fails deep inside `make install` + with 'Permission denied' -- so verify it here, where the fix is obvious, + instead of 20 minutes into a compile.""" + lfs = _lfs_dir() + if not lfs: + return + import pwd as _pwd + try: + want = _pwd.getpwnam("lfs").pw_uid + except KeyError: + return + bad = [] + for sub in ("usr", "lib64", "var", "etc", "tools"): + p = os.path.join(lfs, sub) + if os.path.isdir(p) and not os.path.islink(p): + try: + if os.stat(p).st_uid != want: + bad.append(p) + except OSError: + pass + if not bad: + print(f"# ownership check: {lfs} build dirs belong to lfs -- OK") + return + print(f"# ownership check: fixing {len(bad)} dir(s) still owned by root ...") + for p in bad: + _run_bash(f'chown -R lfs "{p}"') + still = [p for p in bad if os.path.isdir(p) and os.stat(p).st_uid != want] + if still: + sys.stderr.write( + f"! could not give the lfs user ownership of: {', '.join(still)}\n" + f" Builds WILL fail with 'Permission denied' during make install.\n" + f" Fix manually as root: chown -R lfs {' '.join(still)}\n") + sys.exit(2) + print("# ownership check: fixed -- OK") + + +def cmd_bs_show(args): + """Print the raw commands of any build-system section id (for inspection).""" + label, title, script = _section_script(args, args.section) + if script is None: + sys.stderr.write(f"section not found: {args.section}\n") + sys.exit(1) + sys.stderr.write(f"# {args.section} -- {title} [{label}]\n") + print(script) + + +def cmd_bs_session(args): + """Start/rejoin an LFS build session: show or ask for the config (partition, + mount point, swap), mount it, set $LFS, and chown the build tree to lfs.""" + cfg = load_config() + mount = cfg.get("lfs_mount") + active = mount and _is_mounted(mount) + + if active and not args.reconfigure: + print(f"Active session: {mount} is already mounted.") + else: + print("Session config:") + for k, desc, _ in _SESSION_KEYS: + print(f" {k:<16} = {cfg.get(k, '') or '(unset)'}") + have_all = all(cfg.get(k) for k, _, d in _SESSION_KEYS + if d != "" or k == "lfs_device") + if sys.stdin.isatty(): + if have_all and not args.reconfigure: + keep = input("\nKeep these settings? [Y/n]: ").strip().lower() + if keep in ("n", "no"): + cfg = _prompt_session_config(cfg, force=True) + else: + print("\nSome settings are missing -- let's set them:") + cfg = _prompt_session_config(cfg, force=args.reconfigure) + mount = cfg.get("lfs_mount") + + if not mount: + sys.stderr.write("no lfs_mount configured.\n") + sys.exit(2) + os.environ["LFS"] = mount + + if not args.run: + print("\n(dry run) would, with --run:") + print(f" mkdir -pv {mount}") + if cfg.get("lfs_device") and not active: + print(f" mount -v -t {cfg.get('lfs_fstype', 'ext4')} " + f"{cfg['lfs_device']} {mount}") + if cfg.get("lfs_swap"): + print(f" /sbin/swapon -v {cfg['lfs_swap']}") + print(f" chown lfs (build dirs under {mount})") + print(f"\nexport LFS={mount} # <-- run this in your shell") + return + + require_lfs_for_root() + if os.geteuid() != 0: + sys.stderr.write("! session --run must be run as root (it mounts " + "partitions and chowns the build tree).\n") + sys.exit(2) + import subprocess + subprocess.run(["mkdir", "-pv", mount], cwd="/") + if cfg.get("lfs_device") and not _is_mounted(mount): + rc = subprocess.run(["mount", "-v", "-t", + cfg.get("lfs_fstype", "ext4"), + cfg["lfs_device"], mount], cwd="/").returncode + if rc != 0: + sys.stderr.write("mount failed.\n") + sys.exit(rc) + if cfg.get("lfs_home_device"): + subprocess.run(["mkdir", "-pv", os.path.join(mount, "home")], cwd="/") + subprocess.run(["mount", "-v", cfg["lfs_home_device"], + os.path.join(mount, "home")], cwd="/") + if cfg.get("lfs_swap"): + subprocess.run(["/sbin/swapon", "-v", cfg["lfs_swap"]], cwd="/") + # re-apply the 4.3 chown so lfs owns the build tree (harmless if already so) + try: + import pwd + pwd.getpwnam("lfs") + _run_bash(f'chown -R lfs "{mount}"/{{usr,lib,lib64,var,etc,tools}} ' + f'2>/dev/null || true') + except (KeyError, ImportError): + pass + print(f"\nSession ready. $LFS={mount}") + print("Put this in your shell (root needs it too):") + print(f" export LFS={mount}") + print("Then become the build user: su - lfs") + + +# ---- chapter 3: sources ---------------------------------------------------- # +def _wget_list_urls(ver): + wl = wget_list_path(ver) + if not os.path.isfile(wl): + return None + urls = [] + with open(wl) as f: + for ln in f: + ln = ln.strip() + if ln and not ln.startswith("#"): + urls.append(ln) + return urls + + +def _book_patch_urls(args): + """Patch URLs listed in 3.3 Needed Patches (belt-and-suspenders on top of the + wget-list, which already includes patches).""" + _l, path = resolve_book(args) + soup = load_soup(path) + urls = set() + for a in soup.find_all("a", href=True): + h = a["href"] + if h.endswith(".patch") and h.startswith("http"): + urls.add(h) + return urls + + +def _human_size(n): + for unit in ("B", "KB", "MB", "GB"): + if n < 1024: + return f"{n:.0f}{unit}" + n /= 1024 + return f"{n:.0f}TB" + + +def cmd_bs_get_sources(args): + """Chapter 3 -- download every package + patch (from wget-list, union the + book's patch links) into $LFS/sources and verify against md5sums.""" + ver = args.book or build_version() + urls = _wget_list_urls(ver) + if urls is None: + sys.stderr.write(f"no wget-list cached for {ver}. Run: lfs fetch {ver}\n") + sys.exit(1) + urls = list(dict.fromkeys(urls + sorted(_book_patch_urls(args)))) + lfs = require_mounted_lfs() if args.run else (_lfs_dir() or "$LFS") + srcdir = os.path.join(lfs, "sources") + # get-sources usually runs BEFORE gen-chroot-scripts, so make sure the + # final-step script exists by now -- otherwise its SOURCES are silently + # skipped and the packages it needs are missing when it runs. + ensure_last_step_script() + extra = last_step_sources() + if extra: + urls = list(urls) + [u for u in extra if u not in urls] + npatch = sum(1 for u in urls if u.endswith(".patch")) + print(f"{len(urls)} files ({npatch} patches) -> {srcdir}") + if not args.run: + # print the WHOLE list: a truncated one cannot be checked, diffed or + # fed to anything else, which is the only reason to run a dry run + for u in urls: + print(f" {u}") + dry_run_note("download") + return + os.makedirs(srcdir, exist_ok=True) + _run_bash(f'chmod -v a+wt "{srcdir}" 2>/dev/null || true') + got, skip, fail = 0, 0, 0 + total = len(urls) + for i, u in enumerate(urls, 1): + fname = u.split("/")[-1] + dest = os.path.join(srcdir, fname) + tag = f"[{i}/{total}]" + if os.path.isfile(dest) and not args.force: + skip += 1 + print(f"{tag} {fname}: already have it (skip)", flush=True) + continue + print(f"{tag} {fname}: downloading ...", end=" ", flush=True) + t0 = time.time() + # try the mirror first (upstream links rot), then the book's own URL + sources = [] + mb = mirror_base(ver) + if mb: + sources.append(mb.rstrip("/") + "/" + fname) + sources.append(u) + n, err = None, None + for src in sources: + try: + n = _download_to(src, dest) + break + except Exception as e: + err = e + if n is not None: + got += 1 + via = " (mirror)" if mb and src == sources[0] else "" + print(f"{_human_size(n)} in {time.time() - t0:.1f}s{via}", flush=True) + else: + fail += 1 + print(f"FAILED: {err}", flush=True) + print(f"\ndownloaded {got}, skipped {skip}, failed {fail}") + md5 = md5sums_path(ver) + if os.path.isfile(md5): + import shutil + shutil.copy(md5, os.path.join(srcdir, "md5sums")) + print("verifying md5sums...", flush=True) + _run_bash(f'cd "{srcdir}" && md5sum -c md5sums | grep -v ": OK" || ' + f'echo "all md5sums OK"') + if fail: + sys.exit(1) + + +# ---- chapter 5: the cross toolchain --------------------------------------- # +# (step name, section id, source-tarball package name) +CROSSCHAIN_STEPS = [ + # ---- chapter 5: cross toolchain ---- + ("binutils-pass1", "ch-tools-binutils-pass1", "binutils"), + ("gcc-pass1", "ch-tools-gcc-pass1", "gcc"), + ("linux-headers", "ch-tools-linux-headers", "linux"), + ("glibc", "ch-tools-glibc", "glibc"), + ("libstdcpp", "ch-tools-libstdcpp", "gcc"), + # ---- chapter 6: temporary tools (built WITH the ch5 cross toolchain) ---- + ("m4", "ch-tools-m4", "m4"), + ("ncurses", "ch-tools-ncurses", "ncurses"), + ("bash", "ch-tools-bash", "bash"), + ("coreutils", "ch-tools-coreutils", "coreutils"), + ("diffutils", "ch-tools-diffutils", "diffutils"), + ("file", "ch-tools-file", "file"), + ("findutils", "ch-tools-findutils", "findutils"), + ("gawk", "ch-tools-gawk", "gawk"), + ("grep", "ch-tools-grep", "grep"), + ("gzip", "ch-tools-gzip", "gzip"), + ("make", "ch-tools-make", "make"), + ("patch", "ch-tools-patch", "patch"), + ("sed", "ch-tools-sed", "sed"), + ("tar", "ch-tools-tar", "tar"), + ("xz", "ch-tools-xz", "xz"), + ("binutils-pass2", "ch-tools-binutils-pass2", "binutils"), + ("gcc-pass2", "ch-tools-gcc-pass2", "gcc"), +] + +CROSSCHAIN_MARK = "### lfs-crosschain-script" + + +def _pkg_glob_for(pkgver, tarpfx): + """Candidate tarball patterns for a package. Book titles are capitalised + ("Python-3.13.7") and so is the real tarball, but plenty of packages ship + lowercase -- so try BOTH spellings rather than guessing (a lowercase-only + glob silently misses Python-3.13.7.tar.xz).""" + stems = [] + if pkgver: + stems.append(pkgver) + if pkgver.lower() != pkgver: + stems.append(pkgver.lower()) + else: + stems.append(f"{tarpfx}-*") + exts = ("tar.*", "tgz", "tbz2", "zip") + return " ".join(f"{st}.{e}" for st in stems for e in exts) + + +def _title_pkgver(title): + """The Name-Version token from a book section title. + + Titles look like "7.12. Util-linux-2.41.1", "5.2. Binutils-2.45 - Pass 1", + "5.4. Linux-6.16.1 API Headers", "5.6. Libstdc++ from GCC-15.2.0". Scanning + for the first "<letters>-<digits>" match is WRONG for hyphenated names -- + in "Util-linux-2.41.1" it skips past "Util-" and returns "linux-2.41.1", + which then looks for the wrong tarball entirely. So: drop the section + number, then take the first whole whitespace-separated token that ends in a + version.""" + t = re.sub(r"^\d+(\.\d+)*\.\s*", "", (title or "").strip()) + tok_re = re.compile(r"^[A-Za-z][A-Za-z0-9+_.]*" # name start + r"(?:(?:::|[-_])[A-Za-z][A-Za-z0-9+_.]*)*" # ::/-/_ parts + r"[-_][0-9][0-9.]*(?:-[0-9]+)*$") # -version + for tok in t.split(): + if tok_re.match(tok): + return tok + return None + + +def _pkg_glob_for(pkgver, tarpfx): + """Candidate tarball patterns for a package, most specific first. + + Book titles are capitalised ("Python-3.13.7", "Util-linux-2.41.1") but the + tarball may be lowercase, and a few packages drop the dash entirely + (Expect-5.45.4 ships as expect5.45.4.tar.gz, Tcl-8.6.16 as + tcl8.6.16-src.tar.gz), so try each spelling rather than guessing.""" + stems, seen = [], set() + + def add(x): + if x and x not in seen: + seen.add(x) + stems.append(x) + + if pkgver: + for base in (pkgver, pkgver.lower()): + add(base) + # Perl modules are written XML::Parser-2.47 but ship as XML-Parser- + add(base.replace("::", "-")) + # Python dists are titled Flit-Core-3.12.0 but ship as flit_core- + add(base.replace("-", "_", base.count("-") - 1) + if base.count("-") > 1 else base) + # dash-less spelling: Expect-5.45.4 -> expect5.45.4 + add(re.sub(r"-(?=[0-9])", "", pkgver.lower(), count=1)) + else: + add(f"{tarpfx}-*") + exts = ("tar.*", "tgz", "tbz2", "zip") + pats = [f"{st}.{e}" for st in stems for e in exts] + # last resort: allow a suffix (tcl8.6.16-src.tar.gz, foo-1.2-source.tar.gz) + pats += [f"{st}*.{e}" for st in stems for e in ("tar.*", "tgz")] + return " ".join(pats) + + +# The books contain PLACEHOLDERS meant for a human to replace: +# PAGE=<paper_size> ./configure --prefix=/usr +# A shell reads "<paper_size>" as an input redirect and the build dies with +# line 4: paper_size: No such file or directory +# Substitute the ones we have settings for, and refuse to generate a script that +# still contains an unresolved one rather than letting it fail at build time. +# Placeholders come in more shapes than lowercase words: +# <paper_size> <locale name> <charmap> (lowercase) +# <FQDN> <HOSTNAME> <CC> <ll> <@modifiers> (upper, short, punctuated) +# <192.168.1.2> (an example address) +# Missing the latter wrote "<ll>_<CC>.UTF-8<@modifiers>" straight into +# /etc/profile, which then breaks every login shell: +# /etc/profile: line 10: syntax error near unexpected token `newline' +# The character class deliberately excludes < and > so shell redirects and +# heredocs (<<EOF, 2>&1) are never touched. +_PLACEHOLDER_RE = re.compile(r'<([A-Za-z0-9_.@ -]{1,40})>') + + +def _fill_placeholders(text, name): + cfg = load_config() + loc = cfg.get("locale") or "en_US.UTF-8" + host = cfg.get("hostname") or "lfs" + # The book spells the locale out piecewise in /etc/profile: + # export LANG=<ll>_<CC>.<charmap><@modifiers> + # Substituting the parts individually leaves a broken hybrid, so replace + # the whole construct first. + text = re.sub(r'<ll>_<CC>\.<charmap><@modifiers>', loc, text) + text = re.sub(r'<ll>_<CC>\.[A-Za-z0-9-]+<@modifiers>', loc, text) + text = re.sub(r'<ll>_<CC>[^\s]*', loc, text) + # /etc/hosts' optional static-IP line is a decision, not a default: comment + # it out rather than writing a made-up address into the file. + text = re.sub(r'^(<\d+\.\d+\.\d+\.\d+>.*)$', + r'# \1 <-- uncomment and edit if this host has a static IP', + text, flags=re.M) + known = { + "paper_size": cfg.get("paper_size") or "A4", + "lfs": cfg.get("hostname") or "lfs", + "locale name": loc, + "charmap": cfg.get("charmap") or "UTF-8", + "HOSTNAME": host, + "FQDN": "%s.localdomain" % host, + "ll": loc.split("_")[0], + "CC": (loc.split("_")[1].split(".")[0] + if "_" in loc else "US"), + "@modifiers": "", + } + def sub(m): + key = m.group(1) + if key in known: + return known[key] + return m.group(0) + out = _PLACEHOLDER_RE.sub(sub, text) + + # Report only things that are really a "fill this in" instruction. The + # angle brackets also enclose perfectly ordinary text: + # sed '/unistd.h/i #include <string.h>' (a C include) + # # maintained by <roryo@roryo.dynup.net> (an email in a comment) + # and anything already commented out needs no action either. + def actionable(ph, line): + if line.lstrip().startswith("#"): + return False + if re.fullmatch(r'[A-Za-z0-9_/.+-]+\.[hc]', ph): # <string.h> + return False + if "@" in ph and "." in ph: # an address + return False + if re.fullmatch(r'[\d.]+', ph): # a bare IP + return False + return True + + left = [] + for line in out.splitlines(): + for ph in _PLACEHOLDER_RE.findall(line): + if actionable(ph, line) and ph not in left: + left.append(ph) + return out, sorted(left) + + +def _crosschain_pkgver(args, sid): + """(version-label, commands) for a crosschain step's section.""" + _l, path = resolve_book(args) + soup = load_soup(path) + for s, t, sec in iter_sections(soup): + if s == sid: + return (_title_pkgver(t) or t), sec["commands"] + return None, [] + + +def _crosschain_all_pkgvers(args): + """(name -> (version-label, commands)) for every crosschain step, parsing + the book exactly ONCE instead of once per step (that repeated re-parse was + what made --list slow).""" + _l, path = resolve_book(args) + soup = load_soup(path) + sid_to_name = {sid: name for name, sid, _tarpfx in CROSSCHAIN_STEPS} + out = {} + for s, t, sec in iter_sections(soup): + name = sid_to_name.get(s) + if name: + out[name] = ((_title_pkgver(t) or t), sec["commands"]) + return out + + +def crosschain_dir(ver): + """Scripts don't hardcode $LFS (they use the shell variable), so they're + reusable across sessions -- stored once per BOOK version in the shared + store, next to the book itself, world-readable/editable.""" + d = os.path.join(store_dir(), "crosschain", ver) + os.makedirs(d, exist_ok=True) + try: + os.chmod(d, 0o755) + except OSError: + pass + return d + + +def crosschain_script_path(ver, name): + return os.path.join(crosschain_dir(ver), f"{name}.sh") + + +CROSSCHAIN_SCRIPT_VERSION = 17 # v17: test scaffolding stays with the tests + +_INSTALL_BLOCK_RE = re.compile(r'\b(make|ninja)\b[^\n]*\binstall\b') + + +# A block that runs the package's test suite. These are pulled OUT of the build +# phase: the books expect some tests to fail ("A few failures out of over 6000 +# tests can generally be ignored" for Glibc), so a non-zero `make check` must not +# abort the build the way a failed compile does. +# Test invocations take several shapes in the books: +# make check +# make -k check +# su tester -c "PATH=$PATH make -k check" +# spawn make tests (inside an expect heredoc) +# ninja test / meson test +# so match the make/ninja/meson call anywhere in the block, not just at the +# start of a line -- anchoring missed everything wrapped in `su tester -c`. +_TEST_BLOCK_RE = re.compile( + r'\b(?:make|ninja|meson)\b[^\n]*?[\s"\']' + # the target may carry a suffix: test_harness, check-TESTS, check-recursive + r'(?:-k\s+)?(?:check|test|tests)(?:[-_][A-Za-z_]+)?(?:\s|"|\'|$)') + +# Diagnostics that belong WITH the test suite, not with the build. Glibc's +# grep "Timed out" $(find -name \*.out) +# inspects test output; when nothing matches -- the normal, healthy case -- +# grep exits 1 and `set -e` kills the build with no message at all. +_TEST_DIAG_RE = re.compile( + r'Timed out|find\s+-name\s+\\?\*\.out|\btest-suite\.log\b' + r'|[\w.-]*check-log\b|\bcheck\.log\b|\btests?\.log\b' + # Preparation and summary for a test run that, for us, never happens. + # GCC's chapter-8 section is: + # make + # ulimit -s -H unlimited + # sed ... ../gcc/testsuite/gcc.dg/plugin/plugin.exp + # sed ... ../gcc/testsuite/gcc.target/i386/... + # chown -R tester . + # su tester -c "PATH=$PATH make -k check" + # ../contrib/test_summary + # Only the chown/su lines were recognised, so the ulimit, the testsuite + # seds and test_summary stayed in the BUILD phase. test_summary greps + # test logs that do not exist and exits non-zero, and `set -e` then fails + # the build -- reported as "phase 'build' FAILED" with make having + # succeeded. + r'|\btestsuite\b|\btest_summary\b|^\s*ulimit\b') + +# Scaffolding that exists only so the test suite can run: the books create a +# `tester` user and a throwaway group, hand the build tree to them, and tear it +# down afterwards. Coreutils does +# groupadd -g 102 dummy -U tester +# chown -R tester . +# su tester -c "... make ... check" +# groupdel dummy +# Left in the build phase, those run as the package user and die with +# groupadd: Permission denied. +# They belong with the tests -- and are skipped entirely unless tests are asked +# for, which is why the build no longer needs to touch /etc/group at all. +_TEST_SETUP_RE = re.compile( + r'^\s*(?:groupadd|groupdel|userdel|usermod)\b[^\n]*\btester\b' + r'|^\s*(?:groupadd|groupdel)\b[^\n]*\bdummy\b' + r'|^\s*chown\b[^\n]*\btester\b' + r'|^\s*(?:useradd|adduser)\b[^\n]*\btester\b', + re.M) + + +# files a test block writes (its log), so the block that reads them back can be +# kept with the tests instead of being left behind in the build +_TEE_TARGET_RE = re.compile(r'\|\s*tee\s+(?:-a\s+)?([\w./-]+)') +_REDIR_TARGET_RE = re.compile(r'&?>\s*([\w./-]+\.(?:log|out|txt))') + + +def _is_test_block(b): + return bool(_TEST_BLOCK_RE.search(b) or _TEST_DIAG_RE.search(b) + or _TEST_SETUP_RE.search(b)) + + +# A lone `grep ...` in a build sequence is informational ("did anything go +# wrong?"), and grep exits 1 when it finds nothing. Under `set -e` that aborts +# the build silently, so such lines get an explicit `|| true`. +_LONE_GREP_RE = re.compile(r'^\s*grep\b[^\n]*$') + + +# Re-running a phase must not fail just because it already did its job. The +# books write `ln -sv target /usr/lib` with no -f, so the second attempt dies +# with "failed to create symbolic link '/usr/lib/cpp': File exists" -- which +# makes retrying an install or configure phase impossible. Adding -f keeps the +# intent (make this symlink point here) and makes the step repeatable. +_LN_FLAGS_RE = re.compile(r'\bln\s+(-[A-Za-z]+)') + + +def _idempotent_links(block): + def fix(m): + flags = m.group(1) + if "s" in flags and "f" not in flags: + return f"ln {flags}f" + return m.group(0) + return _LN_FLAGS_RE.sub(fix, block) + + +def _soften_diagnostics(block): + out = [] + for ln in block.split("\n"): + if _LONE_GREP_RE.match(ln) and not ln.rstrip().endswith(("\\", "|| true")): + out.append(ln.rstrip() + " || true # informational: no match is fine") + else: + out.append(ln) + return "\n".join(out) + + +def _split_crosschain_phases(cmds): + """Split a step's command BLOCKS (the book gives us discrete <pre> blocks) + into build / install / configure / test -- the same phases BLFS uses. + + Everything up to the first 'make/ninja ... install' block is build; that + block is install; anything after it (sanity checks, symlink/doc fixups) is + configure. Test-suite blocks are lifted out of whichever phase they landed + in and returned separately. If no install-style block exists (e.g. Linux + API headers, which install with a plain `cp`), everything stays in build -- + the same convention BLFS uses for packages that install during their build. + """ + tests = [b for b in cmds if _is_test_block(b)] + # A test block usually writes a log, and a LATER block reads it back to + # summarise the run -- GMP does + # make check 2>&1 | tee gmp-check-log + # awk '/# PASS:/{total+=$3} ; END{print total}' gmp-check-log + # If only the first moves to the test phase, the awk stays in the build and + # dies with "cannot open file `gmp-check-log'". So anything referring to a + # file the tests produced belongs with the tests. + artifacts = set() + for b in tests: + artifacts |= set(_TEE_TARGET_RE.findall(b)) + artifacts |= set(_REDIR_TARGET_RE.findall(b)) + artifacts = {a for a in artifacts if len(a) > 3 and not a.startswith("/")} + if artifacts: + extra = [b for b in cmds + if b not in tests and any(a in b for a in artifacts)] + tests = [b for b in cmds if b in tests or b in extra] + rest = [_idempotent_links(_soften_diagnostics(b)) + for b in cmds if b not in tests] + idx = next((i for i, b in enumerate(rest) if _INSTALL_BLOCK_RE.search(b)), + None) + test_cmds = "\n\n".join(tests) + if idx is None: + return "\n\n".join(rest), "", "", test_cmds + return ("\n\n".join(rest[:idx]), rest[idx], + "\n\n".join(rest[idx + 1:]), test_cmds) + + +def _indent(text, tab="\t"): + return "\n".join((tab + ln if ln.strip() else ln) + for ln in text.splitlines()) + + +def _crosschain_script_body(name, sid, pkgver, glob, cmds): + ts = time.strftime("%Y-%m-%d %H:%M:%S") + build_cmds, install_cmds, config_cmds, test_cmds = \ + _split_crosschain_phases(cmds) + L = [] + w = L.append + w("#!/bin/bash") + w("############################################") + w(f"### {name} ({pkgver})") + w(f"{CROSSCHAIN_MARK} v{CROSSCHAIN_SCRIPT_VERSION}: {name}") + w("### phased build script -- run a single phase with its name as $1:") + w("### all (default) | unpack | build | install | configure | test") + w("### re-run just 'install' after fixing a failure, without recompiling.") + w("###") + w("### This is YOUR editable script -- lfs reuses it as-is on every --run.") + w(f"### To pull a fresh copy from the book (discarding edits):") + w(f"### lfs build-system crosschain {name} --regenerate\n") + w(f'section="{sid}"') + w(f'name_version="{pkgver}"') + w(f'pkg_glob="{glob}"') + w("") + w("# safety net: these are normally embedded by 'lfs build-system crosschain") + w("# --run' itself, but if you run this script directly (e.g. from inside your") + w("# own 'su - lfs' session) it still notices and fixes an unset LFS_TGT/PATH,") + w("# and refuses rather than silently building against the wrong toolchain.") + w('if [ -z "$LFS" ]; then') + w('\techo "!! \\$LFS is not set -- refusing to guess. ' + 'export LFS=/mnt/lfs (see book 2.6) and re-run." >&2') + w('\texit 2') + w("fi") + w('if [ -z "$LFS_TGT" ]; then') + w('\tLFS_TGT=$(uname -m)-lfs-linux-gnu') + w('\texport LFS_TGT') + w('\techo "note: \\$LFS_TGT was unset -- defaulting to $LFS_TGT (book 4.4)" >&2') + w("fi") + w('case ":$PATH:" in') + w('\t*":$LFS/tools/bin:"*) ;;') + w('\t*) PATH="$LFS/tools/bin:$PATH"; export PATH') + w('\t echo "note: added $LFS/tools/bin to \\$PATH (book 4.4)" >&2 ;;') + w("esac") + w('if [ -z "$MAKEFLAGS" ]; then') + w('\texport MAKEFLAGS="-j$(nproc)"') + w('\techo "note: \\$MAKEFLAGS was unset -- defaulting to $MAKEFLAGS "' + '"(book 4.4, use all cores)" >&2') + w("fi") + + w('\n# exported: the phase bodies below run as separate `bash -s` processes,') + w('# so anything they reference must be in the ENVIRONMENT, not just a shell') + w('# variable of this script.') + w('export LFS') + w('BUILD_ROOT="${BUILD_ROOT:-$LFS/sources}"') + w('export BUILD_ROOT') + w('pkg_dir=""') + w("") + w("# Run a phase body as its OWN process, from a FILE -- never piped in on") + w("# stdin. A book command that reads stdin (Expect's PTY check does:") + w("# python3 -c 'from pty import spawn; spawn([\"echo\", \"ok\"])' )") + w("# would otherwise swallow the rest of the script and silently skip the") + w("# real build. stdin is /dev/null so nothing can consume it or hang.") + w("_phase_body() {") + w('\tlocal _f; _f="$(mktemp)" || return 1') + w('\tcat > "$_f"') + w('\tbash "$_f" </dev/null') + w('\tlocal _r=$?') + w('\trm -f "$_f"') + w('\treturn $_r') + w("}\n") + w("_enter_build() {") + w('\tcd "$BUILD_ROOT" || return 1') + w(f'\t[ -z "$pkg_dir" ] && pkg_dir="$(cat .cc-dir-{name} 2>/dev/null)"') + w('\tif [ -n "$pkg_dir" ] && [ -d "$pkg_dir" ]; then cd "$pkg_dir" || return 1') + w('\telse echo "no unpacked source in $BUILD_ROOT -- run: $0 unpack" >&2; return 1; fi') + w("}\n") + + w("# resume in the SAME subdir build_pkg finished in (e.g. pkg_dir/build),") + w("# not just pkg_dir -- the book's own post-build commands (install,") + w("# post-install fixups) assume they're still there, same as reading the") + w("# book top-to-bottom would put you.") + w("_resume_build_dir() {") + w(f'\tif [ -f "$BUILD_ROOT/.cc-build-{name}" ] && ' + f'[ -d "$(cat "$BUILD_ROOT/.cc-build-{name}")" ]; then') + w(f'\t\tcd "$(cat "$BUILD_ROOT/.cc-build-{name}")"') + w('\t\treturn 0') + w('\tfi') + w('\t# No marker: build_pkg records the build directory as its LAST step, so') + w('\t# a build that died part-way (a failed test suite, an interrupted') + w('\t# compile) never wrote one. Fall back to the usual out-of-tree layout') + w('\t# -- without this, `make install` runs in the source root and glibc') + w('\t# stops with "objdir must be defined by the build-directory Makefile".') + w('\tlocal _d') + w('\tfor _d in build builddir _build obj; do') + w('\t\tif [ -d "$_d" ] && { [ -f "$_d/Makefile" ] || [ -f "$_d/build.ninja" ] ' + '|| [ -f "$_d/config.status" ]; }; then') + w('\t\t\techo "# resuming in ./$_d (no build marker was recorded)" >&2') + w('\t\t\tcd "$_d"; return 0') + w('\t\tfi') + w('\tdone') + w('\treturn 0') + w("}\n") + + w("unpack_pkg() {") + w("#### UNPACK ####") + w('\tcd "$BUILD_ROOT" || return 1') + w('\t# Pick the source tarball. Two traps here:') + w('\t# * do NOT use `ls $pkg_glob` -- patterns that match nothing make ls') + w('\t# fail and swallow the one that did match;') + w('\t# * several packages ship docs beside the source (tcl8.6.16-html.tar.gz') + w('\t# sorts BEFORE tcl8.6.16-src.tar.gz), so prefer an explicit -src') + w('\t# archive and never pick a documentation one.') + w('\tpkg=""') + w('\tfor _c in $pkg_glob; do') + w('\t\t[ -e "$_c" ] || continue') + w('\t\tcase "$_c" in *-src.*|*-source.*) pkg="$_c"; break ;; esac') + w('\tdone') + w('\tif [ -z "$pkg" ]; then') + w('\t\tfor _c in $pkg_glob; do') + w('\t\t\t[ -e "$_c" ] || continue') + w('\t\t\tcase "$_c" in') + w('\t\t\t\t*-html.*|*-doc.*|*-docs.*|*-man.*|*-manual.*|*-tests.*) continue ;;') + w('\t\t\tesac') + w('\t\t\tpkg="$_c"; break') + w('\t\tdone') + w('\tfi') + w('\t[ -n "$pkg" ] || { echo "no $pkg_glob in $BUILD_ROOT -- run: ' + 'lfs build-system get-sources --run" >&2; return 1; }') + w('\tdir=$(tar tf "$pkg" | head -n1 | cut -d/ -f1)') + w('\t# --no-same-owner: as root, tar would restore the UIDs recorded IN the') + w('\t# archive, littering /sources with owners like 8282 or 15399 that do') + w('\t# not exist here. The unpacking user should own what it unpacks.') + w('\trm -rf "$dir"; tar --no-same-owner -xf "$pkg"') + w('\tpkg_dir="$dir"') + w(f'\techo "$pkg_dir" > "$BUILD_ROOT/.cc-dir-{name}"') + w('\techo "unpacked into $BUILD_ROOT/$pkg_dir"') + w("#### UNPACK DONE ####") + w("}\n") + + w("# Each phase body runs as a SEPARATE 'bash -s' process. This matters:") + w("# bash disables 'set -e' inside any function reached from a tested") + w("# context (an && chain, an || handler), and that suppression is even") + w("# inherited by nested subshells -- so a failing 'make' would be silently") + w("# ignored and the step would wrongly report success. A separate process") + w("# has its own -e state that nothing upstream can suppress, so failures") + w("# always propagate.") + w("build_pkg() {") + w("#### BUILD ####") + w("\t_enter_build") + w("\t_phase_body <<'__LFS_PHASE__'") + w("set -e") + # record the build directory as soon as the commands have cd'd into it, so + # a later failure still leaves install/configure able to find it + w("_mark_build_dir() { pwd > \"$BUILD_ROOT/.cc-build-%s\"; }" % name) + w("trap _mark_build_dir EXIT") + w(build_cmds if build_cmds.strip() else ': # (nothing to build)') + w(f'pwd > "$BUILD_ROOT/.cc-build-{name}"') + w("__LFS_PHASE__") + w("\t_rc=$?; [ $_rc -eq 0 ] || return $_rc") + w("#### BUILD DONE ####") + w("}\n") + + w("install_pkg() {") + w("#### INSTALL ####") + w("\t_enter_build") + w("\t_resume_build_dir") + w("\t_phase_body <<'__LFS_PHASE__'") + w("set -e") + w(install_cmds if install_cmds.strip() + else ': # (this step installs during the build phase)') + w("__LFS_PHASE__") + w("\t_rc=$?; [ $_rc -eq 0 ] || return $_rc") + w("#### INSTALL DONE ####") + w("}\n") + + if config_cmds.strip(): + w("configure_pkg() {") + w("#### CONFIGURE ####") + w("\t_enter_build") + w("\t_resume_build_dir") + w("\t_phase_body <<'__LFS_PHASE__'") + w("set -e") + w(config_cmds) + w("__LFS_PHASE__") + w("\t_rc=$?; [ $_rc -eq 0 ] || return $_rc") + w("#### CONFIGURE DONE ####") + w("}\n") + else: + w("configure_pkg() { : ; }\n") + + w("test_pkg() {") + if test_cmds.strip(): + w("#### TEST ####") + w("\t_enter_build") + w("\t_resume_build_dir") + w("\t# The book expects some tests to fail, so this reports the result") + w("\t# instead of aborting -- read the summary and judge for yourself.") + w("\t_phase_body <<'__LFS_PHASE__'") + w(test_cmds) + w("__LFS_PHASE__") + w('\t_rc=$?') + w('\tif [ $_rc -ne 0 ]; then') + w('\t\techo "!! test suite exited $_rc -- check the summary above." >&2') + w('\t\techo " Some failures are normal (the book lists the usual"' + ' >&2') + w('\t\techo " suspects); this does NOT stop the install." >&2') + w('\tfi') + w('\treturn 0') + w("#### TEST DONE ####") + else: + w('\t: # the book defines no test suite for this step') + w('\techo "no test suite for this step"') + w("}\n") + + w('_phase_fail() { echo "!! phase \'$1\' FAILED (exit $2)" >&2; exit "$2"; }') + w('case "${LFS_CC_PHASE:-${1:-all}}" in') + w("\tall) unpack_pkg || _phase_fail unpack $?") + w("\t build_pkg || _phase_fail build $?") + w('\t [ "${LFS_RUN_TESTS:-0}" = "1" ] && test_pkg') + w("\t install_pkg || _phase_fail install $?") + w("\t configure_pkg || _phase_fail configure $? ;;") + w("\tunpack) unpack_pkg || _phase_fail unpack $? ;;") + w("\tbuild) build_pkg || _phase_fail build $? ;;") + w("\tinstall) install_pkg || _phase_fail install $? ;;") + w("\tconfigure) configure_pkg || _phase_fail configure $? ;;") + w("\ttest) test_pkg || _phase_fail test $? ;;") + w('\t*) echo -e "usage: $0 {all|unpack|build|install|configure|test}\\n' + ' all full run (default)\\n' + ' unpack fetch tarball from $LFS/sources + extract\\n' + ' build configure + compile (needs unpack)\\n' + ' install install step only -- re-run after fixing a failure\\n' + ' configure post-install fixups (symlinks, sanity checks)\\n' + ' test run this step\'s tests (edit test_pkg first)" ;;') + w("esac") + + header_tag = f"{CROSSCHAIN_MARK} v{CROSSCHAIN_SCRIPT_VERSION}: {name}" + body = "\n".join(L) + "\n" + # Stamp which book this came from: the scripts outlive the book version, + # and "where did this come from?" is otherwise unanswerable. + book = "LFS %s" % (load_config().get("default") or "?") + body = body.replace(header_tag, + f"{header_tag}\n### book : {book}" + f"\n### generated : {ts}", 1) + return body + + +def ensure_crosschain_script(args, ver, name, sid, tarpfx, force_regen=False, + book_info=None): + """Return the on-disk script path for a step, writing/refreshing it from the + book if it doesn't exist yet or --regenerate was asked for. Prints which one + it's using so it's never ambiguous. `book_info` (name -> (pkgver, cmds)), + if given, avoids re-parsing the book per step (see _crosschain_all_pkgvers).""" + path = crosschain_script_path(ver, name) + if os.path.isfile(path) and not force_regen: + print(f"# using SAVED script (edit it freely): {path}") + return path + if book_info is not None and name in book_info: + pkgver, cmds = book_info[name] + else: + pkgver, cmds = _crosschain_pkgver(args, sid) + glob = _pkg_glob_for(pkgver, tarpfx) + text = _crosschain_script_body(name, sid, pkgver, glob, cmds) + if os.path.isfile(path): + import shutil + shutil.copy(path, path + ".bak") + print(f"# regenerated from the book (previous version saved as " + f"{path}.bak)") + with open(path, "w") as f: + f.write(text) + _make_world_readable(path) + print(f"# wrote new script -> {path}") + return path + + +# ---- crosschain resume: remember progress across steps so a cancelled run can +# continue where it left off instead of restarting from step 1 ---------------- +def _crosschain_progress_path(lfs): + d = os.path.join(lfs, PKGUSR_DIR) + os.makedirs(d, exist_ok=True) + return os.path.join(d, "crosschain-progress.json") + + +def _chapter56_has_output(lfs): + """Is there real chapter 5-6 output in this tree? + + The cross toolchain leaves unmistakable traces: a cross-gcc in tools/bin, + or a populated $LFS/usr/bin. An empty tree that root happens to own is + just `layout` having run.""" + tools_bin = os.path.join(lfs, "tools", "bin") + if os.path.isdir(tools_bin): + for f in os.listdir(tools_bin): + if f.endswith(("-gcc", "-ld", "-as")): + return True + usr_bin = os.path.join(lfs, "usr", "bin") + try: + if len(os.listdir(usr_bin)) > 20: + return True + except OSError: + pass + return os.path.isfile(_crosschain_progress_path(lfs)) + + +def _load_crosschain_progress(lfs): + try: + with open(_crosschain_progress_path(lfs)) as f: + return json.load(f) + except Exception: + return None + + +def _mark_crosschain_done(lfs, ver, name): + """Record ONE completed step. Progress is a global set of finished steps, + not a snapshot of whatever list the last command happened to run -- so + building a single package in between (crosschain ncurses --run) adds to the + record instead of wiping the full run's progress.""" + prog = _load_crosschain_progress(lfs) or {} + if prog.get("book") != ver: + prog = {"book": ver, "done": []} + done = [d for d in prog.get("done", []) if d != name] + done.append(name) + with open(_crosschain_progress_path(lfs), "w") as f: + json.dump({"book": ver, "done": done}, f, indent=2) + + +def _clear_crosschain_progress(lfs): + try: + os.remove(_crosschain_progress_path(lfs)) + except OSError: + pass + + +def _require_lfs_writable_by_lfs(lfs): + """Fail fast if the lfs user can't write where the build installs. Without + this, binutils/gcc compile happily for minutes and only die inside `make + install` with a wall of 'mkdir: Permission denied' -- the real cause (the + book's 4.3 chown never applied, or a root-created subdir) being far off + screen by then.""" + import pwd as _pwd + try: + want = _pwd.getpwnam("lfs").pw_uid + except KeyError: + sys.stderr.write("! the 'lfs' user doesn't exist yet.\n" + " Run: lfs build-system add-user --run\n") + sys.exit(2) + # After book 7.2 the whole tree is handed to root on purpose, and chapters + # 5-6 are finished -- don't demand it be given back to lfs. + # + # But root owning the tree is NOT by itself evidence of that: `layout` + # runs as root and creates the tree as root, so a brand-new tree looks + # identical. Only refuse when chapter 5-6 work actually exists; otherwise + # this is a fresh tree that simply needs handing to the lfs user. + if _owner_is_root(lfs) and not _chapter56_has_output(lfs): + # a fresh tree that layout made as root: just hand it over + if _chown_tree_to_lfs(lfs): + pass + if _owner_is_root(lfs) and _chapter56_has_output(lfs): + sys.stderr.write( + f"\n! {lfs} is owned by root -- chapter 7 (chroot) has already been\n" + f" prepared, so the chapter 5-6 steps are done and shouldn't run\n" + f" as the lfs user again.\n" + f" Work inside the chroot instead:\n" + f" {sudo_prefix()}lfs build-system chroot enter\n" + f" (to deliberately go back, run: {sudo_prefix()}lfs build-system " + f"fix-ownership --run)\n") + sys.exit(2) + bad = [] + for sub in ("tools", "usr", "var", "etc", "lib64"): + p = os.path.join(lfs, sub) + if os.path.isdir(p) and not os.path.islink(p): + try: + if os.stat(p).st_uid != want: + bad.append(p) + except OSError: + pass + # also catch root-owned dirs NESTED inside those trees: a single root-owned + # $LFS/var/lib is enough to break glibc's install (var/lib/nss_db), even + # though $LFS/var itself looks fine. + nested = [] + for sub in ("var", "usr", "etc", "tools"): + root_p = os.path.join(lfs, sub) + if not os.path.isdir(root_p) or os.path.islink(root_p): + continue + for dirpath, dirnames, _files in os.walk(root_p): + dirnames[:] = [d for d in dirnames + if not os.path.islink(os.path.join(dirpath, d))] + try: + if os.stat(dirpath).st_uid != want and dirpath not in bad: + nested.append(dirpath) + except OSError: + pass + if len(nested) > 20: + break + problem = bad + nested + if problem: + shown = problem[:10] + sys.stderr.write( + f"\n! these build dirs are not owned by the 'lfs' user:\n" + f" {', '.join(shown)}" + + (f" (and {len(problem) - len(shown)} more)" if len(problem) > len(shown) else "") + + f"\n The build would fail inside 'make install' with 'Permission " + f"denied'.\n" + f" Fix all of them with:\n" + f" {sudo_prefix()}lfs build-system fix-ownership --run\n") + sys.exit(2) + + +def cmd_bs_fix_ownership(args): + """Give the lfs user ownership of the whole build tree again. Useful after + something ran as root and created a root-owned dir inside $LFS (which then + blocks packages installing as lfs).""" + lfs = require_mounted_lfs() + targets = [os.path.join(lfs, s) for s in + ("tools", "usr", "var", "etc", "lib64", "lib", "bin", "sbin", + "sources", PKGUSR_DIR)] + targets = [p for p in targets if os.path.exists(p) and not os.path.islink(p)] + print(f"# would give 'lfs' ownership of:\n" + + "\n".join(f" {p}" for p in targets)) + if not args.run: + dry_run_note("apply") + return + if os.geteuid() != 0: + sys.stderr.write("! must be run as root.\n") + sys.exit(2) + for p in targets: + _run_bash(f'chown -R lfs "{p}"') + print("# ownership restored to lfs.") + + +# Every user-settable key, with a description and how the default is derived. +# Which settings belong to which job. Two different things share this file: +# building a system (a one-off, with a partition and a target triplet), and +# installing packages onto one (ongoing). Showing them in one flat list made +# it impossible to tell which was which -- particularly for the two book +# settings, where "which book?" has two different answers. +CONFIG_SECTIONS = [ + ("installing packages on this system", [ + "default", "mirror", "editor", "sudo", + ]), + ("building a system: the disk", [ + "lfs_mount", "lfs_device", "lfs_fstype", "lfs_swap", "lfs_home_device", + "esp", "bootloader", + ]), + ("building a system: how it is built", [ + "build_book", "lfs_tgt", "makeflags", "collector_prefix", + "collector_import_file", "last_step_script", + ]), + ("the system being built: how it is set up", [ + "hostname", "locale", "charmap", "paper_size", "locales", "network", + "timezone", "keymap", "console_font", + ]), +] + + +# Which part of the tool each setting belongs to. Two different jobs share +# one config file: building a system from the book, and installing packages on +# a machine afterwards. Printing them in one flat list made it impossible to +# tell which setting affected which -- and "set-default" ambiguous. +CONFIG_SECTIONS = [ + ("building a system", [ + "build_book", "lfs_mount", "lfs_device", "lfs_fstype", "lfs_swap", + "lfs_home_device", "lfs_tgt", "makeflags", "bootloader", "esp", + "last_step_script", "snapshot_dir", + ]), + ("the system being built (its settings, not yours)", [ + "locale", "charmap", "paper_size", "hostname", "network", + "static_ip", "gateway", "dns", "domain", "fqdn", "timezone", + ]), + ("installing packages", [ + "default", "mirror", "collector_prefix", "collector_import_file", + ]), + ("this tool", [ + "editor", "sudo", + ]), +] + + +CONFIG_KEYS = [ + ("default", "book used for installing packages (when --book isn't given)"), + ("build_book", "book used to BUILD the system; falls back to 'default'"), + ("lfs_mount", "where the LFS partition is mounted ($LFS)"), + ("lfs_device", "the LFS root partition device"), + ("lfs_fstype", "filesystem type of that device"), + ("lfs_swap", "swap device (optional)"), + ("lfs_home_device", "separate /home device (optional)"), + ("lfs_tgt", "cross-compile target triplet ($LFS_TGT); " + "default: $(uname -m)-lfs-linux-gnu"), + ("makeflags", "parallelism for builds; default: -j$(nproc)"), + ("mirror", "package mirror base URL ('off' to disable); " + "{book} is replaced by the version"), + ("editor", "editor for --edit; default: $EDITOR or vim"), + ("sudo", "prefix suggested root commands with 'sudo' " + "(on/off; default off)"), + ("bootloader", "which bootloader to set up: refind (default), grub, " + "or none. GRUB's book instructions run " + "'grub-install /dev/sda', which writes to a disk's boot " + "sector -- with 'refind' that section is skipped " + "entirely and a rEFInd script is generated instead"), + ("esp", "path of the EFI System Partition inside the built " + "system (e.g. /boot/efi). rEFInd installs there; it is " + "never formatted, and an existing installation is left " + "alone unless you ask otherwise"), + ("locale", "system locale, e.g. en_US.UTF-8 or de_DE.UTF-8 " + "(default en_US.UTF-8). The book writes " + "'LC_ALL=<locale name>', which a shell would read as an " + "input redirect. Check what is available with 'locale -a'"), + ("charmap", "character map for that locale (default UTF-8)"), + ("paper_size", "paper size for groff and friends: A4 (default) or " + "letter. The book writes 'PAGE=<paper_size>', which a " + "shell would read as an input redirect"), + ("hostname", "hostname for the built system (default: lfs)"), + ("locales", "languages to keep for translated man pages and " + "message catalogues (space separated; English is " + "always kept). Used by 'lfs-helper prune-locales'. " + "Default: en de"), + ("last_step_script", "your own script, run as the LAST step of the build " + "(default /etc/lfs/last_build_step.sh). Its SOURCES " + "array is downloaded by get-sources, and the rest runs " + "as root in the chroot after every book package. It " + "ships with wget, since LFS has no download tool"), + ("network", "how the built system gets its network: dhcp " + "(default) or static. With dhcp no static interface or " + "resolv.conf files are generated -- you install a DHCP " + "client (dhcpcd) from BLFS afterwards"), + ("collector_import_file", "path to a collector-group export (from " + "'lfs-helper export-groups'). When set, it is copied " + "into the chroot and imported automatically, so a " + "rebuild reuses the group names you already chose " + "instead of asking again"), + ("collector_prefix", "prefix for COLLECTOR groups. Packages sometimes " + "install into a directory another package owns (gcc " + "writes gcc.mo into glibc's /usr/share/locale/*/" + "LC_MESSAGES). Such a directory keeps its owner but is " + "handed to a group named <prefix>_<owner> and made " + "group-writable, and both packages join it -- so the " + "installer gets exactly the access it needs and nothing " + "more. Default: sysgroup"), +] + + +REQUIRED_CONFIG = [ + ("default", "which book to build", "lfs fetch <version>"), + ("lfs_mount", "where $LFS is mounted", "lfs config lfs_mount /mnt/lfs"), + ("collector_prefix", "prefix for collector groups", + "lfs config collector_prefix sysgroup"), +] + + +def require_complete_config(): + """Refuse to go further with a half-configured session. A missing + collector_prefix in particular only shows up much later, as a failed + install deep inside the chroot, so it's checked up front.""" + cfg = load_config() + missing = [(k, why, how) for k, why, how in REQUIRED_CONFIG if not cfg.get(k)] + if not missing: + return + sys.stderr.write("! the build session is not fully configured:\n") + for k, why, how in missing: + sys.stderr.write(f" {k:<18} -- {why}\n" + f" {'':<18} set it with: {sudo_prefix()}{how}\n") + sys.stderr.write("\n See every setting with: lfs config\n") + sys.exit(2) + + +def extra_packages(): + v = load_config().get("extra_packages") + if v is None: + v = "wget" + return [p for p in v.split() if p] + + +def network_mode(): + return (load_config().get("network") or "dhcp").lower() + + +def bootloader(): + return (load_config().get("bootloader") or "refind").lower() + + +def collector_prefix(): + return (load_config().get("collector_prefix") or "sysgroup").rstrip("_") + + +def sudo_prefix(): + """'sudo ' or '' -- some systems (and root shells) have no sudo at all, so + suggested commands are shown bare unless you turn this on.""" + v = str(load_config().get("sudo", "")).lower() + return "sudo " if v in ("1", "on", "true", "yes") else "" + + +def cmd_config(args): + """Show or change every setting, and show where things live on disk.""" + cfg = load_config() + + if args.edit: + ed = cfg.get("editor") or os.environ.get("EDITOR") or "vim" + _ensure_store() + if not os.path.isfile(config_path()): + save_config(cfg) + import subprocess + subprocess.run([ed, config_path()], cwd="/") + return + + if args.unset: + for k in args.unset: + cfg.pop(k, None) + save_config(cfg) + print(f"unset: {', '.join(args.unset)}") + return + + if args.key and args.value is not None: + known = [k for k, _d in CONFIG_KEYS] + if args.key not in known: + sys.stderr.write(f"unknown key '{args.key}'. Known keys:\n " + + "\n ".join(known) + "\n") + sys.exit(2) + cfg[args.key] = args.value + save_config(cfg) + print(f"{args.key} = {args.value}") + return + + if args.key: + print(cfg.get(args.key, "")) + return + + print(f"Settings ({config_path()})\n") + shown = set() + for title, keys in CONFIG_SECTIONS: + rows = [(k, d) for k, d in CONFIG_KEYS if k in keys] + if not rows: + continue + print(" %s%s%s" % (_C_GROUP if _tty_out() else "", title, + _C_OFF if _tty_out() else "")) + for k, desc in rows: + shown.add(k) + val = cfg.get(k) + # one line per setting: the value is what people are looking for, + # so it goes first and the explanation is only shown with --long + print(" %-22s %s" % (k, val if val not in (None, "") else "-")) + if getattr(args, "long", False): + for line in _wrap(desc, 60): + print(" %-22s %s" % ("", line)) + print() + rest = [(k, d) for k, d in CONFIG_KEYS if k not in shown] + if rest: + print(" other") + for k, desc in rest: + val = cfg.get(k) + print(" %-22s %s" % (k, val if val not in (None, "") else "-")) + print() + + b, d = build_version(), default_version() + if b or d: + print("\nWhich book is used where:") + print(" building the system : %s%s" + % (b or "(none chosen)", + " (from 'default')" if b and not cfg.get("build_book") else "")) + print(" installing packages : %s" % (d or "(none chosen)")) + + print("\nEffective build environment (what the lfs user gets):") + lfs = _lfs_dir() or "(unset -- run: lfs build-system session)" + tgt = cfg.get("lfs_tgt") or f"{os.uname().machine}-lfs-linux-gnu" + mk = cfg.get("makeflags") or "-j$(nproc)" + print(f" LFS = {lfs}") + print(f" LFS_TGT = {tgt}") + print(f" MAKEFLAGS = {mk}") + print(f" PATH = $LFS/tools/bin:/usr/bin[:/bin]") + print(f" CONFIG_SITE = $LFS/usr/share/config.site") + print("\nPaths:") + print(f" store {store_dir()}") + print(f" books {books_dir()}") + # the paths below belong to the book the SYSTEM is built from + ver = build_version() + if ver: + print(f" book file {book_path(ver)}") + print(f" wget-list {wget_list_path(ver)}") + print(f" md5sums {md5sums_path(ver)}") + print(f" crosschain dirs {crosschain_dir(ver)}") + if mirror_base(ver): + print(f" mirror {mirror_base(ver)}") + if _lfs_dir(): + d = _lfs_dir() + print(f" sources {os.path.join(d, 'sources')}") + print(f" manifests {os.path.join(d, PKGUSR_DIR, 'manifests')}") + print(f" build progress {_crosschain_progress_path(d)}") + print("\nChange a value: lfs config <key> <value>" + "\nEdit everything: lfs config --edit" + "\nScript paths: lfs build-system crosschain --list") + + +# ---- chapter 7: the chroot environment ------------------------------------ # +# Virtual kernel filesystems, in mount order. Teardown reverses this. +_KERNFS = ["dev/pts", "proc", "sys", "run", "dev/shm", "dev"] + + +def _chroot_mounts(lfs): + """Which of our virtual filesystems are currently mounted, in mount order.""" + live = [] + try: + with open("/proc/mounts") as f: + mounted = {ln.split()[1] for ln in f if len(ln.split()) > 1} + except OSError: + return live + for sub in _KERNFS: + p = os.path.realpath(os.path.join(lfs, sub)) + if p in mounted: + live.append(sub) + return live + + +def _owner_is_root(lfs): + try: + return os.stat(os.path.join(lfs, "usr")).st_uid == 0 + except OSError: + return False + + +def _owner_name_of(path): + """Who owns this, by name where possible. + + `chown --from lfs` matched only files owned by exactly that user, so a + tree owned by a uid with no matching name -- or by an `lfs` account whose + uid changed since -- was silently left alone, and 7.2 appeared to run + while changing nothing. Book 7.2 chowns unconditionally; so do we now.""" + try: + st = os.stat(path) + except OSError: + return "?" + try: + import pwd + return pwd.getpwuid(st.st_uid).pw_name + except (KeyError, ImportError): + return "uid %d (no such user here)" % st.st_uid + + +# ---- chapter 7 (in chroot) + chapter 8: scripts for lfs-helper ------------ # +# Chapter 7's in-chroot packages, in book order. Chapter 8 is appended from the +# book's own package list, so this tracks whatever release you're building. +# These are TEMPORARY tools -- chapter 8 rebuilds every one of them for real. +# They get a -tmp suffix so they don't collide with the chapter 8 package of the +# same name, and the book builds them as root (the package-user system only +# takes over for the real chapter 8 system). +# Chapter 7 housekeeping that must happen INSIDE the chroot before any package +# is built: 7.5 creates the directory tree, 7.6 creates /etc/passwd, /etc/group +# and friends (until 7.6 runs the shell shows "I have no name!", because there +# is no passwd file yet). These are plain root command blocks, not package +# builds, so they're generated as simple scripts. +CHROOT_INIT_STEPS = [ + ("init-dirs", "ch-tools-creatingdirs"), + ("init-files", "ch-tools-createfiles"), +] + +# Chapter 9: configuring the system. These are root housekeeping steps that +# write config files -- no package is built -- so they are generated as plain +# root scripts, like 7.5 and 7.6. Several ask for a decision (hostname, network, +# locale), so they are placed after the packages and flagged where the book +# leaves a placeholder for you to fill in. +CHROOT_CONFIG_STEPS_9 = [ + ("cfg-bootscripts", "ch-config-bootscripts"), + ("cfg-network", "ch-config-network"), + ("cfg-resolv", "resolv.conf"), + ("cfg-hostname", "ch-config-hostname"), + ("cfg-hosts", "ch-config-hosts"), + ("cfg-sysvinit", "conf-sysvinit"), + ("cfg-clock", "ch-config-clock"), + ("cfg-console", "ch-config-console"), + ("cfg-sysklogd", "ch-config-sysklogd"), + ("cfg-rcsite", "ch-config-site"), + ("cfg-locale", "ch-config-locale"), + ("cfg-inputrc", "ch-config-inputrc"), + ("cfg-shells", "ch-config-shells"), +] + +CHROOT_STEPS_7 = [ + ("gettext-tmp", "ch-tools-gettext"), + ("bison-tmp", "ch-tools-bison"), + ("perl-tmp", "ch-tools-perl"), + ("python-tmp", "ch-tools-Python"), + ("texinfo-tmp", "ch-tools-texinfo"), + ("util-linux-tmp", "ch-tools-util-linux"), +] + + +def write_chroot_env(lfs, quiet=False): + """Persist the configured build variables INSIDE the tree, so lfs-helper + (which runs in the chroot, where our config file isn't reachable and the + environment has been reset) uses the same LFS_TGT/MAKEFLAGS you set here. + Without this a custom triplet silently reverts to the default.""" + cfg = load_config() + tgt = cfg.get("lfs_tgt") or f"{os.uname().machine}-lfs-linux-gnu" + mkf = cfg.get("makeflags") or "-j$(nproc)" + d = os.path.join(lfs, PKGUSR_DIR) + os.makedirs(d, exist_ok=True) + path = os.path.join(d, "env") + with open(path, "w") as f: + f.write("# written by `lfs` -- sourced by lfs-helper inside the chroot\n" + f'export LFS_TGT="{tgt}"\n' + f'export MAKEFLAGS="{mkf}"\n' + f'# where this tree is mounted OUTSIDE the chroot. Manifests\n' + f'# written before chrooting hold host paths ({lfs}/usr/bin/...)\n' + f'# which are simply /usr/bin/... in here, so lfs-helper strips\n' + f'# this prefix when it reads them.\n' + f'export LFS_HOST_MOUNT="{lfs}"\n' + f'# prefix for collector groups -- see `lfs config`\n' + f'export LFS_COLLECTOR_PREFIX="{collector_prefix()}"\n' + f'# languages to keep -- see `lfs-helper prune-locales`\n' + f'export LFS_LOCALES="{cfg.get("locales") or "en de"}"\n') + _make_world_readable(path) + if not quiet: + print(f"# build env -> {path} (LFS_TGT={tgt}, MAKEFLAGS={mkf})") + return path + + +def _find_helper_source(): + """The lfs-helper script shipped next to this tool.""" + here = os.path.dirname(os.path.abspath(__file__)) + for cand in (os.path.join(here, "lfs-helper"), + os.path.join(here, "..", "lfs-helper"), + "/usr/bin/lfs-helper", "/usr/share/lfs/lfs-helper"): + if os.path.isfile(cand): + return os.path.abspath(cand) + return None + + +def install_helper(lfs, quiet=False): + """Put lfs-helper inside the LFS tree so it's on PATH in the chroot.""" + src = _find_helper_source() + if not src: + if not quiet: + sys.stderr.write("! lfs-helper not found next to this tool -- the " + "chroot won't have it.\n") + return None + dest_dir = os.path.join(lfs, "usr", "bin") + os.makedirs(dest_dir, exist_ok=True) + dest = os.path.join(dest_dir, "lfs-helper") + import shutil + shutil.copy(src, dest) + os.chmod(dest, 0o755) + if not quiet: + print(f"# installed lfs-helper -> {dest} (available as 'lfs-helper' " + f"in the chroot)") + return dest + + +def _chrootify(text): + """Turn a cross-compile script into an IN-CHROOT one. Inside the chroot the + final system is simply '/', so $LFS must not be required (the cross-build + guard would refuse to run), sources live at /sources, and there's no + cross-compiler prefix to put on PATH.""" + text = text.replace('BUILD_ROOT="${BUILD_ROOT:-$LFS/sources}"', + 'BUILD_ROOT="${BUILD_ROOT:-/sources}"') + text = text.replace('export LFS\n', 'export LFS="${LFS:-/}"\n') + # drop the "refusing to guess without $LFS" guard: correct outside, wrong here + text = re.sub(r'if \[ -z "\$LFS" \]; then\n.*?\nfi\n', '', text, count=1, + flags=re.S) + # $LFS/tools/bin doesn't exist inside the chroot -- the real toolchain is on + # the normal PATH + text = re.sub(r'case ":\$PATH:" in\n.*?\nesac\n', '', text, count=1, + flags=re.S) + return text + + +def _plain_root_script(name, sid, title, cmds): + """A straight root command block (book 7.5 / 7.6) -- no unpack/build/install + phases, because nothing is being compiled.""" + ts = time.strftime("%Y-%m-%d %H:%M:%S") + # same treatment as the package phases: re-runnable symlinks, and a bare + # informational grep must not abort the step + body = "\n\n".join(_idempotent_links(_soften_diagnostics(c)) for c in cmds) + # `exec /usr/bin/bash --login` in 7.6 restarts the reader's shell so the + # prompt picks up the new /etc/passwd. Running it here would replace this + # script's shell and swallow the rest of the step, so drop it and tell the + # user instead. + body = re.sub(r'^\s*exec /usr/bin/bash --login\s*$', '', body, flags=re.M) + # Substitute FIRST, then look for what is left. Checking the raw text + # flags placeholders we can perfectly well fill in from config. + body, left = _fill_placeholders(body, name) + guard = "" + if left: + # Refuse to run rather than misfire. "LC_ALL=<locale name>" is an input + # redirect to bash, and the failure it produces + # line 13: locale: No such file or directory + # says nothing about the real problem. + names = ", ".join("<%s>" % x for x in left) + guard = ( + 'cat >&2 <<"__PH__"\n' + '\n== this step needs a decision from you ==\n' + 'The book leaves a placeholder here for you to fill in: %s\n' + '\nEither set it once and regenerate:\n' + ' lfs config locale en_US.UTF-8 # outside the chroot\n' + ' lfs build-system gen-chroot-scripts --run --overwrite\n' + '\nor edit this script directly, then:\n' + ' lfs-helper build %s --force\n' + '__PH__\nexit 3\n' % (names, name)) + + book = "LFS %s" % (load_config().get("default") or "?") + return f"""#!/bin/bash +############################################ +### {name} -- {title} +{CROSSCHAIN_MARK} v{CROSSCHAIN_SCRIPT_VERSION}: {name} +### section : {sid} +### book : {book} +### generated : {ts} +### +### Book housekeeping run as root inside the chroot (no package is built). +### Editable -- lfs-helper runs this file as-is. +set -e +{guard} +{body} +""" + + +REFIND_SCRIPT = r'''#!/bin/bash +############################################ +### refind -- install the rEFInd boot manager +{MARK} v{VER}: refind +### section : (not from the LFS book -- generated by `lfs`) +### generated : {TS} +### +### rEFInd instead of GRUB. The book's chapter 10 runs +### grub-install /dev/sda +### which writes a disk's boot sector. This script never does anything of the +### sort. It only ever ADDS files to an existing, already-formatted EFI System +### Partition: +### +### * it refuses to run if the ESP is not mounted; +### * it never formats, partitions, or writes to a raw device; +### * it never deletes an existing bootloader -- your current one keeps +### working, and rEFInd is added alongside it; +### * it refuses to overwrite an existing rEFInd unless REFIND_OVERWRITE=1. +### +### Set the ESP with: lfs config esp /boot/efi +### Homepage: https://www.rodsbooks.com/refind/ +set -e + +ESP="${{ESP:-{ESP}}}" +KERNEL_VERSION="${{KERNEL_VERSION:-}}" + +# exit 3 = "this needs YOU", not a build error: the tooling must not try to +# fix it by handing out permissions. +need_you() {{ echo "" >&2; echo "== rEFInd needs you to do something first ==" >&2 + echo "$*" >&2; echo "" >&2; exit 3; }} +die() {{ echo "!! $*" >&2; exit 1; }} + +# ---------------------------------------------------------------- safety ---- +[ -n "$ESP" ] || need_you "No ESP configured. Set it with: + lfs config esp /boot/efi" +# Show the user their actual candidates instead of a generic instruction. +suggest_esp() {{ + local found="" + # an ESP is a FAT partition, usually flagged as "EFI System" + if command -v lsblk >/dev/null 2>&1; then + found="$(lsblk -rno NAME,FSTYPE,SIZE,PARTTYPENAME,LABEL 2>/dev/null \ + | awk '$2 ~ /^(vfat|fat32|fat16|msdos)$/ {{ print }}' || true)" + fi + if [ -z "$found" ] && command -v blkid >/dev/null 2>&1; then + found="$(blkid -t TYPE=vfat -o device 2>/dev/null || true)" + fi + if [ -n "$found" ]; then + echo " FAT partitions on this machine (your ESP is almost certainly one):" >&2 + printf '%s\n' "$found" | sed 's|^| /dev/|' >&2 + else + echo " Could not list partitions from in here. Run this OUTSIDE the chroot:" >&2 + echo " lsblk -o NAME,SIZE,FSTYPE,PARTTYPENAME,MOUNTPOINT" >&2 + fi +}} + +if [ ! -d "$ESP" ]; then + {{ + echo "" + echo "== rEFInd needs you to do something first ==" + echo "$ESP does not exist, so there is nowhere to install to." + echo "" + echo "rEFInd goes on your EFI System Partition -- the one your firmware" + echo "already boots from. It is ADDED alongside what is there; nothing is" + echo "formatted and no existing bootloader is removed." + echo "" + suggest_esp + echo "" + echo " Then, inside the chroot:" + echo " mkdir -pv $ESP" + echo " mount -v -t vfat /dev/<esp> $ESP" + echo " lfs-helper build refind --force" + echo "" + echo " Not ready to deal with the bootloader yet? Skip it:" + echo " lfs-helper done refind" + }} >&2 + exit 3 +fi + +if ! mountpoint -q "$ESP" 2>/dev/null && ! grep -qs " ${{ESP}} " /proc/mounts; then + need_you "$ESP exists but nothing is mounted there. + + Writing into an unmounted directory would put the bootloader on the wrong + filesystem, so this stops here. + + mount -v /dev/<your-esp-partition> $ESP" +fi + +# An ESP is FAT; if this looks like anything else, we are in the wrong place. +esp_fs="$(findmnt -no FSTYPE --target "$ESP" 2>/dev/null || true)" +case "$esp_fs" in + vfat|msdos|fat|fat32|"") : ;; + *) need_you "$ESP is a $esp_fs filesystem, not FAT -- that is not an ESP. + + Refusing to touch it. Check which partition is your ESP: + lsblk -o NAME,SIZE,FSTYPE,PARTTYPENAME,MOUNTPOINT" ;; +esac + +echo "# ESP: $ESP ($esp_fs)" +echo "# existing EFI entries (these are NOT touched):" +ls -1 "$ESP/EFI" 2>/dev/null | sed 's/^/# /' || echo "# (none)" + +dest="$ESP/EFI/refind" + +# Add a menu entry for this system to an existing rEFInd. +# +# If rEFInd is already installed -- as it will be if this machine already boots +# with it -- reinstalling is the wrong move. What is actually needed is an +# entry so the firmware menu can boot LFS. That is additive: the config is +# backed up first, nothing else in it is changed, and other bootloaders on the +# ESP are untouched. +add_lfs_entry() {{ + local conf="$1" + [ -f "$conf" ] || return 1 + if grep -q '^menuentry "LFS"' "$conf" 2>/dev/null; then + return 2 # already there + fi + local bak="$conf.bak-$(date +%Y%m%d%H%M%S)" + cp -p "$conf" "$bak" + + local root_dev root_uuid root_spec kern + root_dev="$(findmnt -no SOURCE / 2>/dev/null || echo /dev/CHANGE-ME)" + root_uuid="$(blkid -s UUID -o value "$root_dev" 2>/dev/null || true)" + if [ -n "$root_uuid" ]; then root_spec="UUID=$root_uuid"; else root_spec="$root_dev"; fi + kern="$(ls -1 /boot/vmlinuz-* 2>/dev/null | sort -V | tail -n1 || true)" + kern="${{kern#/boot/}}" + + cat >> "$conf" <<RENTRY + +# --- added by lfs on $(date +%Y-%m-%d) --------------------------------- +menuentry "LFS" {{ + icon /EFI/refind/icons/os_linux.png + loader /boot/${{kern:-vmlinuz}} + options "root=$root_spec ro" +}} +RENTRY + echo "# added an \"LFS\" entry to $conf" + echo "# root: $root_spec" + echo "# kernel: /boot/${{kern:-vmlinuz (NO KERNEL BUILT YET -- edit this)}}" + echo "# backup: $bak" + return 0 +}} + +if [ -d "$dest" ] && [ "${{REFIND_OVERWRITE:-0}}" != "1" ]; then + echo "# rEFInd is already installed at $dest -- not reinstalling it." + conf="$dest/refind.conf" + # `set -e` aborts on a non-zero return, so capture the status rather than + # letting the function's "2 = already there" kill the script + rc_entry=0 + add_lfs_entry "$conf" || rc_entry=$? + case $rc_entry in + 0) exit 4 ;; # entry added; nothing lands in the LFS tree + 2) echo "# an \"LFS\" entry is already in $conf -- nothing to do." + exit 4 ;; # 4 = deliberately nothing to do + *) echo "# no $conf found; rEFInd is installed but unconfigured." + echo "# To install this rEFInd build over it: REFIND_OVERWRITE=1 $0" + exit 4 ;; + esac +fi + +# ------------------------------------------------------------- the files ---- +# rEFInd ships as a binary zip. Point REFIND_SRC at it, or drop it in /sources. +src="${{REFIND_SRC:-}}" +if [ -z "$src" ]; then + src="$(ls -1 /sources/refind-bin-*.zip /sources/refind-bin-*.tar.gz 2>/dev/null | head -n1 || true)" +fi +if [ -z "$src" ]; then + src="$(ls -1 /sources/refind-src-*.tar.gz 2>/dev/null | head -n1 || true)" +fi +[ -n "$src" ] && [ -e "$src" ] || need_you "No rEFInd archive in /sources. + + Binary build (simplest -- no compiler or gnu-efi needed): + https://www.rodsbooks.com/refind/getting.html + -> refind-bin-<version>.zip + + Or the source: + wget -O /sources/refind-src-0.14.2.tar.gz \\ + 'https://sourceforge.net/projects/refind/files/0.14.2/refind-src-0.14.2.tar.gz/download' + + Then: lfs-helper build refind --force" + +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT +case "$src" in + *.zip) unzip -q "$src" -d "$work" ;; + *.tar.gz) tar -xf "$src" -C "$work" ;; + *) die "don't know how to unpack $src" ;; +esac +root="$(find "$work" -maxdepth 1 -type d -name 'refind-bin-*' | head -n1)" +if [ -z "$root" ]; then + # a source tree: build it (needs gnu-efi) + srcroot="$(find "$work" -maxdepth 1 -type d -name 'refind-*' | head -n1)" + if [ -n "$srcroot" ] && [ -f "$srcroot/Makefile" ]; then + echo "# source archive -- building rEFInd (needs gnu-efi)" + if ! make -C "$srcroot" gnuefi >/dev/null 2>&1; then + need_you "Could not build rEFInd from source. + + It needs gnu-efi, which LFS does not install. The binary build avoids this + entirely -- download refind-bin-<version>.zip into /sources instead: + https://www.rodsbooks.com/refind/getting.html" + fi + root="$srcroot" + fi +fi +[ -n "$root" ] || root="$work" + +arch=x64 +case "$(uname -m)" in + x86_64) arch=x64 ;; + aarch64) arch=aa64 ;; + i?86) arch=ia32 ;; +esac + +mkdir -p "$dest/drivers_$arch" "$dest/tools_$arch" +install -m 644 "$root/refind/refind_$arch.efi" "$dest/" +[ -d "$root/refind/icons" ] && cp -r "$root/refind/icons" "$dest/" +for d in "$root/refind/drivers_$arch"/*.efi; do + [ -e "$d" ] && install -m 644 "$d" "$dest/drivers_$arch/" +done + +# ------------------------------------------------------------ refind.conf --- +if [ ! -f "$dest/refind.conf" ]; then + if [ -f "$root/refind/refind.conf-sample" ]; then + install -m 644 "$root/refind/refind.conf-sample" "$dest/refind.conf" + else + : > "$dest/refind.conf" + fi + cat >> "$dest/refind.conf" <<'RCONF' + +# --- added by lfs ------------------------------------------------------- +# rEFInd scans for other bootloaders by default, so anything already on this +# ESP keeps working. The entry below boots this LFS system explicitly. +timeout 10 +RCONF +fi + +# a manual stanza for this system, appended only once +if ! grep -q "^menuentry \"LFS\"" "$dest/refind.conf" 2>/dev/null; then + root_dev="$(findmnt -no SOURCE / 2>/dev/null || echo /dev/CHANGE-ME)" + root_uuid="$(blkid -s UUID -o value "$root_dev" 2>/dev/null || true)" + if [ -n "$root_uuid" ]; then root_spec="UUID=$root_uuid"; else root_spec="$root_dev"; fi + kern="$(ls -1 /boot/vmlinuz-* 2>/dev/null | sort -V | tail -n1 || true)" + kern="${{kern#/boot/}}" + cat >> "$dest/refind.conf" <<RENTRY + +menuentry "LFS" {{ + icon /EFI/refind/icons/os_linux.png + volume LFS + loader /boot/${{kern:-vmlinuz}} + options "root=$root_spec ro" +}} +RENTRY +fi + +echo "# rEFInd installed to $dest" +echo "#" +echo "# NOT done automatically (they change your firmware's boot order):" +echo "# efibootmgr --create --disk /dev/<esp-disk> --part <n> \\" +echo "# --loader '\\EFI\\refind\\refind_$arch.efi' --label rEFInd" +echo "# Run that yourself once you have checked the entry above is correct." +# Everything rEFInd writes goes to the ESP, never into the LFS filesystem, so +# there is nothing for the file tracker to see. Exit 4 says "success, but by +# design nothing was installed into the system tree". +exit 4 +''' + + +def _refind_script(): + import time as _t + return REFIND_SCRIPT.format( + MARK=CROSSCHAIN_MARK, VER=CROSSCHAIN_SCRIPT_VERSION, + TS=_t.strftime("%Y-%m-%d %H:%M:%S"), + ESP=(load_config().get("esp") or "/boot/efi")) + +def _install_collector_groups(lfs): + """Copy a collector-group export into the tree so lfs-helper can import it. + + Without this a rebuild asks the same "which group should share this + directory?" question for every package all over again.""" + src = load_config().get("collector_import_file") + if not src: + # fall back to one sitting in the store, so a rebuild picks up the + # groups from the previous system without any configuration at all + default = os.path.join(store_dir(), "collector-groups.export") + if os.path.isfile(default): + src = default + else: + return + src = os.path.expanduser(src) + if not os.path.isfile(src): + sys.stderr.write("! collector_import_file is set but %s does not " + "exist\n" % src) + return + import shutil + dst_dir = os.path.join(lfs, PKGUSR_DIR) + os.makedirs(dst_dir, exist_ok=True) + dst = os.path.join(dst_dir, "collector-groups.import") + shutil.copy(src, dst) + _make_world_readable(dst) + print("# collector groups -> %s" % dst) + print("# import them inside the chroot with:") + print("# lfs-helper import-groups --run") + + +def _gen_blfs_extra(outdir, order, unresolved): + """Generate install scripts for the BLFS packages we want during the build. + + LFS ships no download tool at all, so a freshly booted system cannot fetch + anything -- not even the sources for its own next package. wget is the + smallest fix, and BLFS already has the script for it; there is no reason to + write a second one here.""" + extras = extra_packages() + if not extras: + return 0 + blfs = _find_blfs() + if not blfs: + sys.stderr.write("! extra_packages is set (%s) but the 'blfs' tool is " + "not next to this one -- skipping\n" % " ".join(extras)) + return 0 + import subprocess + written = 0 + for pkg in extras: + # `blfs script` WRITES install_<Name-Version> into the current + # directory and prints only a summary, so run it in a scratch dir and + # pick up the file rather than trying to read it from stdout. + import tempfile + tmp = tempfile.mkdtemp() + try: + # honour the same book selectors blfs takes, so a pinned book + # file or version is used here too + sel = [] + if os.environ.get("BLFS_BOOK_FILE"): + sel = ["--book-file", os.environ["BLFS_BOOK_FILE"]] + elif os.environ.get("BLFS_BOOK"): + sel = ["--book", os.environ["BLFS_BOOK"]] + r = subprocess.run(blfs + sel + ["script", pkg], + capture_output=True, text=True, + timeout=600, cwd=tmp) + except (OSError, subprocess.SubprocessError) as e: + sys.stderr.write("! blfs script %s: %s\n" % (pkg, e)) + continue + made = [f for f in os.listdir(tmp) if f.startswith("install_")] + if r.returncode != 0 or not made: + detail = "" + for line in (r.stderr or r.stdout or "").strip().splitlines()[::-1]: + if line.strip(): + detail = ": " + line.strip() + break + sys.stderr.write("! blfs has no script for '%s'%s\n" % (pkg, detail)) + continue + with open(os.path.join(tmp, made[0])) as f: + raw = f.read() + import shutil as _sh + _sh.rmtree(tmp, ignore_errors=True) + text = _chrootify(raw) + text, left = _fill_placeholders(text, pkg) + if left: + unresolved.append((pkg, left)) + dest = os.path.join(outdir, "%s.sh" % pkg) + with open(dest, "w") as f: + f.write(text) + os.chmod(dest, 0o755) + if pkg not in order: + order.append(pkg) + written += 1 + print("# %s: install script from the BLFS book" % pkg) + return written + + +LAST_STEP_DEFAULT = r'''#!/bin/bash +# last_build_step.sh -- the final step of an LFS build. +# +# EDIT THIS FILE. It is yours: whatever a freshly built system should have +# that the book does not provide goes here, and it runs as the last step of +# `lfs-helper build-all`. +# +# * SOURCES below is read by `lfs build-system get-sources`, so anything you +# add is downloaded along with the book's own packages. +# * The rest runs as root inside the chroot, after every book package. +# * It is copied into the chroot by `lfs build-system gen-chroot-scripts`, +# so edit it HERE (on the host) and regenerate -- not in the chroot. +# +# It ships with wget because LFS has no download tool at all: without one a +# freshly booted system cannot fetch even the sources for its next package. + +# --- extra sources -------------------------------------------------------- # +# Downloaded into $LFS/sources by `lfs build-system get-sources`. +SOURCES=( + # wget, so the new system can fetch anything at all + "https://ftpmirror.gnu.org/wget/wget-1.25.0.tar.gz" + + # What `lfs` and `blfs` need to parse the books. WHEELS, not source + # tarballs: a modern sdist needs its build backend (beautifulsoup4 wants + # hatchling, requests wants setuptools) and with no package index reachable + # pip cannot fetch one -- + # BackendUnavailable: Cannot import 'hatchling.build' + # A wheel is already built, so pip just unpacks it. Both are pure Python, + # so the "any" wheel works on every architecture. + "https://files.pythonhosted.org/packages/py3/r/requests/requests-2.32.3-py3-none-any.whl" + "https://files.pythonhosted.org/packages/py3/b/beautifulsoup4/beautifulsoup4-4.12.3-py3-none-any.whl" + # urllib3, charset-normalizer, idna and certifi are what requests imports + "https://files.pythonhosted.org/packages/py3/u/urllib3/urllib3-2.2.3-py3-none-any.whl" + "https://files.pythonhosted.org/packages/py3/c/charset_normalizer/charset_normalizer-3.4.0-py3-none-any.whl" + "https://files.pythonhosted.org/packages/py3/i/idna/idna-3.10-py3-none-any.whl" + "https://files.pythonhosted.org/packages/py3/c/certifi/certifi-2024.8.30-py3-none-any.whl" + "https://files.pythonhosted.org/packages/py3/s/soupsieve/soupsieve-2.6-py3-none-any.whl" +) + +# --- the work ------------------------------------------------------------- # +set -e + +# Build one of the SOURCES tarballs as its own package user. +# build_as_package_user <user> <tarball-glob> <configure options...> +build_as_package_user() { + local user="$1" glob="$2"; shift 2 + local src="/sources" + + local tarball="" + local c + for c in $src/$glob; do [ -e "$c" ] && { tarball="$c"; break; }; done + if [ -z "$tarball" ]; then + echo "!! no $glob in $src -- add it to SOURCES and run:" >&2 + echo " lfs build-system get-sources --run" >&2 + return 1 + fi + + lfs-helper add-user "$user" >/dev/null 2>&1 || true + + local dir + dir="$(tar tf "$tarball" | head -n1 | cut -d/ -f1)" + rm -rf "${src:?}/${dir:?}" + tar --no-same-owner -xf "$tarball" -C "$src" + chown -R "$user:$user" "$src/$dir" + + echo "# building $user from $(basename "$tarball") as package user '$user'" + su - "$user" -c "cd '$src/$dir' && ./configure $* && make && make install" + + # record what was installed, the same way lfs-helper does + printf '%s\n# installed %s by last_build_step.sh\n' \ + "$dir" "$(date '+%Y-%m-%d %H:%M:%S')" > "/usr/src/$user/VERSION" \ + 2>/dev/null || true + chown "$user:$user" "/usr/src/$user/VERSION" 2>/dev/null || true +} + +# Install a Python module as ITS OWN package user, the same as any other +# package. requests and beautifulsoup4 are what `lfs` and `blfs` need to parse +# the books, so without them the tools cannot run on the new system. +# Let a package user install into site-packages. +# +# site-packages belongs to the python package and is shared through a collector +# group. Ask lfs-helper for access, exactly as a failed build would: it joins +# the collector group the directory already carries, or asks which group should +# own it. It must NOT be handed to the install group -- that means "anyone may +# install here", and site-packages is python's tree, not shared infrastructure. +_grant_site_packages() { + local user="$1" sp + sp="$(python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])' \ + 2>/dev/null)" + if [ -z "$sp" ] || [ ! -d "$sp" ]; then + echo "note: cannot find site-packages -- skipping $user" >&2 + return 1 + fi + lfs-helper grant-dir "$sp" "$user" --run +} + +install_python_module() { + local user="$1" module="$2" glob="$3" + local src="/sources" wheel="" c + for c in $src/$glob; do [ -e "$c" ] && { wheel="$c"; break; }; done + if [ -z "$wheel" ]; then + echo "note: no $glob in $src -- skipping $module" >&2 + echo " add it to SOURCES and run: lfs build-system get-sources --run" >&2 + return 0 + fi + command -v python3 >/dev/null 2>&1 || { + echo "note: no python3 yet -- skipping $module" >&2; return 0; } + + lfs-helper add-user "$user" >/dev/null 2>&1 || true + chown "$user:$user" "$wheel" 2>/dev/null || true + _grant_site_packages "$user" || return 0 + + echo "# installing $module from $(basename "$wheel") as package user '$user'" + # A wheel needs no build backend and no network: pip unpacks it. + # PIP_USER=0 / PYTHONNOUSERSITE keep it out of the package user's ~/.local, + # where nothing else would ever find it. + su - "$user" -c "PIP_USER=0 PYTHONNOUSERSITE=1 \ + python3 -m pip install --no-index --no-deps '$wheel'" \ + || echo "!! $module did not install -- do it later with: + packagemanager pip install $module" >&2 +} + +# wget: the one thing a new system cannot bootstrap without. +# Deliberately minimal -- reinstall it properly once the system boots: +# packagemanager install wget --recursive --run +build_as_package_user wget "wget-*.tar.*" \ + --prefix=/usr --sysconfdir=/etc --with-ssl=openssl + +# The two Python modules the tools themselves need. Each gets its own package +# user, exactly like every other package -- there is no "last-step" package and +# nothing here is owned by one. +# Dependencies first -- --no-deps means pip will not pull them in itself. +install_python_module urllib3 urllib3 "urllib3-*.whl" +install_python_module charset-normalizer charset-normalizer "charset_normalizer-*.whl" +install_python_module idna idna "idna-*.whl" +install_python_module certifi certifi "certifi-*.whl" +install_python_module soupsieve soupsieve "soupsieve-*.whl" +install_python_module requests requests "requests-*.whl" +install_python_module beautifulsoup4 beautifulsoup4 "beautifulsoup4-*.whl" + +# --- wget without certificates -------------------------------------------- # +# A freshly built LFS has no CA certificates -- they come from BLFS' make-ca -- +# so every HTTPS fetch fails with "cannot verify ... certificate". That is a +# chicken-and-egg problem: you need wget to fetch make-ca. +# +# Turn verification off in /etc/wgetrc rather than with a wrapper script in +# $PATH. wget reads /etc/wgetrc no matter how it is invoked or what PATH the +# caller has, so this works for builds run through `su -` with the package +# user's own environment -- where a $PATH wrapper is simply never seen. +# +# This is a deliberate, TEMPORARY weakening: downloads are unverified until +# make-ca is installed. The marker below is how the tools find and remove it. +disable_wget_verification() { + local rc=/etc/wgetrc + grep -q 'lfs-temporary-no-verify' "$rc" 2>/dev/null && return 0 + cat >> "$rc" <<'WRC' + +# --- lfs-temporary-no-verify --------------------------------------------- +# This system has no CA certificates yet, so wget cannot verify anything. +# Downloads are UNVERIFIED until you install make-ca: +# packagemanager install make-ca --recursive --run +# The tools remove these two lines automatically once certificates exist. +check_certificate = off +# --- end lfs-temporary-no-verify ----------------------------------------- +WRC + echo "# certificate checking disabled in $rc until make-ca is installed" + echo "# (downloads are UNVERIFIED until then -- the tools re-enable it" + echo "# automatically as soon as certificates exist)" +} + +disable_wget_verification + +# --- add your own steps below --------------------------------------------- # +# e.g. a package the book does not carry, a config file, a user account. + +# --- accounts you can actually log in with -------------------------------- # +# A freshly built system has root with NO password. Depending on the login +# manager that is either "anyone can log in as root" or "nobody can log in at +# all" -- and it is discovered at the worst moment, after a reboot, with no +# way in. Ask now, while there is still a working shell here. +set_up_login_accounts() { + # No terminal (a scripted or resumed run): say what is missing rather than + # blocking, and leave the system unbootable-but-known instead of hanging. + if [ ! -t 0 ]; then + echo + echo "!! Not running interactively, so no passwords were set." + echo " Before rebooting, set at least the root password:" + echo " chroot /mnt/lfs /usr/bin/passwd root" + return 0 + fi + + echo + echo "--- accounts ------------------------------------------------" + + # root + if grep -qE '^root:[^:]*:' /etc/shadow 2>/dev/null && + ! grep -qE '^root:(\*|!|)?:' /etc/shadow 2>/dev/null; then + echo "root already has a password." + else + echo "root has NO password yet. Without one you may not be able to" + echo "log in after rebooting." + passwd root || echo "!! setting the root password failed -- do it before rebooting" >&2 + fi + + # a login account for a person: the name is already configured, so this + # only has to create it and set a password + local main + main="$(sed -n 's/^main_user=\(.*\)$/\1/p' /etc/pkgusr/packagemanager.conf \ + 2>/dev/null | head -n1)" + if [ -z "$main" ]; then + echo + echo "No login account is configured. You can add one now, or later" + echo "with: useradd -m -G users <name> && passwd <name>" + printf "Create a login account now? [name, or blank to skip]: " + read -e -r main + fi + [ -n "$main" ] || { echo; return 0; } + + if id "$main" >/dev/null 2>&1; then + echo "Account '$main' already exists." + else + echo "Creating login account '$main' ..." + useradd -m -k /etc/skel -G users -s /bin/bash "$main" \ + || { echo "!! could not create $main" >&2; return 0; } + fi + if grep -qE "^$main:(\*|!|)?:" /etc/shadow 2>/dev/null; then + echo "Set a password for '$main':" + passwd "$main" || echo "!! setting the password for $main failed" >&2 + fi + + # so the account can reach the application users created later + echo + echo " '$main' can be given access to application users with:" + echo " packagemanager user create <app> --shared --launcher" +} + +set_up_login_accounts + +echo +echo "==============================================================" +echo " The base system is built." +echo +echo " wget was installed with a minimal configure line so the system" +echo " can fetch things at all. Reinstall it properly -- with its" +echo " dependencies, from the BLFS book -- once you have booted:" +echo +echo " packagemanager install wget --recursive --run" +echo +echo " Anything else you want on a fresh system belongs in" +echo " last_build_step.sh on the host." +echo "==============================================================" +''' + + +def last_step_path(): + """Where the user's final-step script lives ON THE HOST.""" + p = load_config().get("last_step_script") + if p: + return os.path.expanduser(p) + return "/etc/lfs/last_build_step.sh" + + +def ensure_last_step_script(verbose=True): + """Create the final-step script if it does not exist yet.""" + path = last_step_path() + if os.path.isfile(path): + return path + try: + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w") as f: + f.write(LAST_STEP_DEFAULT) + os.chmod(path, 0o755) + if verbose: + print("# created %s -- edit it to change what a finished system" + " gets" % path) + except OSError as e: + sys.stderr.write("! could not create %s: %s\n" % (path, e)) + return None + return path + + +def last_step_sources(): + """The SOURCES array from the final-step script. + + Parsed rather than executed: this file is meant to be edited freely, and + running it just to read a list of URLs would be a poor trade.""" + path = last_step_path() + if not os.path.isfile(path): + return [] + try: + text = open(path).read() + except OSError: + return [] + m = re.search(r'^SOURCES=\(\s*(.*?)^\)', text, re.M | re.S) + if not m: + return [] + urls = [] + for line in m.group(1).splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + for tok in re.findall(r'"([^"]+)"|\'([^\']+)\'|(\S+)', line): + u = tok[0] or tok[1] or tok[2] + if u.startswith(("http://", "https://", "ftp://")): + urls.append(u) + return urls + + +def _write_tool_stamps(lfs, tools): + """Record the md5 of every tool we copied in. + + The chroot keeps running whatever was copied last; without a record, a fix + made on the host silently does not apply in there and the same bug appears + to come back.""" + import hashlib + stamps = {} + for name, src in tools: + try: + with open(src, "rb") as f: + stamps[name] = hashlib.md5(f.read()).hexdigest() + except OSError: + continue + path = os.path.join(lfs, "usr", "share", "lfs", "tool-stamps.json") + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w") as f: + json.dump(stamps, f, indent=2) + + +def _sync_report(title, items, note=None): + """One consistent shape for 'here is what I copied'. + + The old output mixed three different layouts, so it was easy to miss that + lfs-helper was being copied at all.""" + print(" %s" % title) + for i in items: + print(" %s" % i) + if note: + print(" (%s)" % note) + + +def _run_choose_book(): + """Ask which book to build, and make it the default. + + The checklist step is "choose and fetch a book" -- a choice, so ask for it + instead of printing sixty versions and stopping.""" + cfg = load_config() + cached = sorted(cached_versions()) + if cached: + print("\nAlready downloaded:") + for i, v in enumerate(cached, 1): + print(" %2d) %s" % (i, v)) + print("\nEnter a number, or any version name to download it") + print("(`lfs books` lists everything available).") + else: + print("\nNothing downloaded yet. Enter a version, e.g. 12.4") + print("(`lfs books` lists everything available).") + try: + ans = input("\nbook> ").strip() + except (EOFError, KeyboardInterrupt): + print() + return False + if not ans: + return False + if ans.isdigit() and cached: + idx = int(ans) - 1 + if not (0 <= idx < len(cached)): + sys.stderr.write("no such number\n") + return False + ans = cached[idx] + argv = [sys.executable, os.path.abspath(__file__), "set-default", ans] + if subprocess.call(argv) != 0: + # not cached yet -- fetch it, then set it + if subprocess.call([sys.executable, os.path.abspath(__file__), + "fetch", ans]) != 0: + return False + if subprocess.call(argv) != 0: + return False + return True + + +def _run_banner(step, steps, cmd=None): + """One consistent header before every step, so it is always clear what is + happening now, how far along it is, and what comes next. Long steps print + nothing for minutes; without this you cannot tell work from a hang.""" + done = sum(1 for s in steps if s["done"]) + total = len(steps) + pending = [s for s in steps if not s["done"]] + nxt = pending[1]["title"] if len(pending) > 1 else "enter the chroot" + + bar_w = 20 + filled = int(bar_w * done / total) if total else 0 + bar = "#" * filled + "." * (bar_w - filled) + + W = 66 + def row(text): + print("| %-*s |" % (W - 4, text[:W - 4])) + + print() + print("+" + "-" * (W - 2) + "+") + row("step %d of %d [%s]" % (done + 1, total, bar)) + row("") + row("NOW: " + step["title"]) + row("NEXT: " + nxt) + if cmd: + row(" $ " + cmd) + print("+" + "-" * (W - 2) + "+") + + +def _refresh_chroot_tools(lfs, quiet=False): + """Copy the tools into the tree if what is in there is out of date. + + Compares content, so an unchanged tool is not touched and there is no + output in the common case.""" + import hashlib + + def _md5(p): + try: + with open(p, "rb") as f: + return hashlib.md5(f.read()).hexdigest() + except OSError: + return None + + src = _find_helper_source() + if not src: + return + dst = os.path.join(lfs, "usr", "bin", "lfs-helper") + if _md5(src) == _md5(dst): + return + install_helper(lfs, quiet=True) + if not quiet: + print("# lfs-helper in the chroot was out of date -- updated to %s " + "(build %s)" % (LFS_VERSION, _build_id())) + + +def cmd_bs_run(args): + """Run the whole outside-the-chroot procedure, from wherever it stands. + + The checklist behind `next` already knows what is done and what comes next; + this executes it instead of printing it. After a host reboot that means: + remount, re-prepare the chroot, and drop you into it -- one command. + + It stops the moment a step needs YOU (an unconfigured setting, a missing + book) or a step fails, and it never repeats a step that made no progress. + The chroot itself is the destination: once everything before it is done, + this execs `chroot enter` and you are inside.""" + if inside_chroot(): + _refuse_inside_chroot("build-system", "run") + + def enter_chroot(): + lfs = require_mounted_lfs() + # Refresh the in-chroot tools FIRST. They are copies: fixing a bug in + # lfs-helper out here changes nothing inside until it is copied in + # again. Relying on the user to remember means they keep hitting bugs + # that were fixed days ago, and every diagnosis starts from the wrong + # version of the evidence. + _refresh_chroot_tools(lfs) + print() + print("+" + "-" * 64 + "+") + print("| %-62s |" % "everything outside the chroot is done") + print("| %-62s |" % "") + print("| %-62s |" % "NOW: entering the chroot") + print("| %-62s |" % "NEXT: lfs-helper next (inside, to build the system)") + print("+" + "-" * 64 + "+") + print() + fake = argparse.Namespace(action="enter", run=False) + cmd_bs_chroot(fake) + sys.exit(0) + + last_title = None + for _ in range(30): + steps, lfs = _next_steps() + pending = [st for st in steps if not st["done"]] + if not pending: + enter_chroot() + nxt = pending[0] + title = nxt["title"] + + # the chroot is the destination, and everything after it is inside work + if title.startswith(("build the system inside the chroot", + "install the Python modules", + "hand over")): + enter_chroot() + + if title == last_title: + sys.stderr.write("\n! '%s' ran but is still pending -- stopping " + "rather than looping.\n Run it by hand to see " + "why:\n %s\n" + % (title, nxt["cmds"][0] if nxt["cmds"] else "?")) + sys.exit(1) + last_title = title + + cmd = nxt["cmds"][0] if nxt["cmds"] else "" + cmd = cmd.strip() + if cmd.startswith("sudo "): + cmd = cmd[5:] + + # Steps that need a DECISION are walked through here rather than + # printed as homework. Resuming a build should not mean reading a + # list and retyping a command. + if title == "choose and fetch a book": + _run_banner(nxt, steps) + if not _run_choose_book(): + sys.exit(1) + last_title = None + continue + if title == "complete the configuration": + _run_banner(nxt, steps) + argv = [sys.executable, os.path.abspath(__file__), + "build-system", "session"] + if subprocess.call(argv) != 0: + sys.exit(1) + last_title = None + continue + + # Anything else must be a command that actually CHANGES something. + # `lfs books` only lists: running it can never complete a step, which + # is how this looped until the guard caught it. + applies = ("--run" in cmd or cmd.startswith("lfs build-system session")) + if not cmd.startswith("lfs ") or "<" in cmd or not applies: + print("\nThis needs you first: %s" % title) + for c in nxt["cmds"]: + print(" %s" % c) + if nxt.get("note"): + print("\n (%s)" % nxt["note"]) + sys.exit(1) + + _run_banner(nxt, steps, cmd) + argv = [sys.executable, os.path.abspath(__file__)] + shlex.split(cmd)[1:] + rc = subprocess.call(argv) + if rc != 0: + sys.stderr.write("\n! step failed (exit %d): %s\n" + " Fix it, then just run `lfs build-system run` " + "again -- it continues from here.\n" % (rc, title)) + sys.exit(rc) + sys.stderr.write("! 30 steps and still not at the chroot -- something is " + "cycling. Run `lfs build-system next --all`.\n") + sys.exit(1) + + +def cmd_bs_restart(args): + """Throw away the built system and start the build again from scratch. + + Deliberately loud. This deletes everything that was built -- weeks of + compilation on a slow machine -- so it says exactly what will go, exactly + what will be kept, prints the commands it will run, and asks the user to + type the mount point before doing any of it.""" + lfs = require_mounted_lfs() + + # Refuse while the chroot's virtual filesystems are mounted: deleting + # through a bind-mounted /dev would take the HOST's /dev with it. + mounted = [d for d in ("dev/pts", "dev/shm", "dev", "proc", "sys", "run") + if _is_mounted(os.path.join(lfs, d))] + if mounted: + sys.stderr.write( + "\n! the chroot is still mounted: %s\n" + " Deleting through those would reach the running system.\n" + " Unmount first:\n %slfs build-system chroot unmount --run\n" + % (", ".join(mounted), sudo_prefix())) + sys.exit(2) + + keep_sources = not args.wipe_sources + entries = [] + try: + entries = sorted(os.listdir(lfs)) + except OSError as e: + sys.stderr.write("cannot read %s: %s\n" % (lfs, e)) + sys.exit(1) + doomed = [e for e in entries + if e not in ("dev", "proc", "sys", "run") + and not (keep_sources and e == "sources")] + + def _du(p): + try: + r = subprocess.run(["du", "-sh", p], capture_output=True, text=True, + timeout=120) + return r.stdout.split()[0] if r.stdout.strip() else "?" + except Exception: + return "?" + + warn("!! This DELETES the system you have built.") + print() + print(" from %s:" % lfs) + for e in doomed: + print(" %-16s %s" % (e, _du(os.path.join(lfs, e)))) + if not doomed: + print(" (nothing -- the tree is already empty)") + print() + print(" kept:") + if keep_sources: + print(" sources the downloaded tarballs " + "(--wipe-sources removes these too)") + print(" the books, your settings and any snapshots (outside the tree)") + print() + print(" Package users and collector groups live in %s/etc, so they go" + % lfs) + print(" with it. Users on the HOST are untouched.") + print() + print(" Then the build starts again from the beginning:") + print(" %slfs build-system run" % sudo_prefix()) + print() + + snaps = _list_snapshots() + if not snaps: + warn(" You have no snapshots. This cannot be undone.") + print(" Consider: %slfs snapshot save before-restart --run" + % sudo_prefix()) + else: + print(" You have %d snapshot(s); the newest is '%s'." + % (len(snaps), snaps[-1][0])) + print(" Restoring one is usually faster than rebuilding:") + print(" %slfs snapshot restore %s --run" + % (sudo_prefix(), snaps[-1][0])) + print() + + if not args.run: + dry_run_note("delete them") + return + + if not args.yes: + warn("This cannot be undone.") + try: + ans = input("Type the mount point (%s) to confirm: " % lfs).strip() + except (EOFError, KeyboardInterrupt): + print() + return + if ans != lfs: + print("Not confirmed -- nothing was deleted.") + return + + import shutil as _sh + removed = 0 + for e in doomed: + p = os.path.join(lfs, e) + try: + if os.path.islink(p) or os.path.isfile(p): + os.remove(p) + else: + _sh.rmtree(p) + removed += 1 + except OSError as err: + sys.stderr.write(" could not remove %s: %s\n" % (p, err)) + print("\nremoved %d item(s) from %s" % (removed, lfs)) + + # the build's own progress lives outside the tree as well + for f in ("crosschain",): + d = os.path.join(store_dir(), f) + if os.path.isdir(d): + for sub in os.listdir(d): + prog = os.path.join(d, sub, "progress.json") + if os.path.isfile(prog): + os.remove(prog) + print("cleared the recorded build progress") + print() + next_step("%slfs build-system run" % sudo_prefix(), label="start again with:") + + +def cmd_bs_set_book(args): + """Choose the book the SYSTEM is built from (config key build_book). + + Separate from `lfs set-default`, which chooses the book packages are + installed from on a running system. Having one command for both made it + impossible to tell which you were setting.""" + ver = args.version + if not os.path.isfile(book_path(ver)): + known = [] + try: + known = list(remote_versions()) + except Exception: + known = [] + if known and ver not in known: + sys.stderr.write("no such book: %s\n see them all: lfs books\n" + % ver) + sys.exit(1) + sys.stderr.write("note: %s isn't downloaded yet -- fetching it now\n" + % ver) + try: + _fetch_version(ver, set_default=False) + except Exception as e: + sys.stderr.write("fetch failed: %s\n" % e) + sys.exit(1) + cfg = load_config() + cfg["build_book"] = ver + save_config(cfg) + print("the system will be built from book %s" % ver) + if cfg.get("default") and cfg["default"] != ver: + print(" (packages are installed from %s -- lfs set-default changes that)" + % cfg["default"]) + + +def cmd_bs_sync_tools(args): + """Copy this machine's package-user environment into the tree. + + Run after `layout` and again whenever anything here changes: the tools are + updated in place, so a chroot built earlier keeps running an old copy until + this is re-run. Copies: + * /etc/pkgusr/skel-package -- the package-user home skeleton + * /etc/pkgusr/{bash_profile,bashrc,build} + * the hint's helper scripts (add_package_user and friends) + * lfs-helper itself + * a collector-group export, if one is configured + Nothing here is destructive: it overwrites the tooling, never the build.""" + lfs = require_mounted_lfs() + print("Sync the package-user environment into %s\n" % lfs) + if not args.run: + _sync_report("package-user skeleton", + ["/etc/pkgusr/skel-package", + "/etc/pkgusr/{bash_profile,bashrc,build}"]) + _sync_report("helper scripts", + ["add_package_user, install_package, list_package, ...", + "the chown/chgrp/chmod/mkdir/install wrappers"]) + _sync_report("the build driver", ["lfs-helper -> /usr/bin/lfs-helper"]) + if load_config().get("collector_import_file"): + _sync_report("collector groups", ["the configured export"]) + dry_run_note("copy them") + return + + # the target directory must exist even when this machine has nothing to + # copy -- lfs-helper writes the default environment into it on first use, + # and the checklist uses its presence as "this step is done" + os.makedirs(os.path.join(lfs, "etc", "pkgusr"), exist_ok=True) + skel_ok = _install_pkgusr_skel(lfs, verbose=False) + _sync_report("package-user skeleton", + ["/etc/pkgusr/skel-package" if skel_ok + else "none on this machine -- a default will be written"]) + + copied = _install_pkgusr_helpers(lfs, verbose=False) + if copied: + _sync_report("helper scripts", copied) + else: + _sync_report("helper scripts", + ["none on this machine"], + note="the tools use Shadow's useradd/groupadd instead") + + install_helper(lfs, quiet=True) + _sync_report("the build driver", + ["lfs-helper -> %s/usr/bin/lfs-helper" % lfs], + note="available as 'lfs-helper' inside the chroot") + + _install_collector_groups(lfs) + + # Everything above was copied in as root. Before the chroot exists the + # tree belongs to the lfs user, so hand these over too -- otherwise the + # next step refuses with "these build dirs are not owned by the 'lfs' + # user: /mnt/lfs/usr/lib/pkgusr, /mnt/lfs/etc/pkgusr, ...". + # (Book 7.2 gives the whole tree back to root when the chroot is prepared.) + _chown_tree_to_lfs(lfs, quiet=True) + print() + next_step("%slfs build-system next" % sudo_prefix()) + + +def _fstab_script(cfg): + """Write /etc/fstab from the session settings (book 10.2). + + Without this the boot scripts cannot remount root read-write, so the very + first thing that tries to write -- /run/bootlog -- fails with + Read-only file system + and everything after it fails too. The device and filesystem were already + given to `lfs build-system session`, so there is no reason to make anyone + write this file by hand.""" + dev = cfg.get("lfs_device") or "" + fstype = cfg.get("lfs_fstype") or "ext4" + swap = cfg.get("lfs_swap") or "" + home = cfg.get("lfs_home_device") or "" + + out = ["#!/bin/bash", "set -e", "### book : LFS 12.4", "", + "# 10.2 -- /etc/fstab, from the settings given to " + "`lfs build-system session`"] + if not dev: + out += [ + 'echo "!! no root device configured -- cannot write /etc/fstab." >&2', + 'echo " Set it, then regenerate:" >&2', + 'echo " lfs config lfs_device /dev/sdXn" >&2', + 'echo " lfs build-system gen-chroot-scripts --run --overwrite" >&2', + "exit 3", + ] + return "\n".join(out) + "\n" + + # btrfs/xfs do their own checking and must not be fsck'd at boot + fsck = "0 0" if fstype in ("btrfs", "xfs", "f2fs") else "1 1" + opts = "defaults,compress=zstd" if fstype == "btrfs" else "defaults" + + rows = ["%-16s %-15s %-9s %-20s %s" % (dev, "/", fstype, opts, fsck)] + if swap: + rows.append("%-16s %-15s %-9s %-20s %s" % (swap, "swap", "swap", "pri=1", "0 0")) + if home: + rows.append("%-16s %-15s %-9s %-20s %s" % (home, "/home", fstype, "defaults", "0 2")) + for a, b, c, d, e in ( + ("proc", "/proc", "proc", "nosuid,noexec,nodev", "0 0"), + ("sysfs", "/sys", "sysfs", "nosuid,noexec,nodev", "0 0"), + ("devpts", "/dev/pts", "devpts", "gid=5,mode=620", "0 0"), + ("tmpfs", "/run", "tmpfs", "defaults", "0 0"), + ("devtmpfs", "/dev", "devtmpfs", "mode=0755,nosuid", "0 0"), + ("tmpfs", "/dev/shm", "tmpfs", "nosuid,nodev", "0 0"), + ("cgroup2", "/sys/fs/cgroup", "cgroup2", "nosuid,noexec,nodev", "0 0")): + rows.append("%-16s %-15s %-9s %-20s %s" % (a, b, c, d, e)) + + out += [ + '# Never clobber an fstab someone else wrote.', + 'if [ -s /etc/fstab ] && ! grep -q "generated by lfs" /etc/fstab; then', + ' echo "# /etc/fstab exists and was not written by us -- left alone."', + ' exit 4', + 'fi', + "", + "cat > /etc/fstab <<'FSTAB_EOF'", + "# Begin /etc/fstab -- generated by lfs; edit freely", + "#", + "# file system mount-point type options dump fsck", + "# order", + "", + ] + rows + [ + "", + "# End /etc/fstab", + "FSTAB_EOF", + "", + 'echo "# wrote /etc/fstab"', + "sed 's/^/ /' /etc/fstab", + "", + '# A device name can change between boots; a UUID cannot.', + 'if command -v blkid >/dev/null 2>&1; then', + ' _uuid="$(blkid -s UUID -o value ' + dev + ' 2>/dev/null || true)"', + ' if [ -n "$_uuid" ]; then', + ' echo ""', + ' echo "# tip: to survive a device rename, replace ' + dev + '"', + ' echo "# with UUID=$_uuid"', + ' fi', + 'fi', + ] + return "\n".join(out) + "\n" + + +def cmd_bs_gen_chroot_scripts(args): + require_complete_config() + """Write phased install scripts for the in-chroot packages (chapter 7's + remainder and chapter 8) into $LFS/.lfs-pkgusr/scripts, where lfs-helper + runs them. Same phased format as the packagemanager scripts, so + packagemanager can reuse them to update these packages later.""" + ver, path = resolve_book(args) + lfs = require_mounted_lfs() + soup = load_soup(path) + secs = {sid: (title, sec) for sid, title, sec in iter_sections(soup)} + + steps = list(CHROOT_INIT_STEPS) + list(CHROOT_STEPS_7) + if not args.chapter7_only: + for sid, (title, sec) in secs.items(): + nv = package_name_version(sid, title) + if nv and sec["commands"]: + steps.append((nv[0], sid)) + + bl = bootloader() + if bl != "grub": + # GRUB's book instructions run `grub-install /dev/sda`, which writes a + # disk's boot sector. If it isn't the chosen bootloader, don't generate + # it at all -- there should be no script lying around that could do that + # to an existing boot partition by accident. + steps = [(n, sid) for n, sid in steps + if "grub" not in n.lower() and "grub" not in sid.lower()] + + outdir = os.path.join(lfs, PKGUSR_DIR, "scripts") + print(f"{len(steps)} script(s) -> {outdir}") + if not args.run: + for name, sid in steps[:10]: + print(f" {name}") + if len(steps) > 10: + print(f" ... and {len(steps) - 10} more") + dry_run_note("write them") + return + + os.makedirs(outdir, exist_ok=True) + written, skipped = 0, 0 + order = [] + unresolved = [] + for name, sid in steps: + entry = secs.get(sid) + if not entry: + continue + title, sec = entry + if not sec["commands"]: + continue + order.append(name) + dest = os.path.join(outdir, f"{name}.sh") + if os.path.isfile(dest) and not args.overwrite: + skipped += 1 + continue + if name in [n for n, _s in CHROOT_INIT_STEPS]: + text = _plain_root_script(name, sid, title, sec["commands"]) + else: + pkgver = _title_pkgver(title) or name + glob = _pkg_glob_for(pkgver, name) + text = _crosschain_script_body(name, sid, pkgver, glob, + sec["commands"]) + text = _chrootify(text) + text, left = _fill_placeholders(text, name) + if left: + unresolved.append((name, left)) + with open(dest, "w") as f: + f.write(text) + os.chmod(dest, 0o755) + written += 1 + # chapter 9 configuration, after all the packages + pkg_like_cfg = [] + cfg_steps_9 = list(CHROOT_CONFIG_STEPS_9) + if network_mode() == "dhcp": + # The book's network section writes a static interface config and a + # hand-made resolv.conf. With DHCP both are wrong -- the client + # supplies them -- so don't generate scripts that would overwrite what + # dhcpcd sets up. + cfg_steps_9 = [(n, sid) for n, sid in cfg_steps_9 + if n not in ("cfg-network", "cfg-resolv")] + if not args.chapter7_only: + secs_all = secs + for cname, csid in cfg_steps_9: + entry = secs_all.get(csid) + if not entry: + continue + ctitle, csec = entry + if not csec["commands"]: + continue + dest = os.path.join(outdir, f"{cname}.sh") + if os.path.isfile(dest) and not args.overwrite: + skipped += 1 + order.append(cname) + continue + # Some chapter-9 "configuration" sections are actually PACKAGES: + # 9.2 LFS-Bootscripts-20250827 is a tarball to unpack and install, + # not a config file to write. Its title carries a version, so + # generate a normal phased build for it -- treating it as a plain + # script runs `make install` with nothing unpacked: + # make: *** No rule to make target 'install'. Stop. + cpkgver = _title_pkgver(ctitle) + if cpkgver and any(ch.isdigit() for ch in cpkgver): + cglob = _pkg_glob_for(cpkgver, cname) + ctext = _crosschain_script_body(cname, csid, cpkgver, cglob, + csec["commands"]) + ctext = _chrootify(ctext) + pkg_like_cfg.append(cname) + else: + ctext = _plain_root_script(cname, csid, ctitle, csec["commands"]) + ctext, cleft = _fill_placeholders(ctext, cname) + if cleft: + unresolved.append((cname, cleft)) + with open(dest, "w") as f: + f.write(ctext) + os.chmod(dest, 0o755) + order.append(cname) + written += 1 + + # /etc/fstab, before the bootloader -- the system cannot boot read-write + # without it, and the bootloader step is where booting starts to matter + if not args.chapter7_only: + dest = os.path.join(outdir, "cfg-fstab.sh") + if not (os.path.isfile(dest) and not args.overwrite): + with open(dest, "w") as f: + f.write(_fstab_script(load_config())) + os.chmod(dest, 0o755) + written += 1 + order.append("cfg-fstab") + + if bl == "refind": + dest = os.path.join(outdir, "refind.sh") + with open(dest, "w") as f: + f.write(_refind_script()) + os.chmod(dest, 0o755) + order.append("refind") + written += 1 + + # the user's own final step, copied in and run last + lstep = ensure_last_step_script() + if lstep and os.path.isfile(lstep): + import shutil as _sh + dest = os.path.join(outdir, "last-step.sh") + _sh.copy(lstep, dest) + os.chmod(dest, 0o755) + if "last-step" in order: + order.remove("last-step") + order.append("last-step") # always last + written += 1 + print("# last build step <- %s" % lstep) + extra_urls = last_step_sources() + if extra_urls: + print("# extra sources it wants (get-sources downloads these):") + for u in extra_urls: + print("# %s" % u) + + + with open(os.path.join(lfs, PKGUSR_DIR, "steporder"), "w") as f: + f.write("\n".join(order) + "\n") + # chapter 7's temporary tools are built as root, like the book does; the + # package-user system takes over for the real chapter 8 packages. + root_steps = [n for n, _s in CHROOT_INIT_STEPS + CHROOT_STEPS_7 + if n in order] + root_steps += [n for n, _s in cfg_steps_9 + if n in order and n not in pkg_like_cfg] + if "last-step" in order: + root_steps.append("last-step") # the user's own final step, as root + if "cfg-fstab" in order: + root_steps.append("cfg-fstab") # writes /etc/fstab + if "refind" in order: + root_steps.append("refind") # writes to the ESP, so root + with open(os.path.join(lfs, PKGUSR_DIR, "rootsteps"), "w") as f: + f.write("\n".join(root_steps) + "\n") + _run_bash(f'chmod -R a+rX "{os.path.join(lfs, PKGUSR_DIR)}" 2>/dev/null; true') + print(f"# wrote {written}, kept {skipped} existing " + f"(use --overwrite to replace)") + if unresolved: + sys.stderr.write( + "\n! these scripts still contain a placeholder the book expects " + "you to fill in;\n edit the script before building them:\n") + for nm, left in unresolved: + sys.stderr.write(" %-18s %s\n" + % (nm, ", ".join("<%s>" % x for x in left))) + print(f"# step order -> {os.path.join(lfs, PKGUSR_DIR, 'steporder')}") + write_chroot_env(lfs) + install_helper(lfs) + _install_collector_groups(lfs) + live = _chroot_mounts(lfs) + print() + if len(live) < len(_KERNFS): + print("Next: prepare the chroot, then enter it:") + print(f" {sudo_prefix()}lfs build-system chroot prepare --run") + print(f" {sudo_prefix()}lfs build-system chroot enter") + else: + print("Next: enter the chroot:") + print(f" {sudo_prefix()}lfs build-system chroot enter") + + +def _copy_resolv_conf(lfs): + """Give the chroot working DNS by copying the host's resolver config. + + Routing works inside the chroot (a bare IP pings fine), but with no + /etc/resolv.conf nothing resolves, and every download fails with the + thoroughly unhelpful + Temporary failure in name resolution + This is a BUILD-TIME convenience only: the finished system gets its own + resolv.conf from dhcpcd (or from the static config, if you chose that).""" + dst = os.path.join(lfs, "etc", "resolv.conf") + src = "/etc/resolv.conf" + if not os.path.isfile(src): + return + def usable(path): + """A resolv.conf is only usable if a nameserver line holds a real IP. + + The book's template ships placeholders: + nameserver <IP address of your primary nameserver> + Checking merely that a line starts with "nameserver" accepts that and + leaves DNS just as broken as before.""" + try: + with open(path) as f: + for line in f: + parts = line.split() + if len(parts) >= 2 and parts[0] == "nameserver": + try: + import ipaddress + ipaddress.ip_address(parts[1]) + return True + except ValueError: + continue + except OSError: + return False + return False + + try: + if os.path.isfile(dst): + if usable(dst): + return # already usable + if os.path.getsize(dst) > 0: + bak = dst + ".bak" + import shutil as _sh + _sh.copyfile(dst, bak) + print("# the chroot's /etc/resolv.conf has no real nameserver " + "(the book's placeholders);") + print("# replacing it with the host's -- old one kept at " + "%s" % bak) + os.makedirs(os.path.dirname(dst), exist_ok=True) + import shutil + # follow a symlink (a systemd-resolved stub, say) and copy the CONTENT, + # since the target does not exist inside the chroot + shutil.copyfile(os.path.realpath(src), dst) + print("# copied the host's /etc/resolv.conf into the chroot (for DNS " + "during the build)") + except OSError as e: + sys.stderr.write("! could not copy /etc/resolv.conf into the chroot: " + "%s\n" % e) + + +def cmd_bs_chroot(args): + require_complete_config() + """7.2-7.4 -- hand the tree to root, mount the virtual kernel filesystems, + and enter (or leave) the chroot.""" + lfs = require_mounted_lfs() + action = args.action or "status" + + if action == "status": + live = _chroot_mounts(lfs) + print(f"LFS : {lfs}") + print(f"owned by root : {'yes' if _owner_is_root(lfs) else 'no (still lfs -- run: chroot prepare)'}") + print(f"virtual fs : {', '.join(live) if live else '(none mounted)'}") + rc_path = os.path.join(lfs, "etc", "resolv.conf") + has_dns = False + if os.path.isfile(rc_path): + import ipaddress + for _l in open(rc_path): + _p = _l.split() + if len(_p) >= 2 and _p[0] == "nameserver": + try: + ipaddress.ip_address(_p[1]) + has_dns = True + break + except ValueError: + pass + print(f"DNS in chroot : {'yes' if has_dns else 'no (fix: lfs build-system chroot resolv)'}") + if len(live) < len(_KERNFS): + missing = [m for m in _KERNFS if m not in live] + print(f" not mounted : {', '.join(missing)}") + print(f"\n prepare: {sudo_prefix()}lfs build-system chroot prepare --run" + f"\n enter : {sudo_prefix()}lfs build-system chroot enter" + f"\n leave : exit, then {sudo_prefix()}lfs build-system chroot unmount --run") + return + + if os.geteuid() != 0: + sys.stderr.write(f"! chroot {action} must be run as root.\n") + sys.exit(2) + + if action == "prepare": + # Handing the tree to root is a one-way door: after it the lfs user can + # no longer build, and chapters 5-6 cannot be resumed. Refuse to do + # that on top of a half-built toolchain -- an interrupted chain used to + # sail straight through here into a chroot that could build nothing. + cc_ok, cc_n = _crosschain_looks_done(lfs) + if not cc_ok and not getattr(args, "force", False): + sys.stderr.write( + "\n! chapters 5-6 are not finished: %d of %d steps built.\n" + " Preparing the chroot gives the tree to root, after which\n" + " they cannot be resumed. Finish them first:\n" + " %slfs build-system crosschain --run\n" + " (it continues from where it stopped)\n" + " To prepare anyway: --force\n" + % (cc_n, len(CROSSCHAIN_STEPS), sudo_prefix())) + why = _crosschain_missing_output(lfs) + if why: + sys.stderr.write("\n %s\n" % why) + sys.exit(2) + + steps = [] + if not _owner_is_root(lfs): + steps.append(("give the tree to root", + f'for d in {" ".join(LFS_HANDOVER_DIRS)}; do ' + f'[ -e "{lfs}/$d" ] && ' + f'chown -h -R root:root "{lfs}/$d"; done; true')) + else: + print("# 7.2 already done (tree owned by root)") + steps.append(("create the mount points", + f'mkdir -pv "{lfs}"/{{dev,proc,sys,run}}')) + live = _chroot_mounts(lfs) + mounts = [ + ("dev", f'mount -v --bind /dev "{lfs}/dev"'), + ("dev/pts", f'mkdir -pv "{lfs}/dev/pts"; ' + f'mount -vt devpts devpts -o gid=5,mode=0620 "{lfs}/dev/pts"'), + ("proc", f'mount -vt proc proc "{lfs}/proc"'), + ("sys", f'mount -vt sysfs sysfs "{lfs}/sys"'), + ("run", f'mount -vt tmpfs tmpfs "{lfs}/run"'), + ("dev/shm", f'if [ -h "{lfs}/dev/shm" ]; then ' + f'install -v -d -m 1777 "{lfs}$(realpath /dev/shm)"; ' + f'else mkdir -pv "{lfs}/dev/shm"; ' + f'mount -vt tmpfs -o nosuid,nodev tmpfs "{lfs}/dev/shm"; fi'), + ] + for name, cmd in mounts: + if name in live: + print(f"# {name} already mounted -- skipping") + continue + steps.append((f"mount {name}", cmd)) + + for label, cmd in steps: + # On a real run the label is enough: the commands are the book's, + # they are about to run, and printing every one of them buries the + # mount output that actually matters. A dry run shows them, since + # seeing exactly what WOULD run is the whole point. + if args.run: + print(f"# {label}") + else: + print(f"# {label}") + print(f" {cmd}") + if not args.run: + dry_run_note("apply") + return + for label, cmd in steps: + rc = _run_bash(cmd) + if rc != 0: + sys.stderr.write(f"\n! failed: {label} (exit {rc})\n") + sys.exit(rc) + write_chroot_env(lfs) + install_helper(lfs) + _copy_resolv_conf(lfs) + have_scripts = os.path.isdir(os.path.join(lfs, PKGUSR_DIR, "scripts")) + print("\n# chroot environment ready.") + if not have_scripts: + next_step(f"{sudo_prefix()}lfs build-system gen-chroot-scripts --run", + f"{sudo_prefix()}lfs build-system chroot enter", + label="next run:") + else: + next_step(f"{sudo_prefix()}lfs build-system chroot enter") + return + + if action == "resolv": + _copy_resolv_conf(lfs) + print("# DNS inside the chroot should work now:") + print("# ping -c1 archlinux.org") + return + + if action == "unmount": + live = _chroot_mounts(lfs) + if not live: + print("# nothing of ours is mounted.") + return + # unmount deepest-first so nested mounts (dev/pts inside dev) come off + order = sorted(live, key=lambda p: p.count("/"), reverse=True) + for sub in order: + print(f"# umount {sub}") + if args.run: + _run_bash(f'umount -v "{lfs}/{sub}" || umount -lv "{lfs}/{sub}"') + if not args.run: + dry_run_note("unmount") + else: + left = _chroot_mounts(lfs) + print("# unmounted." if not left + else f"# still mounted: {', '.join(left)}") + return + + if action == "enter": + # same reason as in `run`: entering with a stale lfs-helper means + # debugging a version that no longer exists out here + _refresh_chroot_tools(lfs) + live = _chroot_mounts(lfs) + missing = [m for m in _KERNFS if m not in live] + if missing: + sys.stderr.write( + f"! the virtual kernel filesystems aren't mounted " + f"({', '.join(missing)}).\n" + f" Run first: {sudo_prefix()}lfs build-system chroot prepare --run\n") + sys.exit(2) + if not _owner_is_root(lfs): + sys.stderr.write( + "! the tree is not owned by root yet (book 7.2): %s owns " + "%s/usr.\n" % (_owner_name_of(os.path.join(lfs, "usr")), lfs) + + f" Run first: {sudo_prefix()}lfs build-system chroot prepare --run\n") + sys.exit(2) + cfg = load_config() + mkf = cfg.get("makeflags") or "-j$(nproc)" + install_helper(lfs, quiet=True) + write_chroot_env(lfs, quiet=True) + scripts_dir = os.path.join(lfs, PKGUSR_DIR, "scripts") + print(f"# entering chroot at {lfs} (type 'exit' to leave)") + print("#") + print("# There's no Python in here yet -- use 'lfs-helper' (bash) to") + print("# drive the build. Start with:") + if not os.path.isdir(scripts_dir): + print("# (no build scripts yet -- leave the chroot and run:") + print(f"# {sudo_prefix()}lfs build-system gen-chroot-scripts --run)") + else: + print("# lfs-helper next # what to do, step by step") + print("# lfs-helper status # where the session stands") + print("# lfs-helper list # all steps + progress") + print("#") + inner = (f'chroot "{lfs}" /usr/bin/env -i ' + f'HOME=/root TERM="$TERM" PS1="(lfs chroot) \\u:\\w\\$ " ' + f'PATH=/usr/bin:/usr/sbin ' + f'MAKEFLAGS="{mkf}" TESTSUITEFLAGS="{mkf}" ' + f'/bin/bash --login') + import subprocess + rc = subprocess.run(["bash", "-c", inner], cwd="/").returncode + print(f"# left the chroot (exit {rc}).") + print(f"# when you're done for now: {sudo_prefix()}lfs build-system chroot unmount --run") + return + + +def cmd_bs_crosschain(args): + """Build the chapter-5 cross toolchain + chapter-6 temporary tools, in order, + AS the lfs user. Each + step is a real, phased script on disk (unpack/build/install/configure/test, + same shape as the BLFS install scripts) you can read and edit before + running, and re-run a single phase (e.g. just 'install') without rebuilding. + If a multi-step run is interrupted, the next run offers to continue where it + left off.""" + ver, _bookpath = resolve_book(args) + + if args.list: + # fast path: read the version straight out of already-saved scripts + # (a plain file read) instead of re-parsing the whole book per step -- + # the book is only touched (once, for ALL missing steps together) if + # something hasn't been generated yet. + need_book = any(not os.path.isfile(crosschain_script_path(ver, n)) + for n, _s, _t in CROSSCHAIN_STEPS) + book_info = _crosschain_all_pkgvers(args) if need_book else {} + outdated = [] + for name, sid, tarpfx in CROSSCHAIN_STEPS: + p = crosschain_script_path(ver, name) + if os.path.isfile(p): + head = open(p, encoding="utf-8", errors="replace").read(2000) + m = re.search(r'^name_version="([^"]+)"', head, re.M) + pkgver = m.group(1) if m else "?" + vm = re.search(rf'{re.escape(CROSSCHAIN_MARK)} v(\d+):', head) + fv = int(vm.group(1)) if vm else 0 + if fv < CROSSCHAIN_SCRIPT_VERSION: + saved = f"saved script -- OUTDATED FORMAT v{fv}, current is v{CROSSCHAIN_SCRIPT_VERSION}" + outdated.append(name) + else: + saved = "saved script" + else: + pkgver = book_info.get(name, (None, None))[0] + saved = "not generated yet" + print(f"{name:<16} {pkgver or '?':<20} [{saved}] {p}") + if outdated: + print(f"\n{len(outdated)} script(s) predate a bug fix -- regenerate " + f"them (this discards any edits; use --edit to preserve " + f"them manually first):") + print(" lfs build-system crosschain " + " ".join(outdated) + + " --regenerate") + return + + chosen = CROSSCHAIN_STEPS if args.step in (None, "all") else \ + [(n, s, p) for n, s, p in CROSSCHAIN_STEPS if n == args.step] + if not chosen: + sys.stderr.write(f"unknown step: {args.step} (see: crosschain --list)\n") + sys.exit(2) + + if args.path: + for name, sid, tarpfx in chosen: + print(crosschain_script_path(ver, name)) + return + + need_book = args.regenerate or any( + not os.path.isfile(crosschain_script_path(ver, name)) + for name, _s, _t in chosen) + book_info = _crosschain_all_pkgvers(args) if need_book else {} + + if args.edit: + editor = os.environ.get("EDITOR", "vim") + for name, sid, tarpfx in chosen: + path = ensure_crosschain_script(args, ver, name, sid, tarpfx, + book_info=book_info) + import subprocess + subprocess.run([editor, path], cwd="/") + return + + if args.run and os.geteuid() != 0: + sys.stderr.write( + "! crosschain --run must be run as ROOT.\n" + " It builds each step AS the lfs user (via su - lfs) -- so run it " + "from a root shell, not as lfs:\n" + f" {sudo_prefix()}lfs build-system crosschain --run\n") + sys.exit(2) + lfs = require_mounted_lfs() if args.run else None + phase = args.phase or "all" + + # ---- resume: skip steps already recorded as built. The record is a global + # set of finished steps for this book, so it survives building an individual + # package in between; only a multi-step run consults it. + chosen_names = [n for n, _s, _t in chosen] + skip = set() + if args.run and phase == "all" and not args.restart and len(chosen) > 1: + prog = _load_crosschain_progress(lfs) if lfs else None + done_all = [d for d in (prog or {}).get("done", []) + if (prog or {}).get("book") == ver and d in chosen_names] + if done_all and len(done_all) < len(chosen_names): + todo = [n for n in chosen_names if n not in done_all] + # Asked the same way round as lfs-helper's version of this + # question, and defaulting the same way: the safe answer is to + # KEEP work that is already done, so "rebuild everything" has to + # be chosen deliberately. The two prompts used to be inverted + # relative to each other -- same decision, opposite default. + print(f"\n{len(done_all)} of the {len(chosen_names)} step(s) in " + f"range are already built (last: {done_all[-1]}).") + print(f" y = rebuild all {len(chosen_names)} step(s) from the " + f"start of the range") + if todo: + print(f" N = keep them, build only the {len(todo)} remaining " + f"step(s), starting with '{todo[0]}'") + else: + print(" N = keep them; nothing would be built " + "(everything is done)") + rebuild = False + if sys.stdin.isatty() and not args.yes: + rebuild = input("Rebuild those too? [y/N]: ").strip().lower() \ + in ("y", "yes") + if rebuild: + print("# rebuilding everything in range") + _clear_crosschain_progress(lfs) + else: + skip = set(done_all) + print(f"# keeping them; building the {len(todo)} remaining " + f"step(s)") + + if args.run: + srcdir = os.path.join(lfs, "sources") + if not os.path.isdir(srcdir): + sys.stderr.write( + f"\n! {srcdir} doesn't exist -- packages haven't been " + f"downloaded yet.\n Run: lfs build-system get-sources --run\n") + sys.exit(2) + _require_lfs_writable_by_lfs(lfs) + + total = len(chosen) + try: + for i, (name, sid, tarpfx) in enumerate(chosen, 1): + if name in skip: + continue + path = ensure_crosschain_script(args, ver, name, sid, tarpfx, + force_regen=args.regenerate, + book_info=book_info) + script = open(path).read() + print(f"\n# ===== STEP {i}/{total}: {name} ({sid}, phase: {phase}) =====") + print(f"# script: {path}") + if not args.run: + print(script) + continue + + m = re.search(r'^pkg_glob="([^"]+)"', script, re.M) + glob = m.group(1) if m else f"{tarpfx}-*.tar.*" + # as ROOT: clear any leftover extracted dir (could be root-owned from + # a previous failed run, which lfs couldn't delete) and fix perms. + if phase in ("all", "unpack"): + print(f"# [root] clearing any old {tarpfx}-* source dir + " + f"fixing perms ...") + preclean_rc = _run_bash( + f'cd "$LFS/sources" && ' + f'(for d in {tarpfx}-*/ ; do [ -d "$d" ] && rm -rf "$d"; done); ' + f'chmod a+r {glob} 2>/dev/null; ' + f'chmod a+wt "$LFS/sources" 2>/dev/null; true') + if preclean_rc != 0: + sys.stderr.write( + f"\n! STEP {i}/{total} ({name}) FAILED before the build " + f"even started -- couldn't prepare $LFS/sources.\n" + f" script: {path}\n" + f" This usually means the session isn't active. Check:\n" + f" lfs build-system session\n") + sys.exit(preclean_rc) + + print(f"# [lfs] running {path} (phase: {phase}) ...") + full = f'export LFS_CC_PHASE="{phase}"\n' + script + rc = _run_tracked(name, full, as_lfs=True, lfs=lfs) + if rc != 0: + bar = "!" * 58 + sys.stderr.write( + f"\n{bar}\n" + f"! STEP {i}/{total} FAILED: {name} (phase '{phase}', " + f"exit {rc})\n" + f"! script: {path}\n" + f"! Open/edit that file to fix it, then retry just this " + f"phase:\n" + f"! lfs build-system crosschain {name} --edit\n" + f"! {sudo_prefix()}lfs build-system crosschain {name} --phase " + f"{phase} --run\n" + f"{bar}\n") + sys.exit(rc) + print(f"# STEP {i}/{total} ({name}): done") + if phase == "all": # a fully-built step -- remember it globally + _mark_crosschain_done(lfs, ver, name) + except KeyboardInterrupt: + sys.stderr.write( + "\n\n! interrupted -- progress has been saved. Re-run the same " + "command to continue from the next step:\n" + f" {sudo_prefix()}lfs build-system crosschain --run\n") + sys.exit(130) + + if args.run: + # Do NOT clear the progress file here. It is the only exact record of + # which of the 22 steps ran, and clearing it made "finished" and + # "never started" indistinguishable -- which is why an interrupted + # chain could be mistaken for a complete one. + print("\ncrosschain: all requested steps done.") + + +# ---- package-user file tracking ------------------------------------------- # +# We record which files each package installs (snapshot diff of $LFS before/after +# each install) so ownership can later be handed to that package's user. The +# data lives under $LFS/var/lib/pkgusr so it persists into the final system. +# Build-tool metadata (manifests, snapshots, resume progress). This deliberately +# does NOT live under $LFS/var/lib: creating it as root would make $LFS/var/lib +# root-owned, and then packages building as the lfs user cannot create their own +# subdirs there (glibc's var/lib/nss_db fails with 'Permission denied'). It's +# our bookkeeping, not part of the LFS system being built, so it goes in its own +# top-level dir that we own outright. +PKGUSR_DIR = ".lfs-pkgusr" + + +def _pkgusr_dir(lfs): + d = os.path.join(lfs, PKGUSR_DIR, "manifests") + existed = os.path.isdir(d) + os.makedirs(d, exist_ok=True) + if not existed: + # we may be running as root; make sure the lfs user can write here too + _run_bash(f'chown -R lfs "{os.path.join(lfs, PKGUSR_DIR)}" 2>/dev/null; true') + return os.path.join(lfs, PKGUSR_DIR) + + +def _snapshot_cmd(lfs): + x = PKGUSR_DIR + return (f'find "{lfs}" -xdev \\( -type f -o -type l \\) ' + f'-not -path "{lfs}/sources/*" -not -path "{lfs}/{x}/*" ' + f'-not -path "{lfs}/dev/*" -not -path "{lfs}/proc/*" ' + f'-not -path "{lfs}/sys/*" -not -path "{lfs}/run/*" ' + f'-not -path "{lfs}/tmp/*" 2>/dev/null | sort') + + +def _snapshot_dirs_cmd(lfs): + x = PKGUSR_DIR + return (f'find "{lfs}" -xdev -type d ' + f'-not -path "{lfs}/sources/*" -not -path "{lfs}/{x}/*" ' + f'-not -path "{lfs}/dev/*" -not -path "{lfs}/proc/*" ' + f'-not -path "{lfs}/sys/*" -not -path "{lfs}/run/*" ' + f'-not -path "{lfs}/tmp/*" 2>/dev/null | sort') + + +def _take_snapshot(lfs, dirs=False): + import subprocess + cmd = _snapshot_dirs_cmd(lfs) if dirs else _snapshot_cmd(lfs) + r = subprocess.run(["bash", "-c", cmd], cwd="/", + capture_output=True, text=True) + return set(r.stdout.splitlines()) + + +def _run_tracked(name, script, as_lfs, lfs): + """Run an install (as lfs or root), then diff the $LFS snapshot to record the + files this package installed into $LFS/var/lib/pkgusr/manifests/<name>.files. + The manifest ACCUMULATES: a re-run of an already-installed step naturally + diffs to "0 new" (the files already exist), so we union with whatever was + already recorded instead of overwriting -- otherwise a harmless re-run would + silently erase a correct manifest from an earlier successful run. + Returns the install's exit code.""" + import subprocess + d = _pkgusr_dir(lfs) + snap = os.path.join(d, ".snapshot") + before = (set(open(snap).read().splitlines()) + if os.path.isfile(snap) else _take_snapshot(lfs)) + # The directory snapshot must be taken HERE, before the build runs -- the + # old code took it afterwards and then discarded the diff whenever there + # was no saved snapshot, so on a first run every package recorded zero + # directories. Its files got the package user while the directories + # holding them stayed root-owned. + snapd = os.path.join(d, ".snapshot-dirs") + before_d = (set(open(snapd).read().splitlines()) + if os.path.isfile(snapd) else _take_snapshot(lfs, dirs=True)) + if as_lfs: + rc = run_as_lfs(script, lfs=lfs) + else: + rc = _run_bash(script) + after = _take_snapshot(lfs) # capture even on failure + new = after - before + # directories too: a package owns the dirs it creates for itself, and + # without this every package looks like it created none + after_d = _take_snapshot(lfs, dirs=True) + new_d = after_d - before_d + mand = os.path.join(d, "manifests", f"{name}.dirs") + existing_d = (set(l for l in open(mand).read().splitlines() if l) + if os.path.isfile(mand) else set()) + with open(mand, "w") as f: + tot = sorted(existing_d | new_d) + f.write(("\n".join(tot) + "\n") if tot else "") + with open(snapd, "w") as f: + f.write("\n".join(sorted(after_d)) + "\n") + man = os.path.join(d, "manifests", f"{name}.files") + existing = set(l for l in open(man).read().splitlines() if l) \ + if os.path.isfile(man) else set() + total = sorted(existing | new) + with open(man, "w") as f: + f.write(("\n".join(total) + "\n") if total else "") + with open(snap, "w") as f: + f.write("\n".join(sorted(after)) + "\n") + print(f"# tracked {len(new)} new file(s) and {len(new_d)} new dir(s) this " + f"run ({len(total)} file(s) total known) for '{name}'") + return rc + + +def cmd_bs_manifests(args): + """List tracked package manifests (or one package's file list).""" + lfs = _lfs_dir() + if not lfs: + sys.stderr.write("no LFS mount set (lfs build-system session)\n") + sys.exit(2) + mdir = os.path.join(lfs, PKGUSR_DIR, "manifests") + if not os.path.isdir(mdir): + print("no manifests yet.") + return + if args.package: + shown = False + for suffix, label in ((".files", "files"), (".dirs", "directories")): + p = os.path.join(mdir, f"{args.package}{suffix}") + if os.path.isfile(p): + body = open(p).read() + if body.strip(): + print(f"# {label}") + sys.stdout.write(body) + shown = True + if not shown: + sys.stderr.write(f"no manifest for {args.package}\n") + sys.exit(1) + return + + def _count(path): + try: + return sum(1 for ln in open(path) if ln.strip()) + except OSError: + return 0 + + # Report directories alongside files. A package owns the directories it + # creates as well as the files in them, so a package showing files but no + # directories is a bug worth seeing -- that is exactly how the C++ header + # tree ended up root-owned while the headers inside it belonged to gcc. + total_f = total_d = 0 + rows = [] + for f in sorted(os.listdir(mdir)): + if not f.endswith(".files"): + continue + name = f[:-6] + nf = _count(os.path.join(mdir, f)) + nd = _count(os.path.join(mdir, name + ".dirs")) + total_f += nf + total_d += nd + rows.append((name, nf, nd)) + for name, nf, nd in rows: + flag = " <- no directories recorded" if (nf and not nd) else "" + print(f" {name:<20} {nf:>6} files {nd:>5} dirs{flag}") + print(f"\n {total_f} file(s) and {total_d} directory(ies) tracked " + f"across {len(rows)} package(s) [{lfs}]") + + +# ---- installing the real toolchain into the chroot ------------------------ # + +# The Python tooling needs two modules that base LFS does not ship: +# requests (blfs fetches the BLFS book) +# beautifulsoup4 (lfs and blfs parse the books) +# Without them the tools import-fail the moment you run them in the chroot. +TOOL_PY_DEPS = [("bs4", "beautifulsoup4"), ("requests", "requests")] + + +# The package-users hint ships helper scripts and command wrappers. A system +# built with this tooling should have the same ones, so `add_package_user`, +# `install_package` and friends work in the chroot exactly as they do outside. +PKGUSR_HELPERS = [ + # (name, where it belongs) + ("add_package_user", "usr/sbin"), + ("install_package", "usr/sbin"), + ("useradd", "usr/lib/pkgusr"), + ("groupadd", "usr/lib/pkgusr"), + ("list_package", "usr/bin"), + ("uninstall_package", "usr/bin"), + ("list_suspicious_files", "usr/bin"), + ("list_suspicious_files_from", "usr/bin"), + ("grep_all_regular_files_for", "usr/bin"), + ("forall_direntries_from", "usr/bin"), + # the five command wrappers, which live first in a package user's PATH + ("chgrp", "usr/lib/pkgusr"), + ("chown", "usr/lib/pkgusr"), + ("chmod", "usr/lib/pkgusr"), + ("mkdir", "usr/lib/pkgusr"), + ("install", "usr/lib/pkgusr"), +] + +_HELPER_SEARCH = ["/usr/lib/pkgusr", "/usr/sbin", "/usr/bin", "/sbin", "/bin", + "/usr/local/sbin", "/usr/local/bin", + "/tools/more_control_helpers/sbin", + "/tools/more_control_helpers/bin", + "/tools/more_control_helpers/lib"] + + +def _find_host_helper(name, subdir): + """Find a hint helper on THIS machine. + + The wrappers share names with the real commands (chown, install, ...), so + only look for those under a pkgusr directory -- copying /usr/bin/chown into + the chroot as a "wrapper" would be a fine way to break the system.""" + if subdir.endswith("pkgusr"): + cands = ["/usr/lib/pkgusr/" + name, + "/tools/more_control_helpers/lib/" + name] + else: + cands = [os.path.join(d, name) for d in _HELPER_SEARCH + if "pkgusr" not in d or name not in + ("chown", "chgrp", "chmod", "mkdir", "install")] + for c in cands: + if os.path.isfile(c) and os.access(c, os.X_OK): + return c + return None + + +# The package-user skeleton. If this machine has one (the hint's +# /etc/pkgusr/skel-package), copy it verbatim -- matching the host exactly +# matters more than any default we could invent, and a mismatch shows up much +# later as a package user whose environment differs from every other one. +PKGUSR_SKEL_SRC = "/etc/pkgusr/skel-package" + + +def _install_pkgusr_skel(lfs, verbose=True): + if not os.path.isdir(PKGUSR_SKEL_SRC): + if verbose: + print("# no %s on this machine -- the tools will write a default " + "package-user environment instead" % PKGUSR_SKEL_SRC) + return False + import shutil + dst = os.path.join(lfs, PKGUSR_SKEL_SRC.lstrip("/")) + try: + if os.path.isdir(dst): + shutil.rmtree(dst) + shutil.copytree(PKGUSR_SKEL_SRC, dst, symlinks=True) + except OSError as e: + sys.stderr.write("! could not copy %s: %s\n" % (PKGUSR_SKEL_SRC, e)) + return False + # the shared files the skeleton's symlinks point at + for name in ("bash_profile", "bashrc", "build"): + src = os.path.join("/etc/pkgusr", name) + if os.path.isfile(src): + d = os.path.join(lfs, "etc", "pkgusr", name) + os.makedirs(os.path.dirname(d), exist_ok=True) + shutil.copy(src, d) + os.chmod(d, 0o755 if name == "build" else 0o644) + if verbose: + print("# package-user skeleton copied from %s" % PKGUSR_SKEL_SRC) + return True + + +def _install_pkgusr_helpers(lfs, verbose=True): + """Copy the hint's helper scripts into the chroot, where present.""" + import shutil + copied, missing = [], [] + for name, subdir in PKGUSR_HELPERS: + src = _find_host_helper(name, subdir) + if not src: + missing.append(name) + continue + dst_dir = os.path.join(lfs, subdir) + os.makedirs(dst_dir, exist_ok=True) + dst = os.path.join(dst_dir, name) + try: + shutil.copy(src, dst) + os.chmod(dst, 0o755) + copied.append("%s -> /%s" % (name, subdir)) + except OSError as e: + sys.stderr.write("! %s: %s\n" % (name, e)) + if verbose: + if copied: + print("# package-user helpers copied from this machine:") + for c in copied: + print("# %s" % c) + if missing: + print("# not on this machine (the tools fall back to Shadow's own" + " useradd/groupadd): %s" % ", ".join(missing)) + return copied + + +def _chroot_missing_tool_deps(lfs): + import glob as _glob + sitedirs = _glob.glob(os.path.join(lfs, "usr", "lib", "python3*", + "site-packages")) + missing = [] + for mod, pkg in TOOL_PY_DEPS: + found = any(os.path.exists(os.path.join(d, mod)) + or _glob.glob(os.path.join(d, mod + "-*")) + or _glob.glob(os.path.join(d, mod + ".py")) + for d in sitedirs) + if not found: + missing.append((mod, pkg)) + return missing + + +def _chroot_has_python(lfs): + """Is there a usable Python inside the tree yet?""" + for p in ("usr/bin/python3", "usr/bin/python"): + if os.path.exists(os.path.join(lfs, p)): + return True + return False + + +def cmd_bs_install_tools(args): + """Put the Python tooling (lfs, packagemanager, blfs) inside the chroot. + + lfs-helper exists as a bash stand-in for exactly one reason: there is no + Python in the chroot until chapter 8 builds it. Once it's there the real + tools can run inside, and packagemanager takes over package management -- + including for BLFS packages later on.""" + lfs = require_mounted_lfs() + if not _chroot_has_python(lfs): + sys.stderr.write( + "! there is no Python inside the chroot yet, so the Python tooling\n" + " cannot run there. Build it first (inside the chroot):\n" + " lfs-helper build Python\n") + sys.exit(2) + + missing = _chroot_missing_tool_deps(lfs) + if missing and not args.force: + sys.stderr.write( + "! the Python tooling needs modules that base LFS does not ship:\n") + for mod, pkg in missing: + sys.stderr.write(" %-16s (pip install %s)\n" % (mod, pkg)) + sys.stderr.write( + "\n Install them inside the chroot -- this needs network access:\n" + " pip3 install %s\n" + "\n No network in the chroot? Download the wheels on this machine\n" + " into %s/sources and install from there:\n" + " pip3 install --no-index --find-links /sources %s\n" + "\n packagemanager itself does not need them; only 'lfs' and 'blfs'\n" + " do (they parse the books). To install the tools anyway:\n" + " %slfs build-system install-tools --run --force\n" + % (" ".join(p for _m, p in missing), lfs, + " ".join(p for _m, p in missing), sudo_prefix())) + sys.exit(2) + + here = os.path.dirname(os.path.abspath(__file__)) + tools = [] + for name in ("lfs", "packagemanager", "blfs", "lfs-helper", + "packagemanager_install"): + src = os.path.join(here, name) + if os.path.isfile(src): + tools.append((name, src)) + if not tools: + sys.stderr.write("! no tools found next to %s\n" % here) + sys.exit(2) + + dest_dir = os.path.join(lfs, "usr", "bin") + cfg_dir = os.path.join(lfs, "etc", "pkgusr") + store = os.path.join(lfs, "usr", "share", "lfs") + + print("Install the Python tooling into %s:" % lfs) + for name, _src in tools: + print(" %-22s -> /usr/bin/%s" % (name, name)) + print(" %-22s -> /usr/share/lfs" % "book + scripts") + print(" %-22s -> /etc/pkgusr/packagemanager.conf" % "packagemanager conf") + if not args.run: + dry_run_note("install") + return + + import shutil + os.makedirs(dest_dir, exist_ok=True) + os.makedirs(cfg_dir, exist_ok=True) + os.makedirs(store, exist_ok=True) + for name, src in tools: + dst = os.path.join(dest_dir, name) + shutil.copy(src, dst) + os.chmod(dst, 0o755) + + # The BLFS book lives in its OWN store (/usr/share/blfs), so copying only + # the LFS store left the chroot with no BLFS book at all -- and every + # `packagemanager install <pkg>` failed with "not found in book" when the + # real problem was that there was no book to look in. + _blfs_src = None + for cand in (os.environ.get("BLFS_STORE"), "/usr/share/blfs", + os.path.expanduser("~/.cache/blfs")): + if cand and os.path.isdir(os.path.join(cand, "books")): + _blfs_src = os.path.join(cand, "books") + break + if _blfs_src: + _blfs_dst = os.path.join(lfs, "usr", "share", "blfs", "books") + os.makedirs(_blfs_dst, exist_ok=True) + _n = 0 + for f in os.listdir(_blfs_src): + if not f.endswith((".html", ".html.gz")): + continue + d = os.path.join(_blfs_dst, f) + if not os.path.isfile(d): + shutil.copy(os.path.join(_blfs_src, f), d) + _make_world_readable(d) + _n += 1 + if _n: + print(" %-22s -> /usr/share/blfs/books" % ("BLFS book" if _n == 1 + else "BLFS books")) + else: + print(" %-22s (none cached here -- `blfs fetch` inside, or" + % "BLFS book:") + print(" %-22s `blfs import <file>`)" % "") + + for sub in ("books", "crosschain"): + s_dir = os.path.join(store_dir(), sub) + if os.path.isdir(s_dir): + d_dir = os.path.join(store, sub) + if os.path.isdir(d_dir): + shutil.rmtree(d_dir) + shutil.copytree(s_dir, d_dir) + cfgsrc = config_path() + if os.path.isfile(cfgsrc): + shutil.copy(cfgsrc, os.path.join(store, "config.json")) + + # packagemanager's own config, matching the settings used out here so the + # collector groups it creates line up with the ones already on the system + pmconf = os.path.join(cfg_dir, "packagemanager.conf") + if not os.path.isfile(pmconf): + # key=value, NOT json: packagemanager parses this file line by line. + # Written as json it parsed to nothing, so the prefix never arrived and + # packagemanager fell back to its default -- every package installed + # afterwards joined `sysgroup_*` groups while the built system used + # whatever prefix was configured out here. + with open(pmconf, "w") as f: + f.write("# packagemanager configuration\n") + f.write("# written by lfs build-system install-tools\n") + f.write("collector_prefix=%s\n" % collector_prefix()) + f.write("user_prefix=u\n") + _install_pkgusr_skel(lfs) + _install_pkgusr_helpers(lfs) + _write_tool_stamps(lfs, tools) + _run_bash('chmod -R a+rX "%s" "%s" 2>/dev/null; true' % (store, cfg_dir)) + + print("\n# installed version %s (build %s). Inside the chroot the real " + "tools are now available:" % (LFS_VERSION, _build_id())) + print(" lfs next # what to do next") + print(" packagemanager --help # package management from here on") + print(" blfs --help # BLFS install scripts") + + +# ---- installing and updating LFS packages --------------------------------- # + +def _find_blfs(): + here = os.path.dirname(os.path.abspath(__file__)) + local = os.path.join(here, "blfs") + if os.path.isfile(local): + # Check the mode bits, not os.access(X_OK): for root that returns True + # for a file with no execute bit at all, and the exec then fails with a + # confusing "Permission denied" on a path that plainly exists. + executable = bool(os.stat(local).st_mode & 0o111) + return [local] if executable else [sys.executable or "python3", local] + import shutil as _sh + return ["blfs"] if _sh.which("blfs") else None + + +def _blfs_knows(name): + """Does the BLFS book have this package? + + BLFS is the better source when both books carry a package: its scripts + carry the dependency information and the configure options that matter for + a package used as a library, where LFS only builds what the base system + needs.""" + blfs = _find_blfs() + if not blfs: + return False + import subprocess + try: + r = subprocess.run(blfs + ["debug", name], capture_output=True, + text=True, timeout=60, cwd="/") + return r.returncode == 0 and bool(r.stdout.strip()) + except (OSError, subprocess.SubprocessError): + return False + + +def _lfs_script_path(lfs, name): + return os.path.join(lfs, PKGUSR_DIR, "scripts", "%s.sh" % name) + + +def _find_lfs_package(args, name): + """Locate a package in the LFS book by the name used for its script.""" + ver, path = resolve_book(args) + soup = load_soup(path) + want = name.lower() + for sid, title, sec in iter_sections(soup): + nv = package_name_version(sid, title) + if nv and nv[0].lower() == want: + return sid, title, sec + # fall back to the chapter 5-7 step names + for n, sid in (list(CHROOT_STEPS_7) + [(a, b) for a, b in CHROOT_CONFIG_STEPS_9]): + if n.lower() == want: + for s2, t2, sec2 in iter_sections(soup): + if s2 == sid: + return sid, t2, sec2 + return None, None, None + + +def cmd_install(args): + """Install or reinstall an LFS package, as its package user. + + The build itself is lfs-helper's job -- it owns the package users and the + file tracking -- so this generates a current script and hands over, rather + than growing a second, subtly different build path.""" + # Are we running INSIDE the chroot? The state directory sits at the root of + # the built system, so it only exists at /.lfs-pkgusr when we are in there. + inside = os.path.isdir(os.path.join("/", PKGUSR_DIR)) + lfs = "/" if inside else require_mounted_lfs() + reinstall = args.reinstall + + for name in args.packages: + # BLFS first: if both books have the package, BLFS' script is the more + # complete one (dependencies, and the options a library build needs). + if not args.lfs_book and _blfs_knows(name): + print("%s is in the BLFS book -- using that (more complete than " + "the LFS one)." % name) + print(" blfs install %s" % name) + print(" (force the LFS book's version with: lfs install " + "--lfs-book %s)" % name) + blfs = _find_blfs() + if inside and blfs: + rc = _run_bash(" ".join(shlex.quote(c) + for c in blfs + ["install", name])) + if rc != 0: + sys.exit(rc) + continue + + sid, title, sec = _find_lfs_package(args, name) + if not sid: + sys.stderr.write("! '%s' is not an LFS package.\n" + " List them with: lfs packages\n" + " BLFS packages are blfs' job: blfs script %s\n" + % (name, name)) + continue + if not sec["commands"]: + sys.stderr.write("! %s has no build commands in the book.\n" % name) + continue + + script = _lfs_script_path(lfs, name) + print("%s (%s)" % (name, title.strip())) + print(" script: %s" % script) + + if args.regenerate or not os.path.isfile(script): + pkgver = _title_pkgver(title) or name + glob = _pkg_glob_for(pkgver, name) + text = _chrootify(_crosschain_script_body(name, sid, pkgver, glob, + sec["commands"])) + text, left = _fill_placeholders(text, name) + os.makedirs(os.path.dirname(script), exist_ok=True) + with open(script, "w") as f: + f.write(text) + os.chmod(script, 0o755) + print(" regenerated from the book") + if left: + sys.stderr.write(" ! still contains: %s -- edit the script " + "before building\n" + % ", ".join("<%s>" % x for x in left)) + + # Reinstalling re-runs the CONFIGURE phase too, and that is not always + # harmless: those are the post-install steps that move files about, + # rewrite config and recreate symlinks. Ask rather than assume. + phases = "all" + if reinstall: + has_cfg = False + if os.path.isfile(script): + has_cfg = "#### CONFIGURE ####" in open(script).read() + if has_cfg: + print() + print(" %s has a configure phase -- the post-install steps" % name) + print(" (moving files, rewriting config, recreating symlinks).") + print(" Re-running it is usually right, but it can overwrite") + print(" changes you made by hand since the first install.") + ans = "" + if args.yes: + ans = "y" + elif sys.stdin.isatty(): + try: + ans = input(" Run the configure phase too? [y/N]: ").strip() + except EOFError: + ans = "" + if ans[:1].lower() != "y": + phases = "install" + print(" -> unpack/build/install only, leaving configure alone") + + cmd = ["lfs-helper", "build", name, "--force"] + if phases != "all": + cmd += ["--phase", phases] + print() + if inside: + print(" running: %s" % " ".join(cmd)) + rc = _run_bash(" ".join(shlex.quote(c) for c in cmd)) + if rc != 0: + sys.exit(rc) + else: + print(" Run this inside the chroot:") + print(" %slfs build-system chroot enter" % sudo_prefix()) + print(" %s" % " ".join(cmd)) + + +def cmd_update(args): + """Rebuild packages whose installed version differs from the book's.""" + inside = os.path.isdir(os.path.join("/", PKGUSR_DIR)) + lfs = "/" if inside else require_mounted_lfs() + ver, path = resolve_book(args) + soup = load_soup(path) + + manifests = os.path.join(lfs, PKGUSR_DIR, "manifests") + installed = set() + if os.path.isdir(manifests): + installed = {f[:-6] for f in os.listdir(manifests) if f.endswith(".files")} + + names = args.packages or sorted(installed) + if not names: + print("nothing is installed yet -- build the system first:") + print(" %slfs build-system next" % sudo_prefix()) + return + + stale, current, unknown = [], [], [] + for name in names: + sid, title, sec = _find_lfs_package(args, name) + if not sid: + unknown.append(name) + continue + book_ver = _title_pkgver(title) or "?" + script = _lfs_script_path(lfs, name) + # What is INSTALLED comes from the package user's VERSION file. The + # script is regenerated whenever the book moves on, so its name_version + # says what the book has -- using it here would compare the book with + # itself and never report anything as out of date. + have, from_book = None, None + vpath = os.path.join(lfs, "usr", "src", name, "VERSION") + if os.path.isfile(vpath): + for line in open(vpath): + line = line.strip() + if line.startswith("# from:"): + from_book = line.split(":", 1)[1].strip() + elif line and not line.startswith("#") and have is None: + have = line + if have is None and os.path.isfile(script): + m = re.search(r'^name_version="([^"]+)"', open(script).read(), re.M) + if m: + have = m.group(1) + if have and have != book_ver: + stale.append((name, have, book_ver)) + else: + current.append((name, have or book_ver)) + + if stale: + print("Out of date -- comparing what is installed with the %s book:\n" + % ver) + for name, have, book_ver in stale: + print(" %-20s %s -> %s" % (name, have, book_ver)) + print() + print("Rebuild them with:") + print(" lfs install --reinstall %s" + % " ".join(n for n, _h, _b in stale)) + else: + print("Everything checked is current with the %s book." % ver) + if unknown: + in_blfs = [n for n in unknown if _blfs_knows(n)] + other = [n for n in unknown if n not in in_blfs] + if in_blfs: + print("\nIn the BLFS book -- update those with blfs:") + print(" blfs update %s" % " ".join(in_blfs)) + if other: + print("\nIn neither book: %s" % ", ".join(other)) + if args.verbose and current: + print("\nCurrent:") + for name, v in current: + print(" %-20s %s" % (name, v)) + + +# ---- what to do next ------------------------------------------------------ # + +def _chroot_progress(lfs): + """Steps already built INSIDE the chroot (written by lfs-helper).""" + p = os.path.join(lfs, PKGUSR_DIR, "chroot-progress") + if os.path.isfile(p): + return [l.strip() for l in open(p) if l.strip()] + return [] + + +def _chroot_steporder(lfs): + p = os.path.join(lfs, PKGUSR_DIR, "steporder") + if os.path.isfile(p): + return [l.strip() for l in open(p) if l.strip()] + return [] + + +def _crosschain_looks_done(lfs): + """Chapters 5-6 finished? + + Returns (done, how_many_of_the_22). + + This must be EXACT. A cross-gcc lands in $LFS/tools/bin after step 2 of + 22, so treating its presence as "chapters 5-6 are done" declared a + toolchain finished when two thirds of it was missing -- and the run then + handed the tree to root and entered the chroot on top of it. Half a + toolchain is not a system you can build in. + + The progress file is the only real record, so it is no longer cleared on + completion (an empty file used to mean both "not started" and "all done"). + """ + names = {n for n, _s, _t in CROSSCHAIN_STEPS} + prog = _load_crosschain_progress(lfs) or {} + done = set(prog.get("done", [])) & names + if done >= names: + return True, len(done) + # Past this stage entirely: chapter 7/8 work exists in the chroot, which + # can only happen after chapters 5-6 completed. + if _chroot_progress(lfs): + return True, len(names) + # No progress file at all, but a full temporary system is present: an + # older run finished before progress was recorded. Require evidence from + # the END of the chain (chapter 6's last steps), not the beginning. + if not prog: + import glob as _glob + # Evidence must come from the LAST step, gcc-pass2. gawk/sed/tar/xz + # are steps 13-20, so they are present at 20 of 22 as well -- a tree + # missing only gcc-pass2 looked complete, and gcc-pass2 is what + # installs the C++ headers, so chapter 8's gcc then died with + # fatal error: bits/c++config.h: No such file or directory + if _glob.glob(os.path.join(lfs, "usr", "include", "c++", "*", + "*", "bits", "c++config.h")): + return True, len(names) + return False, len(done) + + +def _crosschain_missing_output(lfs): + """A short, checkable reason why chapters 5-6 look unfinished, or ''.""" + import glob as _glob + if not _glob.glob(os.path.join(lfs, "usr", "include", "c++", "*", + "*", "bits", "c++config.h")): + return ("the C++ headers are missing (gcc-pass2, the last step of " + "chapter 6, did not finish) -- chapter 8's gcc cannot build " + "without them") + return "" + + +def _step(done, title, *cmds, note=None): + return {"done": done, "title": title, "cmds": list(cmds), "note": note} + + +def _next_steps(): + """Work out where the build stands, as an ordered list of steps.""" + cfg = load_config() + ver = build_version() # the checklist builds a system + steps = [] + + steps.append(_step(bool(ver), "choose and fetch a book", + "lfs books", "lfs fetch <version>")) + missing = [k for k, _w, _h in REQUIRED_CONFIG if not cfg.get(k)] + steps.append(_step(not missing, "complete the configuration", + *["%s%s" % (sudo_prefix(), h) + for k, _w, h in REQUIRED_CONFIG if k in missing] + or ["lfs config"], + note="see everything with: lfs config")) + + lfs = _lfs_dir() + mounted = bool(lfs) and (not cfg.get("lfs_device") or _is_mounted(lfs)) + steps.append(_step(mounted, "mount the LFS partition", + "%slfs build-system session --run" % sudo_prefix())) + if not lfs: + return steps, None + + steps.append(_step(os.path.isdir(os.path.join(lfs, "tools")), + "create the directory layout", + "%slfs build-system layout --run" % sudo_prefix())) + steps.append(_step(os.path.isdir(os.path.join(lfs, "etc", "pkgusr")), + "copy the package-user environment into the tree", + "%slfs build-system sync-tools --run" % sudo_prefix(), + note="skel-package, the helper scripts and lfs-helper; " + "re-run it whenever they change here")) + + import pwd as _pwd + try: + _pwd.getpwnam("lfs") + have_lfs_user = True + except KeyError: + have_lfs_user = False + steps.append(_step(have_lfs_user, "create the lfs user", + "%slfs build-system add-user --run" % sudo_prefix())) + + srcdir = os.path.join(lfs, "sources") + have_sources = os.path.isdir(srcdir) and any( + f.endswith((".tar.xz", ".tar.gz", ".tar.bz2", ".tgz")) + for f in os.listdir(srcdir)) if os.path.isdir(srcdir) else False + steps.append(_step(have_sources, "download the sources", + "%slfs build-system get-sources --run" % sudo_prefix())) + + cc_done, cc_n = _crosschain_looks_done(lfs) + steps.append(_step(cc_done, + "build the cross toolchain and temporary tools " + "(chapters 5-6, %d/%d)" % (cc_n, len(CROSSCHAIN_STEPS)), + "%slfs build-system crosschain --run" % sudo_prefix())) + + live = _chroot_mounts(lfs) + steps.append(_step(len(live) == len(_KERNFS) and _owner_is_root(lfs), + "prepare the chroot (chapter 7)", + "%slfs build-system chroot prepare --run" % sudo_prefix())) + steps.append(_step(os.path.isdir(os.path.join(lfs, PKGUSR_DIR, "scripts")), + "generate the in-chroot build scripts", + "%slfs build-system gen-chroot-scripts --run" % sudo_prefix())) + + # inside the chroot + order = _chroot_steporder(lfs) + built = set(_chroot_progress(lfs)) + inside_done = bool(order) and set(order) <= built + steps.append(_step(inside_done, + "build the system inside the chroot (chapters 7-9%s)" + % (", %d/%d" % (len(built & set(order)), len(order)) + if order else ""), + "%slfs build-system chroot enter" % sudo_prefix(), + "lfs-helper next # (inside the chroot)", + note="lfs-helper is the bash stand-in used until Python " + "exists in the chroot")) + + have_py = _chroot_has_python(lfs) + dep_missing = _chroot_missing_tool_deps(lfs) if have_py else TOOL_PY_DEPS + steps.append(_step(have_py and not dep_missing, + "install the Python modules the tooling needs (%s)" + % ", ".join(p for _m, p in TOOL_PY_DEPS), + "pip3 install %s # inside the chroot" + % " ".join(p for _m, p in TOOL_PY_DEPS), + note="base LFS has no requests/beautifulsoup4; " + "'lfs' and 'blfs' parse the books with them")) + + tools_in = os.path.exists(os.path.join(lfs, "usr", "bin", "packagemanager")) + steps.append(_step(tools_in and have_py, + "hand over from lfs-helper to the real tooling", + "%slfs build-system install-tools --run" % sudo_prefix(), + note="Python is in the chroot now, so packagemanager can " + "take over from lfs-helper")) + return steps, lfs + + +def cmd_next(args): + """Show where the build stands and what to do next.""" + steps, lfs = _next_steps() + pending = [s for s in steps if not s["done"]] + done_n = len(steps) - len(pending) + + if args.all: + print("Build progress:\n") + for s in steps: + mark = "[x]" if s["done"] else "[ ]" + print(" %s %s" % (mark, s["title"])) + print() + + if not pending: + print("All %d steps are done -- the system is built.\n" % len(steps)) + print("\nFrom here package management is packagemanager's job:") + print(" packagemanager list") + print(" packagemanager install <package>") + print(" blfs script <package> # BLFS install scripts") + return + + nxt = pending[0] + # Say plainly how much is done and how much is left. "Step 11 of 11" reads + # like "finished" when it actually means "one still to go". + print("%d of %d done -- %d still to do.\n" + % (done_n, len(steps), len(pending))) + print("NEXT: %s" % nxt["title"]) + for c in nxt["cmds"]: + print(" %s" % c) + if nxt["note"]: + print("\n (%s)" % nxt["note"]) + if len(pending) > 1: + print("\n after that: %s" % pending[1]["title"]) + if len(pending) > 2: + print(" and %d more -- see: lfs build-system next --all" + % (len(pending) - 2)) + elif not args.all: + print("\n full checklist: lfs build-system next --all") + +LFS_HELP_EPILOG = """\ +commands, by what you are doing: + + build a system -- everything about building lives under build-system + build-system run do all of it, from wherever you are, then + enter the chroot -- the only one you need + build-system next what is done, what comes next + build-system list every step and its state + build-system snapshot save the build, or put it back + build-system session mount the partition, set $LFS, ask for settings + build-system set-book which book to BUILD from + build-system chroot prepare / enter / resolv / unmount + build-system install-tools hand over to packagemanager + (session/layout/sync-tools/add-user/ + get-sources/crosschain/gen-chroot-scripts + are the individual steps `run` performs) + + manage packages on a machine (BLFS ones hand over to blfs) + install [--reinstall] build a package as its package user + update what the book has a newer version of + packages / commands list packages; print a section's commands + set-default which book to INSTALL from + + books + books / fetch / import list, download, or add a book + sources a package's download URLs + + settings and repair + config [--long] every setting, grouped by what it affects + reset clear the settings and start fresh + +Inside the chroot, lfs-helper drives the build; `lfs` will tell you so. +Dry run is the default everywhere; --run applies. +""" + + +# Inside the chroot, `lfs` is the wrong tool for most things. +# +# The build is driven from OUTSIDE: `lfs` mounts the partition, creates the +# layout, builds the cross toolchain and generates the in-chroot scripts. From +# inside, none of that applies -- $LFS does not even exist as a path -- so +# `lfs build-system next` reads the empty state and cheerfully reports "2 of 12 +# done", sending you back to the beginning of a finished build. +# +# Inside, lfs-helper is the tool. +_INSIDE_ONLY_ELSEWHERE = { + "next", "session", "layout", "add-user", "get-sources", "crosschain", + "chroot", "gen-chroot-scripts", "install-tools", "fix-ownership", + "version-check", "show", +} + +_INSIDE_EQUIVALENT = { + "next": "lfs-helper next", + "show": "lfs-helper list", + "fix-ownership": "lfs-helper fix-ownership --run", + "manifests": "lfs-helper manifests", +} + + +def inside_chroot(): + """Are we running inside the LFS chroot? + + The state directory lives at the root of the built system, so it is only at + /.lfs-pkgusr when we are in there.""" + return os.path.isdir(os.path.join("/", PKGUSR_DIR)) + + +def _refuse_inside_chroot(cmd, sub=None): + name = sub or cmd + sys.stderr.write( + "You are inside the LFS chroot, where `lfs %s` does not apply.\n" + "\n" + " The build is driven from OUTSIDE the chroot: `lfs` mounts the\n" + " partition, builds the toolchain and writes the scripts. From in\n" + " here there is no $LFS to work on, so this command would report\n" + " nonsense about a build that is actually finished.\n" + "\n" % (("build-system " + sub) if sub else cmd)) + eq = _INSIDE_EQUIVALENT.get(name) + if eq: + sys.stderr.write(" In here, use: %s\n" % eq) + else: + sys.stderr.write(" In here, lfs-helper drives the build:\n" + " lfs-helper next # what to do next\n" + " lfs-helper status # where things stand\n" + " lfs-helper list # every step\n") + sys.stderr.write("\n To run this, leave the chroot first (type 'exit').\n") + sys.exit(2) + + +def main(): + ap = argparse.ArgumentParser(prog="lfs", description="LFS book tool " + "(cache + follow one book)") + ap.add_argument("--version", action="version", version=f"lfs {LFS_VERSION} (build {_build_id()})") + ap.add_argument("--book", help="use this cached version for this run " + "(overrides the default)") + ap.add_argument("--book-file", help="use a local nochunks HTML file directly") + ap.epilog = _colour_epilog(LFS_HELP_EPILOG) + ap.formatter_class = argparse.RawDescriptionHelpFormatter + sub = ap.add_subparsers(dest="cmd", required=True, metavar="<command>") + + p = sub.add_parser("next", help="-> lfs build-system next") + q = sub.add_parser("list", help="-> lfs build-system list", description="every build step and its state " + "(same as 'lfs-helper list' inside the chroot)") + q.set_defaults(func=cmd_next, all=True) + + q = sub.add_parser("run", help="-> lfs build-system run", description="alias for 'lfs build-system run': do " + "everything up to the chroot, then enter it") + q.set_defaults(func=cmd_bs_run) + p.add_argument("--all", action="store_true", + help="show the whole checklist, not just the next step") + p.set_defaults(func=cmd_next) + + # kept at the top level as a shorthand, like `next` and `run` + p = sub.add_parser("snapshot", help="-> lfs build-system snapshot") + p.add_argument("action", nargs="?", + choices=["list", "save", "restore", "remove"], + help="default: list") + p.add_argument("name", nargs="?", help="snapshot name") + p.add_argument("--note", help="what this snapshot is (shown in the list)") + p.add_argument("--run", action="store_true", help="apply (default: dry run)") + p.add_argument("--yes", action="store_true", help="skip the confirmation") + p.add_argument("--force", action="store_true", help="replace an existing one") + p.set_defaults(func=cmd_snapshot) + + p = sub.add_parser("reset", help="clear the tools' configuration and start " + "fresh (does NOT touch the built system)") + p.add_argument("--run", action="store_true", help="delete (default: dry run)") + p.add_argument("--yes", action="store_true", help="skip the confirmation") + p.set_defaults(func=cmd_reset) + + p = sub.add_parser("config", help="show or change settings (paths, $LFS, " + "LFS_TGT, mirror, ...)") + # values can start with '-' (e.g. makeflags -j4), so don't let argparse + # try to parse them as flags + p.add_argument("key", nargs="?", help="setting to show, or set with a value") + p.add_argument("value", nargs="?", help="new value") + p.usage = ("lfs config [key] [value] | lfs config --edit | " + "lfs config --unset KEY [KEY ...]") + p.add_argument("--edit", action="store_true", + help="open the config file in $EDITOR") + p.add_argument("--unset", nargs="+", metavar="KEY", + help="remove setting(s), reverting to the default") + p.add_argument("--long", action="store_true", + help="explain what each setting does") + p.set_defaults(func=cmd_config) + + p = sub.add_parser("books", help="list cached books + all versions on the site") + p.add_argument("--local", action="store_true", + help="only show cached books (skip the site listing)") + p.set_defaults(func=cmd_books) + + p = sub.add_parser("set-default", + help="which book to INSTALL PACKAGES from " + "(the build book: lfs build-system set-book)") + p.add_argument("version", help="e.g. 12.4 or 13.0") + p.set_defaults(func=cmd_set_default) + + p = sub.add_parser("fetch", help="download + cache a book and its wget-list") + p.add_argument("version", help="e.g. 12.4, 13.0, stable, development") + p.add_argument("--set-default", action="store_true", + help="also make it the default") + p.set_defaults(func=cmd_fetch) + + p = sub.add_parser("import", help="cache a LOCAL book file (offline)") + p.add_argument("file", help="path to an LFS ...-NOCHUNKS.html") + p.add_argument("--version", required=True, help="label to store it under") + p.add_argument("--wget-list", help="also import a wget-list file") + p.add_argument("--set-default", action="store_true") + p.set_defaults(func=cmd_import) + + p = sub.add_parser("sources", help="print package source URLs (from wget-list)") + p.set_defaults(func=cmd_sources) + + p = sub.add_parser("sections", help="list sect ids + titles of the main book") + p.add_argument("book_pos", nargs="?", metavar="BOOK", + help="version or file (default: the main book)") + p.set_defaults(func=cmd_sections) + + p = sub.add_parser("install", help="install or reinstall an LFS package " + "as its package user") + p.add_argument("packages", nargs="+") + p.add_argument("--reinstall", action="store_true", + help="rebuild a package that is already installed") + p.add_argument("--regenerate", action="store_true", + help="regenerate the install script from the book first") + p.add_argument("--yes", "-y", action="store_true", + help="answer yes to the configure-phase question") + p.add_argument("--lfs-book", action="store_true", + help="use the LFS book even when BLFS also has the package") + p.set_defaults(func=cmd_install, action="install") + + p = sub.add_parser("update", help="show which installed packages the book " + "has a newer version of") + p.add_argument("packages", nargs="*", + help="packages to check (default: everything installed)") + p.add_argument("--verbose", "-v", action="store_true", + help="list up-to-date packages too") + p.set_defaults(func=cmd_update) + + p = sub.add_parser("packages", help="list chapter-8 build packages " + "(name<TAB>version<TAB>id) of the main book") + p.set_defaults(func=cmd_packages) + + p = sub.add_parser("commands", help="print a section's shell commands") + p.add_argument("section", help="section id or exact title") + p.add_argument("book_pos", nargs="?", metavar="BOOK", + help="version or file (default: the main book)") + p.set_defaults(func=cmd_commands) + + # ---- build-system: create the LFS system, step by step ---- + bs = sub.add_parser("build-system", + help="create the LFS system (chapters 2 & 4 for now)") + bsub = bs.add_subparsers(dest="step", required=True) + + q = bsub.add_parser("version-check", + help="2.2 run version-check.sh on the host (read-only)") + q.add_argument("--show", action="store_true", help="print the script, don't run") + q.set_defaults(func=cmd_bs_version_check) + + q = bsub.add_parser("layout", help="4.2 create the $LFS directory layout") + q.add_argument("--run", action="store_true", help="execute (default: dry run)") + q.set_defaults(func=cmd_bs_layout) + + q = bsub.add_parser("add-user", help="4.3 add the 'lfs' build user + 4.4 " + "write its environment (as lfs)") + q.add_argument("--run", action="store_true", help="execute (default: dry run)") + q.set_defaults(func=cmd_bs_add_user) + + q = bsub.add_parser("session", help="start/rejoin a build session: configure " + "+ mount the partition, set $LFS, chown to lfs") + q.add_argument("--run", action="store_true", help="actually mount/chown") + q.add_argument("--reconfigure", action="store_true", + help="re-ask every setting") + q.set_defaults(func=cmd_bs_session) + + q = bsub.add_parser("get-sources", help="3. download all packages + patches " + "into $LFS/sources and verify md5sums") + q.add_argument("--run", action="store_true", help="download (default: dry run)") + q.add_argument("--force", action="store_true", help="re-download existing files") + q.set_defaults(func=cmd_bs_get_sources) + + q = bsub.add_parser("crosschain", help="5-6. build the cross toolchain + temporary tools " + "(binutils/gcc, glibc, m4..xz, binutils/gcc pass2) -- " + "each step is a saved, editable script") + q.add_argument("step", nargs="?", + help="one step name, or 'all' (default); see --list") + q.add_argument("--list", action="store_true", help="list the steps + versions") + q.add_argument("--run", action="store_true", help="build (default: dry run)") + q.add_argument("--phase", choices=["all", "unpack", "build", "install", + "configure", "test"], help="run just this phase (default: all; " + "use 'install' to retry after a failure without rebuilding)") + q.add_argument("--edit", action="store_true", + help="open the step's script in $EDITOR (creates it from the " + "book first if it doesn't exist yet)") + q.add_argument("--regenerate", action="store_true", + help="overwrite the saved script with a fresh one from the " + "book (old one kept as .bak)") + q.add_argument("--path", action="store_true", + help="print the script path(s) and exit") + q.add_argument("--restart", action="store_true", + help="ignore any saved resume progress and start over") + q.add_argument("--yes", action="store_true", + help="auto-confirm resuming a previously interrupted run " + "(skip the prompt)") + q.set_defaults(func=cmd_bs_crosschain) + + q = bsub.add_parser("gen-chroot-scripts", + help="7-8. write phased install scripts for the packages " + "built inside the chroot (used by lfs-helper)") + q.add_argument("--run", action="store_true", help="write them (default: dry run)") + q.add_argument("--overwrite", action="store_true", + help="replace scripts that already exist") + q.add_argument("--chapter7-only", action="store_true", + help="only chapter 7's packages, not all of chapter 8") + q.set_defaults(func=cmd_bs_gen_chroot_scripts) + + q = bsub.add_parser("run", help="do everything up to the chroot, from " + "wherever the build stands, then enter it") + q.set_defaults(func=cmd_bs_run) + + q = bsub.add_parser("next", help="where the build stands and what to do next") + q.add_argument("--all", action="store_true", + help="show the whole checklist, not just the next step") + q.set_defaults(func=cmd_next) + + q = bsub.add_parser("snapshot", help="save the build state, or put it back " + "(so a bad step can be undone)") + q.add_argument("action", nargs="?", + choices=["list", "save", "restore", "remove"], + help="default: list") + q.add_argument("name", nargs="?", help="snapshot name") + q.add_argument("--note", help="what this snapshot is (shown in the list)") + q.add_argument("--run", action="store_true", help="apply (default: dry run)") + q.add_argument("--yes", action="store_true", help="skip the confirmation") + q.add_argument("--force", action="store_true", help="replace an existing one") + q.set_defaults(func=cmd_snapshot) + + # kept as an alias: `set-default` reads naturally under build-system too, + # and someone will type it + q = bsub.add_parser("set-default", help="-> lfs build-system set-book") + q.add_argument("version") + q.set_defaults(func=cmd_bs_set_default) + + q = bsub.add_parser("list", help="every build step and its state " + "(same as 'lfs-helper list' inside the chroot)") + q.set_defaults(func=cmd_next, all=True) + + q = bsub.add_parser("restart", help="delete the built system and start the " + "whole build again (asks first)") + q.add_argument("--run", action="store_true", help="do it (default: dry run)") + q.add_argument("--yes", action="store_true", help="skip the confirmation") + q.add_argument("--wipe-sources", action="store_true", + help="delete the downloaded tarballs too (they re-download)") + q.set_defaults(func=cmd_bs_restart) + + q = bsub.add_parser("set-book", help="which book to BUILD THE SYSTEM from") + q.add_argument("version") + q.set_defaults(func=cmd_bs_set_book) + + q = bsub.add_parser("sync-tools", + help="3b. copy this machine's package-user environment " + "(skel-package, helper scripts, lfs-helper) into the tree") + q.add_argument("--run", action="store_true", help="copy (default: dry run)") + q.set_defaults(func=cmd_bs_sync_tools) + + q = bsub.add_parser("install-tools", + help="8. install the Python tooling (lfs, packagemanager, " + "blfs) inside the chroot, once Python exists there") + q.add_argument("--run", action="store_true", help="install (default: dry run)") + q.add_argument("--force", action="store_true", + help="install even if the Python modules the tools need " + "(requests, beautifulsoup4) are missing") + q.set_defaults(func=cmd_bs_install_tools) + + q = bsub.add_parser("chroot", help="7. prepare / enter / leave the chroot " + "(chown to root, mount virtual kernel filesystems)") + q.add_argument("action", nargs="?", + choices=["status", "prepare", "enter", "unmount", "resolv"], + help="default: status") + q.add_argument("--run", action="store_true", + help="apply (prepare/unmount default to a dry run)") + q.add_argument("--force", action="store_true", + help="prepare even if chapters 5-6 are unfinished") + q.set_defaults(func=cmd_bs_chroot) + + q = bsub.add_parser("fix-ownership", help="give the lfs user ownership of " + "the whole build tree again (after something ran as root)") + q.add_argument("--run", action="store_true", help="apply (default: dry run)") + q.set_defaults(func=cmd_bs_fix_ownership) + + q = bsub.add_parser("manifests", help="show which files each package " + "installed (for later package-user ownership)") + q.add_argument("package", nargs="?", help="one package's file list") + q.set_defaults(func=cmd_bs_manifests) + + q = bsub.add_parser("show", help="print any section's commands by id") + q.add_argument("section", help="section id (e.g. ch-partitioning-mounting)") + q.set_defaults(func=cmd_bs_show) + + # `lfs config makeflags -j4`: the value legitimately starts with '-', which + # argparse would otherwise reject as an unknown option. Insert the '--' + # separator for that one case so values pass through untouched. + argv = sys.argv[1:] + if len(argv) >= 3 and argv[0] == "config" and argv[2].startswith("-") \ + and argv[2] != "--" and not argv[1].startswith("-"): + argv = argv[:2] + ["--"] + argv[2:] + args = ap.parse_args(argv) + + if inside_chroot(): + cmd = getattr(args, "cmd", None) + sub = getattr(args, "step", None) + if cmd == "build-system" and sub in _INSIDE_ONLY_ELSEWHERE: + _refuse_inside_chroot(cmd, sub) + if cmd in _INSIDE_ONLY_ELSEWHERE: + _refuse_inside_chroot(cmd) + + # Build-system commands read the BUILD book; everything else reads the + # default one. Set it in one place so a new subcommand cannot forget. + if getattr(args, "cmd", None) == "build-system": + setattr(args, "_use_build_book", True) + + args.func(args) + + +if __name__ == "__main__": + try: + main() + except BrokenPipeError: + sys.exit(0) + except KeyboardInterrupt: + sys.exit(130) diff --git a/lfs-completion.bash b/lfs-completion.bash new file mode 100644 index 0000000..c88f12e --- /dev/null +++ b/lfs-completion.bash @@ -0,0 +1,134 @@ +# Bash completion for the LFS package-user toolchain. +# +# install: cp lfs-completion.bash /usr/share/bash-completion/completions/lfs +# ln -sf lfs /usr/share/bash-completion/completions/blfs +# ln -sf lfs /usr/share/bash-completion/completions/packagemanager +# ln -sf lfs /usr/share/bash-completion/completions/lfs-helper +# or just: source lfs-completion.bash +# +# Subcommands come from each tool's own --help, so this does not go stale when +# a command is added -- there is no second list to keep in sync. + +_lfs_tool_subcommands() { + local tool="$1" parent="${2:-}" + # argparse prints the subcommand list as a {a,b,c} choices line -- that is + # exact, unlike scraping indented help text, where a wrapped description + # line looks just like a command name. + local help out + + # lfs-helper is a bash script: its usage lists commands as + # lfs-helper <command> ... + # so take the word after the tool's own name. + case "$tool" in + *lfs-helper) + "$tool" --help 2>/dev/null \ + | grep -oE '^ lfs-helper [a-z][a-z0-9-]*' \ + | awk '{print $2}' | sort -u + return ;; + esac + + help="$("$tool" $parent --help 2>/dev/null)" + + # argparse usually prints the subcommands as a {a,b,c} choices line, which + # is exact. A parser with metavar= set (the top level here) shows the + # metavar instead, so fall back to the entry lines: a command is indented + # exactly four spaces, while a wrapped description is indented much more. + out="$(printf '%s\n' "$help" | grep -oE '\{[a-z][a-z0-9,_-]*\}' | head -n1 \ + | tr -d '{}' | tr ',' '\n')" + if [ -z "$out" ]; then + out="$(printf '%s\n' "$help" \ + | sed -n '/^positional arguments:/,/^options:/p' \ + | grep -E '^ {4}[a-z][a-z0-9_-]*( |$)' \ + | awk '{print $1}')" + fi + printf '%s\n' "$out" +} + +_lfs_packages() { + # package users own /usr/src/<name>; that is the cheapest list of what is + # installed, and it works without invoking python + local d + for d in /usr/src/*/; do + [ -d "$d" ] || continue + basename "$d" + done +} + +_lfs_book_versions() { + local d + for d in /usr/share/lfs/books/LFS-BOOK-*-NOCHUNKS.html; do + [ -e "$d" ] || continue + basename "$d" | sed 's/^LFS-BOOK-//; s/-NOCHUNKS\.html$//' + done +} + +_lfs_snapshots() { + local d + for d in /usr/share/lfs/snapshots/*.tar*; do + [ -e "$d" ] || continue + basename "$d" | sed 's/\.tar\(\.[a-z]*\)\?$//' + done +} + +_lfs_complete() { + local cur prev cmd tool + cur="${COMP_WORDS[COMP_CWORD]}" + prev="${COMP_WORDS[COMP_CWORD-1]}" + tool="${COMP_WORDS[0]}" + cmd="${COMP_WORDS[1]:-}" + + # an option's argument + case "$prev" in + --book|--book-file) + COMPREPLY=($(compgen -W "$(_lfs_book_versions)" -- "$cur")); return ;; + --phase) + COMPREPLY=($(compgen -W "all unpack build install configure test" \ + -- "$cur")); return ;; + --jobs) return ;; + esac + + # options, whenever the word starts with a dash + if [[ "$cur" == -* ]]; then + local opts + opts="$("$tool" ${cmd:+$cmd} --help 2>/dev/null \ + | grep -oE '(^|\s)--[a-z][a-z-]*' | tr -d ' ' | sort -u)" + COMPREPLY=($(compgen -W "$opts" -- "$cur")) + return + fi + + # first word: the tool's own subcommands + if [ "$COMP_CWORD" = 1 ]; then + COMPREPLY=($(compgen -W "$(_lfs_tool_subcommands "$tool")" -- "$cur")) + return + fi + + # second word: either a nested subcommand, or an argument + case "$cmd" in + build-system|user|nimgnu|pip|script|snapshot) + if [ "$COMP_CWORD" = 2 ]; then + COMPREPLY=($(compgen -W \ + "$(_lfs_tool_subcommands "$tool" "$cmd")" -- "$cur")) + return + fi ;; + esac + + case "$cmd" in + install|update|remove|info|deps|rdeps|order|sources|which-package|script) + COMPREPLY=($(compgen -W "$(_lfs_packages)" -- "$cur")) ;; + set-default|fetch|set-book) + COMPREPLY=($(compgen -W "$(_lfs_book_versions)" -- "$cur")) ;; + build|done|undone|fix-ownership|fix-perms|add-user) + COMPREPLY=($(compgen -W "$(_lfs_packages)" -- "$cur")) ;; + *) + if [ "${COMP_WORDS[1]}" = "snapshot" ]; then + COMPREPLY=($(compgen -W "$(_lfs_snapshots)" -- "$cur")) + else + COMPREPLY=($(compgen -f -- "$cur")) + fi ;; + esac +} + +complete -F _lfs_complete lfs +complete -F _lfs_complete blfs +complete -F _lfs_complete packagemanager +complete -F _lfs_complete lfs-helper diff --git a/lfs-helper b/lfs-helper new file mode 100644 index 0000000..4c0aa9c --- /dev/null +++ b/lfs-helper @@ -0,0 +1,4250 @@ +#!/bin/bash +# +# lfs-helper -- manage the LFS build from INSIDE the chroot. +# +# The `lfs` tool is Python, and the chroot has no Python until chapter 8 builds +# it. This is the bash stand-in: it tracks session state, sets up the +# package-user system, creates package users, runs the phased install scripts, +# and records which files each package installed. +# +# State lives in /.lfs-pkgusr (the same dir `lfs` uses from outside, so both +# tools see one shared session). +# +# The install scripts it runs are the SAME phased format `packagemanager` uses +# (unpack_pkg / build_pkg / install_pkg / configure_pkg / test_pkg + a +# name_version variable), so once the system is up, `packagemanager` can update +# these packages with the scripts already in each package user's home. +# +# GPLv2-or-later. Part of the pkgusr toolchain. + +set -u + +# Bump on every change that alters behaviour. `lfs-helper --version` makes it +# possible to tell at a glance whether the copy inside the chroot is the one +# that was just fixed -- guessing at that has wasted a lot of time. +LFS_HELPER_VERSION="1.7.0" + +# A fingerprint of this file's own contents. A hand-maintained version number +# goes stale the moment someone forgets to bump it -- and the reason for +# printing a version at all is to answer "am I running the code that was just +# fixed?". This changes whenever the file does, so it cannot lie. +_build_id() { + local f="${BASH_SOURCE[0]:-$0}" + if [ -r "$f" ] && command -v md5sum >/dev/null 2>&1; then + md5sum "$f" 2>/dev/null | cut -c1-7 + else + echo unknown + fi +} + +STATE="${LFS_PKGUSR_DIR:-/.lfs-pkgusr}" +SCRIPTS="$STATE/scripts" +MANIFESTS="$STATE/manifests" +PROGRESS="$STATE/chroot-progress" +SNAPSHOT="$STATE/.snapshot" +SNAPDIRS="$STATE/.snapshot-dirs" +INSTALLDIRS="$STATE/installdirs.lst" +SRCROOT="${LFS_SRC_ROOT:-/usr/src}" +# Root of the filesystem being tracked. Inside the chroot that is "/"; the +# test harness points it elsewhere. Defined here because several paths below +# are derived from it. +SNAP_ROOT="${LFS_SNAP_ROOT:-/}" +ETC="${LFS_ETC:-/etc}" # overridable so the test suite can exercise + # user creation without touching the real /etc + +INSTALL_GID=9999 # 'install' group; package users start at 10000 +PKG_UID_MIN=10000 + +# Collector groups get their OWN range, well clear of the package users. +# Sharing one range means a collector group can take the gid that the next +# package user's group wants, and then user 10042 has group 10043 while some +# unrelated collector group holds 10042 -- the ids stop lining up and a +# `chown pkg:pkg` starts meaning something different from what it reads like. +COLLECTOR_GID_MIN=90000 + +# The build user from outside the chroot. Files it created -- /sources, the +# tree it handed over at 7.2 -- carry its uid, and the chroot has no passwd +# entry for it, so they show as a bare number and every tool calls them +# UNKNOWN. Recreating the account here (just below the install group, clear of +# the package users at 10000+) makes those files resolve to a name again. +LFS_BUILD_UID=9998 + +# The next free gid in the collector range. +next_collector_gid() { + local g="$COLLECTOR_GID_MIN" + while getent group "$g" >/dev/null 2>&1 \ + || { [ -r "$ETC/group" ] && cut -d: -f3 "$ETC/group" | grep -qx "$g"; }; do + g=$((g + 1)) + done + printf '%s' "$g" +} + +# Create a collector group in its own id range. +create_collector_group() { + local grp="$1" gid + getent group "$grp" >/dev/null 2>&1 && return 0 + gid="$(next_collector_gid)" + if have_shadow_tools; then + groupadd -g "$gid" "$grp" 2>/dev/null && return 0 + # the gid raced or is taken: let groupadd choose rather than fail + groupadd "$grp" 2>/dev/null && return 0 + else + # no Shadow yet -- same by hand, the way init-pkgusr does it + printf '%s:x:%s:\n' "$grp" "$gid" >> "$ETC/group" && return 0 + fi + getent group "$grp" >/dev/null 2>&1 +} + +C_OK=$'\033[0;32m'; C_WARN=$'\033[0;33m'; C_ERR=$'\033[0;31m' +C_DIM=$'\033[2m'; C_OFF=$'\033[0m' +[ -t 1 ] || { C_OK=; C_WARN=; C_ERR=; C_DIM=; C_OFF=; } + +say() { echo "$*"; } +ok() { echo "${C_OK}$*${C_OFF}"; } +warn() { echo "${C_WARN}$*${C_OFF}" >&2; } +die() { echo "${C_ERR}!! $*${C_OFF}" >&2; exit 1; } + +need_root() { [ "$(id -u)" = 0 ] || die "must be run as root (inside the chroot)"; } + +# --------------------------------------------------------------------------- # +# state +# --------------------------------------------------------------------------- # +mkstate() { mkdir -p "$STATE" "$SCRIPTS" "$MANIFESTS"; } + +is_done() { [ -f "$PROGRESS" ] && grep -qxF "$1" "$PROGRESS"; } +mark_done() { mkstate; is_done "$1" || echo "$1" >> "$PROGRESS"; } +unmark() { [ -f "$PROGRESS" ] && sed -i "\|^$1\$|d" "$PROGRESS"; } + +# The ordered list of steps. Each is "name" and the script is +# $SCRIPTS/<name>.sh -- generated by `lfs build-system gen-chroot-scripts`. +steps() { + if [ -f "$STATE/steporder" ]; then + grep -v '^[[:space:]]*$' "$STATE/steporder" + else + ls -1 "$SCRIPTS" 2>/dev/null | sed -n 's/\.sh$//p' + fi +} + +# --------------------------------------------------------------------------- # +# package-user system +# --------------------------------------------------------------------------- # +# getent comes from glibc (present), but useradd/groupadd come from Shadow, +# which is chapter 8 package #26 -- so for most of the build there is no user +# management at all. /etc/passwd and /etc/group are plain text though, so we +# create accounts by appending to them directly when the tools are missing. +have_shadow_tools() { command -v useradd >/dev/null 2>&1 \ + && command -v groupadd >/dev/null 2>&1; } + +# The passwd/group FILES are authoritative: inside the chroot they ARE the user +# database, and getent can otherwise answer from another NSS source and mask +# what's really in /etc. Only fall back to getent if the file is unreadable. +group_exists() { + if [ -r "$ETC/group" ]; then grep -q "^$1:" "$ETC/group"; + else getent group "$1" >/dev/null 2>&1; fi +} +user_exists() { + if [ -r "$ETC/passwd" ]; then grep -q "^$1:" "$ETC/passwd"; + else getent passwd "$1" >/dev/null 2>&1; fi +} + +id_taken() { + # Ask the real database as well as the files. A uid can be in use without + # matching the ":x:<id>:" pattern (a different password field, an entry from + # another source), and picking it anyway ends in + # useradd: UID 10067 is not unique + getent passwd "$1" >/dev/null 2>&1 && return 0 + getent group "$1" >/dev/null 2>&1 && return 0 + grep -q ":x:$1:" "$ETC/passwd" 2>/dev/null && return 0 + grep -q ":x:$1:" "$ETC/group" 2>/dev/null && return 0 + return 1 +} + +# append a group / user by hand (no Shadow needed) +# remove a user from the install group's member list (handles the member being +# first, last, only, or in the middle -- chained seds get this wrong) +drop_from_install_group() { + local u="$1" line members out="" + line="$(grep "^install:" "$ETC/group" 2>/dev/null)" || return 0 + members="${line##*:}" + local IFS=',' + for m in $members; do + [ "$m" = "$u" ] && continue + [ -n "$m" ] || continue + out="${out:+$out,}$m" + done + unset IFS + sed -i "s|^install:.*|install:x:$INSTALL_GID:$out|" "$ETC/group" +} + +raw_groupadd() { echo "$1:x:$2:" >> "$ETC/group"; } +raw_useradd() { + local name="$1" id="$2" home="$3" + echo "$name:x:$id:$id:package $name:$home:/bin/bash" >> "$ETC/passwd" + # supplementary membership in the install group + if grep -q "^install:" "$ETC/group"; then + local line members + line="$(grep "^install:" "$ETC/group")" + members="${line##*:}" + if [ -n "$members" ]; then members="$members,$name"; else members="$name"; fi + sed -i "s|^install:.*|install:x:$INSTALL_GID:$members|" "$ETC/group" + fi +} + +# The shared "install directories" -- the dirs packages install INTO. These +# belong to root:install and are group-writable so any package user can add +# files; they are NOT owned by individual packages. (Modelled on +# more_control_helpers/installdirs.lst from the package-users hint.) +install_dirs_list() { + if [ -f "$INSTALLDIRS" ]; then + grep -v '^[[:space:]]*\(#\|$\)' "$INSTALLDIRS" + return + fi + cat <<'IDLIST' +/bin +/sbin +/lib +/lib64 +/opt +/boot +/boot/efi +/efi +/sources +/usr/src +/tmp +/var/tmp +/root +/home +/etc +/etc/opt +/etc/sysconfig +/usr +/usr/bin +/usr/sbin +/usr/lib +/usr/lib64 +/usr/lib/pkgconfig +/usr/libexec +/usr/include +/usr/src +/usr/share +/usr/share/aclocal +/usr/share/color +/usr/share/dict +/usr/share/doc +/usr/share/info +/usr/share/locale +/usr/share/man +/usr/share/man/man1 +/usr/share/man/man2 +/usr/share/man/man3 +/usr/share/man/man4 +/usr/share/man/man5 +/usr/share/man/man6 +/usr/share/man/man7 +/usr/share/man/man8 +/usr/share/misc +/usr/share/pkgconfig +/usr/share/terminfo +/usr/share/zoneinfo +/usr/local +/usr/local/bin +/usr/local/sbin +/usr/local/lib +/usr/local/include +/usr/local/share +/usr/local/share/man +/usr/local/src +/var +/var/cache +/var/lib +/var/local +/var/log +/var/mail +/var/opt +/var/spool +IDLIST +} + +# /usr/include itself is shared -- every package drops headers in it. Its +# SUBTREES are not: /usr/include/c++/<version> belongs to gcc. Listing those +# here made them "shared install directories", so a package that could not +# write there got the `install` group applied automatically instead of being +# asked which collector group should own it -- silently handing gcc's headers +# to everyone. Adoption of that tree is prevented by the contents check in +# cmd_adopt_dirs, which is the right mechanism for it. +# +# /tmp and /var/tmp are world-writable scratch space with the sticky bit +# already set -- a package that happens to write there must not come to OWN +# them (man-pages ended up owning /tmp, which would break every other user of +# it). /root and /home belong to people, not packages. +# +# Note the shape of these: /usr/libexec is shared (every package drops a +# subdirectory in it) but /usr/libexec/p11-kit is NOT -- that belongs to +# p11-kit. The same goes for /usr/share/zsh/site-functions, which IS shared: +# every package drops a completion file straight into it. +# +# Directories that are shared by their very nature, matched as PATTERNS rather +# than listed one by one. Translated man pages and message catalogues create a +# directory per language (/usr/share/man/de/man1, /usr/share/locale/pl/ +# LC_MESSAGES, ...) and practically every package writes into them, so listing +# them individually is hopeless -- there are hundreds, and they appear as +# packages are installed. +install_dir_patterns() { + cat <<'IPAT' +*/usr/share/man/man[1-9] +*/usr/share/man/*/man[1-9] +*/usr/share/man/* +*/usr/share/locale/*/LC_MESSAGES +*/usr/share/locale/* +*/usr/share/info +*/usr/share/doc +*/usr/lib/pkgconfig +*/usr/share/pkgconfig +*/usr/share/terminfo/* +*/usr/include +*/usr/share/bash-completion/completions +*/usr/share/zsh/site-functions +*/usr/share/zsh/vendor-completions +*/usr/share/fish/completions +*/usr/share/fish/vendor_completions.d +*/usr/share/applications +*/usr/share/mime/packages +*/usr/share/dbus-1/services +*/usr/share/dbus-1/system-services +*/usr/share/metainfo +*/usr/share/appdata +*/usr/share/icons/hicolor/* +*/usr/share/pixmaps +*/usr/share/glib-2.0/schemas +*/usr/share/vala/vapi +*/usr/share/gir-1.0 +*/usr/lib/girepository-1.0 +*/usr/share/aclocal +*/usr/share/gtk-doc/html +*/usr/lib/systemd/system +*/usr/lib/udev/rules.d +*/etc/xdg/autostart +*/usr/libexec +IPAT +} + +matches_install_pattern() { + local p="${1%/}" pat + while IFS= read -r pat; do + # shellcheck disable=SC2254 + case "$p" in $pat) return 0 ;; esac + done < <(install_dir_patterns) + return 1 +} + +# is this path one of the shared install dirs (root:install), rather than +# something the package created for itself? +is_install_dir() { + local raw="${1%/}" p d + p="${raw#${SNAP_ROOT%/}}" # make it absolute-in-target + [ -n "$p" ] || return 0 + while IFS= read -r d; do + [ "$p" = "$d" ] && return 0 + done < <(install_dirs_list) + matches_install_pattern "$raw" && return 0 + return 1 +} + +pkgusr_ready() { group_exists install; } + +cmd_init_pkgusr() { + need_root; mkstate + local run=0; [ "${1:-}" = "--run" ] && run=1 + + say "Package-user system setup:" + say " 1. create the 'install' group (gid $INSTALL_GID)" + say " 2. give install directories to group 'install', group-writable" + say " NOT sticky yet: an install directory is finally group-writable" + say " AND sticky, but during the build the sticky bit would stop package" + say " users replacing the temporary-system files still owned by root." + say " Add it at the very end with: lfs-helper seal-install-dirs --run" + if [ "$run" = 0 ]; then + warn "(dry run -- re-run with --run to apply)" + return 0 + fi + + if group_exists install; then + say "# 'install' group already exists" + elif have_shadow_tools; then + # Tolerate "already exists": init-pkgusr is run automatically now, and + # a step that cannot be repeated safely would break every later run. + if groupadd -g "$INSTALL_GID" install 2>/dev/null; then + ok "# created group install ($INSTALL_GID)" + elif getent group install >/dev/null 2>&1; then + say "# 'install' group already exists" + else + die "groupadd install failed" + fi + else + raw_groupadd install "$INSTALL_GID" + ok "# created group install ($INSTALL_GID) by editing /etc/group" + say " (Shadow isn't built yet -- no groupadd, so this was done by hand)" + fi + + # Directories packages install into. Anything missing is skipped quietly. + local d n=0 failed="" + while IFS= read -r d; do + d="${SNAP_ROOT%/}$d" + [ -d "$d" ] || continue + if ! chgrp install "$d" 2>/dev/null; then + # `|| continue` here reported "0 install directories" with no hint + # that anything went wrong -- and the build then failed much later + # with permission errors nobody could trace back to this. + failed="$failed $d" + continue + fi + # the hint's definition: chgrp install <dir> && chmod g+w,o+t <dir>. + # o+t (sticky) is deliberately deferred to seal-install-dirs, because + # it would stop package users replacing files the temporary system + # installed as root. + chmod g+w "$d" 2>/dev/null || true + n=$((n+1)) + done < <(install_dirs_list) + ok "# $n install directories now belong to group 'install' (group-writable)" + if [ -n "$failed" ]; then + warn "# could not set the group on:" + local f + for f in $failed; do warn "# $f"; done + warn "# The build will hit permission errors in these later." + fi + mark_done "pkgusr-init" + say "" + say "Next: lfs-helper next" +} + +# add-user <name> -- create a package user the packagemanager way: +# own group, home /usr/src/<name>, member of the supplementary 'install' group +# The shared package-user environment (the hint's /etc/pkgusr). A user without +# it exists but cannot build: no .bash_profile, no .bashrc, no build helper, and +# the wrong umask. +PKGUSR_ETC="${LFS_PKGUSR_ETC:-${SNAP_ROOT%/}/etc/pkgusr}" + +ensure_pkgusr_etc() { + mkdir -p "$PKGUSR_ETC" 2>/dev/null || return 1 + if [ ! -f "$PKGUSR_ETC/bash_profile" ]; then + cat > "$PKGUSR_ETC/bash_profile" <<'EOF' +# /etc/pkgusr/bash_profile -- shared by every package user (symlinked as +# ~/.bash_profile), so every build sees the same environment. +umask 022 +export LC_ALL=POSIX +# /usr/local/bin comes FIRST after the wrappers: locally installed +# overrides belong ahead of /usr/bin, and a temporary wrapper there +# (the no-verify wget, say) is useless if nothing ever finds it. +PATH=/usr/lib/pkgusr:/usr/local/bin:/usr/local/sbin:/usr/bin:/bin:/usr/sbin:/sbin +export PATH +export PS1='\u:\w\$ ' +export PKG_HOME="$HOME" +cd "$HOME" 2>/dev/null || true +[ -f "$HOME/build.conf" ] && . "$HOME/build.conf" +EOF + fi + if [ ! -f "$PKGUSR_ETC/bashrc" ]; then + cat > "$PKGUSR_ETC/bashrc" <<'EOF' +# /etc/pkgusr/bashrc -- symlinked as ~/.bashrc. Package users never log in, so +# login and non-login shells should behave identically: just source the profile. +[ -f "$HOME/.bash_profile" ] && . "$HOME/.bash_profile" +EOF + fi + if [ ! -f "$PKGUSR_ETC/build" ]; then + cat > "$PKGUSR_ETC/build" <<'EOF' +#!/bin/bash +# /etc/pkgusr/build -- symlinked as ~/build. Runs the usual sequence using the +# package user's build.conf. +set -e +conf="${1:-$HOME/build.conf}" +[ -f "$conf" ] || { echo "no build.conf at $conf" >&2; exit 1; } +. "$conf" +: "${SRC_DIR:=$HOME}" +: "${CONFIGURE_OPTS:=--prefix=/usr}" +: "${MAKE_OPTS:=}" +cd "$SRC_DIR" +[ -x ./configure ] && ./configure $CONFIGURE_OPTS +make $MAKE_OPTS +make install +EOF + chmod 755 "$PKGUSR_ETC/build" + fi + return 0 +} + +init_package_user_home() { + local name="$1" home="${2:-$SRCROOT/$1}" + ensure_pkgusr_etc || return 1 + mkdir -p "$home" + local link target + for link in .bash_profile:bash_profile .bashrc:bashrc build:build; do + target="${link#*:}"; link="${link%%:*}" + ln -sfn "$PKGUSR_ETC/$target" "$home/$link" + done + if [ ! -f "$home/build.conf" ]; then + cat > "$home/build.conf" <<EOF +# build.conf for package user '$name' -- read by ~/build. +#SRC_DIR="\$HOME/$name-1.0" +CONFIGURE_OPTS="--prefix=/usr" +MAKE_OPTS="" +EOF + fi + [ -f "$home/.project" ] || printf '%s\n\nPackage user for %s.\n' "$name" "$name" > "$home/.project" + chown -R -h "$name:$name" "$home" 2>/dev/null || true + return 0 +} + +# Steps that are not packages, and must never get a package user. +# +# last-step, init-dirs, cfg-* and refind are actions, not software: they own no +# files of their own and installing "as" them makes no sense. A `last-step` +# user appeared because a manifest existed under that name, and everything the +# step touched -- including wget's binary, installed as the wget user -- was +# then handed to it. +_is_not_a_package() { + case "$1" in + last-step|init-*|cfg-*|refind) return 0 ;; + esac + return 1 +} + +# Keep /etc/passwd and /etc/group in id order as users are created. +# +# They are appended to, so without this the files drift into build order and +# have to be tidied up afterwards -- which only happens if someone remembers. +# The files are a few kilobytes; sorting them on each creation costs nothing. +_sort_users_quietly() { + cmd_sort_users --run >/dev/null 2>&1 || true +} + +cmd_add_user() { + need_root + local name="${1:-}"; [ -n "$name" ] || die "usage: lfs-helper add-user <name>" + if _is_not_a_package "$name"; then + warn "'$name' is a build step, not a package -- no user created." + return 0 + fi + pkgusr_ready || die "run 'lfs-helper init-pkgusr --run' first" + + if user_exists "$name"; then + say "# user '$name' already exists" + else + if have_shadow_tools; then + # Let Shadow allocate the ids. Choosing them by hand means racing + # its own idea of what is free -- and losing: + # useradd: UID 10067 is not unique + # -K UID_MIN/GID_MIN keeps package users in their own range without + # us having to track which numbers are already used. + if group_exists "$name"; then + useradd -c "package $name" -d "$SRCROOT/$name" -g "$name" \ + -G install -s /bin/bash -K "UID_MIN=$PKG_UID_MIN" "$name" \ + || die "useradd $name failed" + else + # -U creates the private group together with the user, so the + # uid and gid match -- creating the group separately lets the + # two counters drift apart (uid 10000 / gid 10001). + useradd -c "package $name" -d "$SRCROOT/$name" -U \ + -G install -s /bin/bash \ + -K "UID_MIN=$PKG_UID_MIN" -K "GID_MIN=$PKG_UID_MIN" \ + "$name" \ + || die "useradd $name failed" + fi + ok "# created package user $name (uid/gid $(id -u "$name")/$(id -g "$name"), +install)" + _sort_users_quietly + else + # No Shadow yet: we write the files ourselves, so we do have to pick + # a free id -- one that is free as BOTH a uid and a gid, so the + # package user gets a matching private group. + local uid=$PKG_UID_MIN + while id_taken "$uid"; do uid=$((uid+1)); done + group_exists "$name" || raw_groupadd "$name" "$uid" + raw_useradd "$name" "$uid" "$SRCROOT/$name" + ok "# created package user $name (uid/gid $uid, +install) via /etc/passwd" + _sort_users_quietly + fi + fi + init_package_user_home "$name" "$SRCROOT/$name" + chmod 755 "$SRCROOT/$name" +} + +# --------------------------------------------------------------------------- # +# file tracking (same manifest layout the `lfs` tool writes) +# --------------------------------------------------------------------------- # +# Build variables (LFS_TGT, MAKEFLAGS) as configured by `lfs config`. The +# chroot resets the environment, so they're read from a file inside the tree +# rather than inherited -- otherwise a custom LFS_TGT silently reverts. +load_env() { [ -f "$STATE/env" ] && . "$STATE/env"; return 0; } +load_env + +# Read a stored snapshot, translating paths recorded OUTSIDE the chroot. +# Chapters 5-6 ran on the host, where this tree was $LFS/usr/bin/...; in here +# the same files are /usr/bin/.... Without translating, the "before" set shares +# no paths with the "after" set and the very first in-chroot build looks like it +# installed the entire system. +load_snapshot() { + local f="$1" m="${LFS_HOST_MOUNT:-}" r="${SNAP_ROOT%/}" + if [ -n "$m" ] && grep -q "^$m/" "$f" 2>/dev/null; then + sed "s|^$m/|$r/|" "$f" + else + cat "$f" + fi +} + +have_su() { command -v su >/dev/null 2>&1; } + +snapshot_dirs() { + local r="${SNAP_ROOT%/}" + find "${SNAP_ROOT:-/}" -xdev -type d \ + -not -path "$r/.lfs-pkgusr/*" -not -path "$SRCROOT/*" \ + -not -path "$r/dev/*" -not -path "$r/proc/*" \ + -not -path "$r/sys/*" -not -path "$r/run/*" \ + -not -path "$r/tmp/*" -not -path "$r/sources/*" \ + 2>/dev/null | sort +} + +snapshot() { + # Exclusions are anchored to the tracked root -- a bare "*/tmp/*" would also + # throw away legitimate files like /usr/share/foo/tmp/bar. + local r="${SNAP_ROOT%/}" + find "${SNAP_ROOT:-/}" -xdev \( -type f -o -type l \) \ + -not -path "$r/.lfs-pkgusr/*" -not -path "$SRCROOT/*" \ + -not -path "$r/dev/*" -not -path "$r/proc/*" \ + -not -path "$r/sys/*" -not -path "$r/run/*" \ + -not -path "$r/tmp/*" -not -path "$r/sources/*" \ + 2>/dev/null | sort +} + +# --------------------------------------------------------------------------- # +# building +# --------------------------------------------------------------------------- # +script_for() { echo "$SCRIPTS/$1.sh"; } + +# Chapter 7's temporary tools are built as root (the book does it that way); +# the package-user system takes over for the real chapter 8 packages. +# The package USER a step's files belong to. Several steps are really the same +# package built more than once -- gcc-pass1, gcc-pass2 and libstdcpp are all +# GCC; binutils-pass1/-pass2 are both Binutils -- and chapter 8 rebuilds each of +# them under the plain name. Giving each pass its own user would leave the +# system with gcc-pass1-owned files that the real `gcc` user can never manage, +# so every pass maps to the base package name. +pkg_owner_name() { + # Lower-case first. User names are case-sensitive on Unix, and the book's + # anchors are not consistent: chapter 7 gives "python-tmp" but chapter 8 + # gives "Python", which would become two different accounts -- and then the + # chapter-8 package cannot overwrite its own chapter-7 files: + # install: cannot remove '/usr/lib/python3.13/.../__init__.py': + # Permission denied (owned by python, installing as Python) + local n; n="$(printf '%s' "$1" | tr 'A-Z' 'a-z')" + # Perl module names ("XML::Parser") are not valid user names + n="${n//:://}"; n="${n//\//-}" + case "$n" in + libstdcpp|libstdc++) echo gcc; return ;; + esac + # strip a trailing -pass<N> or -tmp + n="${n%-tmp}" + case "$n" in + *-pass[0-9]) n="${n%-pass[0-9]}" ;; + esac + echo "$n" +} + +# Per-phase progress. A step is only "built" after a full run, but people +# legitimately run one phase at a time (retrying just `install` after a failed +# test suite, say), so remember which phases finished and let `next` pick up +# from there instead of suggesting the whole package again. +PHASES="$STATE/phases" +LOGS="$STATE/logs" + +# Pull the actual error out of a build log. With a parallel make the failing +# job's message can be thousands of lines above the end, so showing the tail +# (what you see on screen) is usually useless. +show_build_errors() { + local log="$1" n=0 + [ -f "$log" ] || return 0 + say "" + say "${C_ERR}--- first errors in the log ---${C_OFF}" + grep -nE "(^|[^-])\\berror\\b:|\\*\\*\\* \\[|Error [0-9]+|No such file or directory|command not found|undefined reference" \ + "$log" 2>/dev/null | grep -vE "Error 1$|Error 2$" | head -12 | while IFS= read -r l; do + say " $l" + done + n=$(grep -cE "\\berror\\b:|\\*\\*\\* \\[" "$log" 2>/dev/null || echo 0) + say "${C_ERR}-------------------------------${C_OFF}" + say " full log: $log" +} + +phase_file() { echo "$PHASES/$1"; } +record_phase() { mkdir -p "$PHASES"; grep -qxF "$2" "$(phase_file "$1")" 2>/dev/null \ + || echo "$2" >> "$(phase_file "$1")"; } +phase_done() { grep -qxF "$2" "$(phase_file "$1")" 2>/dev/null; } +clear_phases() { rm -f "$(phase_file "$1")"; } + +# which phases does this package's script actually define? +script_phases() { + local f; f="$(script_for "$1")" + [ -f "$f" ] || return 0 + echo unpack + echo build + echo install + grep -q '^#### CONFIGURE ####' "$f" && echo configure + return 0 +} + +# The phase to run next: the one AFTER the furthest phase already finished. +# (Not simply the first unfinished one -- after running only `install`, the +# earlier unpack/build had clearly happened too, and suggesting "unpack" would +# send you backwards.) +next_phase_for() { + local name="$1" p last="" seen=0 + while read -r p; do + phase_done "$name" "$p" && { last="$p"; seen=1; } + done < <(script_phases "$name") + [ "$seen" = 1 ] || { script_phases "$name" | head -n1; return 0; } + local after=0 + while read -r p; do + [ "$after" = 1 ] && { echo "$p"; return 0; } + [ "$p" = "$last" ] && after=1 + done < <(script_phases "$name") + return 1 +} + +# =========================================================================== # +# SHARED WITH packagemanager (kept in sync by hand for now) +# +# These functions exist in both tools -- lfs-helper (bash, drives the chroot +# build) and packagemanager/packagemanager_install (drives everything after). +# They must behave identically; when one changes, change the other: +# +# here there +# ---------------------------- ------------------------------------------- +# unwritable_dirs_from_log packagemanager_install: same name +# grant_dir_access packagemanager_install: grant_dir_to_user +# is_install_dir + patterns packagemanager: config install_dirs +# init_package_user_home packagemanager: init_package_user_home +# ensure_pkgusr_etc packagemanager: ensure_pkgusr_etc +# count_lines (bash-only) +# +# The rules they implement, in one place: +# * a package's files belong to its package user -- always +# * shared install dirs are root:install, group-writable, later sticky +# * a directory another package owns is shared via a collector group; +# if it already HAS one, join it -- never make a second +# * NEVER a collector group for root; root-owned trees are adopted or left +# * paths to fix come only from genuine permission-error lines, never from +# tracebacks or incidental log text +# =========================================================================== # + +# Count non-empty lines safely. +# +# `grep -c .` prints 0 AND exits non-zero when there is no match, so the common +# `grep -c . file || echo 0` produces "0\n0" and every later numeric test fails +# with "integer expression expected". +count_lines() { + local n + if [ -f "$1" ]; then + n=$(grep -c . "$1" 2>/dev/null) || n=0 + else + n=0 + fi + printf '%s' "${n:-0}" +} + +count_stdin_lines() { + local n + n=$(grep -c . 2>/dev/null) || n=0 + printf '%s' "${n:-0}" +} + +is_root_step() { [ -f "$STATE/rootsteps" ] && grep -qxF "$1" "$STATE/rootsteps"; } + +# Remove a stale unpacked source tree BEFORE the build, as root. +# +# /sources is sticky (1777), so only the owner may delete a directory in it. +# Chapters 5-6 unpacked their sources as the `lfs` user, and chapter 8 builds +# the same packages as their own package user -- which therefore cannot remove +# the leftover tree. The script's own `rm -rf` fails, tar extracts on top of +# it, and the build dies with a wall of +# tar: gcc-15.2.0/README: Cannot open: File exists +# tar: gcc-15.2.0: Cannot utime: Operation not permitted +# Doing it here, as root, keeps the package user out of the problem entirely. +clean_stale_source() { + local name="$1" script="$2" srcdir="${SNAP_ROOT%/}/sources" + [ -d "$srcdir" ] || return 0 + local glob; glob="$(sed -n 's/^pkg_glob="\(.*\)"$/\1/p' "$script" | head -n1)" + [ -n "$glob" ] || return 0 + + local pkg="" _c + ( cd "$srcdir" || exit 0 + for _c in $glob; do + [ -e "$_c" ] || continue + case "$_c" in *-src.*|*-source.*) echo "$_c"; exit 0 ;; esac + done + for _c in $glob; do + [ -e "$_c" ] || continue + case "$_c" in *-html.*|*-doc.*|*-docs.*|*-man.*|*-manual.*|*-tests.*) continue ;; esac + echo "$_c"; exit 0 + done ) > /tmp/.lfs-pkg.$$ 2>/dev/null + pkg="$(cat /tmp/.lfs-pkg.$$ 2>/dev/null)"; rm -f /tmp/.lfs-pkg.$$ + [ -n "$pkg" ] || return 0 + + # The build markers are written by whoever built last. /sources is sticky, + # so a marker left by the `lfs` user (chapters 5-6) cannot be replaced by a + # chapter-8 package user, and the build dies with + # /sources/.cc-build-ncurses: Permission denied + # Clear them here, as root; the build recreates them. + rm -f "$srcdir/.cc-build-$name" "$srcdir/.cc-dir-$name" 2>/dev/null + + local dir + dir="$(tar tf "$srcdir/$pkg" 2>/dev/null | head -n1 | cut -d/ -f1)" + [ -n "$dir" ] || return 0 + [ -d "$srcdir/$dir" ] || return 0 + say "${C_DIM}# [root] removing stale source tree $srcdir/$dir${C_OFF}" + rm -rf "${srcdir:?}/${dir:?}" || warn "could not remove $srcdir/$dir" +} + +# --------------------------------------------------------------------------- +# collector groups +# +# The package-user scheme gives every package its own user, but packages do +# install into directories another package created -- gcc drops gcc.mo into +# /usr/share/locale/<lang>/LC_MESSAGES, which belongs to whichever package made +# those directories. Without help the installing user simply cannot write +# there: +# install: cannot create regular file '.../gcc.mo': Permission denied +# +# The fix (same as the packagemanager script uses) is a COLLECTOR GROUP: the +# owning package's directory is handed to a group named <prefix>_<owner>, made +# group-writable, and every package that needs to install there joins it. +# Ownership stays with the owner; the others get exactly the access they need. +# One source of truth, in this order: +# 1. LFS_COLLECTOR_PREFIX -- set by the generated build env +# 2. the lfs config -- what the system was actually built with +# 3. packagemanager's own conf -- for a booted system with no lfs config +# Disagreeing prefixes mean two parallel sets of groups over the same +# directories, with nothing to say so, which is why this is resolved once here +# rather than defaulted separately in each tool. +_read_shared_prefix() { + local v="" + for f in /usr/share/lfs/config.json "$HOME/.local/share/lfs/config.json"; do + [ -r "$f" ] || continue + v="$(sed -n 's/.*"collector_prefix"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' \ + "$f" | head -n1)" + [ -n "$v" ] && { printf '%s' "$v"; return 0; } + done + for f in /etc/pkgusr/packagemanager.conf; do + [ -r "$f" ] || continue + v="$(sed -n 's/^collector_prefix=\(.*\)$/\1/p' "$f" | head -n1)" + [ -n "$v" ] && { printf '%s' "$v"; return 0; } + done + return 1 +} + +COLLECTOR_PREFIX="${LFS_COLLECTOR_PREFIX:-$(_read_shared_prefix || true)}" +GRANTED="$STATE/granted-groups" +# dir|group, one per line: the decisions made so far. Exported so a rebuild +# can reuse them instead of asking the same questions again. +COLLECTOR_MAP="$STATE/collector-map" + +require_collector_prefix() { + [ -n "$COLLECTOR_PREFIX" ] && return 0 + die "no collector-group prefix configured. + + Packages sometimes install into a directory another package owns -- gcc + writes gcc.mo into glibc's /usr/share/locale/*/LC_MESSAGES. Such a + directory keeps its owner but is handed to a group named <prefix>_<owner> + and made group-writable, and both packages join it. + + Set the prefix OUTSIDE the chroot, then regenerate: + lfs config collector_prefix sysgroup + lfs build-system gen-chroot-scripts --run --overwrite" +} + +collector_group_for() { echo "${COLLECTOR_PREFIX}_$1"; } + +ensure_group() { + local grp="$1" + group_exists "$grp" && return 0 + if have_shadow_tools; then + # groupadd fails if it already exists in the real database (which + # group_exists may not see when $ETC is redirected) -- treat that as fine + create_collector_group "$grp" || return 1 + else + local gid=$(( INSTALL_GID - 1 )) + while getent group "$gid" >/dev/null 2>&1 \ + || grep -q ":x:$gid:" "$ETC/group" 2>/dev/null; do + gid=$(( gid - 1 )) + done + echo "$grp:x:$gid:" >> "$ETC/group" + fi +} + +add_to_group() { + local user="$1" grp="$2" + if have_shadow_tools && command -v usermod >/dev/null 2>&1; then + usermod -a -G "$grp" "$user" 2>/dev/null && return 0 + fi + local line members + line="$(grep "^$grp:" "$ETC/group" 2>/dev/null)" || return 1 + members="${line##*:}" + case ",$members," in *",$user,"*) return 0 ;; esac + [ -n "$members" ] && members="$members,$user" || members="$user" + local gid; gid="$(echo "$line" | cut -d: -f3)" + sed -i "s|^$grp:.*|$grp:x:$gid:$members|" "$ETC/group" +} + +# Group names may only sensibly contain [a-z0-9_-]; keep them short too, since +# many tools cap group names at 32 characters. +sanitise_group_name() { + # `tr -c` turns EVERY other character into a dash, including spaces -- so + # an answer typed with a leading space became "-name", and one with two + # became "--name", which groupadd rejects as an option: + # could not create group --nimgnu_gcc + # Trim first, then collapse runs of dashes, then strip them from the ends: + # a group name may not start with a dash, and a trailing one is just noise. + local n="$1" + n="${n#"${n%%[![:space:]]*}"}" # leading whitespace + n="${n%"${n##*[![:space:]]}"}" # trailing whitespace + printf '%s' "$n" \ + | tr 'A-Z' 'a-z' \ + | tr -c 'a-z0-9_-' '-' \ + | sed -E 's/-+/-/g; s/^-+//; s/-+$//' \ + | cut -c1-32 +} + +# Which collector group should own this directory? +# +# The default names it after the OWNING package (<prefix>_binutils), which +# groups every directory that package owns under one group. Naming it after +# the DIRECTORY instead (<prefix>_bfd-plugins) is often clearer when the point +# is "anyone who installs a linker plugin", so offer both and let the user +# decide -- with enough context to choose sensibly. +# Is <user> already a member of <group>? +user_in_group() { + local u="$1" g="$2" line + if [ -r "$ETC/group" ]; then + line="$(grep "^$g:" "$ETC/group" 2>/dev/null | head -n1)" + else + line="$(getent group "$g" 2>/dev/null)" + fi + [ -n "$line" ] || return 1 + case ",${line##*:}," in + *",$u,"*) return 0 ;; + esac + return 1 +} + +choose_collector_group() { + local dir="$1" owner="$2" user="$3" + local by_owner by_dir + by_owner="$(sanitise_group_name "$(collector_group_for "$owner")")" + by_dir="$(sanitise_group_name "${COLLECTOR_PREFIX}_$(basename "$dir")")" + + # already handled? reuse whatever group the directory has + local cur; cur="$(stat -c %G "$dir" 2>/dev/null)" + case "$cur" in + "${COLLECTOR_PREFIX}_"*) echo "$cur"; return 0 ;; + esac + + # Already a member of the owner's collector group? Then the decision was + # made when that group was created: this package may install into the + # owner's directories. Asking again for every directory is just noise -- + # a package in <prefix>_python installing into a python-owned directory is + # precisely what that group is for. The directory still changes one at a + # time; membership does not hand over python's whole tree. + if group_exists "$by_owner" && user_in_group "$user" "$by_owner"; then + echo "$by_owner" + return 0 + fi + + # decided before (this run or an imported list)? don't ask again + # An export copied into the tree is a decision already made, even if + # `import-groups` has not been run. Consulting it here means the build + # never stops to ask a question whose answer is sitting in the tree. + if [ -f "$STATE/collector-groups.import" ]; then + local _pre + _pre="$(awk -F'|' -v d="$dir" \ + '$1 == "dir" && $2 == d { print $3; exit }' \ + "$STATE/collector-groups.import" 2>/dev/null)" + if [ -n "$_pre" ]; then + echo "$_pre" + return 0 + fi + fi + + if [ -f "$COLLECTOR_MAP" ]; then + local mapped + mapped="$(awk -F'|' -v d="${dir#${SNAP_ROOT%/}}" \ + '$1 == d { print $2; exit }' "$COLLECTOR_MAP" 2>/dev/null)" + if [ -n "$mapped" ]; then + echo "$mapped"; return 0 + fi + fi + + # Ask on /dev/tty, not stdin: this runs inside a `while ... done <<< list` + # loop, so stdin is the directory list and a plain `read` would silently + # consume that instead of waiting for an answer. + if [ ! -r /dev/tty ] || [ ! -w /dev/tty ] || [ "${LFS_ASSUME_YES:-0}" = 1 ]; then + echo "$by_owner"; return 0 + fi + + { + say "" + say " '$user' needs to install into: $dir" + say " that directory belongs to package '$owner'" + say " contents: $(ls -A "$dir" 2>/dev/null | wc -l) entry(ies)" + say " Which group should share it?" + say " 1) $by_owner (everything owned by '$owner' -- fewer groups)" + say " 2) $by_dir (just this directory -- more precise)" + say " or type another name" + } > /dev/tty + # Ask until the answer is usable. Quietly substituting a different group + # than the one asked for is worse than asking again -- the whole point of + # this prompt is that the choice is the user's. + local ans picked attempts=0 + while :; do + attempts=$((attempts + 1)) + printf ' [1/2/name] (default 1): ' > /dev/tty + # -e: readline editing, so arrow keys and backspace work while + # typing an answer instead of inserting escape codes + if ! read -e -r ans < /dev/tty; then + # no terminal (a script, a pipe): take the default rather than + # looping forever on input that will never come + echo "$by_owner" + return 0 + fi + case "$ans" in + ""|1) echo "$by_owner"; return 0 ;; + 2) echo "$by_dir"; return 0 ;; + esac + picked="$(sanitise_group_name "$ans")" + if [ -n "$picked" ]; then + # say so when the name had to be adjusted, so it is never a + # surprise which group ends up owning the directory + [ "$picked" = "$ans" ] \ + || echo " using '$picked' (adjusted from '$ans')" > /dev/tty + echo "$picked" + return 0 + fi + echo " '$ans' has nothing usable in it -- a group name needs" > /dev/tty + echo " letters, digits, '_' or '-'. Try again, or 1 / 2." > /dev/tty + if [ "$attempts" -ge 5 ]; then + echo " giving up after $attempts tries -- using $by_owner" > /dev/tty + echo "$by_owner" + return 0 + fi + done +} + +# The top of the tree a directory belongs to: walk up while the parent has the +# same owner and is not one of the shared install directories. Python's +# standard library is ~200 directories under /usr/lib/python3.13, so granting +# one failure at a time never finishes -- share from the top instead. +collector_tree_top() { + local top="$1" parent owner + owner="$(stat -c %U "$top" 2>/dev/null)" || { echo "$top"; return 0; } + while :; do + parent="${top%/*}" + [ -n "$parent" ] || break + [ "$parent" = "${SNAP_ROOT%/}" ] && break + [ -d "$parent" ] || break + is_install_dir "$parent" && break + [ "$(stat -c %U "$parent" 2>/dev/null)" = "$owner" ] || break + top="$parent" + done + echo "$top" +} + +# give <user> the right to install into <dir> +grant_dir_access() { + local dir="$1" user="$2" owner grp + [ -d "$dir" ] || return 0 + owner="$(stat -c %U "$dir" 2>/dev/null)" || return 0 + + # Grant exactly what was asked for. Promoting the request to "the top of + # the owner's tree" turned one directory into a whole hierarchy -- and for + # a root-owned tree that meant offering to share /sources. The package + # asks again for the next directory it needs, which is cheap now that an + # existing membership grants immediately (below). + + # Fast path: the directory already carries a collector group this user + # belongs to. The decision was made when the group was created, so making + # it again is just noise -- open the directory to the group and move on. + local curgrp + curgrp="$(stat -c %G "$dir" 2>/dev/null)" + if [ -n "$curgrp" ] && [ "$curgrp" != root ] \ + && case "$curgrp" in "$COLLECTOR_PREFIX"*) true ;; *) false ;; esac \ + && id -nG "$user" 2>/dev/null | tr ' ' '\n' | grep -qx "$curgrp"; then + chmod g+rwx "$dir" 2>/dev/null || true + say " $dir -> already group $curgrp, and '$user' is a member" + return 0 + fi + + # A directory that really is shared infrastructure -- /usr/lib, + # /usr/share/man/de/man1 and the rest of installdirs.lst -- gets the install + # group and stays root-owned. Nothing else changes here. + if is_install_dir "$dir"; then + # ...but never on a directory a PACKAGE USER owns. The install group + # means "anyone may install here", and applying it to a package's own + # directory hands that package's tree to everyone without asking. + # If a package owns it, the answer is a collector group, and that is a + # decision to put to the user -- fall through. + local _cur_owner + _cur_owner="$(stat -c %U "$dir" 2>/dev/null)" + if [ -n "$_cur_owner" ] && [ "$_cur_owner" != root ] \ + && [ "$_cur_owner" != "$user" ] && user_exists "$_cur_owner"; then + say " $dir is owned by '$_cur_owner' -- asking rather than" + say " opening it to every package." + else + chgrp install "$dir" 2>/dev/null + chmod g+w "$dir" 2>/dev/null + say " $dir -> group install (shared install directory)" + return 0 + fi + # owned by another package: fall through to the collector-group path, + # which asks rather than deciding on the user's behalf + fi + + # A ROOT-OWNED tree that is not an install directory is a leftover: chapter + # 7 built the -tmp package as root, so /usr/lib/python3.13 and its ~190 + # subdirectories all belong to root. That is a package's private tree, not + # shared infrastructure -- it should simply belong to the package. Hand the + # whole leftover subtree to the package installing into it; if some OTHER + # package needs to add files there later, it gets a collector group then, + # which is the normal mechanism. + if [ "$owner" = "root" ]; then + # ONLY this directory -- not the tree under it. + # + # This used to chown the whole subtree, so a package installing one + # file into /usr/lib/python3.13/site-packages took all ~190 python + # directories with it. Needing to write into a directory says nothing + # about the directories below it, and now that packages record the + # directories they create, those already have the right owner. + chown "$user:$user" "$dir" 2>/dev/null + say " $dir -> owned by '$user' (was root, from the temporary system)" + return 0 + fi + if [ "$owner" = "$user" ]; then + return 0 # already ours + fi + + + # Never invent a collector group for root. "<prefix>_root" would mean "may + # write anywhere root owns", which undoes the whole scheme -- and there is + # no user 'root' to put in a collector group anyway. A root-owned + # directory is either shared infrastructure or a leftover to adopt, both + # handled above; it is never a package's to share. + if [ "$owner" = "root" ]; then + warn " $dir is owned by root -- not creating a collector group for it" + return 1 + fi + + grp="$(choose_collector_group "$dir" "$owner" "$user")" + [ -n "$grp" ] || grp="$(collector_group_for "$owner")" + if ! ensure_group "$grp"; then + # A bad name should not end the build: say what was wrong with it and + # fall back to the default, which is always valid. + warn " '$grp' is not a usable group name" + grp="$(collector_group_for "$owner")" + warn " using '$grp' instead" + ensure_group "$grp" || { + warn " could not create group $grp" + return 1 + } + fi + add_to_group "$owner" "$grp" + add_to_group "$user" "$grp" + chgrp "$grp" "$dir" 2>/dev/null + chmod g+rwx "$dir" 2>/dev/null + + # Share exactly the directory that was asked for -- nothing below it. + # + # A collector group named after a package does NOT mean "may write anywhere + # that package owns". It means: when some package needs to install into a + # directory this one owns, that directory can be shared. Walking the + # owner's whole subtree turned one legitimate request into blanket access + # over hundreds of directories, so `nimgnu_python` ended up meaning "full + # run of every Python directory". + # + # Installing into a second directory asks again -- and because the package + # is already a member of the group by then, that grant is automatic (see + # the fast path at the top of this function). + chgrp "$grp" "$dir" 2>/dev/null || return 1 + chmod g+rwx "$dir" 2>/dev/null || true + say " $dir -> group $grp (owner $owner, + $user)" + mkdir -p "$(dirname "$GRANTED")" + grep -qxF "$grp|$user|$owner" "$GRANTED" 2>/dev/null \ + || echo "$grp|$user|$owner" >> "$GRANTED" + local rel="${dir#${SNAP_ROOT%/}}" + grep -qxF "$rel|$grp" "$COLLECTOR_MAP" 2>/dev/null \ + || echo "$rel|$grp" >> "$COLLECTOR_MAP" +} + +# Read a failed build's log, find every directory the package could not write +# to, and grant access via collector groups. Then the same phase can simply be +# re-run -- no editing, no manual chgrp. +# Directories mentioned in a log that <user> genuinely cannot write into. +unwritable_dirs_from_log() { + local log="$1" user="$2" p d + [ -f "$log" ] || return 0 + # Take paths both quoted ('...' / "...") and bare -- bash and perl report + # them unquoted ("...core_perl/perllocal.pod: Permission denied"), and only + # matching quoted ones made the recovery silently find nothing to do. + # Read only the TAIL of the log, and only lines that mention a permission + # problem. A build log is tens of megabytes; extracting every path from all + # of it and then running `su ... test -w` on each is minutes of CPU for an + # answer that is always in the last few lines. + local _tail + _tail="$(tail -c 524288 "$log" 2>/dev/null \ + | grep -aE "Permission denied|Operation not permitted|cannot (create|remove|touch|open)|couldn't copy|No such file or directory" \ + | grep -avE "^[[:space:]]*(File \"|Traceback|at .* line )" \ + | head -n 100)" + { printf '%s\n' "$_tail" | grep -aoE "'[^']+'|\"[^\"]+\"" | tr -d "'\"" + printf '%s\n' "$_tail" | grep -aoE "/[A-Za-z0-9._/+-]{6,}"; } \ + | while IFS= read -r p; do + case "$p" in /*) ;; *) continue ;; esac + case "$p" in + "${SNAP_ROOT%/}"/*) ;; + /usr/*|/etc/*|/var/*|/opt/*|/lib/*|/lib64/*|/bin/*|/sbin/*|\ + /home/*|/root/*|/srv/*|/boot/*|/tools/*|/sources/*) ;; + *) continue ;; + esac + if [ -d "$p" ]; then echo "$p"; else echo "${p%/*}"; fi + done | sort -u \ + | while IFS= read -r d; do + [ -n "$d" ] && [ -d "$d" ] || continue + su -s /bin/bash "$user" -c "test -w '$d'" 2>/dev/null && continue + echo "$d" + done +} + +# Grant <user> access to everything in the log it could not write, and say +# which groups that took. Returns 0 if anything was granted. +auto_grant_from_log() { + local log="$1" user="$2" need d granted=0 + need="$(unwritable_dirs_from_log "$log" "$user")" + [ -n "$need" ] || return 1 + require_collector_prefix + say "" + say "${C_WARN}# '$user' could not write into some directories -- granting" + say "# access via collector groups and retrying:${C_OFF}" + # Many failing paths walk up to the SAME tree top, so without this the same + # directory is granted once per path -- nine identical grants in one round. + local seen="" + while IFS= read -r d; do + [ -n "$d" ] || continue + local top; top="$(collector_tree_top "$d")" + case "$seen" in *"|$top|"*) continue ;; esac + seen="$seen|$top|" + grant_dir_access "$d" "$user" && granted=1 + done <<< "$need" + [ "$granted" = 1 ] +} + +# Repair /sources after the fact. +# +# Two kinds of debris accumulate there: +# * tarballs and trees owned by UIDs that only exist on the HOST (tar run as +# root restores the ownership recorded in the archive -- 8282, 15399, ...); +# * build markers left by a previous package user which, under the sticky +# /sources, nobody else can replace. +# Keep only the languages you actually want. +# +# Packages install translated man pages and message catalogues for every +# language they ship -- often 30+ directories each, none of which you will read. +# LFS itself suggests trimming them. The set is configurable; the default keeps +# English (always) plus German. +LOCALES_KEEP="${LFS_LOCALES:-en de}" + +cmd_prune_locales() { + need_root + local run=0; [ "${1:-}" = "--run" ] && run=1 + local r="${SNAP_ROOT%/}" + + say "Keep only these languages: $LOCALES_KEEP (plus English, always)" + say " configure with: lfs config locales \"en de fr\" (outside the chroot)" + say "" + + local keep_re="^(en|C|POSIX)" + local l + for l in $LOCALES_KEEP; do keep_re="$keep_re|^${l}\$|^${l}[._@]"; done + + local d base victims="" + for d in "$r"/usr/share/locale/*/ "$r"/usr/share/man/*/; do + [ -d "$d" ] || continue + base="$(basename "${d%/}")" + # man/man1..man9 are the untranslated pages -- never touch them + case "$base" in man[1-9]|man[1-9]x) continue ;; esac + printf '%s\n' "$base" | grep -qE "$keep_re" && continue + victims="$victims${d%/} +" + done + victims="$(printf '%s' "$victims" | grep -v '^$' || true)" + + local n; n=$(printf '%s\n' "$victims" | count_stdin_lines) + if [ "$n" = 0 ]; then + ok " nothing to remove -- only the wanted languages are installed" + return 0 + fi + say " $n language director(ies) would be removed, e.g.:" + printf '%s\n' "$victims" | head -6 | sed 's/^/ /' + say "" + if [ "$run" = 0 ]; then + warn "(dry run -- nothing was changed. Add --run to remove them.)" + return 0 + fi + printf '%s\n' "$victims" | while IFS= read -r d; do + [ -n "$d" ] && rm -rf "$d" + done + ok " removed $n language director(ies)" + say " (packages will recreate them on reinstall; re-run this when you like)" +} + +# Find (and optionally remove) per-user Python installs. +# +# pip falls back to ~/.local when the system site-packages is not writable. The +# package that installed it can still import it, so its build looks successful, +# but nothing else on the system can -- which surfaces much later as +# BackendUnavailable: Cannot import 'flit_core.buildapi' +# Export / import the collector-group decisions. +# +# Building a second system means answering the same "which group should share +# this directory?" questions again. Exporting the map makes the next build +# reuse the names you already chose -- no prompts, same layout. +# Remove duplicate entries from an export. +# +# Two sources feed it: a filesystem scan (what really carries a group now) and +# the recorded map (what was decided, possibly not yet applied). A directory +# that is in both appeared twice, because each source was only deduplicated +# against itself. `sort -u` across everything would also reorder the file; +# keeping first occurrences preserves the scan-then-map order, which reads +# more naturally and keeps a re-export stable. +_dedupe_collector_export() { + awk '!seen[$0]++' +} + +cmd_export_groups() { + local out="${1:-$STATE/collector-groups.export}" + : > "$out" + { + echo "# lfs collector groups -- import with: lfs-helper import-groups <file>" + echo "# prefix|$COLLECTOR_PREFIX" + # The directories that actually carry a collector group right now. + # The recorded map only holds the tree TOPS we were asked about; the + # filesystem is the authority on what really ended up in each group. + local r="${SNAP_ROOT%/}" + find "${r:-/}" -xdev -type d \ + -not -path "${r}/proc/*" -not -path "${r}/sys/*" \ + -not -path "${r}/dev/*" -not -path "${r}/run/*" \ + -not -path "${r}/tmp/*" -not -path "${r}/sources/*" \ + -printf '%g|%p\n' 2>/dev/null \ + | awk -F'|' -v p="${COLLECTOR_PREFIX}_" -v r="$r" ' + index($1, p) == 1 { + d = $2 + if (r != "" && index(d, r) == 1) d = substr(d, length(r) + 1) + print "dir|" d "|" $1 + }' + # plus anything decided but not yet applied + if [ -f "$COLLECTOR_MAP" ]; then + sed 's/^/dir|/' "$COLLECTOR_MAP" + fi + # group membership, so the same packages end up in the same groups + local line grp members + while IFS=: read -r grp _x _gid members; do + case "$grp" in + "${COLLECTOR_PREFIX}_"*) echo "group|$grp|$members" ;; + esac + done < "$ETC/group" + } | _dedupe_collector_export >> "$out" + ok "exported $(grep -c '^dir|' "$out" 2>/dev/null || echo 0) directory mapping(s)" + say " and $(grep -c '^group|' "$out" 2>/dev/null || echo 0) group(s) to $out" + say "" + say " Copy that file to the next system and run:" + say " lfs-helper import-groups $out" +} + +cmd_import_groups() { + need_root + # Parse the arguments ONCE, in one place. This used to set run=1, then + # declare `local run=0` below it (wiping the flag), then reassign $in to + # the export path -- so `import-groups --run` read the wrong file and + # reported "imported 0 directory mapping(s)" while the decisions sat + # unapplied in collector-groups.import. + local in="" run=0 a + for a in "$@"; do + case "$a" in + --run) run=1 ;; + -*) die "usage: lfs-helper import-groups [<file>] [--run]" ;; + *) in="$a" ;; + esac + done + if [ -z "$in" ]; then + # prefer a file placed here by `lfs build-system gen-chroot-scripts` + if [ -f "$STATE/collector-groups.import" ]; then + in="$STATE/collector-groups.import" + else + in="$STATE/collector-groups.export" + fi + fi + [ -f "$in" ] || die "no such file: $in" + say "reading $in" + + local ndir=0 ngrp=0 kind a b + while IFS='|' read -r kind a b; do + case "$kind" in + dir) ndir=$((ndir+1)) + [ "$run" = 1 ] && { mkdir -p "$(dirname "$COLLECTOR_MAP")" + grep -qxF "$a|$b" "$COLLECTOR_MAP" 2>/dev/null \ + || echo "$a|$b" >> "$COLLECTOR_MAP"; } ;; + group) ngrp=$((ngrp+1)) + if [ "$run" = 1 ]; then + ensure_group "$a" || continue + local m + for m in ${b//,/ }; do + user_exists "$m" && add_to_group "$m" "$a" + done + fi ;; + esac + # Dedupe on read as well as on write: an export made before the writer + # was fixed can contain the same mapping twice, and importing it would + # report and apply it twice. + done < <(grep -vE '^\s*(#|$)' "$in" | _dedupe_collector_export) + + if [ "$run" = 1 ]; then + ok "imported $ndir directory mapping(s) and $ngrp group(s)" + say " Those directories will not be asked about again." + else + say " would import $ndir directory mapping(s) and $ngrp group(s)" + warn "(dry run -- nothing was changed. Add --run to apply.)" + fi +} + +# Repair package-user homes and VERSION files after the fact. +# +# Users created before the skeleton existed have no .bash_profile/.bashrc/build +# links, or links copied from the wrong place; and packages built before +# VERSION was recorded have no record of what is installed. This rebuilds both +# from what is already on disk. +cmd_fix_users() { + need_root + local run=0 skel="${LFS_PKGUSR_SKEL:-${SNAP_ROOT%/}/etc/pkgusr/skel-package}" + while [ $# -gt 0 ]; do + case "$1" in + --run) run=1; shift ;; + --skel) skel="${2:?--skel needs a directory}"; shift 2 ;; + *) die "usage: lfs-helper fix-users [--run] [--skel <dir>]" ;; + esac + done + + say "Repair every package user's home:" + say " * .bash_profile, .bashrc and build -> $PKGUSR_ETC" + say " * VERSION, from the install script's name_version" + [ -d "$skel" ] && say " * other files from $skel" + say "" + + local n_link=0 n_ver=0 name home nv script + while IFS= read -r name; do + [ -n "$name" ] || continue + home="$SRCROOT/$name" + [ -d "$home" ] || continue + + # links + local need_links=0 l t + for l in .bash_profile:bash_profile .bashrc:bashrc build:build; do + t="${l#*:}"; l="${l%%:*}" + [ "$(readlink "$home/$l" 2>/dev/null)" = "$PKGUSR_ETC/$t" ] || need_links=1 + done + if [ "$need_links" = 1 ]; then + n_link=$((n_link+1)) + printf ' %-24s links\n' "$name" + if [ "$run" = 1 ]; then + init_package_user_home "$name" "$home" + if [ -d "$skel" ]; then + # skeleton files the host uses, without clobbering the links + cp -a "$skel"/. "$home"/ 2>/dev/null || true + init_package_user_home "$name" "$home" + fi + fi + fi + + # VERSION + if [ ! -f "$home/VERSION" ]; then + script="$SCRIPTS/$name.sh" + nv="" + [ -f "$script" ] && \ + nv="$(sed -n 's/^name_version="\(.*\)"$/\1/p' "$script" | head -n1)" + if [ -n "$nv" ]; then + n_ver=$((n_ver+1)) + printf ' %-24s VERSION -> %s\n' "$name" "$nv" + if [ "$run" = 1 ]; then + { + echo "$nv" + echo "# recorded $(date '+%Y-%m-%d %H:%M:%S') by fix-users" + } > "$home/VERSION" + chown "$name:$name" "$home/VERSION" 2>/dev/null || true + fi + fi + fi + done < <(pkg_users_list) + + say "" + if [ "$run" = 1 ]; then + ok " fixed $n_link home(s) and wrote $n_ver VERSION file(s)" + else + say " $n_link home(s) need links, $n_ver VERSION file(s) missing" + warn "(dry run -- nothing was changed. Add --run to apply.)" + fi +} + +# every package user we know about +pkg_users_list() { + local d name + for d in "$SRCROOT"/*; do + [ -d "$d" ] || continue + name="$(basename "$d")" + user_exists "$name" && echo "$name" + done +} + +cmd_find_user_site() { + need_root + local run=0; [ "${1:-}" = "--run" ] && run=1 + local d n=0 + say "Per-user Python installs under $SRCROOT:" + say " (pip put these in a package user's home because the system" + say " site-packages was not writable; only that user can import them)" + say "" + while IFS= read -r d; do + [ -n "$d" ] || continue + printf " %s\n" "$d" + n=$((n+1)) + [ "$run" = 1 ] && rm -rf "$d" + done < <(find "$SRCROOT" -maxdepth 4 -type d -path '*/.local/lib' 2>/dev/null | sort) + say "" + if [ "$n" = 0 ]; then + ok " none -- every package installed system-wide" + elif [ "$run" = 1 ]; then + ok " removed $n per-user install(s)" + say " Rebuild the affected packages so they install system-wide:" + say " lfs-helper build <package> --force" + else + say " $n found" + warn "(dry run -- nothing was changed. Add --run to remove them.)" + fi +} + +cmd_clean_sources() { + need_root + local run=0; [ "${1:-}" = "--run" ] && run=1 + local srcdir="${SNAP_ROOT%/}/sources" + [ -d "$srcdir" ] || die "no $srcdir" + + say "Tidy $srcdir:" + say " * files owned by a UID that does not exist here -> root" + say " * leftover build markers (.cc-build-*, .cc-dir-*) -> removed" + say "" + + local orphans markers + orphans="$(find "$srcdir" -maxdepth 1 \( -nouser -o -nogroup \) 2>/dev/null | sort)" + markers="$(find "$srcdir" -maxdepth 1 -name '.cc-*' 2>/dev/null | sort)" + + local n_o n_m + n_o=$(printf '%s\n' "$orphans" | count_stdin_lines) + n_m=$(printf '%s\n' "$markers" | count_stdin_lines) + say " $n_o path(s) with an unknown owner" + say " $n_m build marker(s)" + if [ "$run" = 0 ]; then + [ "$n_o" -gt 0 ] && printf '%s\n' "$orphans" | head -6 | sed 's/^/ /' + say "" + warn "(dry run -- nothing was changed. Add --run to apply.)" + return 0 + fi + [ "$n_o" -gt 0 ] && chown -R root:root $orphans 2>/dev/null + [ "$n_m" -gt 0 ] && rm -f $markers + say "" + ok " tidied: $n_o ownership(s) reset, $n_m marker(s) removed" + say " (markers are recreated by the next build; nothing else is lost)" +} + +cmd_fix_perms() { + need_root + local name="" phase="all" run=0 + while [ $# -gt 0 ]; do + case "$1" in + --run) run=1; shift ;; + --phase) phase="$2"; shift 2 ;; + -*) die "usage: lfs-helper fix-perms <package> [--phase P] [--run]" ;; + *) name="$1"; shift ;; + esac + done + [ -n "$name" ] || die "usage: lfs-helper fix-perms <package> [--run]" + local owner; owner="$(pkg_owner_name "$name")" + local log="$LOGS/$name-$phase.log" + [ -f "$log" ] || die "no log for $name (phase $phase) at $log" + + # "install: cannot create regular file '/a/b/c': Permission denied" + # "mkdir: cannot create directory '/a/b': Permission denied" + # "/a/b/c: Permission denied" + local dirs + dirs="$(grep -oE "'[^']+'|\"[^\"]+\"" "$log" 2>/dev/null \ + | tr -d "'\"" \ + | while IFS= read -r p; do + case "$p" in /*) ;; *) continue ;; esac + if [ -d "$p" ]; then echo "$p"; else echo "${p%/*}"; fi + done | sort -u)" + # only keep ones that really are unwritable for the package user + local need="" d + while IFS= read -r d; do + [ -n "$d" ] || continue + [ -d "$d" ] || continue + su -s /bin/bash "$owner" -c "test -w '$d'" 2>/dev/null && continue + need="$need$d +" + done <<< "$dirs" + need="$(printf '%s' "$need" | grep -v '^$' || true)" + + if [ -z "$need" ]; then + say "Nothing to fix: '$owner' can already write every directory" + say "mentioned in $log." + return 0 + fi + + say "Directories '$owner' cannot write into (from $log):" + printf ' %s\n' $need + say "" + if [ "$run" = 0 ]; then + warn "(dry run -- add --run to grant access via collector groups)" + return 0 + fi + while IFS= read -r d; do + [ -n "$d" ] && grant_dir_access "$d" "$owner" + done <<< "$need" + say "" + ok "Access granted. Re-run the build:" + say " lfs-helper build $name --phase $phase --force" +} + +# --------------------------------------------------------------------------- +# command wrappers (the package-users hint's /usr/lib/pkgusr) +# +# Packages routinely try to set ownership or permissions a package user simply +# cannot: the LFS book's GCC section runs +# chown -v -R root:root /usr/lib/gcc/<triplet>/<ver>/include{,-fixed} +# and as the `gcc` user that fails with "Operation not permitted", killing the +# install for no good reason -- the files are already owned correctly for this +# scheme. The hint's answer is to put wrappers for the five troublesome +# commands (mkdir, chgrp, chown, chmod, install) first in the package user's +# PATH. They neutralise the impossible parts, report what they skipped, and +# otherwise behave normally. +WRAPPERS="$STATE/wrappers" + +make_wrappers() { + mkdir -p "$WRAPPERS" + + cat > "$WRAPPERS/chown" <<'EOF' +#!/bin/bash +# Ownership is decided by the package-user scheme, not by the package. +echo "*** chown $* (skipped: package users cannot change ownership)" >&2 +exit 0 +EOF + + cat > "$WRAPPERS/chgrp" <<'EOF' +#!/bin/bash +# Group changes (e.g. setgid tty) need root; do them yourself afterwards. +echo "*** chgrp $* (skipped: package users cannot change group)" >&2 +exit 0 +EOF + + cat > "$WRAPPERS/install" <<'EOF' +#!/bin/bash +# Drop -o/-g (ownership), refuse setuid/setgid modes, and never re-create or +# re-mode a directory that already exists. Packages routinely do +# install -m 0755 -d /usr/sbin +# for no good reason; as a package user that fails with +# install: cannot change permissions of '/usr/sbin': Operation not permitted +# The directory is already there and correctly set up, so there is nothing to do. +real=/usr/bin/install +args=(); skipped="" + +# -d mode: create directories. Skip the ones that already exist. +want_dirs=0 +for a in "$@"; do case "$a" in -d|--directory) want_dirs=1 ;; esac; done +if [ "$want_dirs" = 1 ]; then + dirs=(); opts=(); skip_next=0 + for a in "$@"; do + if [ "$skip_next" = 1 ]; then skip_next=0; continue; fi + case "$a" in + -d|--directory) ;; + -m|--mode|-o|--owner|-g|--group) skip_next=1 ;; + -*) ;; + *) dirs+=("$a") ;; + esac + done + rc=0 + for d in "${dirs[@]}"; do + if [ -d "$d" ]; then + echo "*** install -d $d (already exists -- not re-created or re-moded)" >&2 + continue + fi + /usr/bin/mkdir -p "$d" || rc=$? + done + exit $rc +fi +while [ $# -gt 0 ]; do + case "$1" in + -o|--owner|-g|--group) skipped="$skipped $1 $2"; shift 2 ;; + -o*|-g*) skipped="$skipped $1"; shift ;; + -m|--mode) + case "$2" in + [2467][0-7][0-7][0-7]) # setuid/setgid/sticky requested + echo "*** install -m $2 (using 755 instead; set the bit as root if you want it)" >&2 + args+=(-m 755); shift 2 ;; + *) args+=("$1" "$2"); shift 2 ;; + esac ;; + *) args+=("$1"); shift ;; + esac +done +[ -n "$skipped" ] && echo "*** install:$skipped (skipped: ownership is the package user's)" >&2 +exec "$real" "${args[@]}" +EOF + + cat > "$WRAPPERS/chmod" <<'EOF' +#!/bin/bash +# Allow everything except setuid/setgid, which a package user must not set. +real=/usr/bin/chmod +me="$(id -un)" +for a in "$@"; do + case "$a" in + [2467][0-7][0-7][0-7]|*[ug]+s*|*+s*) + echo "*** chmod $* (skipped: refuses to set setuid/setgid; do it as root if needed)" >&2 + exit 0 ;; + esac +done +# Never change the mode of something owned by someone else. Packages try to +# re-mode shared directories they merely install into; the package-user scheme +# says those belong to their owner and stay as they are. +# The first non-option argument is the MODE; everything after it is a target. +args=(); skipped=""; targets=0; seen_mode=0 +for a in "$@"; do + case "$a" in + -[!0-9]*|--*) args+=("$a"); continue ;; + esac + if [ "$seen_mode" = 0 ]; then + seen_mode=1; args+=("$a"); continue # the mode itself + fi + if [ -e "$a" ] && [ "$(stat -c %U "$a" 2>/dev/null)" != "$me" ]; then + skipped="$skipped $a" + continue + fi + args+=("$a"); targets=$((targets+1)) +done +[ -n "$skipped" ] && \ + echo "*** chmod:$skipped (skipped: not owned by '$me' -- left as it is)" >&2 +# every target was skipped? then there is nothing to do, and that is fine +[ "$targets" -eq 0 ] && exit 0 +exec "$real" "${args[@]}" +EOF + + cat > "$WRAPPERS/cp" <<'EOF' +#!/bin/bash +# `cp -a dest/* /` is how several packages install a staged tree. As a package +# user, copying the FILES works, but cp then tries to stamp the destination +# DIRECTORIES (/usr, /usr/bin -- owned by root) with the source's timestamps +# and permissions, and fails: +# cp: preserving times for '/usr/bin': Operation not permitted +# The copy itself succeeded, so treat that specific complaint as harmless and +# fail only on real errors. +real=/usr/bin/cp +err="$(mktemp)" +# capture stderr to a file first: with process substitution the check below can +# run before tee has flushed, and the result becomes a coin toss +"$real" "$@" 2>"$err" +rc=$? +cat "$err" >&2 +if [ $rc -ne 0 ] && [ -s "$err" ]; then + if ! grep -qvE "preserving (times|permissions|ownership) for" "$err"; then + echo "*** cp: could not stamp existing directories (harmless: the files copied)" >&2 + rc=0 + fi +fi +rm -f "$err" +exit $rc +EOF + + cat > "$WRAPPERS/mkdir" <<'EOF' +#!/bin/bash +# A package recreating an existing system directory must not fail the build. +real=/usr/bin/mkdir +"$real" "$@" && exit 0 +rc=$? +for a in "$@"; do + case "$a" in -*) continue ;; esac + [ -d "$a" ] || exit $rc +done +echo "*** mkdir $* (directory already exists -- continuing)" >&2 +exit 0 +EOF + + chmod 755 "$WRAPPERS"/* + chmod a+rx "$WRAPPERS" 2>/dev/null || true +} + +# Re-apply the install-directory permissions. +# +# Packages reset modes on directories they touch -- /usr/share/man/man1 ends up +# root:install but drwxr-xr-x, and the next package then cannot write its man +# page: +# ln: failed to create symbolic link '/usr/share/man/man1/cc.1': Permission denied +# Re-asserting group-writability before each build costs nothing and stops that +# drift from becoming a failed install. +ensure_install_dirs_writable() { + pkgusr_ready || return 0 + local d fixed=0 + while IFS= read -r d; do + d="${SNAP_ROOT%/}$d" + [ -d "$d" ] || continue + # Check the GROUP write bit numerically. Matching "w" in the symbolic + # mode is wrong: drwxr-xr-x contains a "w" -- the owner's. + local _grp _mode + _grp="$(stat -c '%G' "$d" 2>/dev/null)" || continue + _mode="$(stat -c '%a' "$d" 2>/dev/null)" || continue + if [ "$_grp" = "install" ] && [ $(( (10#$_mode / 10) % 10 & 2 )) -ne 0 ] + then + continue # already install + group-writable + fi + chgrp install "$d" 2>/dev/null || continue + chmod g+w "$d" 2>/dev/null || continue + fixed=$((fixed+1)) + done < <(install_dirs_list) + + # ...and the per-language man/locale directories, which come and go as + # packages are installed. Without this every translated man page turns + # into another "grant access?" prompt. + local r="${SNAP_ROOT%/}" + while IFS= read -r d; do + [ -d "$d" ] || continue + local _g _m + _g="$(stat -c '%G' "$d" 2>/dev/null)" || continue + _m="$(stat -c '%a' "$d" 2>/dev/null)" || continue + [ "$_g" = "install" ] && [ $(( (10#$_m / 10) % 10 & 2 )) -ne 0 ] && continue + chgrp install "$d" 2>/dev/null || continue + chmod g+w "$d" 2>/dev/null || continue + fixed=$((fixed+1)) + done < <( + find "$r/usr/share/man" "$r/usr/share/locale" "$r/usr/share/terminfo" \ + -mindepth 1 -maxdepth 2 -type d 2>/dev/null + ) + + [ "$fixed" -gt 0 ] && \ + say "${C_DIM}# re-applied group-writability to $fixed install dir(s)${C_OFF}" + return 0 +} + +# When an install fails, inspect the paths the errors mention and report what +# is actually there. "No such file or directory" on a destination can mean the +# parent directory is missing, or that the target is a dangling symlink -- and +# "Permission denied" can mean the file exists but belongs to another package. +# Guessing from the message alone wastes a lot of time; this shows the facts. +diagnose_paths_from_log() { + local log="$1" user="$2" p shown=0 + [ -f "$log" ] || return 0 + local paths + # tail only, and cap the results: see unwritable_dirs_from_log for why + paths="$(tail -c 524288 "$log" 2>/dev/null | grep -aoE "/[A-Za-z0-9._/+-]{6,}" \ + | grep -vE "^/(proc|sys|dev|tmp)/" | sort -u \ + | while IFS= read -r p; do + case "$p" in + */lib/perl5/*|*/usr/*) echo "$p" ;; + esac + done)" + # only the ones named on an error line + local errpaths + # Require at least two slashes: a bare "/GDBM_File.pm" is the tail of a + # RELATIVE path in the message (lib/GDBM_File.pm), not a real target. + errpaths="$(grep -iE "couldn't|cannot|no such file|permission denied|failed" "$log" 2>/dev/null \ + | grep -oE "/[A-Za-z0-9._/+-]{6,}" \ + | sort -u \ + | while IFS= read -r p; do + # keep real targets: either inside the tree we track, or + # under a known top-level dir. This drops the tails of + # RELATIVE paths in messages ("lib/GDBM_File.pm" -> + # "/GDBM_File.pm", "auto/B/B.so" -> "/auto/B/B.so"). + case "$p" in + "${SNAP_ROOT%/}"/*) echo "$p" ;; + /usr/*|/etc/*|/var/*|/opt/*|/lib/*|/lib64/*|/bin/*|\ + /sbin/*|/home/*|/root/*|/srv/*|/boot/*|/tools/*|/sources/*) + echo "$p" ;; + esac + done | head -8)" + [ -n "$errpaths" ] || return 0 + + say "" + say "${C_WARN}--- what those paths actually are ---${C_OFF}" + while IFS= read -r p; do + [ -n "$p" ] || continue + if [ -L "$p" ] && [ ! -e "$p" ]; then + printf " %-52s DANGLING SYMLINK -> %s\n" "$p" "$(readlink "$p")" + elif [ -e "$p" ]; then + printf " %-52s %s %s:%s\n" "$p" \ + "$(stat -c %A "$p" 2>/dev/null)" \ + "$(stat -c %U "$p" 2>/dev/null)" "$(stat -c %G "$p" 2>/dev/null)" + # "cannot remove"/"cannot create" is really about the DIRECTORY the + # file sits in -- replacing a file needs write permission there, not + # on the file itself. Show it, or the report is misleading. + local pd="${p%/*}" + [ -d "$pd" ] && printf " %-52s %s %s:%s (parent -- this is what matters)\n" \ + " in $pd" "$(stat -c %A "$pd" 2>/dev/null)" \ + "$(stat -c %U "$pd" 2>/dev/null)" "$(stat -c %G "$pd" 2>/dev/null)" + else + local d="${p%/*}" + if [ -d "$d" ]; then + printf " %-52s missing; parent %s is %s %s:%s\n" "$p" "$d" \ + "$(stat -c %A "$d" 2>/dev/null)" \ + "$(stat -c %U "$d" 2>/dev/null)" "$(stat -c %G "$d" 2>/dev/null)" + else + printf " %-52s missing, and so is its parent %s\n" "$p" "$d" + fi + fi + shown=$((shown+1)) + done <<< "$errpaths" + [ "$shown" -gt 0 ] && say "${C_WARN}-------------------------------------${C_OFF}" + return 0 +} + +# Give a pip-based package write access to Python's site-packages BEFORE it +# runs. +# +# `pip3 install` writes into /usr/lib/pythonX.Y/site-packages, which belongs to +# the python package. If that isn't writable, pip does not always fail loudly: +# depending on the build it can fall back to a per-user location under the +# package user's home, where nothing else can import it. The next package that +# needs it then fails with a baffling +# BackendUnavailable: Cannot import 'flit_core.buildapi' +# even though the earlier install "succeeded". Granting up front avoids the +# whole class of problem. +# does this package install with pip? +uses_pip() { + grep -qE '\bpip[0-9]*\s+install|\bpip[0-9]*\s+wheel' "$1" 2>/dev/null +} + +pregrant_python_sitedirs() { + local user="$1" script="$2" r="${SNAP_ROOT%/}" d + uses_pip "$script" || return 0 + + # Remove a stale per-user install. + # + # When the system site-packages is not writable, pip installs into the + # package user's ~/.local instead -- and then reports + # Requirement already satisfied: flit_core in + # /usr/src/flit-core/.local/lib/python3.13/site-packages + # on every later run, so it never installs system-wide. Only that one user + # can import it; every other package fails with + # BackendUnavailable: Cannot import 'flit_core.buildapi' + local ulocal="$SRCROOT/$user/.local" + if [ -d "$ulocal/lib" ]; then + say "${C_WARN}# removing a stale per-user install in $ulocal${C_OFF}" + say "${C_DIM}# (pip put it there because site-packages was not writable;" + say "# nothing outside '$user' can import from it)${C_OFF}" + rm -rf "$ulocal/lib" + fi + for d in "$r"/usr/lib/python*/site-packages "$r"/usr/lib/python*; do + [ -d "$d" ] || continue + su -s /bin/bash "$user" -c "test -w '$d'" 2>/dev/null && continue + say "${C_DIM}# pip package: making sure '$user' can write $d${C_OFF}" + grant_dir_access "$d" "$user" + break + done + return 0 +} + +# A build that was interrupted part-way leaves the package half-installed. +_note_interrupted() { + warn "" + warn "!! interrupted while building '$1'." + warn " The package may be HALF-INSTALLED: some files on disk, some not." + warn " Finish or undo it before building anything else --" + warn " a half-installed toolchain package breaks every build after it:" + warn " lfs-helper build $1 --phase install --force # finish it" + warn " lfs-helper check-toolchain # is cc/c++ ok?" +} + +# Packages recorded as mid-install and never finished. +_interrupted_packages() { + [ -f "$STATE/interrupted" ] || return 1 + sort -u "$STATE/interrupted" 2>/dev/null | grep -v '^$' +} + +_clear_interrupted() { + local name="$1" f tmp + f="$STATE/interrupted" + [ -f "$f" ] || return 0 + tmp="$f.$$" + grep -vx "$name" "$f" > "$tmp" 2>/dev/null + mv -f "$tmp" "$f" 2>/dev/null || rm -f "$tmp" + [ -s "$f" ] || rm -f "$f" +} + +# Keep out of a step's manifest anything that belongs to a DIFFERENT package. +# +# A step can install packages under other users: last-step builds wget as the +# package user 'wget'. Everything written during the step was recorded as +# last-step's, so /usr/bin/wget landed in last-step's manifest -- and the next +# thing to act on that manifest (adoption, fix-ownership) took the file from +# wget and gave it to last-step. wget could then not reinstall itself. +# +# A manifest should list what this step owns, not everything that happened +# while it ran. +_drop_other_packages_files() { + local owner="$1" p cur + while IFS= read -r p; do + [ -n "$p" ] || continue + cur="$(stat -c %U "$p" 2>/dev/null)" + if [ -n "$cur" ] && [ "$cur" != "$owner" ] && [ "$cur" != root ] \ + && [ "$cur" != UNKNOWN ] && user_exists "$cur"; then + continue # another package installed this + fi + printf '%s\n' "$p" + done +} + +cmd_build() { + need_root + local name="" phase="all" force=0 run_tests=0 jobs="" auto_fix=1 + while [ $# -gt 0 ]; do + case "$1" in + --phase) phase="$2"; shift 2 ;; + --force) force=1; shift ;; + --tests) run_tests=1; shift ;; + --jobs|-j) jobs="${2:?--jobs needs a number}"; shift 2 ;; + --no-auto-fix) auto_fix=0; shift ;; + -*) die "unknown option: $1" ;; + *) name="$1"; shift ;; + esac + done + [ -n "$name" ] || die "usage: lfs-helper build <package> [--phase P] [--force]" + local s; s="$(script_for "$name")" + [ -f "$s" ] || die "no script for '$name' at $s + Generate them from outside the chroot: + lfs build-system gen-chroot-scripts --run" + + # Clear a stale source tree first, as root -- independent of everything + # below, and it must happen even if staging later has trouble. + case "$phase" in + all|unpack) clean_stale_source "$name" "$s" ;; + esac + + if is_done "$name" && [ "$force" = 0 ]; then + if [ -t 0 ]; then + printf "'%s' is already built. Rebuild it? [y/N]: " "$name" + local ans; read -e -r ans + case "$ans" in [Yy]*) ;; *) say "# leaving '$name' as it is"; return 0 ;; esac + else + say "# '$name' is already built (skipping; --force to rebuild)" + return 0 + fi + fi + + local chown_after=0 + local as_root=0 staged owner="$name" + if is_root_step "$name"; then + as_root=1 + staged="$SRCROOT/$name/install_$name" + mkdir -p "$SRCROOT/$name" + install -m 0755 "$s" "$staged" || die "could not stage the install script" + # A *-tmp temporary tool is the SAME package chapter 8 rebuilds, so its + # files should belong to that package user from the outset -- otherwise + # chapter 8 inherits a root-owned tree it cannot write into. The user + # may not exist this early (init-pkgusr runs later), in which case + # adopt-existing assigns them afterwards. + case "$name" in + *-tmp) + owner="$(pkg_owner_name "$name")" + # subshell: a `die` inside would kill the build, not just + # fall back to building this step as root + if pkgusr_ready && ( cmd_add_user "$owner" ) >/dev/null 2>&1; then + chown_after=1 + else + owner="$name" # no package users yet; adopt later + fi ;; + esac + else + pkgusr_ready || die "run 'lfs-helper init-pkgusr --run' first" + owner="$(pkg_owner_name "$name")" + cmd_add_user "$owner" + staged="$SRCROOT/$owner/install_$name" + install -m 0755 -o "$owner" -g "$owner" "$s" "$staged" \ + || die "could not stage the install script" + fi + + mkstate + local before after before_dirs + if [ -f "$SNAPSHOT" ]; then before="$(load_snapshot "$SNAPSHOT")" + else before="$(snapshot)"; fi + if [ -f "$SNAPDIRS" ]; then before_dirs="$(load_snapshot "$SNAPDIRS")" + else before_dirs="$(snapshot_dirs)"; fi + + local _book_of_script + _book_of_script="$(sed -n 's/^### book *: *//p' "$s" 2>/dev/null | head -n1)" + say "${C_DIM}# script : $s${C_OFF}" + [ -n "$_book_of_script" ] && \ + say "${C_DIM}# book : $_book_of_script${C_OFF}" + if [ "$as_root" = 1 ]; then + say "${C_DIM}# user : root (temporary chapter-7 tool) phase: $phase${C_OFF}" + else + say "${C_DIM}# user : $owner (package user) phase: $phase${C_OFF}" + fi + [ "$phase" = "all" ] && clear_phases "$name" + # Record that this package is MID-INSTALL. Cancelling here (Ctrl-C during + # `make install`) leaves the package half on disk: gcc, interrupted during + # its install, replaces /usr/bin/gcc with a native compiler but has not yet + # installed the C++ headers -- so the whole toolchain stops working and + # nothing on disk says why. The marker is removed when the build finishes, + # whether it succeeded or failed cleanly. + local _interrupted="$STATE/interrupted" + mkdir -p "$STATE" 2>/dev/null || true + echo "$name" >> "$_interrupted" 2>/dev/null || true + # Interrupting during an install leaves the package half on disk, and for + # a toolchain package that breaks every later build. We cannot stop you + # pressing Ctrl-C, so record it plainly and make the next run fix it -- + # see the auto-finish in cmd_build_all. + trap '_note_interrupted "$name"; exit 130' INT TERM + + say "# building $name ..." + local rc=0 + make_wrappers + ensure_install_dirs_writable + [ "$as_root" = 1 ] || pregrant_python_sitedirs "$owner" "$s" + local envpass="LFS_CC_PHASE='$phase'" + # The wrappers exist to stop a PACKAGE USER doing things it may not: chown, + # setuid modes, re-moding shared directories. A ROOT step must not get + # them. Root legitimately runs `chown -R wget:wget /sources/wget-1.25.0` + # to hand a source tree to a package user, and the wrapper silently skipped + # it -- leaving the tree root-owned, so the build then failed with + # ./configure: line 4389: config.log: Permission denied + if [ "$as_root" = 1 ]; then + envpass="$envpass PATH='/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin'" + else + # wrappers first in PATH -- see make_wrappers() for why + envpass="$envpass PATH='$WRAPPERS:/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin'" + fi + # the books expect some test failures, so suites only run when asked + [ "$run_tests" = 1 ] && envpass="$envpass LFS_RUN_TESTS=1" + [ -n "${LFS_TGT:-}" ] && envpass="$envpass LFS_TGT='$LFS_TGT'" + # MAKEFLAGS: --jobs wins over the configured value (it must be appended + # LAST -- `env A=1 A=2` keeps the last one, so ordering decides). A package + # that races under -j12 usually builds fine with --jobs 1. + if [ -n "$jobs" ]; then + envpass="$envpass MAKEFLAGS='-j$jobs'" + elif [ -n "${MAKEFLAGS:-}" ]; then + envpass="$envpass MAKEFLAGS='$MAKEFLAGS'" + fi + # Keep pip out of the package user's ~/.local. PIP_USER=0 stops it + # installing there, and PYTHONNOUSERSITE=1 stops it *seeing* an existing + # per-user copy and concluding the requirement is "already satisfied" -- + # which is how flit_core ended up importable by nobody but flit-core. + if uses_pip "$s"; then + envpass="$envpass PIP_USER=0 PYTHONNOUSERSITE=1" + fi + + # How the build runs: + # * chapter-7 temp tools -> as root, no package user at all + # * package user + su present -> as that user (the real package-user way) + # * package user, NO su yet -> as root, then hand the freshly installed + # files to the package user. `su` comes from Shadow (chapter 8 #26), + # so for the first 25 packages there is no way to drop privileges -- + # but we know exactly which files appeared, so ownership still ends up + # correct. + mkdir -p "$LOGS" + local log="$LOGS/$name-$phase.log" + # Mark the start of the build. The snapshot diff finds files that are NEW, + # but a package very often OVERWRITES files an earlier one installed (the + # chapter-8 bison replaces the chapter-7 bison-tmp binaries). Those aren't + # new, so without a timestamp to compare against they'd keep the previous + # owner -- which is how bison's files stayed owned by root. + # The stamp is the reference for "what did this build touch". It must NOT + # live in /tmp: init-dirs is the step that CREATES /tmp (book 7.5), so + # mktemp fails there with + # mktemp: failed to create file via template '/tmp/tmp.XXXXXXXXXX' + # and the empty path that follows silently disables file tracking for the + # whole step. $STATE always exists -- it is where the scripts came from. + local stamp + mkdir -p "$STATE/tmp" 2>/dev/null || true + stamp="$STATE/tmp/stamp.$$" + : > "$stamp" 2>/dev/null || stamp="$(mktemp 2>/dev/null || echo "")" + [ -n "$stamp" ] && touch "$stamp" 2>/dev/null || true + if [ -z "$stamp" ] || [ ! -e "$stamp" ]; then + warn "# cannot create a build stamp -- file tracking is off for this step" + fi + if [ "$as_root" = 1 ]; then + ( cd "$SRCROOT/$name" && eval "env $envpass bash '$staged' '$phase'" ) \ + 2>&1 | tee "$log"; rc=${PIPESTATUS[0]} + elif have_su; then + # `su -` resets the environment, so the vars go on the command line + su - "$owner" -c "env $envpass bash '$staged' '$phase'" \ + 2>&1 | tee "$log"; rc=${PIPESTATUS[0]} + else + say "${C_DIM}# no 'su' yet (Shadow not built) -- building as root, then" + say "# giving the installed files to '$owner'${C_OFF}" + chown_after=1 + ( cd "$SRCROOT/$name" && eval "env $envpass bash '$staged' '$phase'" ) \ + 2>&1 | tee "$log"; rc=${PIPESTATUS[0]} + fi + + after="$(snapshot)" + local newfiles newdirs + newfiles="$(comm -13 <(printf '%s\n' "$before") <(printf '%s\n' "$after"))" + newdirs="$(comm -13 <(printf '%s\n' "$before_dirs") <(snapshot_dirs))" + printf '%s\n' "$after" > "$SNAPSHOT" + printf '%s\n' "$(snapshot_dirs)" > "$SNAPDIRS" + + # Everything this build wrote: created OR modified since the stamp. + # Always work out what the build TOUCHED, even when we are not going to + # change ownership. A config step rewrites files that already exist + # (/etc/hosts was created back in 7.6), so the snapshot diff sees nothing + # new and the step looks like it did nothing at all. + local touched="" touched_dirs="" + if true; then + local r="${SNAP_ROOT%/}" + # "${r:-/}": SNAP_ROOT is "/" inside the chroot, and "${SNAP_ROOT%/}" + # trims that to the empty string -- `find ""` then fails silently and + # NOTHING is ever seen as touched, so every config step looks like it + # did nothing at all. + # a missing stamp would make `find -newer` fail and silently report + # that the build touched nothing at all + [ -n "$stamp" ] && [ -e "$stamp" ] && \ + # DIRECTORIES this build wrote into, not just the ones it created. + # A directory that already existed -- made by root in chapter 7, or by + # an earlier run -- kept root ownership while every file inside it + # became the package's, so a package owned its files but not the + # directory holding them. Shared install dirs and directories another + # package owns are filtered out below, exactly as for files. + touched_dirs="$(find "${r:-/}" -xdev -type d -newer "$stamp" \ + -not -path "$r/.lfs-pkgusr*" -not -path "$SRCROOT/*" \ + -not -path "$r/sources/*" -not -path "$r/dev/*" \ + -not -path "$r/proc/*" -not -path "$r/sys/*" \ + -not -path "$r/run/*" -not -path "$r/tmp/*" \ + 2>/dev/null | sort -u)" + touched="$(find "${r:-/}" -xdev \( -type f -o -type l \) -newer "$stamp" \ + -not -path "$r/.lfs-pkgusr/*" -not -path "$SRCROOT/*" \ + -not -path "$r/sources/*" -not -path "$r/dev/*" \ + -not -path "$r/proc/*" -not -path "$r/sys/*" \ + -not -path "$r/run/*" -not -path "$r/tmp/*" \ + 2>/dev/null | sort -u)" + fi + rm -f "$stamp" + + local man="$MANIFESTS/$name.files" mand="$MANIFESTS/$name.dirs" + # Record what was touched regardless of whether ownership changed -- a + # config step that rewrites /etc/hosts installs no NEW file but has very + # much done its job. + { [ -f "$man" ] && cat "$man"; printf '%s\n' "$newfiles" + printf '%s\n' "$touched"; } \ + | grep -v '^$' | sort -u \ + | _drop_other_packages_files "$owner" > "$man.tmp" \ + && mv "$man.tmp" "$man" + { [ -f "$mand" ] && cat "$mand"; printf '%s\n' "$newdirs"; } \ + | grep -v '^$' | sort -u > "$mand.tmp" && mv "$mand.tmp" "$mand" + say "# tracked $(wc -l < "$man") file(s) for '$name'" + + + # A FAILED build must not take ownership of anything. + # + # gcc failing half-way still "tracked 828 file(s)" and chowned them -- so a + # broken, partial install claimed files from the working temporary system, + # and the next attempt inherited the damage. Ownership is a statement that + # this package installed these files successfully; if it did not, say + # nothing. The manifest is still written (it is the record of what the + # attempt touched, needed for cleanup) but nothing changes hands. + if [ "${rc:-0}" != "0" ] && [ "$chown_after" = 1 ]; then + warn "# build failed -- not changing ownership of the $(printf '%s\n%s\n' "$newfiles" "$touched" | grep -cv '^$') path(s) it touched" + chown_after=0 + fi + + if [ "$chown_after" = 1 ] && [ -n "$newfiles$newdirs$touched" ]; then + # hand everything this package just installed to its package user. + # Directories that belong to the install group are shared infrastructure + # (root:install) and must stay that way -- only take the ones this + # package created itself. + local n=0 p cur skipped_foreign=0 + while IFS= read -r p; do + [ -n "$p" ] || continue + # NEVER take a file that already belongs to another package user. + # + # "touched" is every file with a newer mtime than the build stamp, + # which is wider than "files this package installed": a build that + # rewrites, re-links or merely re-times something belonging to + # another package would claim it. That is how zstd came to own + # /usr/include/c++/15.2.0/bits/memoryfwd.h + # -- gcc's own C++ headers -- after which gcc could not build. + # Files owned by root are fair game (the temporary system's), files + # owned by another package are not. + cur="$(stat -c %U "$p" 2>/dev/null)" + if [ -n "$cur" ] && [ "$cur" != "$owner" ] && [ "$cur" != "root" ] \ + && [ "$cur" != "UNKNOWN" ] && user_exists "$cur"; then + skipped_foreign=$((skipped_foreign+1)) + continue + fi + chown -h "$owner:$owner" "$p" 2>/dev/null && n=$((n+1)) + done <<< "$(printf '%s\n%s\n' "$newfiles" "$touched" | grep -v '^$' | sort -u)" + while IFS= read -r p; do + [ -n "$p" ] || continue + # shared by nature (/usr/bin, /usr/include, ...): stays root:install + is_install_dir "$p" && continue + cur="$(stat -c %U "$p" 2>/dev/null)" + [ "$cur" = "$owner" ] && continue + # another package's directory: leave it, and let the collector-group + # machinery decide -- never take it silently + if [ -n "$cur" ] && [ "$cur" != root ] && user_exists "$cur"; then + skipped_foreign=$((skipped_foreign+1)) + continue + fi + chown "$owner:$owner" "$p" 2>/dev/null && n=$((n+1)) + done <<< "$(printf '%s\n%s\n' "$newdirs" "$touched_dirs" \ + | grep -v '^$' | sort -u)" + say "# gave $n newly installed path(s) to '$owner'" + [ "$skipped_foreign" -gt 0 ] && say "# ($skipped_foreign path(s) left with the package that owns them)" + fi + + # Automatic recovery: a "Permission denied" here almost always means the + # package needs to install into a directory another package owns. Grant it + # (collector groups) and retry the phase once, rather than making you run + # fix-perms by hand. + # Do not key this on "Permission denied": perl's installperl reports an + # unwritable destination directory as "No such file or directory", so the + # message is unreliable. auto_grant_from_log TESTS writability as the + # package user and does nothing when there is nothing to grant. + # An install reveals only the FIRST directory it cannot write; granting that + # one and retrying then reveals the next (perl modules span core_perl/, + # site_perl/ and more). So keep granting and retrying while each round + # actually grants something -- bounded, so a genuine build error can't loop. + # Only attempt a permission fix when the failure LOOKS like one. A script + # that stops because the USER must do something -- "mount your EFI System + # Partition first" -- must never be answered by handing directories away. + local _perm_ish=0 + grep -qiE "permission denied|operation not permitted|cannot (create|remove|change|touch)|couldn't copy|read-only file system" \ + "$log" 2>/dev/null && _perm_ish=1 + + # Grants are per-directory now (a collector group is permission to install + # into a directory, not ownership of a tree), so a package installing into + # several directories needs several rounds. The loop still stops as soon + # as a round grants nothing new, so a higher cap costs nothing when things + # are going well and only helps a package with a deep install layout. + local _round=0 _max_rounds=20 _last_need="" _need_now + while [ "$rc" != 0 ] && [ "$auto_fix" = 1 ] && [ "$_perm_ish" = 1 ] \ + && [ "$_round" -lt "$_max_rounds" ]; do + # If this round would grant exactly what the last one did, granting is + # not the answer -- stop rather than burn the whole retry budget. + _need_now="$(unwritable_dirs_from_log "$log" "$owner" | sort -u)" + if [ -n "$_last_need" ] && [ "$_need_now" = "$_last_need" ]; then + warn "the same directories keep coming up -- permissions are not the problem" + break + fi + _last_need="$_need_now" + auto_grant_from_log "$log" "$owner" || break # nothing left to grant + _round=$((_round + 1)) + say "" + say "# retrying $name (phase $phase), round $_round ..." + if [ "$as_root" = 1 ]; then + ( cd "$SRCROOT/$name" && eval "env $envpass bash '$staged' '$phase'" ) \ + 2>&1 | tee "$log"; rc=${PIPESTATUS[0]} + elif have_su; then + su - "$owner" -c "env $envpass bash '$staged' '$phase'" \ + 2>&1 | tee "$log"; rc=${PIPESTATUS[0]} + else + ( cd "$SRCROOT/$name" && eval "env $envpass bash '$staged' '$phase'" ) \ + 2>&1 | tee "$log"; rc=${PIPESTATUS[0]} + fi + after="$(snapshot)" + done + if [ "$_round" -ge "$_max_rounds" ] && [ "$rc" != 0 ]; then + warn "gave up after $_max_rounds permission grants -- something else is wrong" + fi + + # An 'all' run executes the phases in order and stops at the first failure, + # so everything before the failing phase did succeed. Record that, or the + # next run wrongly reports unpack/build as "never completed". + if [ "$phase" = "all" ]; then + local _failed _p + _failed="$(grep -oE "phase '[a-z]+' FAILED" "$log" 2>/dev/null \ + | head -n1 | cut -d"'" -f2)" + if [ -n "$_failed" ]; then + for _p in unpack build install configure; do + [ "$_p" = "$_failed" ] && break + record_phase "$name" "$_p" + done + fi + fi + + # Exit 4 means "there was deliberately nothing to do" -- e.g. rEFInd is + # already installed and already has an LFS entry. That is a success, and + # the "installed no files" check below must not treat it as suspicious. + local _nothing_to_do=0 + if [ "$rc" = 4 ]; then _nothing_to_do=1; rc=0; fi + + # Exit 3 means "this needs the user to do something", not a build error. + # Report it plainly: no failure banner, no permission fixing, no retrying. + if [ "$rc" = 3 ]; then + say "" + say "${C_WARN}$name is waiting on you -- see the message above.${C_OFF}" + say " When it's ready: lfs-helper build $name --force" + trap - INT TERM + _clear_interrupted "$name" + # Pass 3 up unchanged. Flattening it to 1 made "needs a decision" + # indistinguishable from "failed", so build-all could not tell the two + # apart -- refind slipped past and the build claimed to be complete + # with that step still pending. + return 3 + fi + + if [ "$rc" != 0 ]; then + echo "${C_ERR}" + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + echo "! $name FAILED (phase '$phase', exit $rc)" + echo "! script: $s" + echo "! staged: $SRCROOT/$name/install_$name" + if grep -qiE "permission denied|couldn't copy|cannot create|cannot remove" \ + "$log" 2>/dev/null; then + echo "! The package could not write into a directory owned by another" + echo "! package. Grant access (collector groups) and retry:" + echo "! lfs-helper fix-perms $name --phase $phase --run" + echo "! lfs-helper build $name --phase $phase --force" + fi + # A later phase often fails because an EARLIER one never finished -- + # gcc's `mv /usr/lib/*gdb.py` finds nothing when `make install` aborted + # partway (e.g. on a permission error) and never installed them. + if [ "$phase" != "all" ] && [ "$phase" != "unpack" ]; then + local _prev _missing="" + for _prev in unpack build install; do + [ "$_prev" = "$phase" ] && break + phase_done "$name" "$_prev" || _missing="$_missing $_prev" + done + if [ -n "$_missing" ]; then + echo "! NOTE: these earlier phases have not completed successfully for" + echo "! this package:$_missing" + echo "! A later phase can fail simply because an earlier one" + echo "! stopped partway and never installed everything. Run:" + for _prev in $_missing; do + echo "! lfs-helper build $name --phase $_prev --force" + done + fi + fi + echo "! Edit the script, then retry just this phase:" + echo "! lfs-helper build $name --phase $phase --force" + if [ -z "$jobs" ] && [ "$phase" != "install" ]; then + echo "! If the output ends with no obvious error, a parallel build may" + echo "! have raced -- the real message is far above the tail. Retry" + echo "! serially (slower, but the error appears where it happens):" + echo "! lfs-helper build $name --phase $phase --force --jobs 1" + fi + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + echo "${C_OFF}" + show_build_errors "$log" + diagnose_paths_from_log "$log" "$owner" + exit "$rc" + fi + # A build that installs NOTHING has almost certainly failed quietly. pip + # in particular can report success while writing nowhere useful -- flit-core + # "succeeded" without ever putting flit_core into site-packages, and the + # next package then failed with a baffling + # BackendUnavailable: Cannot import 'flit_core.buildapi' + # Say so loudly rather than marking the step done and moving on. + # grep -c prints 0 AND exits non-zero on no match, so a `|| echo 0` fallback + # produces "0\n0" and breaks the numeric test. Count with wc instead. + local _tracked; _tracked=$(count_lines "$man") + if [ "$_tracked" -eq 0 ] && [ "$_nothing_to_do" = 0 ] \ + && [ "$phase" != "unpack" ] && [ "$phase" != "test" ]; then + warn "" + warn "!! $name reported success but installed NO files." + warn " That normally means the install silently did nothing." + warn " Check the log before trusting this step:" + warn " $log" + warn " The step is NOT marked as built." + warn "" + return 1 + fi + + # Record what is installed, in the package user's own home. + # + # The scripts are regenerated from the book whenever it moves on, so their + # name_version tells you what the BOOK has, not what is on disk. Without a + # record of the latter, "is this up to date?" is unanswerable. + local _nv _home + _nv="$(sed -n 's/^name_version="\(.*\)"$/\1/p' "$s" 2>/dev/null | head -n1)" + _home="$SRCROOT/$owner" + if [ -n "$_nv" ] && [ -d "$_home" ]; then + { + echo "$_nv" + echo "# installed $(date '+%Y-%m-%d %H:%M:%S') by lfs-helper" + [ -n "${_book_of_script:-}" ] && echo "# from: $_book_of_script" + } > "$_home/VERSION" + chown "$owner:$owner" "$_home/VERSION" 2>/dev/null || true + fi + + if [ "$phase" = "all" ]; then + mark_done "$name" + ok "# $name: done" + report_granted_groups + trap - INT TERM + _clear_interrupted "$name" + else + record_phase "$name" "$phase" + local nxt; nxt="$(next_phase_for "$name" || true)" + if [ -n "$nxt" ]; then + ok "# $name: phase '$phase' finished" + say " still to do: $nxt -> lfs-helper build $name --phase $nxt" + else + # every phase the script defines has now run + mark_done "$name" + # This build already gave its files to the package user, so there + # is nothing for the adoption pass to do. Recording it here is + # what stops "Give the already-built packages their own users" + # running again on every build-all. + [ -f "$MANIFESTS/$name.files" ] && \ + _mark_adopted "$name" "$(_manifest_size "$MANIFESTS/$name.files")" + ok "# $name: done (all phases complete)" + report_granted_groups + fi + fi + # finished one way or another: no longer mid-install + trap - INT TERM + _clear_interrupted "$name" +} + +# Tell the user which collector groups this run had to create. These are real +# changes to the system's group database, so they should never be silent. +report_granted_groups() { + [ -f "$GRANTED" ] || return 0 + local n; n=$(count_lines "$GRANTED") + [ "$n" -gt 0 ] || return 0 + say "" + say "${C_WARN}Collector groups used during this run:${C_OFF}" + local grp user owner + while IFS='|' read -r grp user owner; do + [ -n "$grp" ] || continue + printf " %-24s lets '%s' install into '%s' directories\n" \ + "$grp" "$user" "$owner" + done < "$GRANTED" + say "" + say " These are permanent: the directory keeps its owner, but the group" + say " owns it group-writable so both packages can install there." + say " Review with: getent group | grep '^${COLLECTOR_PREFIX}_'" + : > "$GRANTED" +} + +# Is there a recorded package that has no package user yet? Cheap enough to +# check on every build-all, and it keeps the adoption pass quiet once done. +_has_unadopted_packages() { + local man + for man in "$MANIFESTS"/*.files; do + [ -e "$man" ] || continue + _needs_adoption "$man" && return 0 + done + return 1 +} + +# Which packages have already been adopted, and at what manifest size. +# +# A whole-manifest signature was useless: every build writes a manifest, so it +# changed constantly and adoption ran every time. Recording "name size" per +# package means a package is re-adopted only when ITS OWN manifest grows -- +# which is exactly when it has new files to hand over. +_ADOPTED_LIST() { echo "$STATE/adopted.list"; } + +_manifest_size() { + wc -c < "$1" 2>/dev/null | tr -d ' ' || echo 0 +} + +_adopted_size() { + local name="$1" f + f="$(_ADOPTED_LIST)" + [ -f "$f" ] || { echo ""; return; } + awk -v n="$name" '$1 == n { print $2; found=1 } END { if (!found) print "" }' \ + "$f" | head -n1 +} + +_mark_adopted() { + local name="$1" size="$2" f tmp + f="$(_ADOPTED_LIST)" + tmp="$f.$$" + { [ -f "$f" ] && grep -v "^$name " "$f"; echo "$name $size"; } > "$tmp" 2>/dev/null + mv -f "$tmp" "$f" 2>/dev/null || rm -f "$tmp" +} + +# A package needs adopting if it has no user yet, or its manifest has grown +# since the last time it was adopted. +_needs_adoption() { + local man="$1" name size + [ -s "$man" ] || return 1 + name="$(basename "${man%.files}")" + _is_not_a_package "$name" && return 1 + + # Adoption exists for ONE thing: packages built before the chroot, whose + # files were recorded but whose ownership could not be assigned at the time + # because no package users existed yet. + # + # Anything in this chroot's step list is built by lfs-helper, which gives + # each file to its package user as part of the build -- there is nothing + # left to adopt, and walking its manifest is pure waste. Checking that + # first is also much cheaper than statting thousands of files. + if steps | grep -qxF "$name"; then + return 1 + fi + + user_exists "$(pkg_owner_name "$name")" || return 0 + size="$(_manifest_size "$man")" + [ "$(_adopted_size "$name")" = "$size" ] && return 1 + return 0 +} + +# Is the compiler in this chroot actually usable? +# +# The book runs a sanity check after gcc-pass2 and again after entering the +# chroot, and it is worth doing before every chapter-8 build: a toolchain that +# cannot link a trivial program produces failures that look like anything but +# a broken compiler -- +# configure: error: cannot compute sizeof (long long) +# fatal error: bits/c++config.h: No such file or directory +# Repair a C++ header tree installed under a different target triplet. +# +# The chroot compiler reports one triplet (say x86_64-pc-linux-gnu) while +# chapter 6 installed its headers under another (x86_64-lfs-linux-gnu), so +# g++ never looks where c++config.h actually is: +# fatal error: bits/c++config.h: No such file or directory +# +# The two trees are the same gcc version on the same machine, so the headers +# are interchangeable -- a symlink from the triplet the compiler expects to +# the one that exists is enough to build the real chapter-8 gcc, which then +# installs a matching set and makes the link redundant. +# Which package installed this path? +# +# The manifests are the record of who installed what, so a file that should +# not exist can be traced to the step that created it. Answers questions like +# "where did this native x86_64-pc-linux-gnu compiler come from?" without +# guessing. +# Sort /etc/passwd and /etc/group by id. +# +# Package users are appended as they are created, so the files end up in build +# order rather than id order -- awkward to read when you are trying to see +# which package owns which id. Sorting is cosmetic and safe: nothing depends +# on the order of these files. System entries (id < 1000) are kept at the top +# in their existing order, because the first few lines of /etc/passwd are +# conventional (root first) and worth not disturbing. +# Files whose owner has no passwd entry in this chroot. +# +# The lfs user exists on the HOST, not in here, so anything it created shows +# as a raw uid. Recreate the account and hand those files to it, rather than +# leaving them owned by nobody in particular. +# Give one user access to one directory, using the normal rules. +# +# lfs-helper grant-dir <dir> <user> [--run] +# +# Exactly what a failed install triggers automatically: if the directory +# already carries a collector group, join it; if not, ask which group should +# own it. Exposed as a command so a script can ask for access BEFORE trying, +# instead of failing first and repairing afterwards. +cmd_grant_dir() { + need_root + local dir="${1:?usage: lfs-helper grant-dir <dir> <user> [--run]}" + local user="${2:?usage: lfs-helper grant-dir <dir> <user> [--run]}" + shift 2 + local run=0 + [ "${1:-}" = "--run" ] && run=1 + + [ -d "$dir" ] || die "no such directory: $dir" + user_exists "$user" || die "no such user: $user" + + if su -s /bin/bash "$user" -c "test -w '$dir'" 2>/dev/null; then + say "'$user' can already write into $dir" + return 0 + fi + if [ "$run" != 1 ]; then + say "'$user' cannot write into $dir" + say " owner: $(stat -c '%U:%G %A' "$dir" 2>/dev/null)" + warn "(dry run -- nothing was changed. Add --run to grant access.)" + return 0 + fi + grant_dir_access "$dir" "$user" +} + +# Is there anything with no passwd entry? Stops at the first hit, so it costs +# nothing on a tree that is already in order. +_has_orphaned_files() { + local r="${SNAP_ROOT%/}" hit + hit="$(find "${r:-/}" -xdev -nouser \ + -not -path "$r/dev/*" -not -path "$r/proc/*" \ + -not -path "$r/sys/*" -not -path "$r/run/*" \ + -print -quit 2>/dev/null)" + [ -n "$hit" ] +} + +cmd_fix_orphans() { + need_root + local run=0 + [ "${1:-}" = "--run" ] && run=1 + + # 1. make sure there IS an lfs user to own them + if ! user_exists lfs; then + say "No 'lfs' user in this chroot -- the build user from outside." + say "Creating it with uid $LFS_BUILD_UID so its files resolve to a name:" + if [ "$run" = 1 ]; then + if have_shadow_tools; then + groupadd -g "$LFS_BUILD_UID" lfs 2>/dev/null || true + useradd -M -u "$LFS_BUILD_UID" -g lfs -s /bin/bash \ + -c "build user from outside the chroot" lfs 2>/dev/null \ + || warn " could not create the lfs user" + else + printf 'lfs:x:%s:\n' "$LFS_BUILD_UID" >> "$ETC/group" + printf 'lfs:x:%s:%s:build user from outside the chroot:/:/bin/bash\n' \ + "$LFS_BUILD_UID" "$LFS_BUILD_UID" >> "$ETC/passwd" + fi + user_exists lfs && ok " created lfs ($LFS_BUILD_UID)" + else + say " would create lfs ($LFS_BUILD_UID)" + fi + say "" + fi + + # 2. hand the orphaned files over + local r="${SNAP_ROOT%/}" + say "Files owned by a uid with no passwd entry:" + # Write the full list to a file rather than capping it. An earlier + # version kept only the first 2000 paths and fixed just those, so /lib and + # /.lfs-pkgusr were still owned by a uid with no name afterwards -- and it + # reported success. + local list="$STATE/tmp/orphans.$$" + mkdir -p "$STATE/tmp" 2>/dev/null || true + find "${r:-/}" -xdev -nouser \ + -not -path "$r/dev/*" -not -path "$r/proc/*" \ + -not -path "$r/sys/*" -not -path "$r/run/*" \ + 2>/dev/null > "$list" + local n + n="$(count_lines "$list")" + if [ "${n:-0}" = 0 ]; then + rm -f "$list" + ok " none -- every file has a real owner" + return 0 + fi + head -n 8 "$list" | sed 's/^/ /' + [ "$n" -gt 8 ] && say " ... and $((n - 8)) more" + say "" + if [ "$run" != 1 ]; then + say " they would be given to 'lfs'" + rm -f "$list" + warn "(dry run -- nothing was changed. Add --run to apply.)" + return 0 + fi + local fixed=0 + while IFS= read -r p; do + [ -n "$p" ] || continue + chown -h lfs:lfs "$p" 2>/dev/null && fixed=$((fixed+1)) + # a few hundred thousand chowns with no output looks like a hang, and + # people cancel things that look hung + [ $((fixed % 5000)) = 0 ] && printf '\r %d of %d ...' "$fixed" "$n" >&2 + done < "$list" + printf '\r%*s\r' 40 '' >&2 + rm -f "$list" + ok " gave $fixed of $n path(s) to 'lfs'" + [ "$fixed" = "$n" ] || warn " ($((n - fixed)) could not be changed)" +} + +cmd_sort_users() { + need_root + local run=0 + [ "${1:-}" = "--run" ] && run=1 + + local f + for f in "$ETC/passwd" "$ETC/group"; do + [ -f "$f" ] || continue + local tmp="$f.sorted.$$" + # field 3 is the uid/gid in both files + { + awk -F: '$3 < 1000' "$f" + awk -F: '$3 >= 1000' "$f" | sort -t: -k3,3n + } > "$tmp" 2>/dev/null || { rm -f "$tmp"; continue; } + + # never write a file with fewer lines than we started with + local a b + a="$(wc -l < "$f")"; b="$(wc -l < "$tmp")" + if [ "$a" != "$b" ]; then + warn " $f: refusing to sort ($a lines in, $b out)" + rm -f "$tmp" + continue + fi + if cmp -s "$f" "$tmp"; then + say " $f is already in id order" + rm -f "$tmp" + continue + fi + if [ "$run" = 1 ]; then + cp -p "$f" "$f.bak" 2>/dev/null || true + cat "$tmp" > "$f" # preserve the inode, mode and owner + rm -f "$tmp" + ok " $f sorted by id (previous kept as $f.bak)" + else + say " $f would be sorted by id" + rm -f "$tmp" + fi + done + [ "$run" = 1 ] || warn "(dry run -- nothing was changed. Add --run to apply.)" +} + +cmd_which_package() { + local target="${1:?usage: lfs-helper which-package <path>}" + local man name hits=0 + + # match the path as recorded (manifests store chroot-absolute paths) + # Manifests store chroot-absolute paths. strip_host_prefix maps a real + # path INTO that form; a path the user typed is usually already in it, so + # try both rather than silently finding nothing. + local needle needle2 + needle="$target" + needle2="$(strip_host_prefix "$target")" + + say "Searching the manifests for: $needle" + say "" + for man in "$MANIFESTS"/*.files "$MANIFESTS"/*.dirs; do + [ -e "$man" ] || continue + if grep -qxF "$needle" "$man" 2>/dev/null \ + || grep -qxF "$needle2" "$man" 2>/dev/null; then + name="$(basename "$man")"; name="${name%.files}"; name="${name%.dirs}" + printf ' %-28s (%s)\n' "$name" "$(basename "$man")" + hits=$((hits+1)) + fi + done + + if [ "$hits" = 0 ]; then + # not an exact entry -- show packages that installed anything beneath it + say " No package recorded that exact path." + say "" + say " Packages that installed something under it:" + local found=0 + for man in "$MANIFESTS"/*.files; do + [ -e "$man" ] || continue + local n + n="$(grep -c -e "^${needle%/}/" -e "^${needle2%/}/" "$man" 2>/dev/null)" + if [ "${n:-0}" -gt 0 ]; then + name="$(basename "${man%.files}")" + printf ' %-28s %s file(s)\n' "$name" "$n" + found=1 + fi + done + if [ "$found" = 0 ]; then + say " (none -- nothing recorded installing it)" + say "" + say " Nothing recorded means it was installed before file tracking" + say " existed, or outside these tools entirely." + fi + fi + + # who owns it now, which may differ from who installed it + if [ -e "$target" ]; then + say "" + say " On disk now: $(stat -c '%A %U:%G' "$target" 2>/dev/null)" + fi +} + +cmd_fix_toolchain() { + need_root + local run=0 + [ "${1:-}" = "--run" ] && run=1 + + local mine have hdr base + mine="$(c++ -dumpmachine 2>/dev/null)" \ + || die "no working c++ -- this cannot be repaired automatically" + [ -n "$mine" ] || die "c++ -dumpmachine said nothing" + + hdr="" + for h in /usr/include/c++/*/*/bits/c++config.h; do + [ -e "$h" ] || continue + hdr="$h"; break + done + [ -n "$hdr" ] || die "no c++config.h anywhere under /usr/include/c++ -- + chapter 6's gcc-pass2 did not install the C++ headers. + Finish it outside the chroot: lfs build-system crosschain --run" + + have="$(basename "$(dirname "$(dirname "$hdr")")")" # the triplet dir + base="$(dirname "$(dirname "$(dirname "$hdr")")")" # .../c++/<version> + + if [ "$have" = "$mine" ]; then + ok "Nothing to repair: the headers are already under '$mine'." + say " If C++ still fails, it is not a triplet mismatch --" + say " run: lfs-helper check-toolchain" + return 0 + fi + + say "The compiler and its headers disagree about the target triplet:" + say "" + say " compiler reports : $mine" + say " headers are under: $have" + say " in : $base" + say "" + say "Same gcc version, same machine, so the headers are interchangeable." + say "Link the name the compiler expects to the one that exists:" + say "" + say " $base/$mine -> $have" + say "" + say "That is enough to build chapter 8's gcc, which installs a matching" + say "set of its own and makes this link unnecessary." + + if [ -e "$base/$mine" ]; then + warn "" + warn "$base/$mine already exists -- not touching it." + return 1 + fi + if [ "$run" != 1 ]; then + say "" + warn "(dry run -- nothing was changed. Add --run to apply.)" + return 0 + fi + + ln -sv "$have" "$base/$mine" || die "could not create the link" + say "" + if cmd_check_toolchain >/dev/null 2>&1; then + ok "C++ works now." + say "" + say "Next: lfs-helper build-all" + else + warn "The link is in place but C++ still fails." + say "Run 'lfs-helper check-toolchain' to see what else is wrong." + return 1 + fi +} + +cmd_check_toolchain() { + local rc=0 t + # not /tmp: it may not exist yet this early in the chroot + mkdir -p "$STATE/tmp" 2>/dev/null || true + t="$(mktemp -d -p "$STATE/tmp" 2>/dev/null || mktemp -d 2>/dev/null)" \ + || die "cannot create a temp dir" + say "Toolchain check:" + say "" + + # Identify the compiler FIRST. Everything else is downstream of which + # binary this is and how it was configured -- and "how it was configured" + # is the only thing that explains a triplet mismatch, since a compiler + # searches for headers under the target it was BUILT for, not the one the + # headers happen to be installed under. + say " The compiler:" + say " c++ : $(command -v c++ 2>/dev/null || echo '(not found)')" + say " gcc : $(command -v gcc 2>/dev/null || echo '(not found)')" + say " dumpmachine: $(c++ -dumpmachine 2>/dev/null || echo '(failed)')" + say " version : $(c++ --version 2>/dev/null | head -n1)" + local cfg + cfg="$(c++ -v 2>&1 | grep -m1 '^Configured with:')" + if [ -n "$cfg" ]; then + # the --host/--target it was built with is the whole story + say " built with :" + printf '%s\n' "$cfg" | tr ' ' '\n' \ + | grep -E '^--(host|target|build|prefix|with-gxx-include-dir)=' \ + | sed 's/^/ /' + fi + say "" + + # 1. C: compile and link + printf 'int main(void){return 0;}\n' > "$t/t.c" + if cc "$t/t.c" -o "$t/t" 2>"$t/cc.err"; then + ok " C compiles and links" + else + warn " C CANNOT compile a trivial program" + sed 's/^/ /' "$t/cc.err" | head -n 5 + rc=1 + fi + + # 2. the interpreter the book checks for + if [ -f "$t/t" ]; then + local interp + interp="$(readelf -l "$t/t" 2>/dev/null | grep -o '/lib64/ld-linux[^]]*' | head -n1)" + if [ -n "$interp" ]; then + say " interpreter: $interp" + fi + fi + + # 3. C++: this is what chapter 8's gcc needs and what fails first + printf '#include <memory>\nint main(void){return 0;}\n' > "$t/t.cc" + if c++ "$t/t.cc" -o "$t/tpp" 2>"$t/cxx.err"; then + ok " C++ compiles and links (headers are complete)" + else + warn " C++ CANNOT compile -- the C++ headers are incomplete" + sed 's/^/ /' "$t/cxx.err" | head -n 5 + rc=1 + say "" + say " Where the C++ headers actually are:" + local d found=0 + for d in /usr/include/c++/*/*/bits/c++config.h \ + /tools/*/include/c++/*/*/bits/c++config.h; do + [ -e "$d" ] || continue + say " $d" + found=1 + done + [ "$found" = 0 ] && say " (none found -- gcc-pass2 did not install them)" + say "" + # A header that exists and is world-readable can still be invisible: + # if any directory on the way to it lacks o+x, the compiler gets + # ENOENT -- "No such file or directory" -- not a permission error. + # That is indistinguishable from a missing file in the message, and it + # is exactly what a mis-assigned package-user directory causes. + local h + for h in /usr/include/c++/*/*/bits/c++config.h; do + [ -e "$h" ] || continue + local part="" comp bad="" + local IFS_SAVE="$IFS"; IFS=/ + for comp in $h; do + [ -n "$comp" ] || continue + part="$part/$comp" + if [ ! -r "$part" ] || { [ -d "$part" ] && [ ! -x "$part" ]; }; then + bad="$part" + break + fi + done + IFS="$IFS_SAVE" + if [ -n "$bad" ]; then + say "" + warn " This header EXISTS but cannot be reached:" + say " $h" + say " blocked at: $bad" + say " $(stat -c '%A %U:%G' "$bad" 2>/dev/null)" + say " Every directory on the way needs o+x, or the compiler" + say " gets 'No such file or directory' even though it is there." + say "" + say " Fix it with:" + say " lfs-helper fix-perms --run" + fi + done + + # The decisive fact: which directories the compiler actually searches. + # A header can exist, be world-readable, and sit on a fully traversable + # path, and still not be found -- because that directory is not in the + # search list at all. Permissions and existence cannot tell you that; + # only the compiler can. + say "" + say " Where this compiler actually looks for C++ headers:" + printf '' | c++ -x c++ -E -v - 2>&1 \ + | sed -n '/#include <...> search starts here/,/End of search list/p' \ + | grep -v 'search starts here\|End of search list' \ + | sed 's/^/ /' + say "" + say " If the directory holding c++config.h is not in that list, the" + say " compiler was built for a different target than the headers were" + say " installed for -- that is a mismatch, not a missing file." + say "" + + say " The compiler looks for them under its OWN target triplet:" + say " $(cc -dumpmachine 2>/dev/null || echo '(cc not working)')" + say " If the headers above are under a different triplet, chapter 6's" + say " gcc-pass2 was built for a different target than this compiler" + say " reports -- rebuild it with the LFS_TGT this tree was made with." + fi + + rm -rf "$t" + say "" + if [ "$rc" = 0 ]; then + ok "Toolchain is usable." + else + warn "Do not start chapter 8 until this is fixed." + fi + return $rc +} + +# Has the copied-in export already been applied? +_groups_already_imported() { + local f="$STATE/collector-groups.import" first + [ -f "$f" ] || return 0 + [ -f "$COLLECTOR_MAP" ] || return 1 + # every dir decision in the file must be in the map + while IFS='|' read -r kind a b; do + [ "$kind" = dir ] || continue + grep -qxF "$a|$b" "$COLLECTOR_MAP" 2>/dev/null || return 1 + done < <(grep -vE '^\s*(#|$)' "$f") + return 0 +} + +cmd_build_all() { + need_root + local force=0 from="" + while [ $# -gt 0 ]; do + case "$1" in + --force) force=1; shift ;; + --from) from="${2:?--from needs a step name}"; shift 2 ;; + *) die "usage: lfs-helper build-all [--from <step>] [--force]" ;; + esac + done + if [ -n "$from" ]; then + steps | grep -qxF "$from" \ + || die "no such step: $from (see: lfs-helper list)" + fi + + # Set the package-user system up NOW, before the first package is built, + # not when we first trip over a package that needs it. The earlier it + # exists, the fewer files are installed as root and have to be given back + # afterwards -- and it is what this whole toolchain is for. + if ! pkgusr_ready; then + say "Setting up the package-user system first (it must exist before" + say "any package is built):" + cmd_init_pkgusr --run || true + say "" + fi + + # A groups export copied into the tree is a set of decisions already made + # -- apply them before anything can ask the same questions again. Leaving + # this to be run by hand meant the file sat in the tree while the build + # stopped to ask about directories it already had an answer for. + if [ -f "$STATE/collector-groups.import" ] && ! _groups_already_imported; then + say "Applying the collector-group decisions copied into this tree:" + cmd_import_groups --run || true + say "" + fi + + # Files carrying the outside build user's uid have no name in here, and + # every ownership check calls them UNKNOWN. Fix that once, up front. + # Gate on the WORK, not on a side effect of starting it. + # + # This used to run only when the lfs user was missing -- but the user is + # created first, so cancelling part-way through the chown left the user in + # place and every later run skipped the repair entirely, with half a + # million files still owned by nobody. + if _has_orphaned_files; then + say "Some files have no owner in this chroot (created outside it)." + cmd_fix_orphans --run || true + say "" + fi + + # Then hand the ALREADY-built packages to their own users, before building + # anything new. Chapters 5-7 ran before package users existed, so their + # files are root-owned and their manifests are the only record of who + # installed what. Doing this now means: + # * the first package users are the packages actually installed first, + # not whichever chapter-8 package happened to come along; + # * bash, coreutils, glibc and the rest own their files from here on; + # * a chapter-8 package that replaces a chapter-7 file writes into a + # directory its own user already owns, instead of hitting root. + # It is idempotent -- a package that already has its user is skipped. + if pkgusr_ready && [ -d "$MANIFESTS" ] && _has_unadopted_packages; then + say "Giving the already-built packages their own users first:" + cmd_adopt_existing --run || true + say "" + fi + + local total done_n=0 s _needs_you="" + total=$(steps | wc -l) + [ "$total" -gt 0 ] || die "no scripts found in $SCRIPTS + Generate them from outside the chroot: + lfs build-system gen-chroot-scripts --run" + # Work out which steps are in range, and whether any are already built. + # Rather than making you remember --force, just ask. + local in_range=() built=() started=1 + [ -n "$from" ] && started=0 + while read -r s; do + if [ "$started" = 0 ]; then + [ "$s" = "$from" ] && started=1 || continue + fi + in_range+=("$s") + is_done "$s" && built+=("$s") + done < <(steps) + + if [ "${#built[@]}" -gt 0 ] && [ "$force" = 0 ]; then + say "${#built[@]} of the ${#in_range[@]} step(s) in range are already built:" + # When the list is long, collapse the OLD ones into a numbered range and + # spell out only the most recent -- those are the ones you care about. + local _show=6 _n="${#built[@]}" + if [ "$_n" -gt "$_show" ]; then + say " steps 1-$(( _n - _show )) (built earlier)" + say " ... most recently:" + printf ' %s\n' "${built[@]:$(( _n - _show ))}" + else + printf ' %s\n' "${built[@]}" + fi + # Spell out what each answer does -- "Rebuild those too? [y/N]" on its + # own doesn't say what happens when you decline. + local _todo=$(( ${#in_range[@]} - ${#built[@]} )) + say "" + say " y = rebuild all ${#in_range[@]} step(s) from the start of the range" + if [ "$_todo" -gt 0 ]; then + local _first="" + local _s + for _s in "${in_range[@]}"; do + is_done "$_s" && continue + _first="$_s"; break + done + say " N = keep them, build only the $_todo remaining step(s)," \ + "starting with '$_first'" + else + say " N = keep them; nothing would be built (everything is done)" + fi + if [ -t 0 ]; then + printf 'Rebuild those too? [y/N]: ' + local ans; read -e -r ans + case "$ans" in [Yy]*) force=1; say "# rebuilding everything in range" ;; + *) say "# keeping them; building the $_todo remaining step(s)" ;; + esac + else + say "# not a terminal -- keeping them, building the $_todo remaining step(s)" + fi + say "" + fi + + local i=0 + started=1; [ -n "$from" ] && started=0 + while read -r s; do + i=$((i+1)) + # --from: skip everything before the named step, then build onward + if [ "$started" = 0 ]; then + if [ "$s" = "$from" ]; then started=1; else + say "${C_DIM}[$i/$total] $s -- before --from $from, skipping${C_OFF}" + continue + fi + fi + if is_done "$s" && [ "$force" = 0 ]; then + say "${C_DIM}[$i/$total] $s -- already built, skipping${C_OFF}" + done_n=$((done_n+1)) + continue + fi + # A package left half-installed by an interrupted build breaks + # everything after it, and the failure never points back here. + if _interrupted_packages >/dev/null 2>&1; then + warn "" + warn "These packages were interrupted mid-install and may be" + warn "half on disk:" + _interrupted_packages | sed 's/^/ /' >&2 + warn "" + warn "Finish one before building anything else:" + warn " lfs-helper build <name> --phase install --force" + warn "Or check what state the toolchain is in:" + warn " lfs-helper check-toolchain" + warn "" + fi + + # The toolchain check belongs HERE, not at the top: chapter 7 builds + # the temporary tools and needs no C++ compiler at all, so checking up + # front blocked the very steps that would fix a broken toolchain -- + # build-all simply did nothing, with nothing to say why. Check once, + # before the first package that actually needs a compiler. + if ! is_root_step "$s" && [ "${_tc_checked:-0}" = 0 ]; then + _tc_checked=1 + if ! cmd_check_toolchain >/dev/null 2>&1; then + # A toolchain package interrupted mid-install is the usual + # cause: `make install` replaced /usr/bin/gcc with the new + # compiler but had not yet installed its headers, so the new + # one shadows the working one and nothing can build. The fix + # is to finish the install that was interrupted -- the tool + # knows which package that was, so do it rather than printing + # a diagnosis and stopping. + local _pkg _fixed=0 _candidates + # The marker only exists if the interruption happened while a + # version of this tool that writes one was running. A package + # that has already installed files but is not marked built is + # in exactly the same state, so treat it the same way -- that + # is precisely what a half-installed gcc looks like. + _candidates="$(_interrupted_packages 2>/dev/null)" + if [ -z "$_candidates" ] && [ -s "$MANIFESTS/$s.files" ] \ + && ! is_done "$s"; then + _candidates="$s" + say "" + say "'$s' has already installed files but is not marked" + say "built -- an earlier attempt stopped part-way." + fi + for _pkg in $_candidates; do + warn "" + warn "The compiler is broken, and '$_pkg' was interrupted" + warn "mid-install -- that is almost certainly why." + say "Finishing its install before going on:" + say "" + if ( cmd_build "$_pkg" --phase install --force ); then + _fixed=1 + fi + if cmd_check_toolchain >/dev/null 2>&1; then + ok "" + ok "# the compiler works again -- continuing" + say "" + _fixed=1 + break + fi + done + if ! cmd_check_toolchain >/dev/null 2>&1; then + warn "" + warn "Stopping before '$s': this is the first package that" + warn "needs a working C++ compiler, and there is not one." + if [ "$_fixed" = 1 ]; then + warn "Finishing the interrupted install did not repair it." + fi + say "" + cmd_check_toolchain || true + return 1 + fi + fi + fi + + if ! is_root_step "$s" && ! pkgusr_ready; then + # Set it up rather than stopping. The package-user system is not + # an optional extra here -- it is the whole point of this toolchain, + # and every chapter-8 package needs it. Asking the user to run one + # obvious command and then re-issue build-all is just a detour. + say "" + say "'$s' is the first package that needs the package-user system." + say "Setting it up now (lfs-helper init-pkgusr):" + if cmd_init_pkgusr --run; then + say "" + ok " package-user system ready -- continuing with $s" + say "" + fi + fi + if ! is_root_step "$s" && ! pkgusr_ready; then + warn "" + warn "Stopping: '$s' is a chapter-8 package and needs the" + warn "package-user system. Set it up, then re-run build-all:" + warn " lfs-helper init-pkgusr --run" + warn " lfs-helper build-all" + return 0 + fi + # Seal immediately BEFORE the final step. By now every package is + # built, so the install directories no longer need to be writable by + # everyone -- and last-step runs after the seal, which is the correct + # order: it installs new files (allowed under the sticky bit) rather + # than replacing other packages' (which is what the seal prevents). + if [ "$s" = "last-step" ] && ! _install_dirs_are_sealed; then + say "" + say "Everything before the final step is built -- sealing the" + say "install directories now, so packages can no longer overwrite" + say "each other's files:" + say "" + cmd_seal_install_dirs --run + fi + + say "" + say "===== [$i/$total] $s =====" + # pass the rebuild decision down -- otherwise the per-package check + # prompts (or skips) again for something we already agreed to rebuild + # The result matters. It used to be discarded, so a step that exits 3 + # ("this one needs you") -- refind, which cannot guess your ESP -- + # slipped past silently and the build reported everything complete + # with that step still pending. + local _rc=0 + if [ "$force" = 1 ]; then cmd_build "$s" --force || _rc=$? + else cmd_build "$s" || _rc=$?; fi + case "$_rc" in + 0|4) ;; # done, or deliberately nothing to do + 3) _needs_you="$_needs_you $s" ;; + *) warn "" + warn "'$s' failed (exit $_rc) -- stopping here." + warn "Fix it, then run build-all again; it continues from '$s'." + return "$_rc" ;; + esac + done < <(steps) + report_granted_groups + + if [ -n "${_needs_you# }" ]; then + say "" + warn "These steps need a decision from you and were not built:" + local _p + for _p in $_needs_you; do + say " lfs-helper build $_p" + done + say "" + say "Everything else is built. Deal with them, then run build-all" + say "again -- or skip one deliberately with: lfs-helper done <name>" + return 0 + fi + + ok "" + ok "all steps complete ($total total)." + + # The genuinely last step of the chroot stage. Until now the install + # directories were group-writable but NOT sticky, so package users could + # replace the temporary system's root-owned files and each other's -- which + # the build needs. Now that everything is built, seal them: a package user + # can still create files there but no longer modify or delete another + # package's. Only the install directories; collector-group directories are + # shared on purpose and are left alone. + if ! _install_dirs_are_sealed; then + say "" + say "Last step: making the install directories sticky, so packages can" + say "no longer overwrite each other's files. (Collector-group" + say "directories are shared deliberately and are left alone.)" + say "" + cmd_seal_install_dirs --run + fi + + # Package users are appended as they are created, so by now /etc/passwd is + # in build order. Sorting by id makes it readable -- and it is purely + # cosmetic, so it is safe to do without asking. + say "" + cmd_sort_users --run || true +} + +# Are the install directories already sticky? Checking a handful of the ones +# that certainly exist is enough to tell whether sealing has been done. +_install_dirs_are_sealed() { + local d probe=0 sealed=0 + while IFS= read -r d; do + d="${SNAP_ROOT%/}$d" + [ -d "$d" ] || continue + probe=$((probe + 1)) + [ -k "$d" ] && sealed=$((sealed + 1)) + [ "$probe" -ge 5 ] && break + done < <(install_dirs_list) + [ "$probe" -gt 0 ] && [ "$sealed" = "$probe" ] +} + +# --------------------------------------------------------------------------- # +# reporting +# --------------------------------------------------------------------------- # +cmd_status() { + local total done_n + total=$(steps | wc -l) + done_n=$(count_lines "$PROGRESS") + say "LFS chroot session" + say " state dir : $STATE" + say " scripts : $SCRIPTS ($(ls -1 "$SCRIPTS" 2>/dev/null | wc -l) available)" + say " package-user: $(pkgusr_ready && echo 'ready' || echo 'NOT set up')" + if pkgusr_ready; then + local probe="${SNAP_ROOT%/}/usr/bin" sticky="no" + [ -k "$probe" ] && sticky="yes" + say " install dirs : group-writable, sticky=$sticky" + if [ "$sticky" = "no" ]; then + say " (correct while building -- sticky would stop one" + say " package replacing another's files. Add it once" + say " every package is built, to complete the chroot" + say " stage: lfs-helper seal-install-dirs --run)" + fi + fi + say " progress : $done_n / $total step(s) built" + if [ -f "$PROGRESS" ] && [ "$done_n" -gt 0 ]; then + say " last built : $(tail -n1 "$PROGRESS")" + fi +} + +cmd_list() { + local total; total=$(steps | wc -l) + [ "$total" -gt 0 ] || { warn "no scripts in $SCRIPTS"; return 0; } + local i=0 s + while read -r s; do + i=$((i+1)) + if is_done "$s"; then + printf " %3d. %-22s ${C_OK}built${C_OFF}\n" "$i" "$s" + elif [ -f "$(phase_file "$s")" ]; then + printf " %3d. %-22s ${C_WARN}partial${C_OFF} (%s)\n" "$i" "$s" \ + "$(tr '\n' ' ' < "$(phase_file "$s")" | sed 's/ $//')" + else + printf " %3d. %-22s ${C_DIM}pending${C_OFF}\n" "$i" "$s" + fi + done < <(steps) +} + +# Are there manifests from the pre-chroot build that nobody owns yet? +needs_adoption() { + is_done "adopt-existing" && return 1 + local man name + for man in "$MANIFESTS"/*.files; do + [ -e "$man" ] || continue + [ -s "$man" ] || continue + name="$(basename "${man%.files}")" + case "$name" in init-*) continue ;; esac + user_exists "$(pkg_owner_name "$name")" && continue + return 0 + done + return 1 +} + +cmd_next() { + local s + # walk the list in order; the first unfinished step IS the next step + while read -r s; do + is_done "$s" && continue + case "$s" in + init-dirs) + say "Next step: create the directory tree (book 7.5)" + say " lfs-helper build init-dirs" ;; + init-files) + say "Next step: create /etc/passwd, /etc/group etc. (book 7.6)" + say " lfs-helper build init-files" + say "" + say " (this is why the prompt says \"I have no name!\" -- after" + say " it runs, type: exec /usr/bin/bash --login)" ;; + *) + # a partially-built step: continue where it left off + local _nxt + if [ -f "$(phase_file "$s")" ]; then + _nxt="$(next_phase_for "$s" || true)" + if [ -n "$_nxt" ]; then + say "Next step: continue '$s' -- already done: $(tr '\n' ' ' < "$(phase_file "$s")" | sed 's/ $//')" + say " lfs-helper build $s --phase $_nxt" + say "" + say " (or redo the whole package: lfs-helper build $s --force)" + return 0 + fi + fi + # step 3: hand the already-built packages to package users, + # before anything new is compiled + if needs_adoption; then + if ! pkgusr_ready; then + say "Next step: set up the package-user system" + say " lfs-helper init-pkgusr --run" + else + say "Next step: give the already-built packages their" + say "package users (chapters 5-6, from their manifests)" + say " lfs-helper adopt-existing --run" + fi + return 0 + fi + if is_root_step "$s"; then + say "Next step: build '$s' (temporary chapter-7 tool, as root)" + say " lfs-helper build $s" + elif ! pkgusr_ready; then + say "Next step: set up the package-user system" + say " lfs-helper init-pkgusr --run" + say "" + say " (chapter 8 starts here -- from now on each package is" + say " built by its own package user)" + return 0 + else + local nxt; nxt="$(next_phase_for "$s" || true)" + if [ -f "$(phase_file "$s")" ] && [ -n "$nxt" ]; then + say "Next step: continue '$s' -- $(tr '\n' ' ' < "$(phase_file "$s")")already done" + say " lfs-helper build $s --phase $nxt" + else + say "Next step: build '$s' (as its own package user)" + say " lfs-helper build $s" + fi + fi ;; + esac + say "" + say " (or build everything remaining: lfs-helper build-all)" + return 0 + done < <(steps) + ok "Everything in the step list is built." + say " review: lfs-helper list" + + # Finishing steps, in order. Skipping these is how a system ends up with + # thousands of root-owned files inside package trees: builds that OVERWROTE + # a chapter-7 file left it owned by root, and only the manifests know who + # it really belongs to. + local _leftover + _leftover="$(find "${SNAP_ROOT%/}/usr/lib" "${SNAP_ROOT%/}/usr/share" \ + -maxdepth 2 -type d -user root 2>/dev/null \ + | while IFS= read -r d; do is_install_dir "$d" || echo "$d"; done \ + | head -n 1)" + if [ -n "$_leftover" ]; then + say "" + say "Next: give every package its files (some are still owned by root," + say "e.g. $_leftover)" + say " lfs-helper fix-ownership --run" + say " lfs-helper adopt-dirs --run" + return 0 + fi + local probe="${SNAP_ROOT%/}/usr/bin" + if pkgusr_ready && [ -d "$probe" ] && [ ! -k "$probe" ]; then + say "" + say "Final step: make the install directories sticky, so a package user" + say "can no longer modify another package's files:" + say " lfs-helper seal-install-dirs --run" + fi +} + +cmd_manifests() { + local name="${1:-}" + if [ -n "$name" ]; then + local shown=0 + if [ -s "$MANIFESTS/$name.files" ]; then + say "# files"; cat "$MANIFESTS/$name.files"; shown=1 + fi + if [ -s "$MANIFESTS/$name.dirs" ]; then + say "# directories"; cat "$MANIFESTS/$name.dirs"; shown=1 + fi + [ "$shown" = 1 ] || die "no manifest for $name" + return 0 + fi + local total=0 total_d=0 f n + for f in "$MANIFESTS"/*.files; do + [ -e "$f" ] || continue + n=$(wc -l < "$f"); total=$((total+n)) + # directories too: a package owns the dirs it creates, and one with + # files but no dirs is a tracking bug worth seeing + local nm nd flag + nm="$(basename "${f%.files}")" + nd=0 + [ -f "$MANIFESTS/$nm.dirs" ] && nd="$(count_lines "$MANIFESTS/$nm.dirs")" + flag="" + [ "$n" -gt 0 ] && [ "$nd" = 0 ] && flag=" <- no directories recorded" + printf " %-22s %6d files %5d dirs%s\n" "$nm" "$n" "$nd" "$flag" + total_d=$((total_d + nd)) + done + say "" + say " $total file(s) and ${total_d:-0} directory(ies) tracked" +} + +# The hint's final chroot-stage step: once everything is built, install +# directories become sticky, so from then on a package user can add files but +# cannot modify or delete files belonging to OTHER packages. Doing this any +# earlier would block package users from overwriting the temporary system. +# Chapters 5-6 were built OUTSIDE the chroot, as the lfs user, and their file +# lists were recorded with host paths ($LFS/usr/bin/...). In here those same +# files are /usr/bin/..., so translate before touching anything. +strip_host_prefix() { + local p="$1" m="${LFS_HOST_MOUNT:-}" r="${SNAP_ROOT%/}" + [ -n "$m" ] && [ "${p#$m/}" != "$p" ] && p="/${p#$m/}" + # anchor to the tracked root: "/" in the real chroot (so unchanged), but the + # test root elsewhere -- without this a stray manifest entry could point at + # the HOST's /usr/bin instead of the tree we're working on + printf '%s\n' "$r$p" +} + +# Hand the already-built packages (chapters 5-6) to package users, using the +# manifests recorded during those builds. Run this once, right after 7.6, so +# every package on the system already belongs to its own user before the first +# chapter-8 package is compiled. +# Undo a bad adoption round. Two things went wrong before the fixes above: +# * the first in-chroot build compared against a snapshot recorded OUTSIDE +# the chroot, so it claimed every pre-existing file as its own (init-dirs +# ended up "owning" ~11000 files that belong to gzip, bash, ...); +# * root-only steps (init-dirs/init-files, the *-tmp tools) were given +# package users they should never have had. +# This removes those users, discards the bogus manifests, and puts the affected +# files back to root so a corrected adopt-existing can assign them properly. +# Chapters 5-6 were tracked before directory recording existed, so those +# packages have file lists but no .dirs list -- which is why they all report +# "0 dirs" even though gcc plainly creates /usr/lib/gcc/<triplet>/<version>/. +# Reconstruct the directories from the file lists: take every parent directory +# of every tracked file, drop the shared install directories, and assign a +# directory only when ALL the tracked files under it come from ONE package. +# A directory shared by two packages stays root-owned, which is the safe answer. +# Does <dir> hold files owned by a package user other than <owner>? +# Only the immediate contents are checked: cheap, and enough to catch a +# package reaching into a tree that is not its own. +_dir_contents_foreign() { + local d="$1" owner="$2" e u + [ -d "$d" ] || return 1 + for e in "$d"/*; do + [ -e "$e" ] || continue + u="$(stat -c %U "$e" 2>/dev/null)" || continue + case "$u" in + "$owner"|root|UNKNOWN|"") continue ;; + esac + # another PACKAGE user owns something in here -- leave the dir alone + user_exists "$u" && return 0 + done + return 1 +} + +infer_dirs() { + local tmp="$STATE/.dirinfer.$$" + : > "$tmp" + local man name owner p d r="${SNAP_ROOT%/}" + for man in "$MANIFESTS"/*.files; do + [ -e "$man" ] || continue + [ -s "$man" ] || continue + name="$(basename "${man%.files}")" + is_root_step "$name" && continue + owner="$(pkg_owner_name "$name")" + while IFS= read -r p; do + [ -n "$p" ] || continue + d="$(strip_host_prefix "$p")" + d="${d%/*}" + # walk up to the tracked root + while [ -n "$d" ] && [ "$d" != "$r" ] && [ "$d" != "/" ]; do + printf '%s\t%s\n' "$d" "$owner" >> "$tmp" + d="${d%/*}" + done + done < "$man" + done + # keep only dirs with exactly one owner + sort -u "$tmp" | awk -F'\t' ' + { if ($1 == last) { multi=1 } else { if (last != "" && !multi) print last "\t" lastowner + last=$1; lastowner=$2; multi=0 } } + END { if (last != "" && !multi) print last "\t" lastowner }' + rm -f "$tmp" +} + +cmd_adopt_dirs() { + need_root + local run=0; [ "${1:-}" = "--run" ] && run=1 + pkgusr_ready || die "run 'lfs-helper init-pkgusr --run' first" + + say "Assign directories that packages created for themselves." + say "Derived from the file lists: a directory goes to a package only when" + say "every tracked file under it belongs to that one package. Shared" + say "directories stay root:install." + say "" + [ "$run" = 1 ] || warn "(dry run -- nothing was changed. Add --run to apply.)" + say "" + + local d owner n=0 skipped=0 + while IFS="$(printf '\t')" read -r d owner; do + [ -n "$d" ] && [ -n "$owner" ] || continue + # Never take a directory whose contents belong to someone else. + # + # infer_dirs walks UP from every installed file, so a package that + # drops one file deep inside another package's tree claims every + # parent on the way -- which is how zstd came to own + # /usr/include/c++/15.2.0/bits, gcc's own C++ headers, and gcc could + # then no longer build. Being the only package with a manifest entry + # under a directory does not make it yours. + _dir_contents_foreign "$d" "$owner" && { skipped=$((skipped+1)); continue; } + [ -d "$d" ] || continue + is_install_dir "$d" && { skipped=$((skipped+1)); continue; } + user_exists "$owner" || continue + if [ "$run" = 1 ]; then + chown "$owner:$owner" "$d" 2>/dev/null && n=$((n+1)) + else + n=$((n+1)) + [ "$n" -le 15 ] && say " $d -> $owner" + fi + done < <(infer_dirs) + + say "" + if [ "$run" = 1 ]; then + ok " $n directory(ies) assigned to their package" + else + say " would assign $n directory(ies) ($skipped shared install dirs left alone)" + fi + return 0 +} + +cmd_repair_adoption() { + need_root + local run=0; [ "${1:-}" = "--run" ] && run=1 + + say "Repair a bad adoption round." + say "" + [ "$run" = 1 ] || warn "(dry run -- nothing was changed. Add --run to apply.)" + say "" + + local man name n p tgt users=0 files=0 owner + for man in "$MANIFESTS"/*.files; do + [ -e "$man" ] || continue + name="$(basename "${man%.files}")" + owner="$(pkg_owner_name "$name")" + # Two kinds of bad user to undo: + # * root-only steps that should own nothing at all + # * per-pass users (gcc-pass1, binutils-pass2, libstdcpp) whose files + # belong to the base package instead + if ! is_root_step "$name" && [ "$owner" = "$name" ]; then + continue + fi + + # 1. hand the files back to root (their real owners get them from their + # own manifests when adopt-existing runs again) + n=0 + while IFS= read -r p; do + [ -n "$p" ] || continue + tgt="$(strip_host_prefix "$p")" + [ -e "$tgt" ] || [ -L "$tgt" ] || continue + if [ "$run" = 1 ]; then + chown -h root:root "$tgt" 2>/dev/null && n=$((n+1)) + else + n=$((n+1)) + fi + done < "$man" + + # 2. init-dirs/init-files create directories and /etc files; they never + # "install" a package, so their manifests are meaningless -- drop + # them. The *-tmp manifests are real and are kept. + case "$name" in + init-*) [ "$run" = 1 ] && : > "$man" + say " $name: cleared bogus manifest ($n file(s) returned to root)" ;; + *) if [ "$owner" != "$name" ]; then + say " $name: $n file(s) will be re-assigned to '$owner' (manifest kept)" + else + say " $name: $n file(s) returned to root (manifest kept)" + fi ;; + esac + files=$((files+n)) + + # 3. remove the package user/group it should never have had + if user_exists "$name"; then + if [ "$run" = 1 ]; then + if have_shadow_tools && command -v userdel >/dev/null 2>&1; then + userdel "$name" 2>/dev/null + groupdel "$name" 2>/dev/null + else + sed -i "/^$name:/d" "$ETC/passwd" + sed -i "/^$name:/d" "$ETC/group" + drop_from_install_group "$name" + fi + fi + say " removed package user '$name'" + users=$((users+1)) + fi + done + + # A user can exist without a manifest (e.g. a step that failed part-way), + # so also sweep the passwd file for package users whose name should never + # have been a user in the first place. + local u + while IFS=: read -r u _x uid _rest; do + [ -n "$u" ] || continue + [ "$uid" -ge "$PKG_UID_MIN" ] 2>/dev/null || continue + if is_root_step "$u" || [ "$(pkg_owner_name "$u")" != "$u" ]; then + if [ "$run" = 1 ]; then + if have_shadow_tools && command -v userdel >/dev/null 2>&1; then + userdel "$u" 2>/dev/null; groupdel "$u" 2>/dev/null + else + sed -i "/^$u:/d" "$ETC/passwd" + sed -i "/^$u:/d" "$ETC/group" + drop_from_install_group "$u" + fi + fi + say " removed stray package user '$u'" + users=$((users+1)) + fi + done < "$ETC/passwd" + + say "" + if [ "$run" = 1 ]; then + ok " $users bogus user(s) removed, $files file(s) returned to root" + unmark "adopt-existing" + say "" + say "Now assign ownership correctly:" + say " lfs-helper adopt-existing --run" + else + say " would remove $users user(s) and return $files file(s) to root" + fi + return 0 +} + +# Manifests oldest-first: the order the packages were built in. +_manifests_in_build_order() { + local f + for f in "$MANIFESTS"/*.files; do + [ -e "$f" ] || continue + # %.Y is nanosecond precision: whole seconds tie for packages built + # in the same second, and the order then falls back to alphabetical + printf '%s\t%s\n' "$(stat -c %.Y "$f" 2>/dev/null || echo 0)" "$f" + done | sort -n -k1,1 | cut -f2- +} + +cmd_adopt_existing() { + need_root + local run=0 verbose=0 + while [ $# -gt 0 ]; do + case "$1" in + --run) run=1; shift ;; + -v|--verbose) verbose=1; shift ;; + *) die "usage: lfs-helper adopt-existing [--run]" ;; + esac + done + pkgusr_ready || die "run 'lfs-helper init-pkgusr --run' first" + + say "Give the CHAPTER 5-6 packages their own package users." + say "" + say "Those were built outside the chroot, before any package user existed," + say "so their files are still root-owned -- only their recorded file lists" + say "say who installed what. Packages built in here already got their" + say "owner as part of the build and are not touched." + say "" + [ "$run" = 1 ] || warn "(dry run -- nothing was changed. Add --run to apply.)" + say "" + + local man name n nd p tgt total_f=0 total_p=0 skipped=0 + # In BUILD order, not alphabetical. + # + # The uid a package gets is the order its user is created in, so walking + # the manifests alphabetically gave gcc (built 2nd of 22) a higher uid than + # man-pages (the first chapter-8 package). Sorting by manifest mtime -- + # written when the package was built -- makes the uids a record of the + # order the system was actually built in. + for man in $(_manifests_in_build_order); do + [ -e "$man" ] || continue + name="$(basename "${man%.files}")" + # init-dirs/init-files are book housekeeping -- they create the + # directory tree and /etc files and own nothing, so they get no user. + # + # The *-tmp temporary tools DO get adopted, to the base package user + # (perl-tmp -> perl). Leaving them root-owned was a mistake: chapter 8 + # rebuilds exactly these packages, and its package user then cannot + # write into the root-owned tree chapter 7 left behind -- which is what + # made perl fail in /usr/lib/perl5/5.42/core_perl. Since it is the same + # package either way, the same user should own it throughout. + case "$name" in + init-*) skipped=$((skipped+1)); continue ;; + esac + [ -s "$man" ] || { skipped=$((skipped+1)); continue; } + # already done, and its manifest has not grown since: nothing to do. + # Without this the whole pass ran again on every build-all, re-checking + # tens of thousands of paths that were already correct. + if ! _needs_adoption "$man"; then + skipped=$((skipped+1)); continue + fi + + # gcc-pass1/-pass2/libstdcpp all belong to 'gcc', etc. + # show which package is being worked on WHILE it is worked on: a big + # manifest takes a few seconds and silence looks like a hang + printf '\r checking %-40s' "$name" >&2 + local owner; owner="$(pkg_owner_name "$name")" + if ! user_exists "$owner"; then + if [ "$run" = 1 ]; then + # In a SUBSHELL: cmd_add_user dies on failure, and `die` + # exits the whole script -- so `|| continue` never ran and one + # unlucky package aborted the entire adoption pass. + ( cmd_add_user "$owner" ) >/dev/null 2>&1 \ + || { warn " $name: could not create package user '$owner' -- skipping"; \ + skipped=$((skipped+1)); continue; } + fi + fi + + n=0; nd=0 + while IFS= read -r p; do + [ -n "$p" ] || continue + tgt="$(strip_host_prefix "$p")" + [ -e "$tgt" ] || [ -L "$tgt" ] || continue + if [ "$run" = 1 ]; then + # already correct: don't re-chown tens of thousands of paths + [ "$(stat -c %U "$tgt" 2>/dev/null)" = "$owner" ] && continue + chown -h "$owner:$owner" "$tgt" 2>/dev/null && n=$((n+1)) + else + n=$((n+1)) + [ "$verbose" = 1 ] && say " $tgt" + fi + done < "$man" + + if [ -f "$MANIFESTS/$name.dirs" ]; then + while IFS= read -r p; do + [ -n "$p" ] || continue + tgt="$(strip_host_prefix "$p")" + [ -d "$tgt" ] || continue + is_install_dir "$tgt" && continue + if [ "$run" = 1 ]; then + [ "$(stat -c %U "$tgt" 2>/dev/null)" = "$owner" ] && continue + chown "$owner:$owner" "$tgt" 2>/dev/null && nd=$((nd+1)) + else + nd=$((nd+1)) + fi + done < "$MANIFESTS/$name.dirs" + fi + + [ "$run" = 1 ] && _mark_adopted "$name" "$(_manifest_size "$man")" + printf '\r%*s\r' 60 '' >&2 + if [ "$owner" != "$name" ]; then + printf " %-22s %6d file(s) %4d dir(s) -> user '%s'\n" \ + "$name" "$n" "$nd" "$owner" + else + printf " %-22s %6d file(s) %4d dir(s)\n" "$name" "$n" "$nd" + fi + total_f=$((total_f+n)); total_p=$((total_p+1)) + done + + say "" + if [ "$run" = 1 ]; then + ok " $total_f path(s) across $total_p package(s) now owned by package users" + mark_done "adopt-existing" + say "" + say "Every package on the system now belongs to its own user." + say "Next: lfs-helper next" + else + say " would assign $total_f path(s) across $total_p package(s)" + fi + [ "$skipped" -gt 0 ] && say " ($skipped manifest(s) skipped -- not built yet, or empty)" + return 0 +} + +cmd_seal_install_dirs() { + need_root + local run=0; [ "${1:-}" = "--run" ] && run=1 + local d n=0 missing=0 + say "Add the sticky bit to the install directories -- the last step of the" + say "chroot stage. Until now they were group-writable but NOT sticky, so" + say "package users could replace the temporary system's root-owned files and" + say "each other's. Once sticky, a package user can still create files here" + say "but can no longer modify or delete another package's." + say "" + say "Run this when every package is built (end of chapter 8) -- doing it" + say "earlier will break installs that legitimately overwrite older files." + say "" + # Two sources: the known install directories, and ANY directory that + # actually carries the install group -- packages create install dirs the + # static list does not know about (/usr/lib/<pkg>, /usr/share/<pkg>), and + # those need sealing just as much. Collector-group directories are shared + # deliberately and are never touched. + while IFS= read -r d; do + [ -n "$d" ] || continue + [ -d "$d" ] || { missing=$((missing+1)); continue; } + [ -k "$d" ] && continue # already sticky + if [ "$run" = 1 ]; then + chmod o+t "$d" 2>/dev/null && n=$((n+1)) + else + n=$((n+1)) + fi + done < <( + { while IFS= read -r d; do + printf '%s\n' "${SNAP_ROOT%/}$d" + done < <(install_dirs_list) + find "${SNAP_ROOT:-/}" -xdev -type d -group install \ + -not -path "${SNAP_ROOT%/}/.lfs-pkgusr/*" \ + -not -path "$SRCROOT/*" \ + -not -path "${SNAP_ROOT%/}/sources/*" \ + -not -path "${SNAP_ROOT%/}/dev/*" \ + -not -path "${SNAP_ROOT%/}/proc/*" \ + -not -path "${SNAP_ROOT%/}/sys/*" \ + -not -path "${SNAP_ROOT%/}/run/*" 2>/dev/null + } | sort -u + ) + if [ "$run" = 1 ]; then + ok " sticky bit set on $n install directory(ies)" + else + say " would set the sticky bit on $n directory(ies)" + warn "(dry run -- nothing was changed. Add --run to apply.)" + fi + [ "$missing" -gt 0 ] && say " ($missing listed dirs don't exist here -- skipped)" + return 0 +} + +cmd_fix_ownership() { + need_root + local run=0 only="" + while [ $# -gt 0 ]; do + case "$1" in + --run) run=1; shift ;; + -*) die "usage: lfs-helper fix-ownership [<package>] [--run]" ;; + *) only="$1"; shift ;; + esac + done + + say "Give each package's tracked files to its package user." + say "Use this once if packages were installed as root before the" + say "package-user system was in place." + say "" + [ "$run" = 1 ] || warn "(dry run -- nothing was changed. Add --run to apply.)" + say "" + + local total_f=0 total_p=0 man name n + for man in "$MANIFESTS"/*.files; do + [ -e "$man" ] || continue + name="$(basename "${man%.files}")" + [ -n "$only" ] && [ "$name" != "$only" ] && continue + # chapter-7 temporaries are root-owned on purpose + if is_root_step "$name"; then + say " ${C_DIM}$name -- temporary chapter-7 tool, leaving as root${C_OFF}" + continue + fi + if ! user_exists "$name"; then + if [ "$run" = 1 ]; then + # subshell: cmd_add_user dies on failure, which would exit + # the whole script instead of skipping this one package + ( cmd_add_user "$name" ) >/dev/null \ + || { warn " $name: cannot create user -- skipping"; continue; } + else + say " $name -- would create the package user first" + fi + fi + n=0 + while IFS= read -r p; do + [ -n "$p" ] || continue + [ -e "$p" ] || [ -L "$p" ] || continue + # Only count files that actually need a change: root-owned or + # mis-owned. And chown to $name -- an earlier version used a + # variable that was never set here, so this command reported + # success while changing nothing, which is exactly how a system + # ends up with thousands of root-owned package files. + local cur + cur="$(stat -c %U "$p" 2>/dev/null)" + [ "$cur" = "$name" ] && continue + if [ "$run" = 1 ]; then + chown -h "$name:$name" "$p" 2>/dev/null && n=$((n+1)) + else + n=$((n+1)) + fi + done < "$man" + # dirs the package created itself (never the shared install dirs) + local nd=0 mand="$MANIFESTS/$name.dirs" + if [ -f "$mand" ]; then + while IFS= read -r p; do + [ -n "$p" ] || continue + [ -d "$p" ] || continue + is_install_dir "$p" && continue + if [ "$run" = 1 ]; then + chown "$name:$name" "$p" 2>/dev/null && nd=$((nd+1)) + else + nd=$((nd+1)) + fi + done < "$mand" + fi + printf " %-22s %6d file(s) %4d dir(s)\n" "$name" "$n" "$nd" + total_f=$((total_f+n)); total_p=$((total_p+1)) + done + say "" + if [ "$run" = 1 ]; then + ok " $total_f path(s) across $total_p package(s) now owned by their package user" + say "" + say " Re-apply the shared install directories too (root:install):" + say " lfs-helper init-pkgusr --run" + else + say " would touch $total_f path(s) across $total_p package(s)" + fi +} + +# Per-command help. `lfs-helper fix-users -h` should say what the command +# does, not print the whole usage page again. +cmd_help_for() { + case "$1" in + fix-users) + cat <<'EOF' +lfs-helper fix-users [--run] [--skel <dir>] + +Repairs package users that were created before the environment existed, or +whose files came from the wrong place. For every package user it: + + * points .bash_profile, .bashrc and build at the shared files in + /etc/pkgusr, replacing any wrong or missing link; + * copies the skeleton files (build.conf, .project) from + /etc/pkgusr/skel-package when that exists; + * writes VERSION from the install script's name_version, so `lfs update` + can tell what is actually installed. + +Nothing is deleted and no package files are touched -- only the user's home. +Without --run it reports what it would change. +EOF + ;; + export-groups) + cat <<'EOF' +lfs-helper export-groups [<file>] + +Saves the collector-group decisions -- which directory is shared under which +group, and who is in each -- so the next build reuses them instead of asking +again. Writes the directories that really carry a group right now, read from +the filesystem, not just the ones you were asked about. +EOF + ;; + import-groups) + cat <<'EOF' +lfs-helper import-groups [<file>] [--run] + +Loads a file written by export-groups: recreates the groups with their +members, and records the directory-to-group decisions so those directories are +never asked about again. With no file it looks for one placed in the tree by +`lfs build-system gen-chroot-scripts`. +EOF + ;; + seal-install-dirs) + cat <<'EOF' +lfs-helper seal-install-dirs [--run] + +The LAST step of the chroot stage. Adds the sticky bit to every shared install +directory, so a package user can still create files there but can no longer +modify or delete another package's. + +Run it only when every package is built: doing it earlier breaks installs that +legitimately overwrite an older version's files. +EOF + ;; + build) + cat <<'EOF' +lfs-helper build <name> [--phase all|unpack|build|install|configure|test] + [--force] [--tests] [--jobs N] [--no-auto-fix] + +Builds one package as its own package user, tracking every file it installs. + + --phase run a single phase instead of all of them + --force rebuild even if the step is already marked built + --tests run the package's test suite (off by default: the books + expect some failures) + --jobs N override MAKEFLAGS for this build + --no-auto-fix do not grant collector-group access on a permission failure +EOF + ;; + *) return 1 ;; + esac + return 0 +} + +# Colour the group headings, matching `lfs --help` and `blfs --help`: the +# command list is what people scan, and an unbroken wall of text is hard to +# search. Dropped when stdout is not a terminal. +_colour_usage() { + if [ ! -t 1 ] || [ -n "${NO_COLOR:-}" ]; then cat; return; fi + sed -E \ + -e "s/^([A-Za-z][^:]*:)\$/$(printf '\033[1m')\1$(printf '\033[0m')/" \ + -e "s/^( )([a-z-]+([ ][a-z<>|-]+)*)/\1$(printf '\033[0;32m')\2$(printf '\033[0m')/" +} + +usage() { + _usage_text | _colour_usage +} + +_usage_text() { + cat <<EOF +lfs-helper $LFS_HELPER_VERSION (build $(_build_id)) -- builds the LFS system +one package user each. + +Follow the build: + lfs-helper next the one command to keep running + lfs-helper status where the session stands + lfs-helper list every step and its state + ('next --all' does the same) + lfs-helper check-toolchain can this compiler build anything? + lfs-helper fix-toolchain --run repair a triplet mismatch in the headers + +Build: + lfs-helper build <name> build one step as its package user + lfs-helper build-all build everything remaining, in order + lfs-helper done|undone <name> mark a step built / clear that mark + build options: --phase <p> --force --tests --jobs N --no-auto-fix + +Ownership and permissions (the package-user model): + lfs-helper init-pkgusr --run FIRST: install group + install dirs + lfs-helper add-user <name> create one package user (with its links) + lfs-helper fix-ownership --run give every tracked file to its package + lfs-helper adopt-dirs --run same for the directories packages made + lfs-helper fix-users --run repair homes: bash links, VERSION files + lfs-helper fix-perms <name> --run grant one package the access it needs + lfs-helper grant-dir <dir> <user> --run + give one user access to one directory + lfs-helper seal-install-dirs --run + LAST: sticky bits on shared dirs + +Collector groups (shared directories): + lfs-helper export-groups [<file>] save the decisions for the next build + lfs-helper import-groups --run load them again + +Look around / tidy up: + lfs-helper manifests [<name>] what each package installed + lfs-helper which-package <path> which package installed this file + lfs-helper clean-sources --run tidy /sources + lfs-helper prune-locales --run drop unwanted translations + lfs-helper find-user-site --run find pip installs hidden in ~/.local + +Any command: -h explains it. Dry run is the default; --run applies. +State: $STATE +EOF +} + +# `lfs-helper <cmd> -h|--help` -> what that command does +if [ $# -ge 2 ]; then + case "$2" in + -h|--help) cmd_help_for "$1" && exit 0 ;; + esac +fi + +case "${1:-}" in + status) shift; cmd_status "$@" ;; + --version|-V) echo "lfs-helper $LFS_HELPER_VERSION (build $(_build_id))"; exit 0 ;; + next) + shift + # `next --all` is how the outside tool spells `list`. Accepting both + # in both tools means never having to remember which one you are in. + if [ "${1:-}" = "--all" ]; then shift; cmd_list "$@"; else cmd_next "$@"; fi + ;; + list) shift; cmd_list "$@" ;; + init-pkgusr) shift; cmd_init_pkgusr "$@" ;; + add-user) shift; cmd_add_user "$@" ;; + build) shift; cmd_build "$@" ;; + build-all) shift; cmd_build_all "$@" ;; + manifests) shift; cmd_manifests "$@" ;; + fix-ownership) shift; cmd_fix_ownership "$@" ;; + fix-perms) shift; cmd_fix_perms "$@" ;; + clean-sources) shift; cmd_clean_sources "$@" ;; + find-user-site) shift; cmd_find_user_site "$@" ;; + check-toolchain) shift; cmd_check_toolchain "$@" ;; + fix-toolchain) shift; cmd_fix_toolchain "$@" ;; + which-package) shift; cmd_which_package "$@" ;; + sort-users) shift; cmd_sort_users "$@" ;; + fix-orphans) shift; cmd_fix_orphans "$@" ;; + grant-dir) shift; cmd_grant_dir "$@" ;; + fix-users) shift; cmd_fix_users "$@" ;; + export-groups) shift; cmd_export_groups "$@" ;; + import-groups) shift; cmd_import_groups "$@" ;; + prune-locales) shift; cmd_prune_locales "$@" ;; + adopt-existing) shift; cmd_adopt_existing "$@" ;; + adopt-dirs) shift; cmd_adopt_dirs "$@" ;; + repair-adoption) shift; cmd_repair_adoption "$@" ;; + seal-install-dirs) shift; cmd_seal_install_dirs "$@" ;; + done) shift; mark_done "${1:?usage: lfs-helper done <name>}"; ok "marked '$1' built" ;; + undone) shift; unmark "${1:?usage: lfs-helper undone <name>}"; ok "cleared '$1'" ;; + -h|--help|help|"") usage ;; + *) die "unknown command '$1' (try: lfs-helper --help)" ;; +esac diff --git a/packagemanager b/packagemanager old mode 100755 new mode 100644 index b7ef98e..2b6546b --- a/packagemanager +++ b/packagemanager @@ -85,7 +85,24 @@ TMP_DIR = "/tmp/packagemanager" CONFIG_PATH = os.environ.get("PKGUSR_CONFIG", "/etc/pkgusr/packagemanager.conf") # Version stamps. Bump PLAN_VERSION when the cached dry-run plan structure # changes (so stale plans are ignored); TOOL_VERSION is the program version. -TOOL_VERSION = "1.0.0" +TOOL_VERSION = "1.7.0" + + +def _build_id(): + """A short fingerprint of this file's own contents. + + A hand-maintained version number goes stale the moment someone forgets to + bump it -- and the whole reason for printing a version is to answer "am I + running the code that was just fixed?". This cannot go stale: it changes + whenever the file does. + """ + import hashlib + try: + with open(os.path.abspath(__file__), "rb") as fh: + return hashlib.md5(fh.read()).hexdigest()[:7] + except OSError: + return "unknown" + PLAN_VERSION = 1 _DEFAULT_CONFIG = {"collector_prefix": "sysgroup", "user_prefix": "u", "main_user": ""} @@ -106,9 +123,48 @@ def load_config(): _config[k.strip()] = v.strip() except OSError: pass + + # Settings that MUST match across the tools come from one place. + # + # collector_prefix decides which groups shared directories carry. The + # system was built with the prefix in the lfs config; if packagemanager + # used its own default instead, every package installed afterwards would + # join `sysgroup_*` groups while the built system uses `nimgnu_*` -- two + # parallel sets of groups over the same directories, and no error to say + # so. The lfs config is authoritative for these. + shared = _shared_lfs_config() + for key in _SHARED_WITH_LFS: + val = shared.get(key) + if not val: + continue + if _config.get(key) and _config[key] != val: + printWarning( + f"note: {key} is '{_config[key]}' here but '{val}' in the lfs " + f"config -- using '{val}', which is what the system was built " + f"with.") + _config[key] = val return _config +# Keys owned by the lfs config, not by packagemanager's own file. +_SHARED_WITH_LFS = ("collector_prefix",) + + +def _shared_lfs_config(): + """The lfs tool's settings, if it has any on this system.""" + for cand in (os.environ.get("LFS_STORE"), "/usr/share/lfs", + os.path.expanduser("~/.local/share/lfs")): + if not cand: + continue + p = os.path.join(cand, "config.json") + try: + with open(p) as f: + return json.load(f) + except (OSError, ValueError): + continue + return {} + + def save_config(cfg): try: os.makedirs(os.path.dirname(CONFIG_PATH) or ".", exist_ok=True) @@ -225,7 +281,11 @@ def find_blfs(): here = os.path.dirname(os.path.abspath(__file__)) local = os.path.join(here, "blfs") if os.path.isfile(local): - return [local] if os.access(local, os.X_OK) else ["python3", local] + # os.access(X_OK) answers "yes" for root even without any execute bit, + # so check the mode bits themselves -- exactly the situation in a + # fresh copy of the tools, and the exec then dies with EACCES. + ex = bool(os.stat(local).st_mode & 0o111) + return [local] if ex else [sys.executable or "python3", local] return ["blfs"] # rely on $PATH @@ -237,9 +297,32 @@ def _blfs_book_flags(): return [] +# what each blfs subcommand is actually doing, for the status line +_BLFS_ACTIVITY = { + "order": "working out the build order for %s", + "debug": "looking up %s in the book", + "deps": "reading %s's dependencies", + "script": "writing the install script for %s", + "versions": "reading versions from the book", + "search": "searching the book for %s", +} + + def run_blfs(args): - """Run blfs; return (returncode, stdout, stderr). rc 127 => blfs missing.""" + """Run blfs; return (returncode, stdout, stderr). rc 127 => blfs missing. + + Each call is a subprocess, and the first one for a given book INDEXES it, + which takes many seconds. Announce what is happening: an install that sits + silent for half a minute before its first output looks frozen.""" cmd = find_blfs() + _blfs_book_flags() + list(args) + verb = args[0] if args else "" + if verb in _BLFS_ACTIVITY: + arg = args[1] if len(args) > 1 and not args[1].startswith("-") else "" + try: + set_status(_BLFS_ACTIVITY[verb] % arg if "%s" in _BLFS_ACTIVITY[verb] + else _BLFS_ACTIVITY[verb]) + except Exception: + pass try: r = subprocess.run(cmd, capture_output=True, text=True) return r.returncode, r.stdout, r.stderr @@ -575,6 +658,32 @@ def _script_name_version(path): return None +def _warn_if_wrong_file_edited(user, chosen): + """Point out an edit made to the wrong file. + + A package user's home holds two files with nearly identical names: + install_<name>-<version> the source -- edit THIS one + install_<name> the engine's runtime copy, overwritten each run + Editing the second is an easy mistake, and the symptom is silent: the + install runs the unedited script and fails in exactly the same way as + before, so it looks as though the edit was thrown away.""" + runtime = os.path.join(BASE_DIR, user, f"install_{user}") + if not os.path.isfile(runtime): + return + try: + if os.path.samefile(runtime, chosen): + return + if os.path.getmtime(runtime) <= os.path.getmtime(chosen): + return # not edited more recently: nothing odd + except OSError: + return + printWarning(f" note: {runtime} was changed more recently than the script " + f"in use.") + printWarning(f" That file is a runtime copy and is overwritten every " + f"run -- edits to it are lost.") + printWarning(f" Edit this instead: {chosen}") + + def resolve_script(user, name_version, script_dir, assume_yes=False, force_regen=False): """Path to the install script to use for `user`. @@ -588,6 +697,7 @@ def resolve_script(user, name_version, script_dir, assume_yes=False, if (not force_regen and os.path.isfile(local) and _script_name_version(local) == name_version): printInfo(f" using LOCAL script (your edits, not the book): {local}") + _warn_if_wrong_file_edited(user, local) try: body = open(local).read() except OSError: @@ -875,6 +985,218 @@ def find_add_package_user(): return os.environ.get("ADD_PACKAGE_USER_BIN", "add_package_user").split() +# useradd/groupadd live in /usr/sbin, which is often absent from a non-login +# PATH -- look there explicitly rather than concluding they do not exist. +_SBIN_PATH = os.environ.get("PATH", "") + ":/usr/sbin:/sbin:/usr/local/sbin" + + +def _have_cmd(name): + return shutil.which(name, path=_SBIN_PATH) is not None + + +def _cmd_path(name): + return shutil.which(name, path=_SBIN_PATH) or name + + +PKGUSR_ETC = os.environ.get("PKGUSR_ETC", "/etc/pkgusr") + +# The shared environment every package user gets, as the package-users hint +# describes it: .bash_profile and .bashrc are SYMLINKS into a central directory +# so all package users share one environment, and `build` is the helper each +# package's build.conf is fed to. Creating a user without these leaves an +# account that cannot build anything. +_PKGUSR_BASH_PROFILE = """\ +# /etc/pkgusr/bash_profile -- shared by every package user. +# Symlinked as ~/.bash_profile. Keep the environment identical for all of +# them, so a build behaves the same whichever package user runs it. + +umask 022 +export LC_ALL=POSIX + +# /usr/local/bin comes FIRST after the wrappers: locally installed +# overrides belong ahead of /usr/bin, and a temporary wrapper there +# (the no-verify wget, say) is useless if nothing ever finds it. +PATH=/usr/lib/pkgusr:/usr/local/bin:/usr/local/sbin:/usr/bin:/bin:/usr/sbin:/sbin +export PATH + +# a short prompt that makes it obvious which package user you are +export PS1='\\u:\\w\\$ ' + +# where this package's sources live +export PKG_HOME="$HOME" +cd "$HOME" 2>/dev/null || true + +[ -f "$HOME/build.conf" ] && . "$HOME/build.conf" +""" + +_PKGUSR_BASHRC = """\ +# /etc/pkgusr/bashrc -- shared by every package user. +# Symlinked as ~/.bashrc. Package users are not used for logging in, so there +# is no reason for login and non-login shells to differ: just source the +# profile, and `su pkg` and `su - pkg` then behave identically. + +[ -f "$HOME/.bash_profile" ] && . "$HOME/.bash_profile" +""" + +_PKGUSR_BUILD = """\ +#!/bin/bash +# /etc/pkgusr/build -- shared by every package user, symlinked as ~/build. +# +# Reads build.conf from the package user's home and runs the usual +# unpack/configure/make/install sequence with it. It is deliberately small: +# anything unusual belongs in the package's own install script. +set -e + +conf="${1:-$HOME/build.conf}" +[ -f "$conf" ] || { echo "no build.conf at $conf" >&2; exit 1; } +. "$conf" + +: "${SRC_DIR:=$HOME}" +: "${CONFIGURE_OPTS:=--prefix=/usr}" +: "${MAKE_OPTS:=}" + +cd "$SRC_DIR" +[ -x ./configure ] && ./configure $CONFIGURE_OPTS +make $MAKE_OPTS +make install +""" + +_PKGUSR_BUILD_CONF = """\ +# build.conf for package user '%(name)s' +# +# Read by ~/build (a symlink to /etc/pkgusr/build). Override anything you need +# and leave the rest alone. + +# where the unpacked source is +#SRC_DIR="$HOME/%(name)s-1.0" + +CONFIGURE_OPTS="--prefix=/usr" +MAKE_OPTS="" +""" + +_PKGUSR_PROJECT = """\ +%(name)s + +Package user for '%(name)s'. Everything this package installed is owned by +this account; `packagemanager info %(name)s` lists it. +""" + + +def ensure_pkgusr_etc(verbose=True): + """Make sure the shared package-user environment exists in /etc/pkgusr. + + Copied from the host when this runs there; written from scratch otherwise + (a fresh LFS system has no /etc/pkgusr at all).""" + made = [] + try: + os.makedirs(PKGUSR_ETC, exist_ok=True) + except OSError as e: + printError(f"cannot create {PKGUSR_ETC}: {e}") + return False + for fname, content, mode in ( + ("bash_profile", _PKGUSR_BASH_PROFILE, 0o644), + ("bashrc", _PKGUSR_BASHRC, 0o644), + ("build", _PKGUSR_BUILD, 0o755)): + path = os.path.join(PKGUSR_ETC, fname) + if os.path.exists(path): + continue + with open(path, "w") as f: + f.write(content) + os.chmod(path, mode) + made.append(fname) + if made and verbose: + printInfo(f"created the shared package-user environment in " + f"{PKGUSR_ETC}: {', '.join(made)}") + return True + + +def init_package_user_home(name, home=None, verbose=True): + """Give a package user the environment the scheme expects. + + Without this the account exists but has no .bash_profile, no .bashrc and no + build helper -- `su - <pkg>` lands in a bare shell with the wrong umask and + no PATH to the wrapper commands.""" + home = home or os.path.join(BASE_DIR, name) + if not ensure_pkgusr_etc(verbose=verbose): + return False + try: + os.makedirs(home, exist_ok=True) + except OSError as e: + printError(f"cannot create {home}: {e}") + return False + + # symlinks to the shared files, so changing one changes them all + for link, target in ((".bash_profile", "bash_profile"), + (".bashrc", "bashrc"), + ("build", "build")): + dst = os.path.join(home, link) + src = os.path.join(PKGUSR_ETC, target) + try: + if os.path.islink(dst) or os.path.exists(dst): + if os.path.islink(dst) and os.readlink(dst) == src: + continue + os.remove(dst) + os.symlink(src, dst) + except OSError as e: + printWarning(f" {dst}: {e}") + + # per-package files + for fname, content in (("build.conf", _PKGUSR_BUILD_CONF % {"name": name}), + (".project", _PKGUSR_PROJECT % {"name": name})): + path = os.path.join(home, fname) + if not os.path.exists(path): + with open(path, "w") as f: + f.write(content) + + _run(["chown", "-R", "-h", f"{name}:{name}", home]) + _run([_cmd_path("chmod"), "0750", home]) if _have_cmd("chmod") else None + if verbose: + printInfo(f" initialised {home} " + f"(.bash_profile, .bashrc, build -> {PKGUSR_ETC}; build.conf)") + return True + + +def create_package_user(name, home=None): + """Create a package user, with or without the hint's add_package_user. + + add_package_user comes from the package-users hint's helper tarball, which + a fresh LFS system does not have -- and without a fallback nothing can be + installed as a package user at all. Shadow's own tools do the same job: + one user, a private group of the same name, plus the supplementary + 'install' group.""" + if _user_exists(name): + return True + home = home or os.path.join(BASE_DIR, name) + + if _have_cmd(find_add_package_user()[0]): + cmd = find_add_package_user() + [name, name, "10000", "20000", + name, "10000", "20000"] + printInfo(f"creating package-user {name}") + if _run(cmd): + init_package_user_home(name, home) + return True + printWarning("add_package_user failed -- falling back to useradd") + + if not _have_cmd("useradd"): + printError("neither add_package_user nor useradd is available; " + "cannot create package users on this system") + return False + + if not _group_exists("install"): + printInfo("creating the 'install' group") + _run([_cmd_path("groupadd"), "-g", "9999", "install"]) + + printInfo(f"creating package-user {name} (useradd)") + # -U makes the private group alongside the user so uid and gid match; + # -K keeps package users in their own id range + ok = _run([_cmd_path("useradd"), "-c", f"package {name}", "-d", home, "-U", + "-G", "install", "-s", "/bin/bash", + "-K", "UID_MIN=10000", "-K", "GID_MIN=10000", name]) + if ok: + init_package_user_home(name, home) + return ok + + def _group_exists(name): try: grp.getgrnam(name) @@ -1095,6 +1417,253 @@ def write_group_manifest(group, background=True): return subprocess.run(cmd).returncode == 0 +# Directories that are shared by nature, matched as patterns. +# +# Kept in step with lfs-helper's list: many packages drop a single file into +# /usr/share/zsh/site-functions or /usr/share/applications, so treating those +# as belonging to whichever package created them first makes every later +# package fail there. /usr/libexec is shared; /usr/libexec/p11-kit is not. +SHARED_DIR_PATTERNS = [ + "/usr/share/man/man[1-9]", "/usr/share/man/*/man[1-9]", "/usr/share/man/*", + "/usr/share/locale/*/LC_MESSAGES", "/usr/share/locale/*", + "/usr/share/info", "/usr/share/doc", + "/usr/lib/pkgconfig", "/usr/share/pkgconfig", + "/usr/share/bash-completion/completions", + "/usr/share/zsh/site-functions", "/usr/share/zsh/vendor-completions", + "/usr/share/fish/completions", "/usr/share/fish/vendor_completions.d", + "/usr/share/applications", "/usr/share/mime/packages", + "/usr/share/dbus-1/services", "/usr/share/dbus-1/system-services", + "/usr/share/metainfo", "/usr/share/appdata", + "/usr/share/icons/hicolor/*", "/usr/share/pixmaps", + "/usr/share/glib-2.0/schemas", "/usr/share/vala/vapi", + "/usr/share/gir-1.0", "/usr/lib/girepository-1.0", + "/usr/share/aclocal", "/usr/share/gtk-doc/html", + "/usr/lib/systemd/system", "/usr/lib/udev/rules.d", + "/etc/xdg/autostart", "/usr/libexec", + "/usr/share/terminfo/*", +] + + +def is_shared_dir(path): + import fnmatch + p = path.rstrip("/") + return any(fnmatch.fnmatch(p, pat) for pat in SHARED_DIR_PATTERNS) + + +# One cheap pre-filter, then the precise patterns. +# +# These run over a whole build log, which for something like p11-kit is tens of +# megabytes. Running five regexes across all of it -- once per retry round -- +# burned ~90 seconds of CPU and made the machine crawl. So: only read the tail +# (errors are at the end), skip any line without the words we care about, and +# anchor the patterns per line so there is nothing to backtrack over. +_PERM_HINT = ("Permission denied", "Operation not permitted", "PermissionError") + +_PERM_PATTERNS = [ + re.compile(r"PermissionError[^']*'([^']+)'"), + re.compile(r"(?:Permission denied|Operation not permitted)[:,]?\s*'([^']+)'"), + re.compile(r"cannot (?:create|remove|touch|open)[^']*'([^']+)'"), + re.compile(r"^\s*(/[^\s:]+):\s*Permission denied"), +] + +_LOG_TAIL_BYTES = 512 * 1024 # the error is never at the start + + +def _paths_from_permission_errors(log_path, max_paths=20): + if not log_path or not os.path.isfile(log_path): + return [] + try: + size = os.path.getsize(log_path) + with open(log_path, errors="replace") as f: + if size > _LOG_TAIL_BYTES: + f.seek(size - _LOG_TAIL_BYTES) + f.readline() # drop the partial first line + lines = f.readlines() + except OSError: + return [] + + seen, out = set(), [] + for line in lines: + # the pre-filter: a plain substring test skips ~all of a build log + if not any(h in line for h in _PERM_HINT): + continue + for pat in _PERM_PATTERNS: + m = pat.search(line) + if m: + path = m.group(1) + if path.startswith("/") and path not in seen: + seen.add(path) + out.append(path) + break + if len(out) >= max_paths: + break + return out + + +def _dir_of(path): + return path if os.path.isdir(path) else os.path.dirname(path) + + +def _user_can_write(user, path): + try: + r = subprocess.run(["su", "-s", "/bin/bash", user, "-c", + "test -w %s" % shlex.quote(path)], + capture_output=True) + return r.returncode == 0 + except OSError: + return False + + +def auto_grant_from_log(user, log_path, verbose=True): + """Give `user` the access a failed install needed, and say what changed. + + This is what lfs-helper does during the base build, and there is no reason + for packagemanager to behave differently afterwards: a package that has to + write into a directory another package owns joins that directory's + collector group. + + Returns the list of directories granted (empty if there was nothing to do). + """ + granted = [] + dirs = [] + for p in _paths_from_permission_errors(log_path): + d = _dir_of(p) + if d and os.path.isdir(d) and d not in dirs: + dirs.append(d) + if not dirs: + return granted + + for d in dirs: + if _user_can_write(user, d): + continue # already fine; something else failed + cur = None + try: + cur = grp.getgrgid(os.stat(d).st_gid).gr_name + except (KeyError, OSError): + pass + + if is_shared_dir(d) and cur != "install": + # shared by nature: it belongs to the install group, and every + # package user is already in that -- no collector group needed + if verbose: + printInfo(f" {d} is a shared directory -> group install") + _assign_group_dir(d, "install", recursive=False) + granted.append((d, "install")) + continue + + if cur == "install" or (cur and cur.startswith(collector_prefix())): + # The directory is ALREADY shared -- the package just isn't in the + # group yet. Adding it is the whole fix; don't touch the directory. + group = cur + if verbose: + printInfo(f" {d} is already group {group} -- adding '{user}' to it") + else: + owner = "root" + try: + owner = pwd.getpwuid(os.stat(d).st_uid).pw_name + except (KeyError, OSError): + pass + group = nimgnu_name(owner if owner != "root" else os.path.basename(d)) + if not _group_exists(group): + if verbose: + printInfo(f" creating collector group {group}") + if not _run([_cmd_path("groupadd"), group]): + continue + if owner not in ("root", "") and _user_exists(owner): + _run([_cmd_path("usermod"), "-a", "-G", group, owner]) + _assign_group_dir(d, group, recursive=False) + if verbose: + printInfo(f" {d} -> group {group} (owner {owner})") + + if not _run([_cmd_path("usermod"), "-a", "-G", group, user]): + continue + granted.append((d, group)) + + if granted and verbose: + printWarning("\nCollector groups changed so '%s' can install:" % user) + for d, g in granted: + print(f" {d:<44} -> {g}") + print(" These are permanent; review with: " + f"getent group | grep '^{collector_prefix()}'") + return granted + + +def _latest_log(name): + """The log the most recent install run wrote.""" + d = _pkg_log_dir(name) + try: + logs = [os.path.join(d, f) for f in os.listdir(d) if f.endswith(".log")] + except OSError: + return None + return max(logs, key=os.path.getmtime) if logs else None + + +def _run_pm_install_once(name, path, env): + """Run the install engine exactly once, with no recovery.""" + cmd = find_pm_install() + [name, path] + printInfo(" $ " + " ".join(shlex.quote(c) for c in cmd)) + try: + return subprocess.run(cmd, env=env) + except FileNotFoundError: + printError(f" {cmd[0]}: command not found") + return subprocess.CompletedProcess(cmd, 127) + + +def run_pm_install(name, path, env, auto_fix=True, max_rounds=4): + """Run the install engine, granting access and retrying on a permission + failure -- the same recovery lfs-helper does during the base build. + + An install reveals only the FIRST directory it cannot write, so one grant + is rarely enough: a package that drops files into several shared places + fails again on the next one. Keep going while each round actually grants + something, bounded so a real build error cannot loop. + + Note the two functions: the retry wrapper must call the RAW runner, never + itself. Calling itself is an infinite recursion that ends in + RecursionError: maximum recursion depth exceeded + before the install is even attempted.""" + result = _run_pm_install_once(name, path, env) + rounds = 0 + while result.returncode != 0 and auto_fix and rounds < max_rounds: + log = _latest_log(name) + granted = auto_grant_from_log(name, log) + if not granted: + break # nothing to grant: a real build error + rounds += 1 + printInfo(f"\nretrying {name} (round {rounds}) ...") + result = _run_pm_install_once(name, path, env) + if result.returncode != 0 and rounds >= max_rounds: + printWarning(f"gave up after {max_rounds} permission grants -- " + f"something else is wrong") + if result.returncode == 0: + warn_if_nothing_installed(name) + return result + + +def warn_if_nothing_installed(name): + """A package that installs nothing has usually failed quietly. + + pip is the worst offender -- it can report success while writing nowhere + useful -- but any build whose install step is a no-op leaves a package user + owning nothing, and the problem only surfaces later when something tries to + use it.""" + pkglist = os.path.join(BASE_DIR, name, "pkg.lst") + n = 0 + if os.path.isfile(pkglist): + try: + with open(pkglist) as f: + n = sum(1 for line in f if line.strip()) + except OSError: + return + else: + return # manifest not written yet: nothing to judge + if n == 0: + printWarning( + f"\n!! {name} reported success but owns no files.\n" + f" That normally means the install silently did nothing.\n" + f" Check before trusting it: packagemanager info {name}\n") + + def cmd_make_group_dir(args): group = _resolve_group_arg(args.group) if group != "install" and not _group_exists(group): @@ -1355,12 +1924,64 @@ def cmd_dependencies(args): # --yes don't ask for confirmation before --run # -o/--outdir DIR where to write the generated scripts +def _blfs_book_available(): + """Is there a BLFS book on disk to look packages up in? + + Asking `blfs books` is no good: it also lists every version available on + the site, so the output is never empty and the answer was always yes.""" + if os.environ.get("BLFS_BOOK_FILE"): + return os.path.isfile(os.environ["BLFS_BOOK_FILE"]) + for cand in (os.environ.get("BLFS_STORE"), "/usr/share/blfs", + os.path.expanduser("~/.cache/blfs")): + if not cand: + continue + d = os.path.join(cand, "books") + try: + if any(f.endswith((".html", ".html.gz")) for f in os.listdir(d)): + return True + except OSError: + continue + return False + + +def _explain_not_in_book(target): + """Say why a package could not be found, and what to do about it. + + "not found in book" is unhelpful when the real problem is that there IS no + book -- a fresh chroot has the LFS book but not the BLFS one, and every + lookup fails the same way.""" + if not _blfs_book_available(): + printError(f"{target}: there is no BLFS book on this system to look " + f"it up in.") + printInfo("") + printInfo(" Download it (needs working networking):") + printInfo(" blfs fetch") + printInfo(" or copy one in from the host and import it:") + printInfo(" blfs import /path/to/BLFS-BOOK-...-nochunks.html") + printInfo("") + return + printError(f"{target}: not in the BLFS book.") + rc, out, _e = run_blfs(["search", target]) + hits = [ln for ln in out.splitlines() if ln.strip().startswith("[")][:5] + if hits: + printInfo(" Did you mean:") + for h in hits: + printInfo(" " + h.strip()) + else: + printInfo(f" Nothing similar found either. Try: blfs search {target}") + + def build_install_plan(args): """Return (plan, missing, why). plan = [(anchor, name_version, reason)] in install order (deps first); reason in {new, update, reinstall}. why maps anchor -> (kind, parent) so we can show 'required by <parent>'.""" ignore = {x for x in (args.ignore or "").split(",") if x} plan, seen, missing_all, why = [], set(), [], {} + _not_found = [] + # Reading the book can take many seconds the first time (it is indexed + # once per version). Without a word on screen that looks like a hang, so + # say what is happening as it happens. + set_status("reading the book ...") def consider(anchor, nv, kind="", parent=""): if anchor not in why: @@ -1379,11 +2000,14 @@ def build_install_plan(args): "update" if u.state == "installed" else "new")) for target in args.packages: + set_status(f"working out what {target} needs ...") if not getattr(args, "recursive", False): # DEFAULT: just this package, no dependency walk nv = book_version(target) if nv is None: - printError(f"{target}: not found in book -- skipping") + clear_status() + _explain_not_in_book(target) + _not_found.append(target) continue consider(target, nv, "requested", "") else: @@ -1398,7 +2022,7 @@ def build_install_plan(args): for danchor, dnv in book_rdeps( target, args.ignore_recommended, args.optional).items(): consider(danchor, dnv, "dependent-of", target) - return plan, missing_all, why + return plan, missing_all, why, _not_found def _select_plan_rows(rows, render): @@ -1495,7 +2119,7 @@ def install_local(name, path, run, yes, verb="install", test=False, clean=False) t0 = time.time() if clean: env = {**env, "PM_NO_REFRESH": "1"} - result = subprocess.run(find_pm_install() + [name, path], env=env) + result = run_pm_install(name, path, env) if result.returncode != 0: printError(f"{verb} failed for {name} (exit {result.returncode}).") printInfo(f" logs: {_pkg_log_dir(name)}") @@ -1517,11 +2141,17 @@ def cmd_install(args): clean=getattr(args, "clean", False)) return - plan, missing, why = build_install_plan(args) + plan, missing, why, not_found = build_install_plan(args) if missing: printWarning("Out-of-book / missing (install these manually): " + ", ".join(sorted(set(missing)))) if not plan: + if not_found: + # NOT success: the package could not be looked up at all. Saying + # "everything is installed and up to date" here told the user the + # opposite of the truth. + printError("Could not install: " + ", ".join(not_found)) + sys.exit(1) printSuccess("Nothing to do -- everything is installed and up to date.") return @@ -1592,7 +2222,7 @@ def cmd_install(args): old_paths = _pkglist_paths(anchor) if cleaning else None t0 = time.time() penv = {**env, "PM_NO_REFRESH": "1"} if cleaning else env - result = subprocess.run(pm + [anchor, path], env=penv) + result = run_pm_install(anchor, path, penv) add_run_entry(run_log, anchor, "install", result.returncode == 0, result.returncode, _pkg_log_dir(anchor)) if result.returncode != 0: @@ -1603,8 +2233,13 @@ def cmd_install(args): printInfo(f" install file: {path}") printInfo(f" logs: {_pkg_log_dir(anchor)} " f"(see all: packagemanager errors)") - printInfo(f" After fixing the problem you can re-run ONLY the " - f"install step (no recompile):\n" + # Name the file to EDIT, not just the command to re-run. A + # package user's home holds install_<name>-<version> (the source) + # and install_<user> (a runtime copy overwritten every run), and + # editing the wrong one fails silently -- the install repeats + # unchanged and the edit looks lost. + printInfo(f" To change what it does, edit:\n {path}") + printInfo(f" Then re-run ONLY the install step (no recompile):\n" f" packagemanager script install {anchor}") sys.exit(result.returncode) if getattr(args, "clean", False): @@ -1947,7 +2582,7 @@ def cmd_update(args): old_paths = _pkglist_paths(name) if cleaning else None t0 = time.time() penv = {**env, "PM_NO_REFRESH": "1"} if cleaning else env - result = subprocess.run(pm + [name, path], env=penv) + result = run_pm_install(name, path, penv) add_run_entry(run_log, name, "update", result.returncode == 0, result.returncode, _pkg_log_dir(name)) if result.returncode != 0: @@ -2334,12 +2969,20 @@ def _run(cmd, show_output=False): own output is shown too; otherwise it's captured and surfaced only on error.""" print(f" {colors.blue}$ " f"{' '.join(shlex.quote(c) for c in cmd)}{colors.normal}") - if _STATE["verbose"] or show_output: - r = subprocess.run(cmd) - else: - r = subprocess.run(cmd, capture_output=True, text=True) - if r.returncode != 0: - printError(f" {' '.join(cmd)} -> {r.stderr.strip() or r.returncode}") + try: + if _STATE["verbose"] or show_output: + r = subprocess.run(cmd) + else: + r = subprocess.run(cmd, capture_output=True, text=True) + if r.returncode != 0: + printError(f" {' '.join(cmd)} -> {r.stderr.strip() or r.returncode}") + except FileNotFoundError: + # a missing helper is an ordinary problem to report, not a stack trace + printError(f" {cmd[0]}: command not found") + return False + except PermissionError as e: + printError(f" {cmd[0]}: {e}") + return False return r.returncode == 0 @@ -2524,15 +3167,108 @@ def report_created_user(name, kind="package-user"): printInfo(f" supplementary groups: {', '.join(groups)}") +def _sanitise_user_name(mod): + """A Python distribution name is not necessarily a valid user name. + + "beautifulsoup4" is fine; "ruamel.yaml", "Jinja2" and "typing_extensions" + are not what we want as accounts. Lower-case and reduce to [a-z0-9_-].""" + n = re.sub(r"[^a-z0-9_-]+", "-", mod.lower()).strip("-") + return n[:32] or "pymodule" + + +def _pip_site_dir(): + """The system site-packages directory pip installs into.""" + import glob as _glob + for pat in ("/usr/lib/python3*/site-packages", + "/usr/lib/python3*/dist-packages"): + hits = sorted(_glob.glob(pat)) + if hits: + return hits[-1] + return None + + +def _pip_owner_of_python(): + """Which package owns Python's tree -- the collector group is named after it.""" + site = _pip_site_dir() + if not site: + return "python" + d = site + while d not in ("/", ""): + try: + import pwd as _pwd + return _pwd.getpwuid(os.stat(d).st_uid).pw_name + except (KeyError, OSError): + d = os.path.dirname(d) + return "python" + + +def cmd_pip(args): + """Install or update a Python module AS A PACKAGE USER. + + A bare `pip3 install beautifulsoup4` installs as whoever ran it -- usually + root -- so the files belong to nobody in particular and no package user can + ever update or remove them. This does it the package-user way: one user per + module, joined to the collector group that owns Python's tree, with pip kept + out of ~/.local so the module lands where every other package can import it. + """ + site = _pip_site_dir() + if not site: + printError("no system site-packages directory found -- is Python installed?") + return 1 + owner = _pip_owner_of_python() + grp = collector_prefix() + owner + + modules = args.modules + if not modules: + printError("usage: packagemanager pip install <module> [<module> ...]") + return 1 + + for mod in modules: + user = _sanitise_user_name(mod) + printHeader(f"{mod} -> package user '{user}'") + + if not _user_exists(user): + if not create_package_user(user): + printError(f"could not create {user}") + continue + report_created_user(user) + + # the collector group is what lets this user write into Python's tree + if not _group_exists(grp): + printInfo(f"creating collector group {grp}") + _run([_cmd_path("groupadd"), grp]) + _run(["usermod", "-a", "-G", grp, user]) + if _group_exists(grp): + _run(["chgrp", "-R", grp, site]) + _run(["chmod", "-R", "g+w", site]) + + # PIP_USER=0 / PYTHONNOUSERSITE=1: without these pip quietly falls back + # to ~/.local when site-packages is not writable, and then reports + # "Requirement already satisfied" from there ever after -- so the module + # is installed for exactly one user and importable by nobody else. + env = ["env", "PIP_USER=0", "PYTHONNOUSERSITE=1"] + pip_args = ["pip3", "install"] + if args.upgrade: + pip_args.append("--upgrade") + if args.find_links: + pip_args += ["--no-index", "--find-links", args.find_links] + pip_args.append(mod) + + cmd = ["su", "-s", "/bin/bash", user, "-c", + " ".join(shlex.quote(x) for x in env + pip_args)] + printInfo(" ".join(pip_args) + f" (as {user})") + if _run(cmd, show_output=True): + printSuccess(f"{mod} installed as package-user {user}") + else: + printError(f"{mod} failed -- see the output above") + return 0 + def cmd_add_user(args): for name in args.packages: if _user_exists(name): printInfo(f"{name}: already exists.") continue - cmd = find_add_package_user() + [name, name, "10000", "20000", - name, "10000", "20000"] - printInfo(f"Creating package-user {name}: {' '.join(cmd)}") - if _run(cmd): + if create_package_user(name): printSuccess(f"Created package-user {name}.") report_created_user(name) @@ -2696,6 +3432,46 @@ def cmd_paths(args): print(f" main user : {main_user() or '(unset)'}") +# Settings worth asking for when they are missing. Only ones that change +# behaviour: a blank main_user means "no human account", which is a decision, +# not an oversight, so it is asked once and then left alone. +_CONFIG_PROMPTS = [ + ("collector_prefix", + "prefix for collector groups (shared install-dir groups)", + "sysgroup"), + ("user_prefix", "prefix for shared application users", "u"), + ("main_user", "your human login account (blank if none)", + lambda: _guess_main_user() or ""), +] + + +def _book_in_use(tool): + """Which book each tool would use -- the thing you actually want to know + when a package cannot be found.""" + if tool == "blfs": + rc, out, _e = run_blfs(["books"]) + if rc != 0: + return "(blfs not available)" + for ln in out.splitlines(): + if "[" in ln and "]" in ln: + return ln.split("[", 1)[1].split("]", 1)[0] + return "(none cached -- blfs fetch, or blfs import <file>)" + # Read the config directly. Parsing `lfs config` needed a mounted LFS + # tree, which does not exist inside the chroot -- so the book the system + # was built from showed as "(unknown)" on the very system it built. + cfg = _shared_lfs_config() + ver = cfg.get("build_book") or cfg.get("default") + if not ver: + return "(no book configured)" + for cand in (os.environ.get("LFS_STORE"), "/usr/share/lfs"): + if not cand: + continue + p = os.path.join(cand, "books", f"LFS-BOOK-{ver}-NOCHUNKS.html") + if os.path.isfile(p): + return os.path.basename(p) + return f"{ver} (not cached here)" + + def cmd_config(args): cfg = load_config() changed = False @@ -2714,6 +3490,22 @@ def cmd_config(args): if args.main_user: cfg["main_user"] = args.main_user changed = True + # Nothing given on the command line and something is unset? Ask. Leaving + # a setting blank and carrying on means it surfaces much later as a failed + # install with no obvious cause. + if not changed and not getattr(args, "show", False): + for key, prompt, default in _CONFIG_PROMPTS: + if cfg.get(key): + continue + d = default() if callable(default) else default + try: + ans = input(f" {prompt} [{d}]: ").strip() + except (EOFError, KeyboardInterrupt): + print() + break + cfg[key] = ans if ans else d + changed = True + if changed: if save_config(cfg): printSuccess(f"Saved {CONFIG_PATH}") @@ -2725,6 +3517,9 @@ def cmd_config(args): print(f" config file : {where}") print(f" collector prefix : {cfg['collector_prefix']}_ " f"(shared install-dir groups)") + for label, tool, args_ in (("LFS book", "lfs", ["config"]), + ("BLFS book", "blfs", ["books"])): + print(f" {label:<16} : {_book_in_use(tool)}") print(f" user prefix : {cfg['user_prefix']}_ (shared users)") print(f" main user : {cfg.get('main_user') or '(not set)'} " f"(your human login account)") @@ -2764,6 +3559,37 @@ def _add_groups(user, groups): printSuccess(f"Added {user} to {g}.") +_APPLICATION_USER_HELP = """\ +Create an application user and make it usable from your own account. + +A package user owns the FILES a package installs. An application user is a +separate idea: an account a program RUNS as, so that a browser or a media +player cannot read your documents, your keys, or another application's data. + +Creating the account is the easy part; what makes it usable is the rest: + + * your home directory stays yours -- the account's home is 0750, so the + application cannot read it + * your login joins the account's group, so a shared directory works + * `su - u_<name>` from your account without a password, where the hint's + /etc/pam.d/su_u_user is installed (skipped where it is not: su then + simply asks for a password) + * with --shared, a directory both accounts can write, symlinked into your + home -- how you get files in and out + * with --launcher, a wrapper in ~/bin so running `<name>` starts the + program as that account + +Your own login is the "main user" setting; without it there is nothing to +grant access to: + + packagemanager config --main-user <your-login> + +Example: + + packagemanager user create firefox --shared --launcher --groups audio,video +""" + + def cmd_user_list(args): up = user_prefix() if pwd is None: @@ -2792,10 +3618,134 @@ def cmd_user_create(args): if not _run(cmd): return printSuccess(f"{u} created.") - report_created_user(u, kind="shared user") + report_created_user(u, kind="application user") if args.groups: _add_groups(u, [g for g in re.split(r"[,\s]+", args.groups) if g]) + # An account on its own is not usable: you cannot become it, its home is + # world-readable, and nothing launches the program. Do the rest. + _setup_application_user(u, args) + + +def _setup_application_user(u, args): + """Make an application user actually usable from your own account. + + Creating the account is the easy part. What makes it useful: + * your account may `su - <user>` without a password (via PAM) + * the home is 0750, so other users cannot read it + * your account joins the user's group, for the shared directory + * an optional shared directory both accounts can write + * an optional launcher in ~/bin so the program runs as that user + """ + home = os.path.join(BASE_DIR, u) + main = main_user() + if not main: + printWarning("\nNo main user is set, so this account cannot be reached " + "from your own.") + printWarning("Set it, then run this again:") + printWarning(" packagemanager config --main-user <your-login>") + return + + # 1. the home is private + if os.path.isdir(home): + _run(["chmod", "0750", home]) + printInfo(f" home {home} is 0750 (private to {u} and its group)") + + # 2. your account joins the user's group -- this is what makes a shared + # directory work, and what lets PAM identify you as entitled + _run(["usermod", "-a", "-G", u, main]) + printInfo(f" {main} joined group '{u}'") + + # 3. su without a password, if this system uses the hint's PAM file + _allow_su_via_pam(u, main) + + # 4. a shared directory, when asked for + if getattr(args, "shared", False): + share = os.path.join(home, "Shared") + _run(["mkdir", "-p", share]) + _run(["chown", f"{u}:{u}", share]) + _run(["chmod", "2770", share]) # setgid: new files keep the group + link = os.path.join(_home_of(main) or "", f"{args.name}-shared") + if link and not os.path.exists(link): + _run(["ln", "-s", share, link]) + printInfo(f" shared directory: {share}") + if link: + printInfo(f" reachable from your account as {link}") + + # 5. a launcher, when the program is on PATH + app = getattr(args, "app", None) or args.name + import shutil as _sh + if getattr(args, "launcher", False): + if not _sh.which(app): + printWarning(f" no '{app}' on PATH -- not writing a launcher") + else: + _write_launcher(main, u, app) + + +def _home_of(user): + try: + import pwd + return pwd.getpwnam(user).pw_dir + except (KeyError, ImportError): + return None + + +def _allow_su_via_pam(u, main): + """Let <main> su to <u> without a password, if the PAM file is present. + + The hint ships /etc/pam.d/su_u_user with a case statement listing which + accounts each user may become. It is not part of stock LFS, so this is a + no-op when the file does not exist -- `su` then simply asks for a + password, which still works.""" + pam = "/etc/pam.d/su_u_user" + if not os.path.isfile(pam): + printInfo(f" ({pam} not present -- `su - {u}` will ask for a password)") + return + try: + text = open(pam).read() + except OSError as e: + printWarning(f" could not read {pam}: {e}") + return + if re.search(r"^\s*%s\)" % re.escape(u), text, re.M): + printInfo(f" {pam} already allows it") + return + marker = 'case "$PAM_USER" in' + if marker not in text: + printWarning(f" {pam} has no '{marker}' block -- not editing it") + return + lines = text.splitlines(True) + out = [] + for line in lines: + out.append(line) + if marker in line: + out.append(' %s) USERS="%s %s"; ;;\n' % (u, u, main)) + try: + with open(pam, "w") as f: + f.writelines(out) + printInfo(f" {main} may now `su - {u}` without a password") + except OSError as e: + printWarning(f" could not write {pam}: {e}") + + +def _write_launcher(main, u, app): + """A wrapper in ~/bin that runs <app> as the application user.""" + home = _home_of(main) + if not home: + return + bindir = os.path.join(home, "bin") + os.makedirs(bindir, exist_ok=True) + path = os.path.join(bindir, app) + if os.path.exists(path): + printInfo(f" {path} already exists -- left alone") + return + with open(path, "w") as f: + f.write('#!/bin/bash\n' + '# Runs %s as the application user %s.\n' + 'exec su - %s -c "%s $(printf \'%%q \' "$@")"\n' % (app, u, u, app)) + os.chmod(path, 0o755) + _run(["chown", f"{main}:{main}", path]) + printInfo(f" launcher: {path}") + def cmd_user_allow(args): user = args.user @@ -3742,14 +4692,279 @@ def cmd_fix_install(args): # cli # --------------------------------------------------------------------------- # +PM_HELP_EPILOG = """\ +commands, by what you are doing: + + finding out what is installed + list every package user on the system + info <pkg> what a package installed, and where it came from + search <pkg> find a package in the book + check <pkg> is it installed, and does it look right + verify [--fix] check every package, optionally repairing + files_with_broken_id files owned by a user that no longer exists + + installing and updating + install <pkg> build and install as a package user + update [<pkg>] rebuild what the book has a newer version of + remove <pkg> uninstall and delete the package user + pip install <module> install a Python module as a package user + dependencies <pkg> what it needs, in build order + + install scripts + script <phase> <pkg> run one phase of a package's script + regenerate-script <pkg> rewrite it from the book + script-status which scripts are edited, current, or outdated + migrate-scripts move scripts to a newer book, keeping edits + template <pkg> skeleton for a package not in the book + + users, groups and permissions + add-user <pkg> create a package user + user / shared-user application users + sysgroup collector groups + make-group-dir <dir> hand a directory to a collector group + add-dir-to-sysgroup <dir> share an existing directory + fix-group-dir / fix-install + repair permissions + + settings and housekeeping + config show or change settings + paths where everything lives + init first-run setup + errors what went wrong in the last run + plan what a run would do, before doing it + reload-pkg-list refresh the cached package list + blfs ... pass a command straight to blfs + +LFS base packages are the `lfs` tool's job; BLFS packages are handled here. +""" + + +# --------------------------------------------------------------------------- # +# housekeeping run on every invocation: cheap checks that stop a temporary +# workaround from quietly becoming permanent +# --------------------------------------------------------------------------- # + +_CA_BUNDLES = ["/etc/ssl/certs/ca-bundle.crt", "/etc/ssl/ca-bundle.crt", + "/etc/pki/tls/certs/ca-bundle.crt", + "/etc/ssl/certs/ca-certificates.crt"] +_WGETRC_MARK = "lfs-temporary-no-verify" + + +def restore_wget_verification(): + """Re-enable certificate checking once CA certificates exist. + + The build disables it in /etc/wgetrc so a certificate-less system can fetch + make-ca at all. Leaving that in place afterwards would mean every download + on this system stays unverified forever, so undo it the moment it is no + longer needed.""" + rc = "/etc/wgetrc" + if not os.path.isfile(rc): + return + try: + text = open(rc).read() + except OSError: + return + if _WGETRC_MARK not in text: + return + have_ca = next((c for c in _CA_BUNDLES if os.path.isfile(c) + and os.path.getsize(c) > 0), None) + if not have_ca: + return + # keep everything outside the marked block, drop the block itself + out, inside = [], False + for line in text.splitlines(True): + if _WGETRC_MARK in line: + inside = "end " not in line + continue + if not inside: + out.append(line) + try: + with open(rc, "w") as f: + f.writelines(out) + except OSError as e: + printWarning(f"could not restore certificate checking in {rc}: {e}") + return + printInfo(f"CA certificates found ({have_ca}) -- certificate checking " + f"re-enabled in {rc}") + + +def check_make_ca_generated(): + """make-ca installed but no certificate bundle? Say what finishes the job. + + `make-ca -g` reports "No update required!" and generates nothing when it + thinks its copy of certdata.txt is current -- which it is, straight after + install, even though no bundle has ever been written. The install then + looks successful while the system still has no certificates, and every + HTTPS fetch keeps warning.""" + import shutil as _sh + if not _sh.which("make-ca") and not os.path.isfile("/usr/sbin/make-ca"): + return + if any(os.path.isfile(c) and os.path.getsize(c) > 0 for c in _CA_BUNDLES): + return + printWarning("\nmake-ca is installed but this system still has no CA " + "certificate bundle.") + printWarning("`make-ca -g` skips generating when it considers its copy of") + printWarning("certdata.txt current, which it is right after installing.") + printWarning("Force it once:") + printWarning(" /usr/sbin/make-ca -g --force") + printWarning("Then HTTPS verification works and the tools re-enable it " + "automatically.\n") + + +def ensure_wget_workaround(): + """Let wget fetch anything at all on a system with no CA certificates. + + This is the other half of restore_wget_verification(). Without it the + first thing anyone tries -- installing make-ca, so that certificates exist + -- fails, because fetching make-ca itself needs a verified connection: + + ERROR: cannot verify ftpmirror.gnu.org's certificate ... + ERROR: md5sum check failed + + /etc/wgetrc is used rather than a $PATH wrapper because builds run through + `su -` with the package user's own environment, where a wrapper directory + on PATH is never seen. + """ + if any(os.path.isfile(c) and os.path.getsize(c) > 0 for c in _CA_BUNDLES): + return # certificates exist: nothing to do + if not shutil.which("wget"): + return # no wget yet, nothing to configure + rc = "/etc/wgetrc" + try: + text = open(rc).read() if os.path.isfile(rc) else "" + except OSError: + return + if _WGETRC_MARK in text: + return # already done + try: + with open(rc, "a") as f: + f.write( + "\n# --- %s ----------------------------------------\n" + "# This system has no CA certificates yet, so wget cannot\n" + "# verify anything. Downloads are UNVERIFIED until make-ca\n" + "# is installed; this block is removed automatically then.\n" + "check_certificate = off\n" + "# --- end %s ------------------------------------\n" + % (_WGETRC_MARK, _WGETRC_MARK)) + except OSError as e: + printWarning(f"could not write {rc}: {e}") + return + printWarning( + "No CA certificates on this system yet, so wget could not verify\n" + "anything and every download would fail. Certificate checking has\n" + "been turned OFF in /etc/wgetrc so that make-ca can be fetched.\n" + "\n" + " Downloads are UNVERIFIED until make-ca is installed.\n" + " This is undone automatically as soon as certificates exist.\n") + + +def _warn_init_mismatch(): + """The BLFS book's init flavour should match the system's. + + A systemd book emits `systemctl` calls in its install scripts; on a SysV + system those fail with "command not found" in the middle of an otherwise + successful install, and every affected package has to be edited by hand. + The two books differ in more than that, so it is worth saying once.""" + have_systemd = (os.path.isdir("/usr/lib/systemd/system") + or bool(shutil.which("systemctl"))) + rc, out, _e = run_blfs(["books"]) + if rc != 0: + return + book = "" + for ln in out.splitlines(): + if "[" in ln and "]" in ln: + book = ln.split("[", 1)[1].split("]", 1)[0] + break + if not book: + return + book_systemd = "systemd" in book.lower() + if book_systemd == have_systemd: + return + printWarning("\nThe BLFS book in use does not match this system's init:") + printInfo(f" book : {book} ({'systemd' if book_systemd else 'SysV'})") + printInfo(f" system : {'systemd' if have_systemd else 'SysV'}") + printInfo("") + printInfo("Install scripts from the wrong book carry commands this system") + printInfo("does not have -- `systemctl: command not found` mid-install is") + printInfo("the usual symptom. Switch with:") + printInfo("") + printInfo(" blfs fetch %s" % ("stable" if have_systemd is False + else "stable-systemd")) + printInfo("") + + +def _hint_generate_ca_bundle(): + """make-ca is installed but the certificate store is empty. + + Installing make-ca does not create any certificates -- it installs the + tool that generates them. Until it is run there is still no CA bundle, + wget still cannot verify anything, and the /etc/wgetrc workaround stays in + place. The step is easy to miss because the install reports success.""" + if any(os.path.isfile(c) and os.path.getsize(c) > 0 for c in _CA_BUNDLES): + return # certificates exist already + if not (os.path.isfile("/usr/sbin/make-ca") + or shutil.which("make-ca")): + return # make-ca not installed yet + printWarning("\nmake-ca is installed but no CA certificates exist yet.") + printInfo("Installing it does not create them -- it installs the tool.") + printInfo("Generate the certificate store now:") + printInfo("") + printInfo(" make-ca -g") + printInfo("") + printInfo("Then HTTPS verification works, and the temporary") + printInfo("'check_certificate = off' in /etc/wgetrc is removed " + "automatically.\n") + + +def _file_md5(path): + try: + with open(path, "rb") as f: + return hashlib.md5(f.read()).hexdigest() + except OSError: + return None + + +def warn_if_tools_are_stale(): + """Tell the user when the chroot is running older tools than the host. + + The tools are copied into the tree; edit them outside and the chroot keeps + running the old copy until they are copied again. That is a genuinely + confusing failure mode -- behaviour that was fixed "yesterday" reappears -- + so say so plainly.""" + stamp = "/usr/share/lfs/tool-stamps.json" + if not os.path.isfile(stamp): + return + try: + stamps = json.load(open(stamp)) + except (OSError, ValueError): + return + stale = [] + for name, want in stamps.items(): + path = shutil.which(name) or os.path.join("/usr/bin", name) + have = _file_md5(path) + if have and want and have != want: + stale.append(name) + if stale: + printWarning( + "\nThese tools differ from the copies on the host: " + + ", ".join(sorted(stale))) + printWarning(" You are probably running an older version in here. " + "On the HOST, run:") + printWarning(" lfs build-system sync-tools --run") + printWarning(" lfs build-system install-tools --run --force\n") + + def main(): ap = argparse.ArgumentParser(prog="packagemanager", description="pkgusr-model package manager") ap.add_argument("-V", "--version", action="version", - version=f"packagemanager {TOOL_VERSION} (plan cache v{PLAN_VERSION})") + version=f"packagemanager {TOOL_VERSION} (build {_build_id()}, " + f"plan cache v{PLAN_VERSION})") ap.add_argument("-v", "--verbose", action="store_true", help="show the system commands being run and their output") - sub = ap.add_subparsers(dest="command", required=True) + ap.epilog = PM_HELP_EPILOG + ap.formatter_class = argparse.RawDescriptionHelpFormatter + sub = ap.add_subparsers(dest="command", required=True, metavar="<command>") # ---- info (implemented) ---- p = sub.add_parser("verify", help="recheck install state of all/named " @@ -3911,6 +5126,19 @@ def main(): help="wait for it instead of running as a disowned task") p.set_defaults(func=cmd_reload_pkg_list) + p = sub.add_parser("pip", help="install or update a Python module as a " + "package user (not as root)") + p.add_argument("action", choices=["install", "update", "upgrade"], + help="install or update") + p.add_argument("modules", nargs="*", help="module names, e.g. beautifulsoup4") + p.add_argument("--find-links", metavar="DIR", + help="install from local wheels in DIR instead of the network " + "(there is no network inside the chroot)") + p.set_defaults(func=lambda a: cmd_pip( + argparse.Namespace(modules=a.modules, + upgrade=(a.action in ("update", "upgrade")), + find_links=a.find_links))) + p = sub.add_parser("add-user", help="create a PACKAGE user (installs files; home /usr/src/<n>)") p.add_argument("packages", nargs="+") @@ -3955,6 +5183,8 @@ def main(): "(default: $EDITOR or vim)") p.add_argument("--difftool", help="diff tool for 'migrate-scripts --vimdiff' " "(default: vimdiff)") + p.add_argument("--show", action="store_true", + help="just print the settings; do not ask for missing ones") p.set_defaults(func=cmd_config) p = sub.add_parser("blfs", help="run the blfs book tool " @@ -3970,9 +5200,19 @@ def main(): q = usub.add_parser("list", help="list shared users and their groups") q.add_argument("filter", nargs="?") q.set_defaults(func=cmd_user_list) - q = usub.add_parser("create", help="create an shared user (u_<name>)") + q = usub.add_parser("create", + help="create an application user (u_<name>) and make " + "it reachable from your own account", + description=_APPLICATION_USER_HELP, + formatter_class=argparse.RawDescriptionHelpFormatter) q.add_argument("name") q.add_argument("--groups", help="also add these groups, e.g. audio,video") + q.add_argument("--shared", action="store_true", + help="a directory both accounts can write, linked into " + "your home") + q.add_argument("--launcher", action="store_true", + help="a wrapper in ~/bin that runs the program as this user") + q.add_argument("--app", help="program the launcher runs (default: <name>)") q.set_defaults(func=cmd_user_create) q = usub.add_parser("delete", help="delete an shared user + group + home") q.add_argument("name") @@ -4127,6 +5367,11 @@ def main(): p.set_defaults(func=cmd_files_with_broken_id) args = ap.parse_args() + restore_wget_verification() + _hint_generate_ca_bundle() + ensure_wget_workaround() + _warn_init_mismatch() + warn_if_tools_are_stale() _STATE["verbose"] = getattr(args, "verbose", False) # first run: ask for the group-name prefixes (once), unless we're already # running the config command or there's no terminal to prompt at. diff --git a/packagemanager_install b/packagemanager_install old mode 100755 new mode 100644 index cf3a1ea..94fd412 --- a/packagemanager_install +++ b/packagemanager_install @@ -26,6 +26,80 @@ date_log="$(date +%Y-%m-%d_%H-%m-%S)" home_user="n76310" home_user_dir="/home/${home_user}" +# --------------------------------------------------------------------------- # +# Command wrappers for the build +# --------------------------------------------------------------------------- # +# Packages routinely try to (re)create and re-mode directories that already +# exist and belong to someone else: +# install -vdm755 /usr/sbin +# install: cannot change permissions of '/usr/sbin': Operation not permitted +# The directory is already there with the right mode; the package just insists +# on saying so. Under the package-user model that is a hard error, and it +# stops installs that would otherwise be completely fine. +# +# So put small wrappers ahead of the real tools for the duration of the build: +# install -d on an existing directory -> succeed, change nothing +# chmod on a directory we do not own -> skip, keep going +# chown/chgrp on anything -> skip (a package user may not) +# Anything else is passed straight through. This is the same approach +# lfs-helper uses during the chroot build. +_make_build_wrappers() { + local d="$1" + mkdir -p "$d" || return 1 + + cat > "$d/install" <<'WRAP' +#!/bin/bash +# only -d (create directories) is special; everything else is the real thing +_dirmode=0 +for a in "$@"; do + case "$a" in + -d|--directory) _dirmode=1 ;; + -*d*) case "$a" in -*[!-]*) _dirmode=1 ;; esac ;; + esac +done +if [ "$_dirmode" = 1 ]; then + _rc=0 + for a in "$@"; do + case "$a" in -*) continue ;; esac + if [ -d "$a" ]; then + continue # already there: nothing to do + fi + /usr/bin/install -d "$a" || _rc=$? + done + exit $_rc +fi +exec /usr/bin/install "$@" +WRAP + + cat > "$d/chmod" <<'WRAP' +#!/bin/bash +# a package user may only re-mode what it owns +_me="$(id -un)" +_rc=0 +for a in "$@"; do + case "$a" in -*) continue ;; esac + [ -e "$a" ] || continue + if [ "$(stat -c %U "$a" 2>/dev/null)" != "$_me" ]; then + echo "*** chmod $a (skipped: owned by $(stat -c %U "$a" 2>/dev/null))" >&2 + continue + fi + /usr/bin/chmod "$@" || _rc=$? + exit $_rc +done +exit 0 +WRAP + + for t in chown chgrp; do + cat > "$d/$t" <<WRAP +#!/bin/bash +echo "*** $t \$* (skipped: package users cannot change ownership)" >&2 +exit 0 +WRAP + done + + chmod 755 "$d"/install "$d"/chmod "$d"/chown "$d"/chgrp +} + echoRed() { echo -e "\033[0;31m$1\033[0m" } @@ -48,6 +122,12 @@ addUser() { fi if [ $UID -ne 0 ]; then echo Please run this script as root. ; exit 1; fi add_package_user "${1}" $2 10000 20000 $3 10000 20000 || exit 1 + # Keep /etc/passwd and /etc/group in id order. They are appended to, so + # without this they drift into install order and someone has to tidy them + # up later -- which only happens if they remember. lfs-helper owns the + # sorting so both halves of the toolchain do it the same way. + command -v lfs-helper >/dev/null 2>&1 \ + && lfs-helper sort-users --run >/dev/null 2>&1 || true } addCollectorGroup() { @@ -71,7 +151,7 @@ addCollectorGroup() { echo "addUser "$group_name"" echo -n "[y/N]: " - read check + read -e -r check [ "$check" != "y" ] && exit addUser "$group_name" echo "Created whyle installing $user_name" > info.txt @@ -140,7 +220,7 @@ checkGroups() { echoOrange "By adding to a group the user gets access to all files of the group" echo -e " usermod -a -G $group $user_name" echo -n "[y/N]: " - read check + read -e -r check [ "$check" == "y" ] && [ "$group $user_name" != " " ] && addUerToCollectorGroup "$group" fi @@ -176,7 +256,7 @@ addUserToGroup() { check="y" else echo -n "[y/N]: " - read check + read -e -r check fi [ "$check" == "y" ] && ( @@ -223,7 +303,7 @@ installNimgnuGroup() { echo "change permission recrusive for $user_name on;" echo "$dir" echo -n "[y/N]: " - read check + read -e -r check [ "$check" == "y" ] && parameter="-R" @@ -433,15 +513,235 @@ installPkg() { chown $user_name: $install_user_dir/install_* chown -R $user_name: $install_user_dir/log/ # pipefail so tee doesn't hide a failing build/install; capture the real code - su - -c "set -o pipefail; bash ~/install_$user_name $parameter 2>&1 | tee ~/log/packagemanager_install-$date_log.log" $user_name + _wrapdir="/tmp/pkgusr-wrappers.$$" + _make_build_wrappers "$_wrapdir" || _wrapdir="" + chmod 755 "$_wrapdir" 2>/dev/null || true + su - -c "set -o pipefail; \ + ${_wrapdir:+PATH='$_wrapdir':\$PATH; export PATH; } \ + bash ~/install_$user_name $parameter 2>&1 | tee ~/log/packagemanager_install-$date_log.log" $user_name install_rc=${PIPESTATUS[0]} + [ -n "${_wrapdir:-}" ] && rm -rf "$_wrapdir" if echo "$user_name" | grep "^u_"; then gpasswd -d $user_name install # remove install group if nimgnu_group fi +# --------------------------------------------------------------------------- # +# Automatic permission repair, the same way lfs-helper does it during the build. +# +# A package that installs into a directory another package owns fails with a +# permission error. That is not a mistake to report -- it is the normal state +# of affairs under the package-user model, and the answer is always the same: +# put both packages in a collector group that owns the directory. So do it, +# and retry, instead of stopping and asking the user to type it out. +# --------------------------------------------------------------------------- # + +collector_prefix_of() { + local conf="${PKGUSR_CONFIG:-/etc/pkgusr/packagemanager.conf}" p="" + [ -f "$conf" ] && p="$(sed -n 's/.*"collector_prefix"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "$conf" | head -n1)" + [ -n "$p" ] || p="sysgroup_" + case "$p" in *_) ;; *) p="${p}_" ;; esac # always ends in one underscore + printf '%s' "$p" +} + +# Directories the log says the package could not write into. +# +# Take paths ONLY from lines that actually report a permission error. A Python +# traceback is full of paths -- meson's own source files, the interpreter's +# stdlib -- and treating those as install targets meant granting a package +# write access to /usr/lib/python3.13. The failing path is on the error line, +# never in the frames above it. +unwritable_dirs_from_log() { + local log="$1" user="$2" p d + [ -f "$log" ] || return 0 + grep -hE "Permission denied|Operation not permitted|Read-only file system" "$log" 2>/dev/null \ + | grep -vE "^[[:space:]]*(File|Traceback|[[:space:]]*[~^]+$)" \ + | { grep -hoE "'[^']+'|\"[^\"]+\"" || true; } | tr -d "'\"" \ + | while IFS= read -r p; do + case "$p" in /*) ;; *) continue ;; esac + if [ -d "$p" ]; then echo "$p"; else echo "${p%/*}"; fi + done | sort -u \ + | while IFS= read -r d; do + [ -n "$d" ] && [ -d "$d" ] || continue + su -s /bin/bash "$user" -c "test -w '$d'" 2>/dev/null && continue + echo "$d" + done +} + +# give <user> the right to install into <dir> +grant_dir_to_user() { + local dir="$1" user="$2" prefix owner grp cur + + # ONE implementation of the collector-group rules, not two. + # + # lfs-helper owns this logic: join the group the directory already carries, + # ask which group should own it when there is none, never create one for + # root, never hand a package's tree to the install group. When it is on + # the system -- which it is, on anything these tools built -- use it, so a + # package installed after boot behaves exactly like one built during the + # chroot stage. The code below is the fallback for a system without it. + if command -v lfs-helper >/dev/null 2>&1; then + lfs-helper grant-dir "$dir" "$user" --run + return $? + fi + + prefix="$(collector_prefix_of)" + owner="$(stat -c %U "$dir" 2>/dev/null)" || return 1 + [ "$owner" = "$user" ] && return 1 + cur="$(stat -c %G "$dir" 2>/dev/null)" + + case "$cur" in + "${prefix}"*) + # the directory is ALREADY shared -- just join the existing group, + # which is the whole point of having one + grp="$cur" + ;; + install) + grp="install" + ;; + *) + # A ROOT-owned directory is never a package's: it is either shared + # system infrastructure or something the distribution installed. + # "<prefix>_root" would be a group meaning "may write anywhere root + # owns", which is the whole scheme undone -- and there is no user + # 'root' to add to a collector group in any case. + if [ "$owner" = "root" ]; then + echo " $dir (owned by root -- NOT shared;" >&2 + echo " a collector group for root would defeat the scheme." >&2 + echo " If this really is a shared install directory:" >&2 + echo " chgrp install '$dir' && chmod g+w '$dir'" >&2 + return 1 + fi + grp="${prefix}${owner}" + # Collector groups live in their own id range (90000+), clear of + # the package users at 10000+ -- otherwise a collector group takes + # the gid the next package user's group wants and the uid/gid + # pairing stops lining up. + if ! getent group "$grp" >/dev/null 2>&1; then + _cgid=90000 + while getent group "$_cgid" >/dev/null 2>&1; do + _cgid=$((_cgid + 1)) + done + groupadd -g "$_cgid" "$grp" >/dev/null 2>&1 \ + || groupadd "$grp" >/dev/null 2>&1 + fi + getent group "$grp" >/dev/null 2>&1 || return 1 + usermod -a -G "$grp" "$owner" 2>/dev/null + chgrp "$grp" "$dir" 2>/dev/null || return 1 + ;; + esac + + usermod -a -G "$grp" "$user" 2>/dev/null + chmod g+w "$dir" 2>/dev/null + echo " $dir -> group $grp (owner $owner, + $user)" + return 0 +} + +# returns 0 if anything was granted +auto_grant_from_log() { + local log="$1" user="$2" need d granted=1 + need="$(unwritable_dirs_from_log "$log" "$user")" + [ -n "$need" ] || return 1 + echo -e "\033[0;33m" + echo "# '$user' could not write into some directories -- granting access" + echo "# via collector groups and retrying:" + while IFS= read -r d; do + [ -n "$d" ] || continue + grant_dir_to_user "$d" "$user" && granted=0 + done <<< "$need" + echo -e "\033[0m" + return $granted +} + +# Say what actually went wrong. +# +# Build systems bury the real error. Meson reports a permission problem as +# thirty lines of Python traceback ending in +# PermissionError: [Errno 13] Permission denied: '/usr/share/zsh/site-functions/_p11-kit' +# and the useful part -- one path this package may not write to -- is easy to +# miss entirely. Pull it out and say what to do about it. +diagnose_install_failure() { + local user="$1" log="$2" + [ -f "$log" ] || return 0 + + # the paths a permission error named, however the tool phrased it + # Only the TAIL of the log: a build log runs to tens of megabytes and the + # error is always at the end. Scanning all of it, then running stat on + # every path it mentions, is what made this take minutes of CPU. + local paths tail_log + tail_log="$(tail -c 524288 "$log" 2>/dev/null)" + paths="$(printf '%s\n' "$tail_log" \ + | grep -aE "Permission denied|Operation not permitted|PermissionError" \ + | grep -aoE "/[A-Za-z0-9._/+-]+" | sort -u | head -n 20)" + + if [ -n "$paths" ]; then + echo -e "\033[0;33m" + echo "--- what went wrong -------------------------------------------" + echo "'$user' could not write to:" + local p d + while IFS= read -r p; do + [ -n "$p" ] || continue + d="$p"; [ -d "$d" ] || d="${p%/*}" + if [ -d "$d" ]; then + printf " %-46s %s %s:%s\n" "$p" \ + "$(stat -c %A "$d" 2>/dev/null)" \ + "$(stat -c %U "$d" 2>/dev/null)" "$(stat -c %G "$d" 2>/dev/null)" + else + printf " %-46s (and its parent does not exist)\n" "$p" + fi + done <<< "$paths" + echo + echo "That directory belongs to another package. Share it, then retry" + echo "just the install (no recompile):" + echo + local first; first="$(printf '%s\n' "$paths" | head -n1)" + d="$first"; [ -d "$d" ] || d="${first%/*}" + echo " packagemanager add-dir-to-sysgroup $d $user" + echo " packagemanager script install $user" + echo "---------------------------------------------------------------" + echo -e "\033[0m" + return 0 + fi + + # otherwise: the last few lines that look like an error, not the whole log + local tail_err + tail_err="$(grep -hiE "^(error|fatal|.*: error:|make.*\*\*\*|.*Error [0-9]+)" "$log" 2>/dev/null \ + | grep -viE "error [0-9]+ \(ignored\)" | tail -n 5)" + if [ -n "$tail_err" ]; then + echo -e "\033[0;33m" + echo "--- what went wrong -------------------------------------------" + printf '%s\n' "$tail_err" | sed 's/^/ /' + echo "---------------------------------------------------------------" + echo -e "\033[0m" + fi + echo " full log: $log" +} + + # A permission failure is repairable and common: grant the access and try + # again, up to a few rounds (each attempt reveals only the FIRST directory + # it could not write, so one round is rarely enough). + _log="$(eval echo ~$user_name)/log/packagemanager_install-$date_log.log" + _round=0 + while [ "$install_rc" != "0" ] && [ "${PM_NO_AUTO_FIX:-0}" != "1" ] \ + && [ "$_round" -lt 20 ]; do + grep -qiE "permission denied|operation not permitted|cannot (create|remove|touch)" \ + "$_log" 2>/dev/null || break + auto_grant_from_log "$_log" "$user_name" || break + _round=$((_round + 1)) + echo -e "\033[0;33m# retrying $user_name ($parameter), round $_round ...\033[0m" + _wrapdir="/tmp/pkgusr-wrappers.$$" + _make_build_wrappers "$_wrapdir" || _wrapdir="" + chmod 755 "$_wrapdir" 2>/dev/null || true + su - -c "set -o pipefail; \ + ${_wrapdir:+PATH='$_wrapdir':\$PATH; export PATH; } \ + bash ~/install_$user_name $parameter 2>&1 | tee ~/log/packagemanager_install-$date_log.log" $user_name + install_rc=${PIPESTATUS[0]} + [ -n "${_wrapdir:-}" ] && rm -rf "$_wrapdir" + done + if [ "$install_rc" != "0" ]; then - echo -e "\033[0;31m!! FAILED: $user_name ($parameter) exited $install_rc. See the log above / ~$user_name/log/.\033[0m" + echo -e "\033[0;31m!! FAILED: $user_name ($parameter) exited $install_rc\033[0m" + diagnose_install_failure "$user_name" "$_log" exit $install_rc fi ################### diff --git a/test_lfs_crosschain.sh b/test_lfs_crosschain.sh new file mode 100644 index 0000000..1646784 --- /dev/null +++ b/test_lfs_crosschain.sh @@ -0,0 +1,4242 @@ +#!/bin/bash +# End-to-end test for lfs crosschain script generation. +# +# Builds a REAL (tiny) package through a REAL generated script: unpack from a +# tarball, configure into a build/ subdir, compile, install, post-install fixup. +# Then checks the failure paths -- because "it printed done" is not evidence +# that anything worked. +# +# Usage: bash test_lfs_crosschain.sh [path-to-lfs-script] +set -u + +LFS_TOOL="${1:-./lfs}" +LFS_BOOK="${2:-/mnt/user-data/uploads/LFS-BOOK-12_4-NOCHUNKS.html}" +T=$(mktemp -d /tmp/lfstest.XXXXXX) +PASS=0; FAIL=0 +ok() { echo " PASS $1"; PASS=$((PASS+1)); } +bad() { echo " FAIL $1"; FAIL=$((FAIL+1)); } + +cleanup() { rm -rf "$T"; } +trap cleanup EXIT + +export LFS="$T/lfs" +export BUILD_ROOT="$LFS/sources" +mkdir -p "$LFS/sources" "$LFS/tools/bin" + +# ---------------------------------------------------------------- fake package +# A package that behaves like a real autotools one: out-of-tree build dir, +# a configure that must run before make, and an install step. +mkdir -p "$T/src/demo-1.0" +cat > "$T/src/demo-1.0/configure" <<'EOF' +#!/bin/bash +prefix=/usr/local +for a in "$@"; do case "$a" in --prefix=*) prefix="${a#--prefix=}";; esac; done +cat > Makefile <<MK +all: + @echo "compiling"; echo "binary" > demo.bin +install: + @mkdir -p "$prefix/bin"; cp demo.bin "$prefix/bin/demo"; echo "installed to $prefix/bin/demo" +MK +echo "configured with prefix=$prefix" +EOF +chmod +x "$T/src/demo-1.0/configure" +( cd "$T/src" && tar czf "$LFS/sources/demo-1.0.tar.gz" demo-1.0 ) + +# ------------------------------------------------- generate a script for real +# Uses the SAME generator the tool uses, with book-shaped command blocks. +python3 - "$LFS_TOOL" "$T/demo.sh" <<'PY' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +cmds = [ + "mkdir -v build\ncd build", + "../configure --prefix=$LFS/tools", + "make", + "make install", + "echo post-install-fixup > $LFS/tools/fixup.txt", +] +body = lfs._crosschain_script_body("demo", "ch-tools-demo", "demo-1.0", + "demo-1.0.tar.* demo-1.0.tgz", cmds) +open(sys.argv[2], "w").write(body) +PY + +bash -n "$T/demo.sh" && ok "generated script is valid bash" || bad "syntax error" + +# phases must be split the way the book reads +grep -q "^\.\./configure" "$T/demo.sh" && ok "configure landed in build phase" \ + || bad "configure not in build phase" +sed -n '/^install_pkg/,/INSTALL DONE/p' "$T/demo.sh" | grep -q "make install" \ + && ok "make install landed in install phase" || bad "install phase wrong" +sed -n '/^configure_pkg/,/CONFIGURE DONE/p' "$T/demo.sh" | grep -q "post-install-fixup" \ + && ok "post-install cmd landed in configure phase" || bad "configure phase wrong" + +# --------------------------------------------------------------- run it fully +out="$T/run.log" +if bash "$T/demo.sh" all > "$out" 2>&1; then + ok "full run exited 0" +else + bad "full run failed (exit $?)"; sed 's/^/ /' "$out" +fi + +[ -f "$LFS/sources/demo-1.0/build/Makefile" ] \ + && ok "built out-of-tree in build/ subdir" || bad "build/ subdir missing" +[ -f "$LFS/tools/bin/demo" ] \ + && ok "package actually installed into \$LFS/tools/bin" \ + || bad "install produced no file" +[ -f "$LFS/tools/fixup.txt" ] \ + && ok "configure (post-install) phase ran" || bad "configure phase did not run" +grep -q "^/.*build$" "$LFS/sources/.cc-build-demo" 2>/dev/null \ + && ok "recorded build dir for resume" || bad "build dir not recorded" + +# ------------------------------------------------- single-phase re-run works +rm -f "$LFS/tools/bin/demo" +if bash "$T/demo.sh" install > "$T/i.log" 2>&1 && [ -f "$LFS/tools/bin/demo" ]; then + ok "re-running ONLY the install phase works (no recompile)" +else + bad "install-only re-run failed"; sed 's/^/ /' "$T/i.log" +fi + +# ----------------------------------------------------- failure MUST propagate +sed 's|^make$|false # simulated compile failure|' "$T/demo.sh" > "$T/failbuild.sh" +if bash "$T/failbuild.sh" all > "$T/f.log" 2>&1; then + bad "FAILING BUILD REPORTED SUCCESS (the bug that wasted a day)" +else + ok "failing build exits non-zero" +fi +grep -q "phase 'build' FAILED" "$T/f.log" \ + && ok "failure names the phase" || bad "failure did not name the phase" +grep -qi "installed to" "$T/f.log" \ + && bad "continued to install after a failed build" \ + || ok "did NOT continue to install after a failed build" + +# missing tarball must fail cleanly, not silently +rm -rf "$T/lfs2"; mkdir -p "$T/lfs2/sources" +if LFS="$T/lfs2" BUILD_ROOT="$T/lfs2/sources" bash "$T/demo.sh" unpack \ + > "$T/m.log" 2>&1; then + bad "missing tarball reported success" +else + ok "missing tarball fails cleanly" +fi + +echo +echo " PASS: $PASS FAIL: $FAIL" +[ "$FAIL" -eq 0 ] + +# ---- tarball name resolution against a real-world /sources listing --------- # +# Regression guard: book titles are messy (Util-linux-2.41.1, XML::Parser-2.47, +# Flit-Core-3.12.0 shipping as flit_core-, Expect-5.45.4 as expect5.45.4) and a +# wrong guess makes unpack fail with "no <wrong-name> in /sources". +python3 - "$LFS_TOOL" <<'PY' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +cases = [ + ("7.12. Util-linux-2.41.1", "util-linux-2.41.1.tar.xz"), + ("7.10. Python-3.13.7", "Python-3.13.7.tar.xz"), + ("8.44. XML::Parser-2.47", "XML-Parser-2.47.tar.gz"), + ("8.52. Flit-Core-3.12.0", "flit_core-3.12.0.tar.gz"), + ("8.x. Expect-5.45.4", "expect5.45.4.tar.gz"), + ("8.x. Tcl-8.6.16", "tcl8.6.16-src.tar.gz"), + ("8.x. Expat-2.7.1", "expat-2.7.1.tar.xz"), + ("6.3. Ncurses-6.5-20250809", "ncurses-6.5-20250809.tgz"), + ("5.2. Binutils-2.45 - Pass 1", "binutils-2.45.tar.xz"), + ("5.4. Linux-6.16.1 API Headers", "linux-6.16.1.tar.xz"), + ("5.6. Libstdc++ from GCC-15.2.0", "gcc-15.2.0.tar.xz"), +] +import fnmatch +bad = 0 +for title, want in cases: + glob = lfs._pkg_glob_for(lfs._title_pkgver(title), "x") + hit = any(fnmatch.fnmatch(want, p) for p in glob.split()) + print((" PASS " if hit else " FAIL ") + f"{title!r} finds {want}") + bad += 0 if hit else 1 +sys.exit(1 if bad else 0) +PY +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi +echo " PASS: $PASS FAIL: $FAIL (incl. tarball-name cases above)" + +# ---- source vs documentation tarballs -------------------------------------- # +# tcl ships tcl8.6.16-html.tar.gz alongside tcl8.6.16-src.tar.gz, and the docs +# sort first -- unpacking those gives a tree with no unix/ dir and the build +# dies with "cd: unix: No such file or directory". +srct="$T/srcsel"; mkdir -p "$srct/src/tcl8.6.16/unix" "$srct/doc/tcl8.6.16/html" +( cd "$srct/src" && tar czf "$srct/tcl8.6.16-src.tar.gz" tcl8.6.16 ) +( cd "$srct/doc" && tar czf "$srct/tcl8.6.16-html.tar.gz" tcl8.6.16 ) +picked=$( cd "$srct" && bash -c ' +pkg_glob="tcl8.6.16.tar.* tcl8.6.16.tgz tcl8.6.16*.tar.* tcl8.6.16*.tgz" +pkg="" +for _c in $pkg_glob; do [ -e "$_c" ] || continue + case "$_c" in *-src.*|*-source.*) pkg="$_c"; break ;; esac; done +if [ -z "$pkg" ]; then for _c in $pkg_glob; do [ -e "$_c" ] || continue + case "$_c" in *-html.*|*-doc.*|*-docs.*|*-man.*|*-manual.*|*-tests.*) continue ;; esac + pkg="$_c"; break; done; fi +echo "$pkg"' ) +[ "$picked" = "tcl8.6.16-src.tar.gz" ] \ + && ok "picks the -src tarball, not the docs" \ + || bad "picked '$picked' instead of tcl8.6.16-src.tar.gz" + +# ---- phase bodies must not be fed on stdin -------------------------------- # +# Expect's book section starts with a PTY sanity check: +# python3 -c 'from pty import spawn; spawn(["echo", "ok"])' +# pty.spawn READS STDIN. When the phase body was piped in via `bash -s`, it +# swallowed the remaining lines -- patch/configure/make were echoed instead of +# executed, and the install then failed with "No rule to make target 'install'". +ptyd="$T/pty"; mkdir -p "$ptyd" +cat > "$ptyd/t.sh" <<'OUTER' +_phase_body() { + local _f; _f="$(mktemp)" || return 1 + cat > "$_f" + bash "$_f" </dev/null + local _r=$? + rm -f "$_f" + return $_r +} +_phase_body <<'__LFS_PHASE__' +set -e +python3 -c 'from pty import spawn; spawn(["echo","ok"])' >/dev/null 2>&1 || true +echo RAN_PATCH +echo RAN_CONFIGURE +echo RAN_MAKE +__LFS_PHASE__ +OUTER +out="$(bash "$ptyd/t.sh" 2>/dev/null | tr -d '\r')" +if echo "$out" | grep -q RAN_PATCH && echo "$out" | grep -q RAN_CONFIGURE \ + && echo "$out" | grep -q RAN_MAKE; then + ok "a stdin-reading command does not swallow the rest of the phase" +else + bad "phase body was consumed by a stdin-reading command" +fi + +# the generated scripts must all use the file-based runner +if grep -l "bash -s <<'__LFS_PHASE__'" "$T"/*.sh >/dev/null 2>&1; then + bad "a generated script still pipes its phase body through stdin" +else + ok "generated scripts run phase bodies from a file" +fi + +# ---- commands inside note/tip boxes must be ignored ------------------------ # +# GMP's section has a note showing "ABI=32 ./configure ..." for 32-bit hosts. +# Running that literally passes an ellipsis to configure: +# Invalid configuration '...': machine '...-unknown' not recognized +# Glibc likewise has a note with an alternative DESTDIR install method. +python3 - "$LFS_TOOL" "$LFS_BOOK" <<'PY2' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +book = sys.argv[2] if len(sys.argv) > 2 else None +if not book: + print(" SKIP no book available for the note-filter check"); sys.exit(0) +soup = lfs.load_soup(book) +# Only flag blocks that genuinely came from a note box. (ncurses uses +# DESTDIR=$PWD/dest as its REAL install method, so that string alone proves +# nothing -- check the specific sections whose notes we know about.) +watch = { + "ch-system-gmp": "ABI=32 ./configure ...", + "ch-system-glibc": "DESTDIR=$PWD/dest", +} +bad = [] +for sid, t, sec in lfs.iter_sections(soup): + needle = watch.get(sid) + for b in sec["commands"]: + if needle and needle in b: + bad.append((sid, b.splitlines()[0][:50])) + if "make distclean" in b: + bad.append((sid, b.splitlines()[0][:50])) +if bad: + for sid, b in bad[:5]: + print(f" FAIL note-box command still extracted: {sid}: {b}") + sys.exit(1) +print(" PASS commands inside note/tip boxes are ignored") +PY2 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- a test's log reader must stay with the test --------------------------- # +# GMP writes a log in the test block and reads it back in the next one: +# make check 2>&1 | tee gmp-check-log +# awk '/# PASS:/{total+=$3} ; END{print total}' gmp-check-log +# Moving only the first to the test phase leaves the awk in the build, where it +# dies with "cannot open file `gmp-check-log'". +python3 - "$LFS_TOOL" <<'PY3' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +cmds = ["./configure --prefix=/usr", + "make", + "make check 2>&1 | tee gmp-check-log", + "awk '/# PASS:/{total+=$3} ; END{print total}' gmp-check-log", + "make install"] +build, install, config, test = lfs._split_crosschain_phases(cmds) +problems = [] +if "gmp-check-log" in build: problems.append("log reader left in the build phase") +if "make check" in build: problems.append("test suite left in the build phase") +if "gmp-check-log" not in test: problems.append("log reader not moved to the test phase") +if "make install" not in install: problems.append("install phase wrong") +if problems: + for p in problems: print(f" FAIL {p}") + sys.exit(1) +print(" PASS a test's log reader travels with the test suite") +PY3 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- a stale source tree from an earlier chapter must be cleared ----------- # +# /sources is sticky (1777), so only the owner may delete a directory there. +# Chapters 5-6 unpack as the `lfs` user; chapter 8 rebuilds the same packages as +# their own package user, which then cannot remove the leftover tree -- the +# script's `rm -rf` fails, tar extracts on top, and the build drowns in +# tar: gcc-15.2.0/README: Cannot open: File exists +# lfs-helper must clear it as root BEFORE handing over to the package user. +stl="$T/stale"; mkdir -p "$stl/sources" "$stl/src/demo-9.9" +echo fresh > "$stl/src/demo-9.9/NEW" +( cd "$stl/src" && tar czf "$stl/sources/demo-9.9.tar.gz" demo-9.9 ) +mkdir -p "$stl/sources/demo-9.9"; echo old > "$stl/sources/demo-9.9/OLD" +chmod 1777 "$stl/sources" +cat > "$stl/script.sh" <<'EOF' +pkg_glob="demo-9.9.tar.* demo-9.9.tgz" +EOF +# run just the cleanup routine out of lfs-helper +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + ( STATE="$stl"; SNAP_ROOT="$stl"; C_DIM=; C_OFF= + say(){ :; }; warn(){ :; } + eval "$(sed -n '/^clean_stale_source() {/,/^}/p' "$helper_src")" + clean_stale_source demo "$stl/script.sh" ) + if [ -d "$stl/sources/demo-9.9" ]; then + bad "stale source tree was not cleared before the build" + else + ok "a stale source tree is cleared before the build" + fi +else + echo " SKIP lfs-helper not found next to $LFS_TOOL" +fi + +# ---- collector groups for cross-package installs --------------------------- # +# gcc installs gcc.mo into /usr/share/locale/<lang>/LC_MESSAGES, directories +# another package owns, so the gcc user gets "Permission denied". The fix is a +# collector group <prefix>_<owner>: the dir keeps its owner but becomes +# group-writable and both packages join the group. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + for fn in collector_group_for ensure_group add_to_group grant_dir_access; do + grep -q "^$fn()" "$helper_src" \ + || { bad "lfs-helper is missing $fn (collector-group support)"; break; } + done + grep -q "^cmd_fix_perms()" "$helper_src" \ + && ok "collector-group support and fix-perms are present" \ + || bad "lfs-helper has no fix-perms command" + # the failure box must point at it + grep -q "lfs-helper fix-perms" "$helper_src" \ + && ok "a Permission-denied failure suggests fix-perms" \ + || bad "failure message does not mention fix-perms" +else + echo " SKIP lfs-helper not found next to $LFS_TOOL" +fi + +# ---- command wrappers (the hint's /usr/lib/pkgusr) ------------------------- # +# The book's GCC section runs `chown -v -R root:root /usr/lib/gcc/.../include`. +# As a package user that fails with "Operation not permitted" and kills the +# install -- even though the files are already owned correctly for this scheme. +# Wrappers for chown/chgrp/install/chmod/mkdir neutralise the impossible parts. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + wt="$T/wrap"; mkdir -p "$wt" + ( STATE="$wt"; WRAPPERS="$wt/wrappers" + eval "$(sed -n '/^make_wrappers() {/,/^}/p' "$helper_src")" + make_wrappers ) + W="$wt/wrappers" + if [ -x "$W/chown" ] && [ -x "$W/install" ] && [ -x "$W/chmod" ] \ + && [ -x "$W/mkdir" ] && [ -x "$W/chgrp" ]; then + ok "the five command wrappers are created" + else + bad "command wrappers missing" + fi + # chown must not fail the build + "$W/chown" -R root:root /nonexistent >/dev/null 2>&1 \ + && ok "chown wrapper never fails the build" \ + || bad "chown wrapper returned non-zero" + # install must still install, minus -o/-g + echo data > "$wt/s" + "$W/install" -m 644 -o root -g root "$wt/s" "$wt/d" >/dev/null 2>&1 + [ -f "$wt/d" ] && ok "install wrapper drops -o/-g but still installs" \ + || bad "install wrapper did not install the file" + # setuid must be refused + "$W/install" -m 4755 "$wt/s" "$wt/suid" >/dev/null 2>&1 + if [ -u "$wt/suid" ]; then bad "install wrapper set the setuid bit" + else ok "install wrapper refuses setuid modes"; fi + # PATH must put them first for package-user builds + grep -q 'PATH=.\$WRAPPERS' "$helper_src" \ + && ok "wrappers go first in the package user's PATH" \ + || bad "wrappers are not put on PATH" +else + echo " SKIP lfs-helper not found next to $LFS_TOOL" +fi + +# ---- install directories stay group-writable ------------------------------- # +# The hint defines an install directory as: chgrp install <dir> && chmod g+w,o+t +# Packages reset modes on dirs they touch, so /usr/share/man/man1 can end up +# root:install but drwxr-xr-x -- and the next package cannot write its man page: +# ln: failed to create symbolic link '/usr/share/man/man1/cc.1': Permission denied +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q "ensure_install_dirs_writable" "$helper_src" \ + && ok "install-dir permissions are re-applied before each build" \ + || bad "no self-healing for install-dir permissions" + # the group-write test must not be fooled by the OWNER's w bit + if grep -q '10#\$_mode' "$helper_src"; then + ok "group-write is detected numerically (drwxr-xr-x is not 'writable')" + else + bad "group-write detection may match the owner's w bit" + fi + # init must use the hint's g+w, and sticky must be deferred to the seal step + grep -q 'chmod g+w "\$d"' "$helper_src" \ + && ok "install dirs get chmod g+w (sticky deferred to seal-install-dirs)" \ + || bad "install dirs are not set group-writable per the hint" +else + echo " SKIP lfs-helper not found next to $LFS_TOOL" +fi + +# ---- re-running a phase must not fail on existing symlinks ----------------- # +# The books write `ln -sv ../bin/cpp /usr/lib` with no -f, so retrying a +# configure/install phase dies with +# ln: failed to create symbolic link '/usr/lib/cpp': File exists +python3 - "$LFS_TOOL" <<'PY4' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +cases = [ + ("ln -sv ../bin/cpp /usr/lib", "ln -svf ../bin/cpp /usr/lib"), + ("ln -sv gcc /usr/bin/cc", "ln -svf gcc /usr/bin/cc"), + ("ln -sfv libncursesw.so /usr/lib/libcurses.so","ln -sfv libncursesw.so /usr/lib/libcurses.so"), + ("ln /usr/bin/foo /usr/bin/bar", "ln /usr/bin/foo /usr/bin/bar"), +] +bad = 0 +for src, want in cases: + got = lfs._idempotent_links(src) + if got != want: + print(f" FAIL {src!r} -> {got!r}, expected {want!r}"); bad += 1 +if bad: sys.exit(1) +print(" PASS symlink steps are made re-runnable (-f added only where missing)") +PY4 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- collector groups are configured, automatic, and reported -------------- # +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q "require_collector_prefix" "$helper_src" \ + && ok "an unset collector prefix stops the build with instructions" \ + || bad "no check for a missing collector prefix" + grep -q "auto_grant_from_log" "$helper_src" \ + && ok "permission failures are granted and retried automatically" \ + || bad "no automatic collector-group recovery" + grep -q "report_granted_groups" "$helper_src" \ + && ok "collector groups created during a run are reported" \ + || bad "collector groups are created silently" +fi +# the config must be complete before the build-system commands run +python3 - "$LFS_TOOL" <<'PY5' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +keys = [k for k, _w, _h in lfs.REQUIRED_CONFIG] +if "collector_prefix" in keys and "lfs_mount" in keys: + print(" PASS collector_prefix is a required setting") +else: + print(f" FAIL REQUIRED_CONFIG is missing keys: {keys}"); sys.exit(1) +PY5 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- collector group name is a choice, not a fixed rule -------------------- # +# Naming after the OWNER (nimgnu_binutils) groups everything that package owns; +# naming after the DIRECTORY (nimgnu_bfd-plugins) is more precise. Both are +# reasonable, so the user picks -- with the context needed to decide. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q "choose_collector_group" "$helper_src" \ + && ok "the collector group name is offered as a choice" \ + || bad "collector group name is not selectable" + # the prompt must use /dev/tty: it runs inside a `while ... <<< list` loop, + # where stdin is the directory list, so a plain read would eat that instead + grep -qE "read (-e )?-r ans < /dev/tty" "$helper_src" \ + && ok "the prompt reads from /dev/tty (stdin is the dir list)" \ + || bad "prompt would consume the directory list instead of user input" + # names must be sanitised for the group database + ( eval "$(sed -n '/^sanitise_group_name() {/,/^}/p' "$helper_src")" + got="$(sanitise_group_name 'nimgnu_BFD Plugins!')" + case "$got" in *' '*|*'!'*) exit 1 ;; esac + [ -n "$got" ] ) \ + && ok "group names are sanitised for the group database" \ + || bad "group names are not sanitised" +fi + +# ---- a later phase failing may mean an earlier one never finished ---------- # +# gcc's configure step runs `mv -v /usr/lib/*gdb.py ...`. Those files come from +# `make install`; if that aborted partway (a permission error, say) they were +# never installed and the mv fails with a confusing "No such file or directory". +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q "these earlier phases have not completed successfully" "$helper_src" \ + && ok "a failure points at earlier phases that never completed" \ + || bad "no hint about incomplete earlier phases" +fi + +# ---- /sources hygiene ------------------------------------------------------ # +# Two kinds of debris break later builds: +# * tar run as root restores the UIDs recorded IN the archive, so /sources ends +# up owned by host-only ids (8282, 15399, ...) that mean nothing here; +# * a build marker left by a previous package user cannot be replaced under the +# sticky /sources: /sources/.cc-build-ncurses: Permission denied +python3 - "$LFS_TOOL" <<'PY6' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +body = lfs._crosschain_script_body("demo", "ch-system-demo", "demo-1.0", + "demo-1.0.tar.*", ["make", "make install"]) +if "--no-same-owner" not in body: + print(" FAIL unpack does not use tar --no-same-owner"); sys.exit(1) +print(" PASS unpack never restores the archive's UIDs") +PY6 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q 'rm -f "\$srcdir/.cc-build-\$name"' "$helper_src" \ + && ok "stale build markers are cleared before a build" \ + || bad "stale build markers are not cleared" + grep -q "^cmd_clean_sources()" "$helper_src" \ + && ok "clean-sources can tidy /sources after the fact" \ + || bad "no clean-sources command" +fi + +# ---- staged installs with `cp -a dest/* /` --------------------------------- # +# ncurses (and others) install a staged tree that way. The FILES copy fine as a +# package user, but cp then tries to stamp the destination DIRECTORIES (/usr, +# /usr/bin -- owned by root) and fails: +# cp: preserving times for '/usr/bin': Operation not permitted +# That is harmless; a real cp error must still fail the build. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + cpt="$T/cpwrap"; mkdir -p "$cpt" + ( STATE="$cpt"; WRAPPERS="$cpt/wrappers" + eval "$(sed -n '/^make_wrappers() {/,/^}/p' "$helper_src")" + make_wrappers ) + if [ -x "$cpt/wrappers/cp" ]; then + ok "a cp wrapper exists for staged installs" + # a genuine failure must still be a failure + if "$cpt/wrappers/cp" /definitely-not-here "$cpt/" >/dev/null 2>&1; then + bad "cp wrapper swallowed a real error" + else + ok "cp wrapper still fails on a real error" + fi + # a normal copy must work + echo hi > "$cpt/a" + "$cpt/wrappers/cp" "$cpt/a" "$cpt/b" >/dev/null 2>&1 + [ -f "$cpt/b" ] && ok "cp wrapper copies normally" \ + || bad "cp wrapper broke a normal copy" + else + bad "no cp wrapper" + fi +fi + +# ---- per-language man/locale dirs are shared, not per-package -------------- # +# Every package ships translated man pages and message catalogues, creating a +# directory per language. Treating those as owned by whichever package made +# them first turns each one into a "grant access?" prompt -- dozens per package. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + ( SNAP_ROOT=/ + eval "$(sed -n '/^install_dir_patterns() {/,/^}/p' "$helper_src")" + eval "$(sed -n '/^matches_install_pattern() {/,/^}/p' "$helper_src")" + for d in /usr/share/man/de/man1 /usr/share/man/man1 \ + /usr/share/locale/pl/LC_MESSAGES /usr/lib/pkgconfig; do + matches_install_pattern "$d" || exit 1 + done + # a package's own directory must NOT be treated as shared + for d in /usr/src/foo /usr/lib/gcc/x86_64/15.2.0; do + matches_install_pattern "$d" && exit 1 + done + exit 0 ) \ + && ok "language man/locale dirs are shared; package dirs are not" \ + || bad "install-dir pattern matching is wrong" + grep -q "^cmd_prune_locales()" "$helper_src" \ + && ok "unwanted languages can be pruned" \ + || bad "no prune-locales command" +fi +python3 - "$LFS_TOOL" <<'PY7' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +keys = [k for k, _d in lfs.CONFIG_KEYS] +print(" PASS 'locales' is a documented setting" if "locales" in keys + else " FAIL no 'locales' setting") +sys.exit(0 if "locales" in keys else 1) +PY7 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- a package owns the files it OVERWRITES, not just new ones ------------- # +# Chapter 8's bison replaces the binaries chapter 7's bison-tmp installed as +# root. Those files are not "new", so a pure snapshot diff never sees them and +# they keep the previous owner -- which is why bison stayed owned by root. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q '\-newer "\$stamp"' "$helper_src" \ + && ok "files modified during a build are reassigned, not just new ones" \ + || bad "only newly-created files get the package user's ownership" + # a stamp must exist; WHERE it lives is checked below (not /tmp) + grep -q 'stamp=' "$helper_src" \ + && ok "there is a build timestamp for detecting modified files" \ + || bad "no build timestamp for detecting modified files" +fi + +# ---- test targets carry suffixes ------------------------------------------- # +# Perl runs `TEST_JOBS=$(nproc) make test_harness`; others use `make check-TESTS` +# or `check-recursive`. Missing those runs a package's whole test suite as part +# of the BUILD -- slow, and a test failure then kills the build. +python3 - "$LFS_TOOL" <<'PY8' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +cases = [("TEST_JOBS=$(nproc) make test_harness", True), + ("make check-TESTS", True), ("make check", True), ("make test", True), + ("make NON_ROOT_USERNAME=tester check-root", True), + ("make install", False), ("make install-html", False), + ("make install-strip", False), ("make", False)] +bad = 0 +for c, want in cases: + got = bool(lfs._TEST_BLOCK_RE.search(c)) + if got != want: + print(f" FAIL {c!r} classified as test={got}"); bad += 1 +if bad: sys.exit(1) +print(" PASS suffixed test targets (test_harness, check-TESTS) are recognised") +PY8 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- failures report what the paths actually are ---------------------------- # +# "No such file or directory" on a destination can mean a missing parent OR a +# dangling symlink; "Permission denied" can mean the file exists but belongs to +# another package. Showing the facts beats guessing from the message. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + dg="$T/diag"; mkdir -p "$dg/real" + ln -s /nowhere/at/all "$dg/real/dangling.pm" + echo x > "$dg/real/owned.pm" + printf "Couldn't copy a to %s: No such file or directory\n" "$dg/real/dangling.pm" > "$dg/log" + printf "Couldn't copy b to %s: Permission denied\n" "$dg/real/owned.pm" >> "$dg/log" + out="$( C_WARN=; C_OFF=; SNAP_ROOT="$dg"; say(){ echo "$*"; } + eval "$(sed -n '/^diagnose_paths_from_log() {/,/^}/p' "$helper_src")" + diagnose_paths_from_log "$dg/log" nobody )" + echo "$out" | grep -q "DANGLING SYMLINK" \ + && ok "a dangling destination symlink is identified" \ + || bad "dangling symlink not reported" + echo "$out" | grep -q "owned.pm" \ + && ok "an existing destination's owner and mode are shown" \ + || bad "destination ownership not reported" + # relative-path tails must not be reported as absolute targets + echo "$out" | grep -qE "^ /[^/]+\.pm " \ + && bad "a relative path tail was treated as a target" \ + || ok "relative path tails are not mistaken for targets" +fi + +# ---- recovery must not depend on the wording of the error ------------------ # +# Perl's installperl reports an unwritable destination directory as +# Couldn't copy lib/GDBM_File.pm to .../GDBM_File.pm: No such file or directory +# not "Permission denied". Keying the auto-recovery on that phrase meant perl +# never recovered. And a root-owned tree left by a chapter-7 -tmp package has +# hundreds of subdirectories the real package installs into, so granting only +# the named one just fails again on the next. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q "couldn't copy|cannot create" "$helper_src" \ + && ok "recovery and hints match more than 'Permission denied'" \ + || bad "recovery still keys only on the literal 'Permission denied'" + # (the earlier root-owned-subtree workaround is gone: -tmp packages are now + # adopted to the base package user, so no root-owned tree is left behind) + # REVERSED deliberately: a collector group means "may install into a + # directory this package owns", NOT "owns that package's tree". Granting + # the subtree in one pass was over-reach -- one file in site-packages took + # ~190 python directories with it. One directory at a time is correct; + # a package that needs several gets several rounds. + grep -q 'find "\$dir" -mindepth 1 -type d -user "\$owner"' "$helper_src" \ + && bad "granting still takes the owner's whole subtree" \ + || ok "only the directory asked for is granted" + grep -q 'SNAP_ROOT%/}"/\*) echo "\$p"' "$helper_src" \ + && ok "diagnostics only report paths inside the tree or a known root" \ + || bad "diagnostics would report relative tails as targets" +fi + +# ---- collector groups share the whole area, iteratively -------------------- # +# An install reveals only the FIRST directory it cannot write. Perl modules +# span core_perl/ and site_perl/ and dozens of subdirs, so granting one and +# retrying once is not enough -- and members must be able to REPLACE each +# other's files, not just add new ones. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q '_round" -lt "\$_max_rounds"' "$helper_src" \ + && ok "grant-and-retry iterates until nothing new needs granting" \ + || bad "only one grant round; later directories would still fail" + grep -q 'find "\$dir" -type f -user "\$owner"' "$helper_src" \ + && bad "granting still rewrites every file in the owner's tree" \ + || ok "files in the owner's tree are left alone" + # unquoted paths in messages must be found too (bash and perl report them + # bare), and the scan must stay bounded + grep -q 'grep -aoE "/\[A-Za-z0-9._/+-\]{6,}"' "$helper_src" \ + && ok "unquoted paths in error messages are recognised" \ + || bad "only quoted paths are extracted; bash/perl messages missed" + # *-tmp packages must be adopted to the base package user + grep -q 'init-\*) skipped=' "$helper_src" \ + && ok "only init-* steps are skipped; -tmp tools get the base user" \ + || bad "-tmp steps are skipped, leaving root-owned trees behind" +fi + +# ---- package user names must be normalised --------------------------------- # +# The book's anchors are inconsistent in case: chapter 7 has "python-tmp" but +# chapter 8 has "Python". Unix user names are case-sensitive, so those become +# two accounts -- and the chapter-8 package then cannot overwrite its OWN +# chapter-7 files: +# install: cannot remove '/usr/lib/python3.13/.../__init__.py': Permission denied +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + out="$( eval "$(sed -n '/^pkg_owner_name() {/,/^}/p' "$helper_src")" + for n in Python python-tmp GCC gcc-pass1 libstdcpp \ + Binutils-pass2 XML::Parser util-linux-tmp; do + printf '%s=%s\n' "$n" "$(pkg_owner_name "$n")" + done )" + fails="" + echo "$out" | grep -q "^Python=python$" || fails="$fails Python" + echo "$out" | grep -q "^python-tmp=python$" || fails="$fails python-tmp" + echo "$out" | grep -q "^GCC=gcc$" || fails="$fails GCC" + echo "$out" | grep -q "^gcc-pass1=gcc$" || fails="$fails gcc-pass1" + echo "$out" | grep -q "^libstdcpp=gcc$" || fails="$fails libstdcpp" + echo "$out" | grep -q "^Binutils-pass2=binutils$" || fails="$fails Binutils-pass2" + echo "$out" | grep -q "^util-linux-tmp=util-linux$" || fails="$fails util-linux-tmp" + # Perl module names are not valid user names + echo "$out" | grep -q "^XML::Parser=xml-parser$" || fails="$fails XML::Parser" + [ -z "$fails" ] \ + && ok "package user names are normalised (case, passes, module names)" \ + || bad "owner name wrong for:$fails" +fi + +# ---- share from the top of a package's tree -------------------------------- # +# Python's standard library is ~200 directories under /usr/lib/python3.13. An +# install reveals one unwritable directory at a time, so granting leaf by leaf +# never finishes -- it exhausts the retry budget and gives up. Walking up to +# the top of the owner's tree shares it in a single pass. The walk must stop at +# a shared install directory (/usr/lib), never above it. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + # REVERSED deliberately: sharing from the top of the owner's tree hands + # over directories the package never asked for. Per-directory granting + # with several retry rounds is the correct trade. + grep -q "the top of \$owner's tree" "$helper_src" \ + && bad "granting still shares from the top of the owner's tree" \ + || ok "grants are per-directory, not whole-tree" + grep -q 'is_install_dir "\$parent" && break' "$helper_src" \ + && ok "the walk up stops at a shared install directory" \ + || bad "the walk could climb past /usr/lib and share too much" + grep -q "parent -- this is what matters" "$helper_src" \ + && ok "diagnostics show the parent dir for 'cannot remove/create'" \ + || bad "diagnostics omit the directory that actually governs access" +fi + +# ---- lfs next / handover to packagemanager --------------------------------- # +# lfs-helper is a bash stand-in only because there is no Python in the chroot +# until chapter 8 builds it. Once it exists, the real tooling takes over. +python3 - "$LFS_TOOL" <<'PY9' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +missing = [n for n in ("cmd_next", "_next_steps", "cmd_bs_install_tools", + "_chroot_has_python") if not hasattr(lfs, n)] +if missing: + print(" FAIL lfs is missing: %s" % ", ".join(missing)); sys.exit(1) +print(" PASS 'lfs next' and the packagemanager handover exist") +PY9 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- bootloader safety ----------------------------------------------------- # +# The book runs `grub-install /dev/sda`, which writes a disk's boot sector. On +# a machine with an existing boot partition that is destructive, so grub is only +# generated when explicitly chosen; the default is rEFInd, which only ever adds +# files to an already-mounted, already-formatted ESP. +python3 - "$LFS_TOOL" <<'PY10' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +if lfs.bootloader() != "refind": + print(" FAIL default bootloader is %r, expected refind" % lfs.bootloader()) + sys.exit(1) +script = lfs._refind_script() +bad = [c for c in ("grub-install /dev/", "mkfs", "sgdisk", "parted", "of=/dev/") + if c in script.replace("### grub-install /dev/sda", "")] +if bad: + print(" FAIL rEFInd script contains destructive commands: %s" % bad) + sys.exit(1) +for guard in ("nothing is mounted there", "not FAT", + "REFIND_OVERWRITE", "does not exist"): + if guard not in script: + print(" FAIL rEFInd script lacks the %r guard" % guard); sys.exit(1) +print(" PASS rEFInd is the default and its script cannot touch a raw device") +PY10 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q 'find "\$dir" -mindepth 1 -type d -user root' "$helper_src" \ + && bad "a root-owned subtree is still taken in one pass" \ + || ok "root-owned adoption is limited to the directory asked for" +fi + +# ---- a package's private tree belongs to the package ----------------------- # +# /usr/lib/python3.13 is python's own tree, not shared infrastructure. Chapter +# 7 built python-tmp as root, so the tree is left root-owned; it must be ADOPTED +# by python, not turned into an install directory. When another package later +# installs modules inside it, that is what collector groups are for. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + # the directory itself is still adopted -- just not its subtree + grep -q "owned by '\$user' (was root, from the temporary system)" "$helper_src" \ + && ok "a root-owned leftover directory is adopted by its package" \ + || bad "root-owned directories are not adopted at all" + grep -q "shared install directory" "$helper_src" \ + && ok "only real install dirs get the install group" \ + || bad "install-group handling missing" + # the two must be distinct branches: install dirs stay root, private trees don't + grep -q 'if is_install_dir "\$dir"; then' "$helper_src" \ + && ok "install dirs and private trees are handled separately" \ + || bad "install dirs and private trees are conflated" +fi + +# ---- pip packages and duplicate grants ------------------------------------- # +# Many failing paths walk up to the SAME tree top, so granting per-path repeats +# the identical grant. And `pip3 install` writes into site-packages: if that is +# not writable up front, pip can quietly install into a per-user location where +# nothing else can import it -- the next package then fails with +# BackendUnavailable: Cannot import 'flit_core.buildapi' +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q "^collector_tree_top()" "$helper_src" \ + && ok "the tree top is computed once and grants are de-duplicated" \ + || bad "the same directory can be granted repeatedly" + grep -q "^pregrant_python_sitedirs()" "$helper_src" \ + && ok "pip packages get site-packages access before they run" \ + || bad "pip packages only get access after failing" +fi + +# ---- a build that installs nothing must not be called successful ----------- # +# pip can report success while writing nowhere useful: flit-core "succeeded" +# without ever putting flit_core into site-packages, and the next package then +# failed with "BackendUnavailable: Cannot import 'flit_core.buildapi'". +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q "reported success but installed NO files" "$helper_src" \ + && ok "a build that installs no files is not marked as built" \ + || bad "an empty install is silently accepted" + # grep -c prints 0 and exits non-zero, so `|| echo 0` yields "0\n0" + grep -q "^count_lines()" "$helper_src" \ + && ok "line counts go through one safe helper" \ + || bad "line counts can yield \"0\\n0\" and break numeric tests" +fi + +# ---- pip must install system-wide, not into ~/.local ----------------------- # +# When site-packages is not writable, pip installs into the package user's +# ~/.local and afterwards reports "Requirement already satisfied" from there -- +# so it never installs system-wide, and only that one user can import it. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q "PIP_USER=0 PYTHONNOUSERSITE=1" "$helper_src" \ + && ok "pip is forced to install system-wide" \ + || bad "pip may still install into the package user's ~/.local" + grep -q "removing a stale per-user install" "$helper_src" \ + && ok "a stale ~/.local install is cleared before a pip build" \ + || bad "a stale ~/.local install would keep shadowing the system one" + grep -q "^cmd_find_user_site()" "$helper_src" \ + && ok "existing per-user installs can be found and removed" \ + || bad "no way to find existing ~/.local installs" +fi + +# ---- package user ids must not collide ------------------------------------- # +# Hand-picking the next free uid races Shadow's own idea of what is free: +# useradd: UID 10067 is not unique +# Once Shadow exists, let it allocate -- and use -U so the private group gets +# the same id as the user instead of drifting apart. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q 'K "UID_MIN=\$PKG_UID_MIN"' "$helper_src" \ + && ok "Shadow allocates package user ids (no hand-picked uid)" \ + || bad "ids are still hand-picked and can collide" + grep -q 'useradd -c "package \$name" -d "\$SRCROOT/\$name" -U' "$helper_src" \ + && ok "the private group is created with the user, so uid == gid" \ + || bad "group and user ids can drift apart" + grep -q 'getent passwd "\$1" >/dev/null 2>&1 && return 0' "$helper_src" \ + && ok "id_taken consults the real database, not just the files" \ + || bad "id_taken only reads the files and can miss a used id" +fi + +# ---- test scaffolding belongs with the tests -------------------------------- # +# Coreutils sets up a throwaway user/group so its suite can run as non-root: +# groupadd -g 102 dummy -U tester ; chown -R tester . ; ... ; groupdel dummy +# Left in the build phase those run as the package user and die with +# groupadd: Permission denied. +python3 - "$LFS_TOOL" <<'PY10' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +cases = [("groupadd -g 102 dummy -U tester", True), + ("chown -R tester .", True), + ("groupdel dummy", True), + ('su tester -c "PATH=$PATH make -k check"', True), + ("make install", False), + ("groupadd install", False), # the install group is not a test + ("chown -R root:root /usr/lib/gcc", False), + ("mv -v /usr/bin/chroot /usr/sbin", False)] +bad = 0 +for c, want in cases: + got = lfs._is_test_block(c) + if got != want: + print(" FAIL %r classified test=%s" % (c, got)); bad += 1 +if bad: sys.exit(1) +print(" PASS test scaffolding travels with the tests, real steps do not") +PY10 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- book placeholders must be filled in ----------------------------------- # +# The book writes PAGE=<paper_size> ./configure for a human to edit. A shell +# reads "<paper_size>" as an input redirect: +# line 4: paper_size: No such file or directory +python3 - "$LFS_TOOL" <<'PY11' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +out, left = lfs._fill_placeholders("PAGE=<paper_size> ./configure --prefix=/usr", "groff") +if "<paper_size>" in out or left: + print(" FAIL <paper_size> not substituted: %r" % out); sys.exit(1) +# an unknown placeholder must be REPORTED, not silently left in +out2, left2 = lfs._fill_placeholders("echo <your name here>", "x") +if "your name here" not in left2: + print(" FAIL an unknown placeholder was not reported"); sys.exit(1) +print(" PASS book placeholders are substituted, unknown ones reported") +PY11 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- packages must not re-mode things they do not own ---------------------- # +# iproute2 does `install -m 0755 -d /usr/sbin` on a directory that already +# exists and belongs to root:install: +# install: cannot change permissions of '/usr/sbin': Operation not permitted +# The directory is already correct, so there is nothing to do -- and a package +# should never re-mode a directory it merely installs into. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + wt="$T/wrap2"; mkdir -p "$wt/existing" "$wt/mine" + ( STATE="$wt"; WRAPPERS="$wt/wrappers" + eval "$(sed -n '/^make_wrappers() {/,/^}/p' "$helper_src")" + make_wrappers ) + W="$wt/wrappers" + before="$(stat -c %A "$wt/existing")" + "$W/install" -m 0700 -d "$wt/existing" >/dev/null 2>&1 + [ "$(stat -c %A "$wt/existing")" = "$before" ] \ + && ok "install -d leaves an existing directory's mode alone" \ + || bad "install -d re-moded an existing directory" + "$W/install" -m 0755 -d "$wt/mine/fresh" >/dev/null 2>&1 + [ -d "$wt/mine/fresh" ] \ + && ok "install -d still creates a missing directory" \ + || bad "install -d no longer creates directories" + # chmod on our own file must still work + echo x > "$wt/mine/f"; "$W/chmod" 600 "$wt/mine/f" >/dev/null 2>&1 + [ "$(stat -c %a "$wt/mine/f")" = "600" ] \ + && ok "chmod still works on files we own" \ + || bad "chmod wrapper broke a legitimate chmod" +fi + +# ---- a failure that needs the USER is not a permission problem -------------- # +# rEFInd stops when the ESP is not mounted. Answering that by handing out +# collector groups chowned /boot to a package user and burned the whole retry +# budget doing it. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q "_perm_ish" "$helper_src" \ + && ok "permission fixes only run on permission-shaped failures" \ + || bad "any failure can trigger permission changes" + grep -q "the same directories keep coming up" "$helper_src" \ + && ok "a retry that makes no progress stops early" \ + || bad "the retry budget can be burned on the same directory" + grep -q 'if \[ "\$rc" = 3 \]' "$helper_src" \ + && ok "exit 3 is treated as 'needs you', not a build error" \ + || bad "a needs-user exit is reported as a build failure" + # /boot and the ESP must never be adoptable by a package + ( SNAP_ROOT=/ + eval "$(sed -n '/^install_dirs_list() {/,/^}/p' "$helper_src")" + eval "$(sed -n '/^install_dir_patterns() {/,/^}/p' "$helper_src")" + eval "$(sed -n '/^matches_install_pattern() {/,/^}/p' "$helper_src")" + eval "$(sed -n '/^is_install_dir() {/,/^}/p' "$helper_src")" + INSTALLDIRS=""; is_install_dir /boot && is_install_dir /boot/efi ) \ + && ok "/boot and the ESP can never be adopted by a package" \ + || bad "/boot could be chowned to a package user" +fi + +# ---- the rEFInd script must survive `set -e` -------------------------------- # +# `found="$(blkid ...)"` aborts the whole script under set -e when blkid finds +# nothing, so the guidance was cut off mid-sentence and the exit code was wrong. +python3 - "$LFS_TOOL" <<'PY12' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +sc = lfs._refind_script() +problems = [] +if "blkid -t TYPE=vfat -o device 2>/dev/null || true" not in sc: + problems.append("blkid assignment can abort under set -e") +if "|| true)\"" not in sc: + problems.append("lsblk assignment can abort under set -e") +if "exit 3" not in sc: + problems.append("no needs-you exit code") +if "lfs-helper done refind" not in sc: + problems.append("no way to skip the bootloader step") +if problems: + for p in problems: print(" FAIL %s" % p) + sys.exit(1) +print(" PASS rEFInd guidance survives set -e and offers a skip") +PY12 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- an existing rEFInd gets an entry, not a reinstall ---------------------- # +# If the machine already boots with rEFInd, reinstalling it is wrong; what is +# needed is a menu entry so the firmware can boot LFS. And "deliberately did +# nothing" must not be reported as a suspicious empty install. +python3 - "$LFS_TOOL" <<'PY13' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +sc = lfs._refind_script() +problems = [] +if "add_lfs_entry" not in sc: problems.append("no LFS menu entry is added") +if "bak=\"$conf.bak-" not in sc: problems.append("the config is not backed up first") +if "exit 4" not in sc: problems.append("no 'nothing to do' exit code") +if "add_lfs_entry \"$conf\" || rc_entry=$?" not in sc: + problems.append("a non-zero return would abort under set -e") +if problems: + for p in problems: print(" FAIL %s" % p) + sys.exit(1) +print(" PASS an existing rEFInd gets an LFS entry, with a backup") +PY13 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q '_nothing_to_do' "$helper_src" \ + && ok "a deliberate no-op is not flagged as an empty install" \ + || bad "'nothing to do' would be reported as a silent failure" +fi + +# ---- chapter 9: system configuration --------------------------------------- # +# Config file creation, not package builds -- generated as plain root scripts +# and ordered after the packages, before the bootloader. +python3 - "$LFS_TOOL" <<'PY14' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +names = [n for n, _s in lfs.CHROOT_CONFIG_STEPS_9] +need = ["cfg-hostname", "cfg-hosts", "cfg-clock", "cfg-inputrc", "cfg-network"] +missing = [n for n in need if n not in names] +if missing: + print(" FAIL chapter 9 is missing: %s" % ", ".join(missing)); sys.exit(1) +# writing a config file installs nothing to compile: they must be root steps +print(" PASS chapter 9 configuration steps are defined") +PY14 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# rEFInd writes only to the ESP, so tracking zero files is expected, not a fault +python3 - "$LFS_TOOL" <<'PY15' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +sc = lfs._refind_script() +if not sc.rstrip().endswith("exit 4"): + print(" FAIL rEFInd does not signal 'nothing installed into the tree'") + sys.exit(1) +print(" PASS rEFInd reports success without tracking files in the tree") +PY15 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- a "configuration" section that is really a package -------------------- # +# 9.2 LFS-Bootscripts-20250827 is a tarball to unpack and install, not a config +# file to write. Generated as a plain script it runs `make install` with +# nothing unpacked: +# make: *** No rule to make target 'install'. Stop. +python3 - "$LFS_TOOL" "$LFS_BOOK" <<'PY16' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +# a versioned title means a package; a plain one means a config step +pv = lfs._title_pkgver("9.2. LFS-Bootscripts-20250827") +if not pv or not any(c.isdigit() for c in pv): + print(" FAIL a versioned config section is not recognised as a package") + sys.exit(1) +glob = lfs._pkg_glob_for(pv, "cfg-bootscripts") +import fnmatch +if not any(fnmatch.fnmatch("lfs-bootscripts-20250827.tar.xz", p) for p in glob.split()): + print(" FAIL the bootscripts tarball would not be found"); sys.exit(1) +for plain in ("9.5.3. Configuring the System Hostname", + "9.8. Creating the /etc/inputrc File"): + p = lfs._title_pkgver(plain) + if p and any(c.isdigit() for c in p): + print(" FAIL %r was mistaken for a package" % plain); sys.exit(1) +print(" PASS versioned config sections build as packages, the rest do not") +PY16 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- config steps that rewrite existing files ------------------------------- # +# /etc/hosts already exists (7.6 created it), so rewriting it adds no NEW file +# and the step looked like it did nothing. What a build TOUCHED must count too. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q "config step that rewrites /etc/hosts" "$helper_src" \ + && ok "a step that rewrites an existing file counts as work done" \ + || bad "rewriting an existing file still looks like an empty install" +fi + +# ---- locale placeholders ---------------------------------------------------- # +# "LC_ALL=<locale name>" is an input redirect to bash: +# line 13: locale: No such file or directory +python3 - "$LFS_TOOL" <<'PY17' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +out, left = lfs._fill_placeholders("LC_ALL=<locale name> locale charmap", "cfg-locale") +if "<locale name>" in out or left: + print(" FAIL <locale name> not substituted: %r" % out); sys.exit(1) +out2, left2 = lfs._fill_placeholders("echo <charmap>", "x") +if "<charmap>" in out2: + print(" FAIL <charmap> not substituted"); sys.exit(1) +# a script whose placeholder cannot be filled must refuse to run +sc = lfs._plain_root_script("demo", "sid", "9.x Demo", ["echo <something unknown>"]) +if "needs a decision from you" not in sc or "exit 3" not in sc: + print(" FAIL an unfillable placeholder does not stop the script"); sys.exit(1) +# ...and one that CAN be filled must not carry a guard +sc2 = lfs._plain_root_script("h", "sid", "9.5.3 Hostname", ['echo "<lfs>" > /etc/hostname']) +if "needs a decision from you" in sc2: + print(" FAIL a filled placeholder still triggered the guard"); sys.exit(1) +print(" PASS locale placeholders fill from config; unfillable ones stop the step") +PY17 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- 'next' must know where the build actually is --------------------------- # +# The crosschain progress file is CLEARED when the chain completes, so an empty +# one means either "not started" or "all done". Reporting 0/22 for a finished +# toolchain sends you back to the beginning. +python3 - "$LFS_TOOL" <<'PY18' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +for fn in ("_crosschain_looks_done", "_chroot_progress", "_chroot_steporder"): + if not hasattr(lfs, fn): + print(" FAIL lfs is missing %s" % fn); sys.exit(1) +print(" PASS build state is inferred from the tree, not just a progress file") +PY18 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- collector groups can be carried to the next build ---------------------- # +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q "^cmd_export_groups()" "$helper_src" && \ + grep -q "^cmd_import_groups()" "$helper_src" \ + && ok "collector-group decisions can be exported and imported" \ + || bad "no export/import for collector groups" + grep -q "COLLECTOR_MAP" "$helper_src" \ + && ok "a decided directory is not asked about twice" \ + || bad "the same directory would be asked about again" +fi + +# ---- placeholders come in more shapes than lowercase words ------------------ # +# "export LANG=<ll>_<CC>.<charmap><@modifiers>" written literally into +# /etc/profile breaks every login shell: +# /etc/profile: line 10: syntax error near unexpected token `newline' +# ...but <string.h> and an email in a comment are NOT placeholders. +python3 - "$LFS_TOOL" <<'PY19' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +out, left = lfs._fill_placeholders( + " export LANG=<ll>_<CC>.<charmap><@modifiers>\n" + "127.0.1.1 <FQDN> <HOSTNAME>\n" + "sed '/unistd.h/i #include <string.h>'\n" + "# maintained by <roryo@roryo.dynup.net>\n", "x") +bad = [] +if "<ll>" in out or "<CC>" in out or "<@modifiers>" in out: + bad.append("the LANG construct was not replaced") +if "<FQDN>" in out or "<HOSTNAME>" in out: + bad.append("hostname placeholders were not replaced") +if "<string.h>" not in out: + bad.append("a C include was mangled") +if "string.h" in left or any("@" in x for x in left): + bad.append("non-placeholders were reported: %s" % left) +if bad: + for b in bad: print(" FAIL %s" % b) + sys.exit(1) +print(" PASS placeholders are filled; includes and emails are left alone") +PY19 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +python3 - "$LFS_TOOL" <<'PY20' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +keys = [k for k, _d in lfs.CONFIG_KEYS] +if "collector_import_file" not in keys or not hasattr(lfs, "_install_collector_groups"): + print(" FAIL no way to supply a collector-group file for a rebuild") + sys.exit(1) +print(" PASS a collector-group export can be supplied via config") +PY20 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- SNAP_ROOT="/" must not break file tracking ----------------------------- # +# Inside the chroot SNAP_ROOT is "/", and "${SNAP_ROOT%/}" trims that to the +# empty string. `find ""` fails silently, so NOTHING was ever seen as touched +# and every config step reported "installed NO files". +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q 'find "\${r:-/}"' "$helper_src" \ + && ok "file tracking works when the tracked root is /" \ + || bad "find would be given an empty path inside the chroot" + grep -q 'find "\${SNAP_ROOT:-/}"' "$helper_src" \ + && ok "snapshots work when the tracked root is /" \ + || bad "snapshot find would be given an empty path" + grep -q "filesystem is the authority" "$helper_src" \ + && ok "the export lists directories that really carry a group" \ + || bad "the export only lists remembered decisions" +fi +python3 - "$LFS_TOOL" <<'PY21' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +if not hasattr(lfs, "network_mode") or lfs.network_mode() != "dhcp": + print(" FAIL network does not default to dhcp"); sys.exit(1) +print(" PASS network defaults to dhcp (no static config generated)") +PY21 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- the tooling's own Python dependencies ---------------------------------- # +# base LFS ships neither requests nor beautifulsoup4, so 'lfs' and 'blfs' fail +# on import the moment they run in the chroot: +# error: this needs BeautifulSoup4 (pip install beautifulsoup4) +# ModuleNotFoundError: No module named 'requests' +python3 - "$LFS_TOOL" <<'PY22' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +if not hasattr(lfs, "_chroot_missing_tool_deps"): + print(" FAIL nothing checks for the tooling's Python modules"); sys.exit(1) +mods = [mod for mod, _pkg in lfs.TOOL_PY_DEPS] +if "bs4" not in mods or "requests" not in mods: + print(" FAIL the dependency list is incomplete: %s" % mods); sys.exit(1) +print(" PASS the tooling's Python modules are checked before handover") +PY22 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- the tools must survive missing optional modules ------------------------ # +# base LFS ships neither requests nor beautifulsoup4. Failing at IMPORT time +# makes the tools useless in exactly the situation they are needed: +# ModuleNotFoundError: No module named 'requests' +# A cached book can be parsed offline, so only commands that fetch should care. +tmpd="$T/nomod"; mkdir -p "$tmpd" +cat > "$tmpd/sitecustomize.py" <<'EOF' +import sys +class _B: + def find_module(self, name, path=None): + return self if name in ('requests', 'bs4') else None + def load_module(self, name): + raise ImportError('missing') +sys.meta_path.insert(0, _B()) +EOF +bad=0 +for tool in blfs packagemanager; do + t="$(dirname "$LFS_TOOL")/$tool" + [ -f "$t" ] || continue + out="$(PYTHONPATH="$tmpd" python3 "$t" --help 2>&1)" || bad=1 + case "$out" in *Traceback*) bad=1 ;; esac +done +[ "$bad" = 0 ] \ + && ok "blfs and packagemanager start without requests/beautifulsoup4" \ + || bad "a tool still fails at import when a module is missing" + +# and there is a package-user-aware way to install those modules +pmt="$(dirname "$LFS_TOOL")/packagemanager" +if [ -f "$pmt" ]; then + grep -q "^def cmd_pip(" "$pmt" \ + && ok "python modules can be installed as package users" \ + || bad "no package-user-aware pip" + grep -q "PIP_USER=0" "$pmt" \ + && ok "pip is kept out of the package user's ~/.local" \ + || bad "pip could install into ~/.local again" +fi + +# ---- package users without the hint's helper -------------------------------- # +# add_package_user comes from the package-users hint's helper tarball, which a +# fresh LFS system does not have. Without a fallback nothing can be installed +# as a package user at all -- and a missing command crashed with a traceback: +# FileNotFoundError: [Errno 2] No such file or directory: 'add_package_user' +pmt="$(dirname "$LFS_TOOL")/packagemanager" +if [ -f "$pmt" ]; then + grep -q "^def create_package_user(" "$pmt" \ + && ok "package users can be created without add_package_user" \ + || bad "package user creation depends on the hint's helper script" + grep -q "except FileNotFoundError:" "$pmt" \ + && ok "a missing command is reported, not raised as a traceback" \ + || bad "a missing command still crashes" + grep -q "_SBIN_PATH" "$pmt" \ + && ok "useradd/groupadd are found in /usr/sbin" \ + || bad "useradd would be missed when /usr/sbin is not on PATH" +fi + +# ---- DNS inside the chroot -------------------------------------------------- # +# Routing works in the chroot (a bare IP pings fine) but without +# /etc/resolv.conf nothing resolves, and every download fails with +# Temporary failure in name resolution +python3 - "$LFS_TOOL" <<'PY23' +import sys, os, tempfile, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +if not hasattr(lfs, "_copy_resolv_conf"): + print(" FAIL nothing gives the chroot a resolver config"); sys.exit(1) +d = tempfile.mkdtemp(); os.makedirs(os.path.join(d, "etc")) +lfs._copy_resolv_conf(d) +p = os.path.join(d, "etc", "resolv.conf") +if os.path.isfile("/etc/resolv.conf") and not os.path.isfile(p): + print(" FAIL resolv.conf was not copied"); sys.exit(1) +# an existing, usable config must not be clobbered +open(p, "w").write("nameserver 9.9.9.9\n") +lfs._copy_resolv_conf(d) +if "9.9.9.9" not in open(p).read(): + print(" FAIL an existing resolv.conf was overwritten"); sys.exit(1) +# ...but the book's PLACEHOLDER file is not a usable config +open(p, "w").write("domain <Your Domain Name>\n" + "nameserver <IP address of your primary nameserver>\n") +lfs._copy_resolv_conf(d) +if "<IP address" in open(p).read(): + print(" FAIL a placeholder resolv.conf was treated as usable"); sys.exit(1) +print(" PASS the chroot gets DNS; real configs kept, placeholders replaced") +PY23 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- package users need the shared environment ------------------------------ # +# The hint's scheme gives every package user symlinks into /etc/pkgusr: +# .bash_profile -> /etc/pkgusr/bash_profile +# .bashrc -> /etc/pkgusr/bashrc +# build -> /etc/pkgusr/build +# Creating a bare account leaves it unable to build anything. +pmt="$(dirname "$LFS_TOOL")/packagemanager" +if [ -f "$pmt" ]; then + grep -q "^def init_package_user_home(" "$pmt" \ + && ok "packagemanager initialises a package user's environment" \ + || bad "packagemanager creates bare accounts" + grep -q "^def ensure_pkgusr_etc(" "$pmt" \ + && ok "the shared /etc/pkgusr environment is created when missing" \ + || bad "no shared package-user environment" +fi +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q "^init_package_user_home()" "$helper_src" \ + && ok "lfs-helper initialises a package user's environment too" \ + || bad "lfs-helper creates bare accounts" +fi + +# ---- installing and updating LFS packages ----------------------------------- # +python3 - "$LFS_TOOL" <<'PY24' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +for fn in ("cmd_install", "cmd_update", "_find_lfs_package"): + if not hasattr(lfs, fn): + print(" FAIL lfs is missing %s" % fn); sys.exit(1) +src = open(sys.argv[1]).read() +if "Run the configure phase too?" not in src: + print(" FAIL reinstall does not ask about the configure phase"); sys.exit(1) +print(" PASS lfs can install/update, and asks before re-running configure") +PY24 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- BLFS is preferred when both books have a package ----------------------- # +# BLFS' script carries the dependency information and the configure options a +# library build needs; LFS only builds what the base system requires. +python3 - "$LFS_TOOL" <<'PY25' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +if not hasattr(lfs, "_blfs_knows"): + print(" FAIL nothing checks whether BLFS has the package"); sys.exit(1) +src = open(sys.argv[1]).read() +if "--lfs-book" not in src: + print(" FAIL no way to force the LFS book's version"); sys.exit(1) +# a network failure must explain itself +msg = lfs._explain_fetch_failure(Exception( + "<urlopen error [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed>")) +if "CA" not in msg and "certificates" not in msg: + print(" FAIL an SSL failure is not explained"); sys.exit(1) +msg2 = lfs._explain_fetch_failure(Exception("Temporary failure in name resolution")) +if "DNS" not in msg2: + print(" FAIL a DNS failure is not explained"); sys.exit(1) +print(" PASS BLFS is preferred, and network failures explain themselves") +PY25 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- help is grouped, not one long list ------------------------------------- # +bad_help="" +for t in lfs blfs packagemanager; do + f="$(dirname "$LFS_TOOL")/$t" + [ -f "$f" ] || continue + grep -q "commands, by what you are doing" "$f" || bad_help="$bad_help $t" +done +h="$(dirname "$LFS_TOOL")/lfs-helper" +[ -f "$h" ] && { grep -q "Follow the build" "$h" || bad_help="$bad_help lfs-helper"; } +[ -z "$bad_help" ] \ + && ok "every tool groups its commands in --help" \ + || bad "ungrouped help in:$bad_help" + +# ---- provenance, versions and the helper scripts ---------------------------- # +python3 - "$LFS_TOOL" <<'PY26' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +problems = [] +if not hasattr(lfs, "_install_pkgusr_helpers"): + problems.append("the hint's helper scripts are never copied into the chroot") +if "wget" not in lfs.LAST_STEP_DEFAULT: + problems.append("wget is not installed by the final build step") +# scripts must say which book they came from +body = lfs._crosschain_script_body("d", "sid", "D-1.0", "d.tar.*", ["make"]) +if "### book" not in body: + problems.append("generated scripts do not record their book") +# the wrappers must never be taken from /usr/bin +if lfs._find_host_helper("chown", "usr/lib/pkgusr") in ("/usr/bin/chown", "/bin/chown"): + problems.append("the real /usr/bin/chown would be copied as a wrapper") +if problems: + for p in problems: print(" FAIL %s" % p) + sys.exit(1) +print(" PASS helpers copied, wget added from BLFS, scripts record their book") +PY26 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q 'VERSION"' "$helper_src" \ + && ok "the installed version is recorded in the package user's home" \ + || bad "nothing records what version is installed" + grep -q '# book :' "$helper_src" \ + && ok "a build says which book the script came from" \ + || bad "the book behind a script is not shown" +fi + +# ---- the user's own final build step ---------------------------------------- # +# A script on the HOST that runs last in the chroot: its SOURCES array is +# downloaded by get-sources, and the rest runs after every book package. It +# ships with wget because LFS has no download tool at all. +python3 - "$LFS_TOOL" <<'PY27' +import sys, os, tempfile, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +problems = [] +for fn in ("last_step_path", "ensure_last_step_script", "last_step_sources"): + if not hasattr(lfs, fn): + problems.append("lfs is missing %s" % fn) +if problems: + for p in problems: print(" FAIL %s" % p) + sys.exit(1) +# the SOURCES array must be PARSED, not executed +d = tempfile.mkdtemp() +p = os.path.join(d, "last.sh") +open(p, "w").write( + '#!/bin/bash\n' + 'SOURCES=(\n' + ' "https://example.org/a-1.0.tar.gz"\n' + ' # a comment\n' + ' "https://example.org/b-2.0.tar.xz"\n' + ')\n' + 'rm -rf / # would be catastrophic if this were executed\n') +old = lfs.load_config +lfs.load_config = lambda: {"last_step_script": p} +urls = lfs.last_step_sources() +lfs.load_config = old +if urls != ["https://example.org/a-1.0.tar.gz", "https://example.org/b-2.0.tar.xz"]: + print(" FAIL SOURCES not parsed correctly: %s" % urls); sys.exit(1) +print(" PASS the final step's SOURCES are parsed (never executed) to fetch") +PY27 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# the dry run must print EVERY url, not a sample +grep -q "print the WHOLE list" "$LFS_TOOL" \ + && ok "get-sources lists every URL in a dry run" \ + || bad "the dry run truncates the list" + +# ---- `lfs` must know when it is inside the chroot ---------------------------- # +# The build is driven from OUTSIDE. Inside, $LFS does not exist as a path, so +# `lfs build-system next` read empty state and reported "2 of 12 done" for a +# FINISHED build -- sending you back to the beginning. +python3 - "$LFS_TOOL" <<'PY28' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +problems = [] +if not hasattr(lfs, "inside_chroot"): + problems.append("nothing detects the chroot") +for c in ("next", "session", "layout", "crosschain", "chroot"): + if c not in lfs._INSIDE_ONLY_ELSEWHERE: + problems.append("'%s' would still run inside the chroot" % c) +# ...but reading the book and managing packages must still work in there +for c in ("books", "config", "packages", "install", "update"): + if c in lfs._INSIDE_ONLY_ELSEWHERE: + problems.append("'%s' should still work inside the chroot" % c) +if lfs._INSIDE_EQUIVALENT.get("next") != "lfs-helper next": + problems.append("the redirect does not name lfs-helper") +if problems: + for p in problems: print(" FAIL %s" % p) + sys.exit(1) +print(" PASS build commands redirect to lfs-helper inside the chroot") +PY28 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# the user's final step must be the LAST entry in the order, after the bootloader +grep -q 'order.append("last-step") # always last' "$LFS_TOOL" \ + && ok "the user's final step is ordered last" \ + || bad "the final step is not last in the build order" + +# ---- the wrappers are for package users, not for root ----------------------- # +# A root step legitimately runs `chown -R wget:wget /sources/wget-1.25.0` to +# hand a source tree to a package user. With the wrappers first in PATH that +# was silently skipped, the tree stayed root-owned, and the build died with +# ./configure: line 4389: config.log: Permission denied +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q "A ROOT step must not get" "$helper_src" \ + && ok "root steps run without the package-user wrappers" \ + || bad "root steps still get the wrappers and cannot chown" + grep -q "envpass=\"\$envpass PATH='/usr/local/bin" "$helper_src" \ + && ok "the root PATH has no wrapper directory" \ + || bad "root's PATH still starts with the wrappers" + # /sources and /usr/src are shared build space, never a package's tree + ( SNAP_ROOT=/; INSTALLDIRS="" + eval "$(sed -n '/^install_dirs_list() {/,/^}/p' "$helper_src")" + eval "$(sed -n '/^install_dir_patterns() {/,/^}/p' "$helper_src")" + eval "$(sed -n '/^matches_install_pattern() {/,/^}/p' "$helper_src")" + eval "$(sed -n '/^is_install_dir() {/,/^}/p' "$helper_src")" + is_install_dir /sources && is_install_dir /usr/src ) \ + && ok "/sources and /usr/src are never given to a package" \ + || bad "/sources could be handed to a collector group" +fi + +# ---- output that is easy to read and to copy -------------------------------- # +python3 - "$LFS_TOOL" <<'PY29' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +problems = [] +for fn in ("warn", "dry_run_note", "next_step"): + if not hasattr(lfs, fn): + problems.append("lfs is missing %s" % fn) +src = open(sys.argv[1]).read() +# no dry-run message may bypass the coloured helper +import re +for m2 in re.finditer(r'sys\.stderr\.write\([^)]*dry run[^)]*\)', src): + problems.append("an uncoloured dry-run message remains: %s" % m2.group(0)[:50]) +# the self-removing wget wrapper +if "check_certificate = off" not in lfs.LAST_STEP_DEFAULT: + problems.append("certificate checking is not disabled for the first fetch") +if "lfs-temporary-no-verify" not in lfs.LAST_STEP_DEFAULT: + problems.append("no marker to find and undo it later") +# the host's skeleton is preferred over any default we invent +if not hasattr(lfs, "_install_pkgusr_skel"): + problems.append("the host's skel-package is not copied") +if problems: + for p in problems: print(" FAIL %s" % p) + sys.exit(1) +print(" PASS dry runs are coloured, hints are copyable, wget wrapper self-removes") +PY29 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q "^cmd_fix_users()" "$helper_src" \ + && ok "existing package users' homes and VERSION files can be repaired" \ + || bad "no way to repair existing package users" +fi + +# ---- /usr/local/bin must precede /usr/bin ----------------------------------- # +# The temporary no-verify wget wrapper lives in /usr/local/bin. With that +# missing from the package user's PATH the wrapper was never found, and every +# HTTPS download failed with "cannot verify ... certificate". +bad_path="" +for f in lfs-helper packagemanager; do + t="$(dirname "$LFS_TOOL")/$f" + [ -f "$t" ] || continue + grep -q "PATH=/usr/lib/pkgusr:/usr/local/bin" "$t" || bad_path="$bad_path $f" +done +[ -z "$bad_path" ] \ + && ok "/usr/local/bin precedes /usr/bin for package users" \ + || bad "a local wrapper would never be found in:$bad_path" + +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q "^cmd_help_for()" "$helper_src" \ + && ok "each command explains itself with -h" \ + || bad "no per-command help" +fi +python3 - "$LFS_TOOL" <<'PY30' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +if not hasattr(lfs, "cmd_bs_sync_tools"): + print(" FAIL no step copies the package-user environment into the tree") + sys.exit(1) +print(" PASS the package-user environment can be synced into the tree") +PY30 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- the certificate workaround must undo itself ---------------------------- # +# A $PATH wrapper was never seen: builds run through `su -` with the package +# user's own environment, whose PATH is the host's /etc/pkgusr/bash_profile. +# /etc/wgetrc is read however wget is invoked, so that is where this belongs -- +# and it must be removed as soon as certificates exist, or every download on +# the system stays unverified forever. +pmt="$(dirname "$LFS_TOOL")/packagemanager" +if [ -f "$pmt" ]; then + grep -q "^def restore_wget_verification" "$pmt" \ + && ok "certificate checking is restored once CA certs exist" \ + || bad "the temporary workaround would never be undone" + grep -q "^def warn_if_tools_are_stale" "$pmt" \ + && ok "a chroot running older tools than the host is flagged" \ + || bad "stale tools in the chroot go unnoticed" +fi +python3 - "$LFS_TOOL" <<'PY31' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +if not hasattr(lfs, "_write_tool_stamps"): + print(" FAIL nothing records which tool versions are in the tree") + sys.exit(1) +if not hasattr(lfs, "_sync_report"): + print(" FAIL sync output has no single consistent shape"); sys.exit(1) +print(" PASS tool versions are stamped and sync output is consistent") +PY31 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- the certificate workaround must apply itself, not wait to be run ------- # +# A system with no CA certificates cannot fetch make-ca -- the very thing that +# would give it certificates. packagemanager has to notice and fix that on its +# own; relying on a build step having run leaves the user stuck. +pmt="$(dirname "$LFS_TOOL")/packagemanager" +if [ -f "$pmt" ]; then + grep -q "^def ensure_wget_workaround" "$pmt" \ + && ok "the certificate workaround applies itself when needed" \ + || bad "the workaround only exists if a build step wrote it" + # both halves must key on the same marker, or one will not find the other + n="$(grep -c '_WGETRC_MARK' "$pmt")" + [ "$n" -ge 3 ] \ + && ok "disable and restore share one marker" \ + || bad "the two halves use different markers" +fi + +# ---- shared drop-in directories --------------------------------------------- # +# Many packages write a file straight into /usr/share/zsh/site-functions, +# /usr/share/applications, /usr/lib/girepository-1.0 and the like. Treating +# those as belonging to whichever package created them first means every later +# package fails there -- meson reports it as a Python traceback ending in +# PermissionError: [Errno 13] Permission denied: '.../site-functions/_p11-kit' +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + ( SNAP_ROOT=/; INSTALLDIRS="" + eval "$(sed -n '/^install_dirs_list() {/,/^}/p' "$helper_src")" + eval "$(sed -n '/^install_dir_patterns() {/,/^}/p' "$helper_src")" + eval "$(sed -n '/^matches_install_pattern() {/,/^}/p' "$helper_src")" + eval "$(sed -n '/^is_install_dir() {/,/^}/p' "$helper_src")" + for d in /usr/share/zsh/site-functions /usr/share/applications \ + /usr/lib/girepository-1.0 /usr/share/glib-2.0/schemas \ + /usr/lib/systemd/system /usr/libexec; do + is_install_dir "$d" || exit 1 + done + # ...but a package's OWN subdirectory is not shared + for d in /usr/libexec/p11-kit /usr/lib/python3.13/site-packages; do + is_install_dir "$d" && exit 1 + done + exit 0 ) \ + && ok "drop-in directories are shared; package subdirs are not" \ + || bad "shared drop-in directories are misclassified" +fi + +pmi="$(dirname "$LFS_TOOL")/packagemanager_install" +if [ -f "$pmi" ]; then + grep -q "^diagnose_install_failure()" "$pmi" \ + && ok "a failed install explains itself instead of dumping a traceback" \ + || bad "install failures are reported as raw build output" +fi + +# ---- packagemanager must recover like lfs-helper does ----------------------- # +# The same permission problem happens after the base build as during it, and it +# has the same answer: put the package in the group that owns the directory. +# Printing a hint and stopping makes the user do by hand what the tool already +# knows how to do. +pmt="$(dirname "$LFS_TOOL")/packagemanager" +if [ -f "$pmt" ]; then + for fn in auto_grant_from_log run_pm_install is_shared_dir \ + warn_if_nothing_installed _paths_from_permission_errors; do + grep -q "^def $fn" "$pmt" || { bad "packagemanager is missing $fn"; break; } + done + grep -q "^def auto_grant_from_log" "$pmt" \ + && grep -q "^def run_pm_install" "$pmt" \ + && grep -q "^def is_shared_dir" "$pmt" \ + && grep -q "^def warn_if_nothing_installed" "$pmt" \ + && ok "packagemanager grants, retries and checks like lfs-helper" \ + || bad "packagemanager still lacks lfs-helper's recovery" + # a directory ALREADY in a collector group: just add the user to it + grep -q "is already group" "$pmt" \ + && ok "an existing collector group is joined, not replaced" \ + || bad "an existing collector group would be overwritten" + # every install path must go through the retrying runner + n_raw="$(grep -c 'subprocess.run(pm + \[\|subprocess.run(find_pm_install()' "$pmt" || true)" + [ "$n_raw" = "0" ] \ + && ok "every install path uses the retrying runner" \ + || bad "$n_raw install call(s) bypass the recovery" +fi + +# ---- log scanning must be bounded ------------------------------------------- # +# A build log runs to tens of megabytes. Extracting every path from all of it +# produced 200,000 candidates, each then given a `stat` and an `su ... test -w` +# -- hundreds of thousands of process spawns, which locked the machine up. +# Read only the tail, only lines that mention a permission problem, capped. +pmt="$(dirname "$LFS_TOOL")/packagemanager" +if [ -f "$pmt" ]; then + grep -q "_LOG_TAIL_BYTES" "$pmt" \ + && ok "packagemanager reads only the tail of a build log" \ + || bad "packagemanager scans whole build logs" + grep -q "_PERM_HINT" "$pmt" \ + && ok "a cheap substring filter runs before the regexes" \ + || bad "regexes run over every line of the log" + # and it must still find the error + python3 - "$pmt" <<'PY32' +import sys, os, tempfile, importlib.machinery as m +pm = m.SourceFileLoader('pm', sys.argv[1]).load_module() +d = tempfile.mkdtemp(); p = os.path.join(d, "big.log") +with open(p, "w") as f: + for i in range(50000): + f.write("[%d/4000] Compiling /very/long/path/file%d.c.o\n" % (i, i)) + f.write("PermissionError: [Errno 13] Permission denied: '/usr/share/zsh/site-functions/_x'\n") +import time +t0 = time.time(); paths = pm._paths_from_permission_errors(p); dt = time.time() - t0 +if paths != ["/usr/share/zsh/site-functions/_x"]: + print(" FAIL wrong paths from a large log: %s" % paths[:3]); sys.exit(1) +if dt > 1.0: + print(" FAIL scanning a large log took %.1fs" % dt); sys.exit(1) +print(" PASS a large log is scanned in %.0f ms and the error is still found" + % (dt * 1000)) +PY32 + if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi +fi +for f in lfs-helper packagemanager_install; do + t="$(dirname "$LFS_TOOL")/$f" + [ -f "$t" ] || continue + grep -q "tail -c 524288" "$t" \ + && ok "$f reads only the tail of a log" \ + || bad "$f scans whole logs" +done + +# ---- packagemanager fixes permissions itself, like lfs-helper --------------- # +# Installing into a directory another package owns is the NORMAL state of +# affairs under the package-user model, not a mistake to report. The answer is +# always the same -- put both packages in a collector group -- so it should be +# done automatically, not printed as a command for the user to copy. +pmi="$(dirname "$LFS_TOOL")/packagemanager_install" +if [ -f "$pmi" ]; then + grep -q "^auto_grant_from_log()" "$pmi" \ + && ok "packagemanager grants collector access on a permission failure" \ + || bad "packagemanager only reports permission failures" + grep -qE '_round\" -lt (6|20)' "$pmi" \ + && ok "it retries until nothing new needs granting" \ + || bad "one grant round is not enough for a multi-directory install" + # an EXISTING collector group must be joined, not duplicated + grep -q '"\${prefix}"\*)' "$pmi" \ + && ok "an existing collector group is joined rather than replaced" \ + || bad "a second group would be made for an already-shared directory" + grep -q "PM_NO_AUTO_FIX" "$pmi" \ + && ok "the automatic repair can be turned off" \ + || bad "no way to opt out of automatic permission changes" +fi + +# ---- a retry wrapper must not call itself ----------------------------------- # +# run_pm_install() gained a retry loop and, in wrapping the old call, ended up +# calling ITSELF instead of the raw runner -- so nothing was ever installed: +# RecursionError: maximum recursion depth exceeded +# Cheap to check, and it aborts before the install is even attempted. +python3 - "$(dirname "$LFS_TOOL")" <<'PY32' +import ast, os, sys +base = sys.argv[1] +problems = [] +for f in ("packagemanager", "lfs", "blfs"): + p = os.path.join(base, f) + if not os.path.isfile(p): + continue + tree = ast.parse(open(p).read()) + for node in ast.walk(tree): + if not isinstance(node, ast.FunctionDef): + continue + # the FIRST statement calling the function itself is always a bug: + # there is no base case before it + first = node.body[0] if node.body else None + if isinstance(first, ast.Expr) and isinstance(first.value, ast.Call): + call = first.value + elif isinstance(first, ast.Assign) and isinstance(first.value, ast.Call): + call = first.value + else: + continue + if isinstance(call.func, ast.Name) and call.func.id == node.name: + problems.append("%s: %s() calls itself first (line %d)" + % (f, node.name, node.lineno)) +if problems: + for p in problems: print(" FAIL %s" % p) + sys.exit(1) +print(" PASS no function recurses before doing any work") +PY32 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +pmt="$(dirname "$LFS_TOOL")/packagemanager" +if [ -f "$pmt" ]; then + grep -q "^def _run_pm_install_once" "$pmt" \ + && ok "the retry wrapper has a raw runner to call" \ + || bad "no raw runner: the wrapper can only call itself" +fi + +# ---- never grant from a traceback, never a group for root ------------------- # +# A Python traceback quotes meson's own source and the interpreter's stdlib. +# Treating those as install targets granted a package write access to +# /usr/lib/python3.13 -- and invented a "<prefix>_root" group, which would mean +# "may write anywhere root owns" and undoes the entire scheme. +pmi="$(dirname "$LFS_TOOL")/packagemanager_install" +if [ -f "$pmi" ]; then + grep -q 'owned by root -- NOT shared' "$pmi" \ + && ok "a collector group is never created for root" \ + || bad "a root collector group could still be created" + grep -q 'grep -vE "\^\[\[:space:\]\]\*(File|Traceback' "$pmi" \ + && ok "traceback frames are excluded when looking for failed paths" \ + || bad "paths from tracebacks could be granted" +fi +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q "not creating a collector group for it" "$helper_src" \ + && ok "lfs-helper also refuses a root collector group" \ + || bad "lfs-helper could create a root collector group" +fi + +# only the erroring path is extracted, not every path in the log +tbd="$T/tb"; mkdir -p "$tbd" +cat > "$tbd/log" <<'EOF' +Traceback (most recent call last): + File "/usr/lib/python3.13/site-packages/mesonbuild/minstall.py", line 336, in copy2 + shutil.copy2(*args, **kwargs) +PermissionError: [Errno 13] Permission denied: '/usr/share/zsh/site-functions/_x' +EOF +found="$(grep -hE "Permission denied" "$tbd/log" \ + | grep -vE "^[[:space:]]*(File|Traceback)" \ + | grep -hoE "'[^']+'" | tr -d "'")" +case "$found" in + */site-functions/_x) ok "only the failing path is taken from a traceback" ;; + *) bad "wrong paths extracted from a traceback: $found" ;; +esac + +# ---- blfs offers the same verbs as lfs, and they are real ------------------- # +# The help advertised install/update/fetch/import before the commands existed. +# A help text that lies is worse than none; every advertised verb must parse. +bt="$(dirname "$LFS_TOOL")/blfs" +if [ -f "$bt" ]; then + missing="" + for c in install update fetch import sources search versions deps rdeps \ + order script books set-default debug; do + python3 "$bt" "$c" --help >/dev/null 2>&1 || missing="$missing $c" + done + [ -z "$missing" ] \ + && ok "every blfs command in the help actually exists" \ + || bad "blfs help advertises commands that do not parse:$missing" + # install/update delegate -- there is exactly ONE install path + grep -q "^def cmd_install" "$bt" && grep -q "os.execvp" "$bt" \ + && ok "blfs install/update hand over to packagemanager" \ + || bad "blfs would grow a second install path" + # sources must read the key the cache actually writes + grep -q 'data.get("additional_links"' "$bt" \ + && ok "blfs sources reads the cache key that exists" \ + || bad "blfs sources reads a key the cache never writes" + # help texts share one shape across the tools + python3 "$bt" --help 2>&1 | grep -q "Dry run is the default" \ + && ok "blfs help ends with the same footer as the other tools" \ + || bad "blfs help has drifted from the shared shape" +fi + +# ---- one command to resume the whole build ---------------------------------- # +# After a host reboot nothing is mounted and the session is gone. `lfs run` +# has to re-mount, continue where it left off, and end inside the chroot -- +# without being told where it stands. +python3 - "$LFS_TOOL" <<'PY33' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +if not hasattr(lfs, "cmd_bs_run"): + print(" FAIL no command runs the procedure end to end"); sys.exit(1) +src = open(sys.argv[1]).read() +for want, why in ( + ('sub.add_parser("run"', "`lfs run` shorthand is not registered"), + ('bsub.add_parser("run"', "`lfs build-system run` is not registered"), + ("still pending -- stopping", "no guard against a step that never completes"), + ("continues from here", "a failure does not say how to resume"), +): + if want not in src: + print(" FAIL %s" % why); sys.exit(1) +print(" PASS one command resumes the build and ends in the chroot") +PY33 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- run must WALK you through, not print homework -------------------------- # +# `lfs books` only lists books: running it can never complete the "choose a +# book" step, so run executed it, nothing changed, and the loop guard fired. +# A step that needs a decision has to ask for the decision. +python3 - "$LFS_TOOL" <<'PY34' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +src = open(sys.argv[1]).read() +problems = [] +if not hasattr(lfs, "_run_choose_book"): + problems.append("nothing asks which book to build") +# only commands that change something may run unattended +if 'applies = ("--run" in cmd' not in src: + problems.append("a listing command could still be auto-run forever") +if 'title == "complete the configuration"' not in src: + problems.append("run does not walk through the configuration") +if problems: + for p in problems: print(" FAIL %s" % p) + sys.exit(1) +print(" PASS run asks for decisions instead of printing them") +PY34 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- nested command blocks, and progress while working ---------------------- # +# extract_commands_phased walked only the DIRECT children of the installation +# div. Packages that build several components (gst-plugins-rs builds +# libgstdav1d and libgstgtk4) put each in a sub-section, so every command sat +# one level down: 5 packages got a completely empty script, 53 lost some +# commands -- and the generated script still passed a bash syntax check. +bt="$(dirname "$LFS_TOOL")/blfs" +if [ -f "$bt" ]; then + grep -q 'inst.find_all(\["pre", "h3", "h4", "div"\])' "$bt" \ + && ok "commands are collected from nested sub-sections too" \ + || bad "only top-level command blocks would be found" + # a stale index must not survive the fix + grep -q "CACHE_VERSION = 4" "$bt" \ + && ok "the cache version was bumped so stale indexes rebuild" \ + || bad "cached books would keep serving the old empty commands" +fi + +pmt="$(dirname "$LFS_TOOL")/packagemanager" +if [ -f "$pmt" ]; then + grep -q "_BLFS_ACTIVITY" "$pmt" \ + && ok "long book operations say what they are doing" \ + || bad "install sits silent while indexing the book" + grep -q 'set_status("reading the book' "$pmt" \ + && ok "planning reports progress" \ + || bad "plan building is silent" +fi + +# ---- /etc/fstab must be generated ------------------------------------------- # +# Nothing wrote /etc/fstab, so the boot scripts could not remount root +# read-write and the first write -- /run/bootlog -- failed with +# Read-only file system +# taking every later service with it. The device and filesystem are already +# known from `session`, so generate it. +python3 - "$LFS_TOOL" <<'PY35' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +if not hasattr(lfs, "_fstab_script"): + print(" FAIL nothing generates /etc/fstab"); sys.exit(1) +problems = [] +# refuses rather than writing a broken fstab +t = lfs._fstab_script({"lfs_fstype": "ext4"}) +if "exit 3" not in t: + problems.append("no root device configured, yet it still writes an fstab") +# journalling checks: btrfs/xfs must NOT be fsck'd at boot +for fs, want in (("ext4", "1 1"), ("btrfs", "0 0"), ("xfs", "0 0")): + row = [l for l in lfs._fstab_script( + {"lfs_device": "/dev/sda2", "lfs_fstype": fs}).splitlines() + if l.startswith("/dev/sda2")] + if not row or not row[0].rstrip().endswith(want): + problems.append("%s got the wrong fsck order" % fs) +# the virtual filesystems the boot scripts need +t = lfs._fstab_script({"lfs_device": "/dev/sda2", "lfs_fstype": "ext4"}) +for need in ("/run", "/dev/shm", "/proc", "/sys", "/dev/pts"): + if need not in t: + problems.append("%s missing from fstab" % need) +# never clobber a hand-written fstab +if "was not written by us" not in t: + problems.append("would overwrite an fstab someone else wrote") +if problems: + for p in problems: print(" FAIL %s" % p) + sys.exit(1) +print(" PASS /etc/fstab is generated correctly for the configured disk") +PY35 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +grep -q 'root_steps.append("cfg-fstab")' "$LFS_TOOL" \ + && ok "writing /etc/fstab runs as root" \ + || bad "cfg-fstab would run as a package user and fail" + +# ---- a book that does not exist must be refused ----------------------------- # +# set-default warned and then saved anyway, so a typo ("ls") became the +# configured book and every later command failed with a 404 far from the +# mistake. A setting that cannot work should not be accepted. +python3 - "$LFS_TOOL" <<'PY36' +import sys, argparse, importlib.machinery as m, tempfile, os, json +store = tempfile.mkdtemp() +os.environ["LFS_STORE"] = store +os.makedirs(os.path.join(store, "books"), exist_ok=True) +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +lfs.remote_versions = lambda: ["12.3", "12.4", "12.4-systemd"] +def try_set(v): + try: + lfs.cmd_set_default(argparse.Namespace(version=v)) + return True # accepted + except SystemExit: + return False # refused +if try_set("ls"): + print(" FAIL a nonexistent book was accepted as the default"); sys.exit(1) +cfg = {} +p = os.path.join(store, "config.json") +if os.path.isfile(p): + cfg = json.load(open(p)) +if cfg.get("default") == "ls": + print(" FAIL the bad book was saved anyway"); sys.exit(1) +print(" PASS a book that does not exist is refused, not saved") +PY36 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- reset clears settings, never the built system -------------------------- # +python3 - "$LFS_TOOL" <<'PY37' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +src = open(sys.argv[1]).read() +if not hasattr(lfs, "cmd_reset"): + print(" FAIL no way to clear the configuration"); sys.exit(1) +body = src[src.index("def cmd_reset"):src.index("def cmd_set_default")] +problems = [] +if "Type 'reset' to confirm" not in body: + problems.append("no confirmation before deleting") +if "args.run" not in body: + problems.append("reset is not dry-run by default") +# Check what it actually DELETES, not what it mentions: the reassurance text +# names sources and books precisely because it spares them. +targets = body[body.index("targets = ["):body.index("present = [")] +for danger in ("sources", "books", "usr/src"): + if danger in targets: + problems.append("reset lists %s among the things it deletes" % danger) +if "rmtree(lfs_mount)" in body or 'rmtree(lfs)' in body: + problems.append("reset would delete the whole LFS tree") +if problems: + for p in problems: print(" FAIL %s" % p) + sys.exit(1) +print(" PASS reset warns, confirms, and spares the built system") +PY37 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- the build book and the install book are separate ----------------------- # +# You may build the system from one book and then install packages on it from a +# newer one. Both used to be the same setting, so choosing a newer book for +# packages silently changed what the chroot scripts would be generated from. +python3 - "$LFS_TOOL" <<'PY38' +import sys, os, json, argparse, tempfile, importlib.machinery as m +store = tempfile.mkdtemp() +os.environ["LFS_STORE"] = store +os.makedirs(os.path.join(store, "books"), exist_ok=True) +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +problems = [] +if not hasattr(lfs, "build_version"): + print(" FAIL there is only one book setting"); sys.exit(1) +# unset build_book falls back to default -- nothing changes for anyone +json.dump({"default": "12.4"}, open(os.path.join(store, "config.json"), "w")) +if lfs.build_version() != "12.4": + problems.append("build_book does not fall back to default") +# set, it wins for building only +json.dump({"default": "12.4", "build_book": "12.3"}, + open(os.path.join(store, "config.json"), "w")) +if lfs.build_version() != "12.3": + problems.append("build_book is ignored when set") +if lfs.default_version() != "12.4": + problems.append("build_book leaked into the install book") +# --book must still override both +src = open(sys.argv[1]).read() +if 'getattr(args, "book", None)' not in src: + problems.append("--book no longer overrides") +# the flag is set in ONE place so a new subcommand cannot forget +if src.count('setattr(args, "_use_build_book", True)') != 1: + problems.append("the build-book flag is set in more than one place") +if problems: + for p in problems: print(" FAIL %s" % p) + sys.exit(1) +print(" PASS building and installing can use different books") +PY38 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- every command must run to completion ----------------------------------- # +# `lfs config` crashed with NameError on a variable removed during an edit -- +# the last few lines of output were never reached by any test. Compiling is +# not the same as running: exercise the read-only commands end to end. +cft="$T/cfgrun"; mkdir -p "$cft/books" +( + export LFS_STORE="$cft" + python3 "$LFS_TOOL" import "$LFS_BOOK" --version 12.4 --set-default >/dev/null 2>&1 + rc=0 + for c in "config" "books --local" "next" "packages" "--help" "reset"; do + out="$(python3 "$LFS_TOOL" $c 2>&1)" || true + case "$out" in + *Traceback*|*NameError*|*AttributeError*|*TypeError*) + echo " crashed: lfs $c" + echo "$out" | tail -3 | sed 's/^/ /' + rc=1 ;; + esac + done + exit $rc +) && ok "the read-only commands all run without crashing" \ + || bad "a command crashes partway through its output" + +# ---- a fresh tree must be handed to the lfs user ---------------------------- # +# `session` applies book 4.3's chown, but it runs BEFORE `layout` creates the +# directories -- so it chowned nothing, the tree stayed root-owned, and +# crosschain then refused with "chapter 7 has already been prepared" on a tree +# where nothing had been built at all. +python3 - "$LFS_TOOL" <<'PY39' +import sys, os, tempfile, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +problems = [] +if not hasattr(lfs, "_chown_tree_to_lfs"): + problems.append("nothing hands the build tree to the lfs user") +if not hasattr(lfs, "_chapter56_has_output"): + problems.append("root ownership alone is still treated as 'chapter 7 done'") +else: + # a tree layout just created must NOT look like finished chapter 5-6 work + t = tempfile.mkdtemp() + os.makedirs(os.path.join(t, "usr", "bin")) + os.makedirs(os.path.join(t, "tools", "bin")) + if lfs._chapter56_has_output(t): + problems.append("an empty tree is mistaken for a built one") + # a cross toolchain in tools/bin must be recognised + open(os.path.join(t, "tools", "bin", "x86_64-lfs-linux-gnu-gcc"), "w").close() + if not lfs._chapter56_has_output(t): + problems.append("a real cross toolchain is not recognised") +src = open(sys.argv[1]).read() +# layout must do the chown itself, since it is what creates the dirs as root +lay = src[src.index("def cmd_bs_layout"):src.index("def _chown_tree_to_lfs")] +if "_chown_tree_to_lfs" not in lay: + problems.append("layout does not hand over the tree it just created") +if problems: + for p in problems: print(" FAIL %s" % p) + sys.exit(1) +print(" PASS a fresh tree is given to the lfs user, not mistaken for a built one") +PY39 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- sync-tools must hand over what it copied ------------------------------- # +# layout chowns the tree to lfs, then sync-tools copies the package-user +# environment in AS ROOT -- re-creating root-owned dirs after the handover: +# ! these build dirs are not owned by the 'lfs' user: +# /mnt/lfs/usr/lib/pkgusr, /mnt/lfs/etc/pkgusr, ... +# Anything copied in before the chroot exists belongs to the lfs user. +python3 - "$LFS_TOOL" <<'PY40' +import sys, importlib.machinery as m +src = open(sys.argv[1]).read() +body = src[src.index("def cmd_bs_sync_tools"):] +body = body[:body.index("\ndef ")] +if "_chown_tree_to_lfs" not in body: + print(" FAIL sync-tools leaves root-owned dirs behind") + sys.exit(1) +print(" PASS sync-tools hands over what it copied in") +PY40 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- every step announces itself -------------------------------------------- # +# Steps can run for many minutes with no output; without a consistent header +# you cannot tell work from a hang. +python3 - "$LFS_TOOL" <<'PY41' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +src = open(sys.argv[1]).read() +if not hasattr(lfs, "_run_banner"): + print(" FAIL steps do not announce what is happening"); sys.exit(1) +run = src[src.index("def cmd_bs_run"):] +run = run[:run.index("\ndef ") if "\ndef " in run else len(run)] +# the interactive steps must use the SAME banner, not their own format +if run.count("_run_banner(") < 3: + print(" FAIL some steps announce themselves differently"); sys.exit(1) +body = src[src.index("def _run_banner"):src.index("def cmd_bs_run")] +for need, why in (("NOW:", "does not say what is running"), + ("NEXT:", "does not say what comes next"), + ("step %d of %d", "does not show progress")): + if need not in body: + print(" FAIL the banner %s" % why); sys.exit(1) +print(" PASS every step shows what is running, and what is next") +PY41 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- the package-user system is not optional -------------------------------- # +# build-all stopped at the first chapter-8 package and asked the user to run +# init-pkgusr and start again. That is the one thing this toolchain exists to +# do -- and every package installed before it exists is another root-owned tree +# to repair later, so set it up FIRST, automatically. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + ba="$(sed -n '/^cmd_build_all() {/,/^}/p' "$helper_src")" + case "$ba" in + *"cmd_init_pkgusr --run"*) + ok "build-all sets up the package-user system before building" ;; + *) bad "build-all still stops and asks for init-pkgusr" ;; + esac + # it must be set up BEFORE the loop, not when we trip over a package + before="$(printf '%s' "$ba" | grep -n "cmd_init_pkgusr --run" | head -1 | cut -d: -f1)" + # the BUILD loop ("while read -r s"), not the argument-parsing one + loop="$(printf '%s' "$ba" | grep -n "while read -r s" | head -1 | cut -d: -f1)" + if [ -n "$before" ] && [ -n "$loop" ] && [ "$before" -lt "$loop" ]; then + ok "it happens before the first package, not at the first failure" + else + bad "the package-user system is still set up too late" + fi + # running it twice must not die + grep -q "getent group install >/dev/null 2>&1" "$helper_src" \ + && ok "init-pkgusr can be run again safely" \ + || bad "a second init-pkgusr would abort the build" +fi + +# ---- an interrupted toolchain must not count as finished -------------------- # +# A cross-gcc appears in $LFS/tools/bin after step 2 of 22, and that alone was +# taken as "chapters 5-6 done". Cancelling at step 11 therefore let `run` +# hand the tree to root and enter a chroot built on a third of a toolchain. +python3 - "$LFS_TOOL" <<'PY42' +import sys, os, json, tempfile, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +names = [n for n, _s, _t in lfs.CROSSCHAIN_STEPS] +problems = [] + +def tree(done=None, tools_gcc=True, temp_system=False): + t = tempfile.mkdtemp() + os.makedirs(os.path.join(t, ".lfs-pkgusr")) + os.makedirs(os.path.join(t, "tools", "bin")) + os.makedirs(os.path.join(t, "usr", "bin")) + if tools_gcc: + open(os.path.join(t, "tools", "bin", "x86_64-lfs-linux-gnu-gcc"), "w").close() + if temp_system: + for b in ("gawk", "sed", "tar", "xz"): + open(os.path.join(t, "usr", "bin", b), "w").close() + if done is not None: + json.dump({"book": "12.4", "done": done}, + open(os.path.join(t, ".lfs-pkgusr", + "crosschain-progress.json"), "w")) + return t + +# interrupted part-way: NOT done, and the count must be honest +d, n = lfs._crosschain_looks_done(tree(names[:10])) +if d or n != 10: + problems.append("a 10-of-22 toolchain is reported as finished") +# a cross-gcc alone proves nothing +d, _ = lfs._crosschain_looks_done(tree(None)) +if d: + problems.append("a cross-gcc alone is taken as a finished toolchain") +# genuinely finished +d, n = lfs._crosschain_looks_done(tree(names)) +if not d or n != len(names): + problems.append("a finished toolchain is not recognised") +# progress must NOT be wiped on completion, or done == never-started again +src = open(sys.argv[1]).read() +tail = src[src.index("crosschain: all requested steps done"):] +head = src[:src.index("crosschain: all requested steps done")] +if "_clear_crosschain_progress(lfs)" in head[head.rindex("if args.run:"):]: + problems.append("progress is cleared on completion, losing the record") +# preparing the chroot is a one-way door: it must check first +if "chapters 5-6 are not finished" not in src: + problems.append("chroot prepare does not verify the toolchain first") +if problems: + for p in problems: print(" FAIL %s" % p) + sys.exit(1) +print(" PASS an unfinished toolchain is detected and blocks the chroot") +PY42 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- snapshots: save the build, put it back --------------------------------- # +# LFS is a long sequence of irreversible steps; without this a bad package +# means starting over. The dangerous parts are what get tested here. +python3 - "$LFS_TOOL" <<'PY43' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +src = open(sys.argv[1]).read() +if not hasattr(lfs, "cmd_snapshot"): + print(" FAIL no way to save or restore the build"); sys.exit(1) +body = src[src.index("def cmd_snapshot"):src.index("def cmd_reset")] +problems = [] +# ownership IS the package-user model: a backup that loses it is worthless +if "--numeric-owner" not in body: + problems.append("uids would be resolved against the host's /etc/passwd") +for need, why in (("--acls", "ACLs"), ("--xattrs", "extended attributes"), + ("--sparse", "sparse files")): + if need not in body: + problems.append("%s are not preserved" % why) +# the virtual filesystems are the HOST's -- capturing or writing them is unsafe +if "_SNAP_EXCLUDE" not in src: + problems.append("the virtual filesystems are not excluded") +for d in ("./dev/*", "./proc/*", "./sys/*"): + if d not in src: + problems.append("%s is not excluded from snapshots" % d) +if "could write through them onto this system" not in body: + problems.append("restore does not refuse while /dev /proc are mounted") +# restore is destructive: dry run by default, and confirmed +if "args.run" not in body or "Type the snapshot name to confirm" not in body: + problems.append("restore is not guarded by a dry run and a confirmation") +# /sources must survive a restore -- it is large and was never captured +if 'keep = "sources"' not in body: + problems.append("a restore would delete the downloaded sources") +if problems: + for p in problems: print(" FAIL %s" % p) + sys.exit(1) +print(" PASS snapshots preserve ownership and restore safely") +PY43 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- already-built packages are adopted before anything new ----------------- # +# Chapters 5-7 run before package users exist, so their files are root-owned +# and the manifests are the only record of who installed what. Nothing ran +# adopt-existing, so the FIRST package user on the system was man-pages -- a +# chapter-8 package -- while bash, coreutils and glibc still owned nothing. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + ba="$(sed -n '/^cmd_build_all() {/,/^}/p' "$helper_src")" + case "$ba" in + *"cmd_adopt_existing --run"*) + ok "already-built packages get their users before new builds" ;; + *) bad "chapter 5-7 packages are never given their package users" ;; + esac + # adoption must come BEFORE the build loop + a="$(printf '%s' "$ba" | grep -n "cmd_adopt_existing --run" | head -1 | cut -d: -f1)" + l="$(printf '%s' "$ba" | grep -n "while read -r s" | head -1 | cut -d: -f1)" + if [ -n "$a" ] && [ -n "$l" ] && [ "$a" -lt "$l" ]; then + ok "adoption happens before the first new package is built" + else + bad "a chapter-8 package could still be the first package user" + fi + # `die` inside cmd_add_user exits the SCRIPT, so `|| continue` never ran: + # one package that could not get a user aborted the whole pass + grep -q '( cmd_add_user "$owner" )' "$helper_src" \ + && ok "one failed adoption no longer aborts the run" \ + || bad "a single failed add-user would kill the whole build" +fi + +# ---- a package must not adopt another package's tree ------------------------ # +# infer_dirs walks UP from every installed file, so a package that drops one +# file deep inside another's tree claimed every parent on the way. zstd ended +# up owning /usr/include/c++/15.2.0/bits -- gcc's own C++ headers -- and gcc +# could no longer build: +# fatal error: bits/c++config.h: No such file or directory +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q "^_dir_contents_foreign()" "$helper_src" \ + && ok "a directory holding another package's files is not adopted" \ + || bad "any package can still claim another package's tree" + + # the header trees are shared by nature and must never be adopted at all + ( SNAP_ROOT=/; INSTALLDIRS="" + eval "$(sed -n '/^install_dirs_list() {/,/^}/p' "$helper_src")" + eval "$(sed -n '/^install_dir_patterns() {/,/^}/p' "$helper_src")" + eval "$(sed -n '/^matches_install_pattern() {/,/^}/p' "$helper_src")" + eval "$(sed -n '/^is_install_dir() {/,/^}/p' "$helper_src")" + is_install_dir /usr/include || exit 1 + exit 0 ) \ + && ok "/usr/include itself is a shared install directory" \ + || bad "/usr/include is not treated as shared" + # The C++ subtree is gcc's, and is protected from adoption by the + # contents check -- NOT by being declared shared, which would hand it to + # every package via the install group. + grep -q "^_dir_contents_foreign()" "$helper_src" \ + && ok "gcc's header subtree is protected by the contents check" \ + || bad "nothing stops another package adopting gcc's headers" + + # and the check must run before the chown, not after + ad="$(sed -n '/^cmd_adopt_dirs() {/,/^}/p' "$helper_src")" + c="$(printf '%s' "$ad" | grep -n "_dir_contents_foreign" | head -1 | cut -d: -f1)" + w="$(printf '%s' "$ad" | grep -n "chown " | head -1 | cut -d: -f1)" + if [ -n "$c" ] && [ -n "$w" ] && [ "$c" -lt "$w" ]; then + ok "the check runs before anything is chowned" + else + bad "directories are chowned before being checked" + fi +fi + +# ---- "nearly finished" is not finished -------------------------------------- # +# The no-progress-file fallback accepted gawk/sed/tar/xz as proof that +# chapters 5-6 were done -- but those are steps 13-20 of 22. A tree missing +# only gcc-pass2 (the LAST step, which installs the C++ headers) looked +# complete, and chapter 8's gcc then died with +# fatal error: bits/c++config.h: No such file or directory +python3 - "$LFS_TOOL" <<'PY44' +import sys, os, tempfile, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +problems = [] + +def tree(with_cxx_headers): + t = tempfile.mkdtemp() + for d in (("tools", "bin"), ("usr", "bin"), + ("usr", "include", "c++", "15.2.0", "bits")): + os.makedirs(os.path.join(t, *d), exist_ok=True) + open(os.path.join(t, "tools", "bin", "x86_64-lfs-linux-gnu-gcc"), "w").close() + for b in ("gawk", "sed", "tar", "xz"): + open(os.path.join(t, "usr", "bin", b), "w").close() + if with_cxx_headers: + p = os.path.join(t, "usr", "include", "c++", "15.2.0", + "x86_64-pc-linux-gnu", "bits") + os.makedirs(p, exist_ok=True) + open(os.path.join(p, "c++config.h"), "w").close() + return t + +d, _ = lfs._crosschain_looks_done(tree(False)) +if d: + problems.append("a tree missing gcc-pass2 is reported as finished") +d, _ = lfs._crosschain_looks_done(tree(True)) +if not d: + problems.append("a genuinely finished toolchain is not recognised") +if not hasattr(lfs, "_crosschain_missing_output"): + problems.append("nothing explains WHAT is missing") +elif "c++config" not in open(sys.argv[1]).read(): + problems.append("the C++ headers are not what is checked") +if problems: + for p in problems: print(" FAIL %s" % p) + sys.exit(1) +print(" PASS a toolchain missing only its last step is not called finished") +PY44 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- a build may not claim files it did not install ------------------------- # +# "touched" is every file with an mtime newer than the build stamp, which is +# wider than "files this package installed". zstd ended up owning +# /usr/include/c++/15.2.0/bits/memoryfwd.h -- gcc's own C++ headers -- and gcc +# could no longer build. And a FAILED gcc still "tracked 828 file(s)" and +# chowned them, so a broken partial install claimed files from the working +# temporary system and the next attempt inherited the damage. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q "NEVER take a file that already belongs to another package user" "$helper_src" \ + && ok "a build cannot take a file another package owns" \ + || bad "a package can still steal another package's files" + grep -q "A FAILED build must not take ownership" "$helper_src" \ + && ok "a failed build changes no ownership" \ + || bad "a failed build still claims what it touched" + # the failure check must come BEFORE the chown block + bl="$(sed -n '/^cmd_build() {/,/^}/p' "$helper_src")" + f="$(printf '%s' "$bl" | grep -n 'rc:-0' | head -1 | cut -d: -f1)" + c="$(printf '%s' "$bl" | grep -n 'gave \$n newly installed' | head -1 | cut -d: -f1)" + if [ -n "$f" ] && [ -n "$c" ] && [ "$f" -lt "$c" ]; then + ok "the failure check runs before anything is chowned" + else + bad "ownership changes before the build result is checked" + fi +fi + +# ---- the compiler must be checked before chapter 8 -------------------------- # +# A toolchain that cannot link a trivial program produces failures that point +# everywhere except at itself: +# configure: error: cannot compute sizeof (long long) +# fatal error: bits/c++config.h: No such file or directory +# Both mean the same thing, and both are cheap to detect up front. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q "^cmd_check_toolchain()" "$helper_src" \ + && ok "there is a toolchain sanity check" \ + || bad "nothing verifies the compiler works" + # C++ specifically -- that is what chapter 8's gcc needs first + grep -q "include <memory>" "$helper_src" \ + && ok "the check compiles C++, not just C" \ + || bad "a missing C++ header set would not be caught" + # and it must report WHERE the headers are, since a triplet mismatch + # looks identical to missing headers + grep -q "dumpmachine" "$helper_src" \ + && ok "it reports the compiler's own target triplet" \ + || bad "a triplet mismatch could not be told from missing headers" + ba="$(sed -n '/^cmd_build_all() {/,/^}/p' "$helper_src")" + case "$ba" in + *cmd_check_toolchain*) ok "build-all checks the compiler before starting" ;; + *) bad "chapter 8 can start on a broken toolchain" ;; + esac +fi + +# ---- a header can exist and still be unreachable ---------------------------- # +# If any directory on the path lacks o+x, the compiler gets ENOENT -- "No such +# file or directory" -- not a permission error. A mis-assigned package-user +# directory therefore looks exactly like a missing header, which is how a +# present, world-readable c++config.h still broke the gcc build. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q "This header EXISTS but cannot be reached" "$helper_src" \ + && ok "an unreachable header is distinguished from a missing one" \ + || bad "a permission problem still reads as a missing file" +fi +# and prove the failure mode itself: mode 644 file, un-traversable parent +tvd="$T/traverse" +mkdir -p "$tvd/a/b" +# the test dir itself must be traversable, or we would be measuring $T +chmod 755 "$T" "$tvd" 2>/dev/null +echo data > "$tvd/a/b/hdr.h" +chmod 644 "$tvd/a/b/hdr.h" +chmod 640 "$tvd/a" +if id nobody >/dev/null 2>&1; then + if su -s /bin/sh nobody -c "cat '$tvd/a/b/hdr.h'" >/dev/null 2>&1; then + bad "the traversal failure mode did not reproduce" + else + ok "a readable file behind a closed directory is unreachable" + fi + chmod 755 "$tvd/a" + su -s /bin/sh nobody -c "cat '$tvd/a/b/hdr.h'" >/dev/null 2>&1 \ + && ok "...and reachable once the directory allows traversal" \ + || bad "still unreachable after opening the directory" +fi + +# ---- existence and permissions are not enough ------------------------------- # +# A header can exist, be world-readable, and sit on a fully traversable path, +# and still not be found -- because the directory holding it is not in the +# compiler's search list at all. Only the compiler can answer that, so the +# check has to ask it rather than inferring from the filesystem. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q "include <...> search starts here" "$helper_src" \ + && ok "the check shows the compiler's real include search path" \ + || bad "a search-path mismatch could not be diagnosed" + # all three explanations must be distinguishable from each other + for phrase in "cannot be reached" "not in that list" "OWN target triplet"; do + grep -q "$phrase" "$helper_src" || bad "missing diagnosis: $phrase" + done + ok "missing, unreachable and not-searched are told apart" +fi + +# ---- repairing a target-triplet mismatch ------------------------------------ # +# The chroot compiler reported x86_64-pc-linux-gnu while chapter 6 installed +# its headers under x86_64-lfs-linux-gnu, so g++ never looked where +# c++config.h actually was. Same gcc version on the same machine, so the two +# trees are interchangeable and a compatibility link unblocks the build. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q "^cmd_fix_toolchain()" "$helper_src" \ + && ok "a triplet mismatch can be repaired" \ + || bad "nothing repairs a triplet mismatch" + fb="$(sed -n '/^cmd_fix_toolchain() {/,/^}/p' "$helper_src")" + case "$fb" in + *'[ "$run" != 1 ]'*) ok "the repair is dry-run by default" ;; + *) bad "the repair changes things without being asked" ;; + esac + case "$fb" in + *'already exists -- not touching it'*) + ok "it refuses rather than clobbering an existing directory" ;; + *) bad "the repair could overwrite a real header directory" ;; + esac + case "$fb" in + *'[ "$have" = "$mine" ]'*) + ok "it does nothing when the triplets already agree" ;; + *) bad "the repair would link a triplet onto itself" ;; + esac +fi + +# ---- tracing a file back to the step that installed it ---------------------- # +# "which step went wrong" should be answerable from the record, not guessed. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q "^cmd_which_package()" "$helper_src" \ + && ok "a file can be traced to the package that installed it" \ + || bad "no way to find out which step installed a file" + wp="$T/whichpkg"; mkdir -p "$wp/.lfs-pkgusr/manifests" "$wp/usr/bin" + printf '/usr/lib/gcc/x86_64-pc-linux-gnu/15.2.0/cc1plus\n' \ + > "$wp/.lfs-pkgusr/manifests/gcc.files" + printf '/usr/bin/grep\n' > "$wp/.lfs-pkgusr/manifests/grep.files" + out="$(LFS_PKGUSR_DIR="$wp/.lfs-pkgusr" LFS_SNAP_ROOT="$wp" \ + LFS_SRC_ROOT="$wp/usr/src" \ + bash "$helper_src" which-package \ + /usr/lib/gcc/x86_64-pc-linux-gnu/15.2.0/cc1plus 2>&1)" + case "$out" in + *gcc*) ok "it names the right package for an exact path" ;; + *) bad "an exactly-recorded path was not traced" ;; + esac + case "$out" in + *grep*) bad "it named a package that did not install the file" ;; + *) ok "it does not name unrelated packages" ;; + esac +fi + +# ---- identify the compiler before diagnosing it ----------------------------- # +# A compiler searches for headers under the target it was BUILT for, not the +# one the headers are installed under. So --host/--target from its configure +# line is the only thing that explains a triplet mismatch; without it the +# check can only say "they differ", not why. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + tc="$(sed -n '/^cmd_check_toolchain() {/,/^}/p' "$helper_src")" + case "$tc" in + *"Configured with:"*) ok "the check reports how the compiler was configured" ;; + *) bad "a triplet mismatch cannot be explained, only observed" ;; + esac + for want in "dumpmachine" "command -v c++"; do + case "$tc" in + *"$want"*) ;; + *) bad "the check does not report: $want" ;; + esac + done + ok "it identifies which binary is being used" + # identity must come first: everything else is downstream of it + i="$(printf '%s' "$tc" | grep -n "The compiler:" | head -1 | cut -d: -f1)" + c="$(printf '%s' "$tc" | grep -n "C compiles" | head -1 | cut -d: -f1)" + if [ -n "$i" ] && [ -n "$c" ] && [ "$i" -lt "$c" ]; then + ok "the compiler is identified before it is tested" + else + bad "the check tests before saying what it is testing" + fi +fi + +# ---- test commands must not run in the build phase -------------------------- # +# GCC's chapter-8 section ends with test scaffolding: +# make / ulimit / sed ../gcc/testsuite/... / chown tester / su tester / +# ../contrib/test_summary +# Only the chown and su lines were recognised, so ulimit, the testsuite seds +# and test_summary stayed in the BUILD phase. test_summary greps test logs +# that do not exist and exits non-zero, and `set -e` then fails the build -- +# reported as "phase 'build' FAILED" with make having succeeded. +python3 - "$LFS_TOOL" "$LFS_BOOK" <<'PY45' +import sys, os, json, tempfile, importlib.machinery as m +store = tempfile.mkdtemp(); os.environ["LFS_STORE"] = store +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +import argparse +args = argparse.Namespace(book=None, _use_build_book=True) +os.makedirs(os.path.join(store, "books"), exist_ok=True) +import shutil +shutil.copy(sys.argv[2], os.path.join(store, "books", + "LFS-BOOK-12.4-NOCHUNKS.html")) +json.dump({"default": "12.4"}, open(os.path.join(store, "config.json"), "w")) +build, test = None, None +try: + r = lfs._section_script_phased(args, "ch-system-gcc") + build, test = r[0], r[1] +except Exception: + src = open(sys.argv[1]).read() + if "test_summary" not in src or "testsuite" not in src: + print(" FAIL test scaffolding is not recognised at all"); sys.exit(1) + print(" PASS test scaffolding patterns are present"); sys.exit(0) +problems = [] +for bad in ("test_summary", "testsuite", "ulimit"): + if build and bad in build: + problems.append("'%s' runs in the BUILD phase" % bad) +if problems: + for p in problems: print(" FAIL %s" % p) + sys.exit(1) +print(" PASS test scaffolding is kept out of the build phase") +PY45 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- the groups import is asked for with the other session settings --------- # +# It has to be known BEFORE the chroot scripts are generated (that is when the +# file is copied into the tree), so asking for it later is too late. +python3 - "$LFS_TOOL" <<'PY46' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +keys = [k for k, _d, _v in lfs._SESSION_KEYS] +if "collector_import_file" not in keys: + print(" FAIL the groups export is never asked for"); sys.exit(1) +src = open(sys.argv[1]).read() +if "no such file: %s -- leaving it unset" not in src: + print(" FAIL a nonexistent export path would be accepted"); sys.exit(1) +print(" PASS the groups import is part of the session setup, and validated") +PY46 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- the compiler check must not block chapter 7 ---------------------------- # +# Requiring a working C++ compiler before ANY step blocked chapter 7 -- which +# builds the temporary tools and needs no compiler at all. build-all then did +# nothing, silently, including the very steps that would have fixed the +# toolchain. The check belongs before the first package that needs it. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + ba="$(sed -n '/^cmd_build_all() {/,/^}/p' "$helper_src")" + # it must be inside the loop, guarded by "not a root step" + case "$ba" in + *'! is_root_step "$s" && [ "${_tc_checked:-0}" = 0 ]'*) + ok "the compiler is checked before the first package that needs it" ;; + *) bad "the compiler check is not tied to the first non-root step" ;; + esac + # and NOT before the loop, where it would block chapter 7 + pre="$(printf '%s' "$ba" | sed -n '1,/while read -r s/p')" + case "$pre" in + *cmd_check_toolchain*) bad "the check still runs before chapter 7" ;; + *) ok "chapter 7 builds without needing a compiler" ;; + esac + # checked once, not once per package + case "$ba" in + *_tc_checked=1*) ok "the check runs once, not per package" ;; + *) bad "the compiler would be re-checked for every package" ;; + esac +fi + +# ---- the build stamp must not depend on /tmp -------------------------------- # +# init-dirs is the step that CREATES /tmp (book 7.5), so mktemp fails there: +# mktemp: failed to create file via template '/tmp/tmp.XXXXXXXXXX' +# touch: cannot touch '': No such file or directory +# and the empty path that follows silently disables file tracking for the whole +# step -- the manifest ends up empty and nothing gets its owner. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q 'stamp="$STATE/tmp/stamp' "$helper_src" \ + && ok "the build stamp lives somewhere that always exists" \ + || bad "the stamp still depends on /tmp existing" + grep -q 'file tracking is off for this step' "$helper_src" \ + && ok "a stamp that cannot be made is reported, not ignored" \ + || bad "a missing stamp would silently disable file tracking" + # and find must not run against an empty stamp path + grep -q '\[ -n "$stamp" \] && \[ -e "$stamp" \] &&' "$helper_src" \ + && ok "no find -newer against a missing stamp" \ + || bad "find -newer would run with an empty reference" + # the toolchain check has the same trap + tc="$(sed -n '/^cmd_check_toolchain() {/,/^}/p' "$helper_src")" + case "$tc" in + *'mktemp -d -p "$STATE/tmp"'*) ok "the toolchain check does not need /tmp either" ;; + *) bad "check-toolchain still requires /tmp" ;; + esac +fi + +# ---- collector-group ids must not collide with package users ---------------- # +# Package users start at 10000 and their groups follow. A collector group +# created with a plain `groupadd` takes the next free gid -- which can be the +# one the next package user's group wants, so user 10042 ends up with group +# 10043 while an unrelated collector group holds 10042. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q "^COLLECTOR_GID_MIN=90000" "$helper_src" \ + && ok "collector groups have their own id range" \ + || bad "collector groups still share the package-user range" + grep -q "^create_collector_group()" "$helper_src" \ + && ok "collector groups are created through one allocator" \ + || bad "collector groups are created ad hoc" + # A bare `groupadd "$grp"` is fine INSIDE the allocator (the fallback when + # the chosen gid is taken); anywhere else it would bypass the range. + outside="$(awk '/^create_collector_group\(\) \{/{inside=1} + inside && /^\}/{inside=0; next} + !inside && /^[[:space:]]*groupadd "\$grp"/{print NR}' \ + "$helper_src")" + [ -z "$outside" ] \ + && ok "no collector group is created outside the allocator" \ + || bad "a bare groupadd bypasses the collector range (line $outside)" +fi +pmi="$(dirname "$LFS_TOOL")/packagemanager_install" +if [ -f "$pmi" ]; then + grep -q "_cgid=90000" "$pmi" \ + && ok "packagemanager uses the same collector range" \ + || bad "packagemanager still allocates collector gids from the user range" +fi + +# ---- the two tools spell the same thing the same way ------------------------ # +python3 - "$LFS_TOOL" <<'PY47' +import sys +src = open(sys.argv[1]).read() +if 'sub.add_parser("list"' not in src: + print(" FAIL `lfs list` does not exist (lfs-helper has `list`)"); sys.exit(1) +print(" PASS `lfs list` matches `lfs-helper list`") +PY47 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi +if [ -f "$helper_src" ]; then + grep -q 'if \[ "\${1:-}" = "--all" \]; then shift; cmd_list' "$helper_src" \ + && ok "\`lfs-helper next --all\` matches \`lfs next --all\`" \ + || bad "the two tools disagree on how to list every step" +fi + +# ---- help is readable ------------------------------------------------------- # +for t in lfs blfs; do + f="$(dirname "$LFS_TOOL")/$t" + [ -f "$f" ] || continue + grep -q "_colour_epilog" "$f" \ + || bad "$t: the command list is not coloured" +done +ok "the command lists are coloured" +# ...but never when piped, or it corrupts logs and pagers +out="$(python3 "$LFS_TOOL" --help 2>&1 | cat)" +case "$out" in + *$'\033'*) bad "colour escapes leak into piped help output" ;; + *) ok "no colour when the output is not a terminal" ;; +esac + +# ---- adoption must not repeat itself ---------------------------------------- # +# A whole-manifest signature was useless as a "have we done this" marker: every +# build writes a manifest, so it changed constantly and the full pass -- tens +# of thousands of paths -- ran again on every single build-all. Adoption is +# now tracked per package, by manifest size, so a package is re-adopted only +# when ITS OWN manifest grows. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q "^_needs_adoption()" "$helper_src" \ + && ok "adoption is decided per package, not for the whole set" \ + || bad "adoption still runs wholesale" + grep -q "^_mark_adopted()" "$helper_src" \ + && ok "each adopted package is recorded" \ + || bad "nothing records which packages are already adopted" + grep -q "_adoption_is_current" "$helper_src" \ + && bad "the whole-manifest signature is still in use" \ + || ok "the useless whole-manifest signature is gone" + na="$(sed -n '/^_needs_adoption() {/,/^}/p' "$helper_src")" + case "$na" in + *'[ -s "$man" ] || return 1'*) ok "empty manifests never look unadopted" ;; + *) bad "an empty manifest would force adoption forever" ;; + esac + case "$na" in + *'_adopted_size "$name"'*) ok "a package that grew is adopted again" ;; + *) bad "new files in an adopted package would be missed" ;; + esac + ae="$(sed -n '/^cmd_adopt_existing() {/,/^}/p' "$helper_src")" + case "$ae" in + *'= "$owner" ] && continue'*) + ok "paths already owned correctly are skipped" ;; + *) bad "every path is re-chowned even when already correct" ;; + esac +fi + +# ---- directories must be recorded, not just files --------------------------- # +# _run_tracked computed the directory diff as `after_d - before_d if before_d +# else set()`, and took the before-snapshot AFTER the build. On a first run +# there was no saved snapshot, so EVERY directory was discarded: the package's +# files got the package user while the directories holding them stayed +# root-owned. +python3 - "$LFS_TOOL" <<'PY48' +import sys, os, tempfile, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +root = tempfile.mkdtemp() +os.makedirs(os.path.join(root, ".lfs-pkgusr", "manifests")) +os.makedirs(os.path.join(root, "usr")) +script = ("mkdir -p %s/usr/include/c++/15.2.0/experimental\n" + "echo hdr > %s/usr/include/c++/15.2.0/experimental/algorithm\n" + % (root, root)) +lfs._run_tracked("gcc-pass2", script, as_lfs=False, lfs=root) +dirs = os.path.join(root, ".lfs-pkgusr", "manifests", "gcc-pass2.dirs") +problems = [] +if not os.path.isfile(dirs) or not open(dirs).read().strip(): + problems.append("no directories recorded on a first run") +else: + rec = open(dirs).read() + for need in ("include/c++/15.2.0", "experimental"): + if need not in rec: + problems.append("%s was not recorded as created" % need) +src = open(sys.argv[1]).read() +body = src[src.index("def _run_tracked"):] +body = body[:body.index("\ndef ")] +if body.index("before_d =") > body.index("rc = run_as_lfs"): + problems.append("the directory snapshot is taken after the build") +if "if before_d else set()" in body: + problems.append("the first-run directory diff is still discarded") +if problems: + for p in problems: print(" FAIL %s" % p) + sys.exit(1) +print(" PASS a package records the directories it creates, first run included") +PY48 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- the collector-group export must not duplicate entries ------------------ # +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q "^_dedupe_collector_export()" "$helper_src" \ + && ok "the export is deduplicated across both of its sources" \ + || bad "the export can contain the same mapping twice" + eg="$(sed -n '/^cmd_export_groups() {/,/^}/p' "$helper_src")" + case "$eg" in + *"| _dedupe_collector_export >>"*) + ok "deduplication happens on the combined output" ;; + *) bad "each source is still deduplicated on its own" ;; + esac + ig="$(sed -n '/^cmd_import_groups() {/,/^}/p' "$helper_src")" + case "$ig" in + *_dedupe_collector_export*) + ok "an already-duplicated export imports once, not twice" ;; + *) bad "importing an old export would apply entries twice" ;; + esac +fi + +# ---- a typed group name must not be mangled --------------------------------- # +# sanitise_group_name used `tr -c` alone, which turns EVERY other character +# into a dash -- including spaces -- so an EXISTING group could not be found: +# could not create group --nimgnu_gcc +if [ -f "$helper_src" ]; then + san="$(sed -n '/^sanitise_group_name() {/,/^}/p' "$helper_src")" + got="$(eval "$san"; sanitise_group_name " nimgnu_gcc")" + [ "$got" = "nimgnu_gcc" ] \ + && ok "a name typed with spaces resolves to the existing group" \ + || bad "whitespace still corrupts the group name (got '$got')" + got="$(eval "$san"; sanitise_group_name "---")" + [ -z "$got" ] \ + && ok "an answer with nothing usable in it yields nothing" \ + || bad "a junk answer produced the group name '$got'" + cg="$(sed -n '/^choose_collector_group() {/,/^}/p' "$helper_src")" + case "$cg" in + *"has nothing usable in it"*"Try again"*) + ok "an unusable answer is asked again, not substituted" ;; + *) bad "an unusable answer is silently replaced" ;; + esac + case "$cg" in + *"read -e -r ans < /dev/tty; then"*) + ok "with no terminal it takes the default instead of looping" ;; + *) bad "a non-interactive run could loop forever on the prompt" ;; + esac + case "$cg" in + *"giving up after \$attempts tries"*) ok "the retry loop is bounded" ;; + *) bad "the retry loop has no bound" ;; + esac +fi + +# ---- one place for building, one for using ---------------------------------- # +python3 - "$LFS_TOOL" <<'PY51' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +src = open(sys.argv[1]).read() +problems = [] +for c in ('"run"', '"next"', '"list"', '"snapshot"', '"set-book"'): + if 'bsub.add_parser(%s' % c not in src: + problems.append("build-system has no %s" % c) +if not hasattr(lfs, "cmd_bs_set_book"): + problems.append("no separate command for the build book") +if "INSTALL PACKAGES from" not in src: + problems.append("set-default does not say which book it sets") +if not hasattr(lfs, "CONFIG_SECTIONS"): + problems.append("config is still one flat list") +asked = [k for k, _d, _v in lfs._SESSION_KEYS] +for k in ("lfs_tgt", "makeflags", "locale", "hostname"): + if k not in asked: + problems.append("session never asks for %s" % k) +if problems: + for p in problems: print(" FAIL %s" % p) + sys.exit(1) +print(" PASS building and using are separated, and the settings are asked for") +PY51 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + + +# ---- manifests report directories, not just files --------------------------- # +# A package owns the directories it creates as well as the files in them. +# Reporting only files hid the tracking bug that left the C++ header tree +# root-owned while the headers inside belonged to gcc. +mfd="$T/manifests-report/.lfs-pkgusr/manifests" +mkdir -p "$mfd" +printf '/usr/bin/a\n/usr/bin/b\n' > "$mfd/good-pkg.files" +printf '/usr/lib/thing\n' > "$mfd/good-pkg.dirs" +printf '/usr/include/x\n' > "$mfd/broken-pkg.files" +out="$(LFS="$T/manifests-report" python3 "$LFS_TOOL" build-system manifests 2>&1)" +case "$out" in + *dirs*) ok "lfs reports directory counts" ;; + *) bad "lfs manifests still reports files only" ;; +esac +case "$out" in + *"no directories recorded"*) + ok "a package with files but no directories is flagged" ;; + *) bad "a package recording zero directories looks healthy" ;; +esac + +# ---- the install group is never applied to a package's own tree ------------- # +# Listing /usr/include/c++/* as a shared install directory made the auto-grant +# apply the `install` group to it -- handing gcc's headers to every package +# without asking. /usr/include is shared; its SUBTREES belong to gcc. +if [ -f "$helper_src" ]; then + ( SNAP_ROOT=/; INSTALLDIRS="" + eval "$(sed -n '/^install_dirs_list() {/,/^}/p' "$helper_src")" + eval "$(sed -n '/^install_dir_patterns() {/,/^}/p' "$helper_src")" + eval "$(sed -n '/^matches_install_pattern() {/,/^}/p' "$helper_src")" + eval "$(sed -n '/^is_install_dir() {/,/^}/p' "$helper_src")" + is_install_dir /usr/include || exit 1 + is_install_dir /usr/include/c++/15.2.0/bits && exit 1 + exit 0 ) \ + && ok "/usr/include is shared, but its c++ subtrees are gcc's" \ + || bad "gcc's header subtrees are still treated as shared" + gd="$(sed -n '/^grant_dir_access() {/,/^}/p' "$helper_src")" + case "$gd" in + *"opening it to every package"*) + ok "a package-owned directory is never opened to everyone" ;; + *) bad "the install group could still be applied to a package's tree" ;; + esac + case "$gd" in + *'find "$dir"'*) bad "granting access still walks the directory tree" ;; + *) ok "granting access touches only the directory asked for" ;; + esac +fi + + +# ---- an interrupted build leaves a trace ------------------------------------ # +# Cancelling during `make install` leaves the package half on disk. gcc, +# interrupted there, replaces /usr/bin/gcc with a native compiler but has not +# yet installed the C++ headers -- so every later build fails with +# fatal error: bits/c++config.h: No such file or directory +# and nothing on disk says why. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q "^_note_interrupted()" "$helper_src" \ + && ok "an interrupted build says what state it left behind" \ + || bad "cancelling a build leaves no trace" + grep -q "trap '_note_interrupted" "$helper_src" \ + && ok "the interrupt is trapped, so the message is not missed" \ + || bad "Ctrl-C exits without a word" + grep -q "^_clear_interrupted()" "$helper_src" \ + && ok "the marker is cleared when the build concludes" \ + || bad "the marker would persist after a completed build" + ba="$(sed -n '/^cmd_build_all() {/,/^}/p' "$helper_src")" + case "$ba" in + *_interrupted_packages*) + ok "a half-installed package is flagged before building more" ;; + *) bad "a half-installed package is silently built over" ;; + esac + # the marker must be written BEFORE the build starts, or a kill loses it + cb="$(sed -n '/^cmd_build() {/,/^}/p' "$helper_src")" + i_mark="$(printf '%s' "$cb" | grep -n 'echo "\$name" >> "\$_interrupted"' | head -1 | cut -d: -f1)" + i_run="$(printf '%s' "$cb" | grep -n 'building \$name' | head -1 | cut -d: -f1)" + if [ -n "$i_mark" ] && [ -n "$i_run" ] && [ "$i_mark" -lt "$i_run" ]; then + ok "the marker is written before the build starts" + else + bad "a kill during the build would leave no marker" + fi +fi + +# ---- an interrupted toolchain package is finished, not just reported -------- # +# Cancelling gcc's `make install` leaves the new compiler in /usr/bin without +# its headers, so it shadows the working one and nothing builds. The tool +# knows which package was interrupted, so the right move is to finish that +# install -- printing a diagnosis and stopping leaves the user to work it out. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + ba="$(sed -n '/^cmd_build_all() {/,/^}/p' "$helper_src")" + case "$ba" in + *'cmd_build "$_pkg" --phase install --force'*) + ok "an interrupted install is finished automatically" ;; + *) bad "a half-installed toolchain package is only reported" ;; + esac + case "$ba" in + *"the compiler works again -- continuing"*) + ok "the build continues once the compiler works" ;; + *) bad "the build stops even after repairing the compiler" ;; + esac + case "$ba" in + *"Finishing the interrupted install did not repair it"*) + ok "it says so when finishing the install was not enough" ;; + *) bad "a failed repair is indistinguishable from no repair" ;; + esac + # the repair must be attempted BEFORE giving up + i_fix="$(printf '%s' "$ba" | grep -n 'phase install --force' | head -1 | cut -d: -f1)" + i_stop="$(printf '%s' "$ba" | grep -n 'Stopping before' | head -1 | cut -d: -f1)" + if [ -n "$i_fix" ] && [ -n "$i_stop" ] && [ "$i_fix" -lt "$i_stop" ]; then + ok "the repair is tried before stopping" + else + bad "it stops before trying to repair" + fi + # and no unverified signal trickery + grep -q "_on_interrupt" "$helper_src" \ + && bad "an unverified interrupt handler is still present" \ + || ok "interrupts are recorded plainly, not intercepted" +fi + +# ---- a partial install is detected without a marker ------------------------- # +# The interrupted-marker only exists if the interruption happened while a +# version of the tool that writes one was running. A package that has already +# installed files but is not marked built is in exactly the same state -- which +# is what a half-installed gcc looks like -- so it must be detected from the +# tree, not only from the marker. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + ba="$(sed -n '/^cmd_build_all() {/,/^}/p' "$helper_src")" + case "$ba" in + *'[ -s "$MANIFESTS/$s.files" ]'*'! is_done "$s"'*) + ok "a partial install is detected from the tree, not just a marker" ;; + *) bad "an interruption from an older run would go undetected" ;; + esac + case "$ba" in + *"has already installed files but is not marked"*) + ok "it says why it thinks the package is half-installed" ;; + *) bad "the repair happens without explaining itself" ;; + esac +fi + +# ---- the build ends by sealing the install directories ---------------------- # +# Until the build is finished the install dirs must be group-writable but NOT +# sticky, so package users can replace the temporary system's root-owned files. +# Once everything is built that has to be undone, or any package can delete +# another's files. It was implemented but never offered, so a finished build +# left the system permanently unsealed. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + ba="$(sed -n '/^cmd_build_all() {/,/^}/p' "$helper_src")" + # it used to only SUGGEST this; a finished build now does it + case "$ba" in + *"cmd_seal_install_dirs --run"*) + ok "a finished build seals the install directories" ;; + *) bad "a finished build leaves the system unsealed" ;; + esac + grep -q "^_install_dirs_are_sealed()" "$helper_src" \ + && ok "it only says so while the dirs are still unsealed" \ + || bad "the reminder would repeat after sealing" + # sealing must touch install dirs only -- collector-group dirs are shared + # deliberately and a sticky bit there would defeat the point + sd="$(sed -n '/^cmd_seal_install_dirs() {/,/^}/p' "$helper_src")" + case "$sd" in + *"done < <(install_dirs_list)"*) + ok "sealing walks the install directories only" ;; + *) bad "sealing could reach collector-group directories" ;; + esac + case "$sd" in + *COLLECTOR*) bad "sealing touches collector-group directories" ;; + *) ok "collector-group directories are left shared" ;; + esac + # and o+t is the right bit: create yes, delete-another's no + case "$sd" in + *'chmod o+t'*) ok "the sticky bit is what gets set" ;; + *) bad "sealing sets something other than the sticky bit" ;; + esac +fi + +# ---- build-all must not lose steps or leave the system unsealed ------------- # +# Three faults in one run: a step that exits 3 ("needs a decision") had its +# code flattened to 1 and then discarded, so refind slipped past and the build +# reported everything complete with it still pending; packages built by this +# very run were re-adopted on the next one; and sealing was only suggested. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + cb="$(sed -n '/^cmd_build() {/,/^}/p' "$helper_src")" + case "$cb" in + *"Pass 3 up unchanged"*) + ok "'needs a decision' is distinguishable from 'failed'" ;; + *) bad "exit 3 is still flattened into a plain failure" ;; + esac + ba="$(sed -n '/^cmd_build_all() {/,/^}/p' "$helper_src")" + case "$ba" in + *'3) _needs_you='*) + ok "a step needing a decision is collected, not skipped" ;; + *) bad "a step that needs you would pass silently" ;; + esac + case "$ba" in + *"These steps need a decision from you and were not built"*) + ok "unbuilt steps are reported at the end" ;; + *) bad "the build could claim to be complete with steps pending" ;; + esac + case "$ba" in + *"cmd_seal_install_dirs --run"*) + ok "sealing is done as the final step, not suggested" ;; + *) bad "sealing is still left to the user" ;; + esac + # a real failure must still stop the build + case "$ba" in + *"failed (exit \$_rc) -- stopping here"*) + ok "a genuine failure stops the build" ;; + *) bad "a failing step would be built over" ;; + esac + # and a package built here is already owned: never re-adopt it + case "$cb" in + *"_mark_adopted \"\$name\""*) + ok "a freshly built package is not re-adopted" ;; + *) bad "every build-all would re-run the adoption pass" ;; + esac +fi + +# ---- the chroot must not run a stale lfs-helper ----------------------------- # +# The in-chroot tools are COPIES. Fixing a bug out here changes nothing inside +# until it is copied in again, so a user who does not know that keeps hitting +# bugs that were fixed days ago -- and every diagnosis starts from the wrong +# version of the evidence. Entering the chroot now refreshes them first. +python3 - "$LFS_TOOL" <<'PY52' +import sys, os, tempfile, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +src = open(sys.argv[1]).read() +problems = [] +if not hasattr(lfs, "_refresh_chroot_tools"): + problems.append("nothing refreshes the in-chroot tools") +else: + root = tempfile.mkdtemp() + os.makedirs(os.path.join(root, "usr", "bin")) + stale = os.path.join(root, "usr", "bin", "lfs-helper") + open(stale, "w").write("old copy\n") + lfs._refresh_chroot_tools(root, quiet=True) + helper = os.path.join(os.path.dirname(os.path.abspath(sys.argv[1])), + "lfs-helper") + if os.path.isfile(helper): + if open(stale, "rb").read() != open(helper, "rb").read(): + problems.append("a stale in-chroot lfs-helper is not replaced") + before = os.path.getmtime(stale) + lfs._refresh_chroot_tools(root, quiet=True) + if os.path.getmtime(stale) != before: + problems.append("an up-to-date copy is rewritten needlessly") +# both ways in must refresh +for fn in ("def enter_chroot", 'if action == "enter"'): + seg = src[src.index(fn):src.index(fn) + 600] + if "_refresh_chroot_tools" not in seg: + problems.append("%s enters without refreshing" % fn) +if problems: + for p in problems: print(" FAIL %s" % p) + sys.exit(1) +print(" PASS entering the chroot refreshes the tools first") +PY52 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- every tool reports its version ----------------------------------------- # +# Without one there is no way to tell whether the copy inside the chroot is the +# one that was just fixed -- and several rounds of diagnosis were spent on +# output from code that no longer existed. +# A hand-maintained number goes stale the moment someone forgets to bump it, +# and the whole point of printing a version is to answer "am I running the +# code that was just fixed?". Each tool also prints a fingerprint of its own +# contents, which cannot go stale. +vers="" +for t in lfs blfs packagemanager; do + f="$(dirname "$LFS_TOOL")/$t" + [ -f "$f" ] || continue + v="$(python3 "$f" --version 2>&1 | head -1)" + case "$v" in + *"build "*) ;; + *) bad "$t does not print a build id: $v" ;; + esac + vers="$vers $(printf '%s' "$v" | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1)" +done +hv="$(bash "$(dirname "$LFS_TOOL")/lfs-helper" --version 2>&1)" +case "$hv" in + *"build "*) ;; + *) bad "lfs-helper does not print a build id: $hv" ;; +esac +vers="$vers $(printf '%s' "$hv" | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1)" +uniq_count="$(printf '%s' "$vers" | tr ' ' '\n' | grep -v '^$' | sort -u | wc -l)" +[ "$uniq_count" = 1 ] \ + && ok "all four tools report the same version, each with a build id" \ + || bad "the tools report different versions:$vers" + +# the fingerprint must actually track the file +tmpcopy="$T/verid"; mkdir -p "$tmpcopy" +cp "$(dirname "$LFS_TOOL")/lfs-helper" "$tmpcopy/lfs-helper" +before="$(bash "$tmpcopy/lfs-helper" --version 2>&1)" +echo "# changed" >> "$tmpcopy/lfs-helper" +after="$(bash "$tmpcopy/lfs-helper" --version 2>&1)" +[ "$before" != "$after" ] \ + && ok "the build id changes when the file changes" \ + || bad "the build id does not track the file contents" + +# ---- sealing covers every install-group directory --------------------------- # +# The static list does not know about install dirs packages create themselves +# (/usr/lib/<pkg>, /usr/share/<pkg>), and those need the sticky bit too. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + sd="$(sed -n '/^cmd_seal_install_dirs() {/,/^}/p' "$helper_src")" + case "$sd" in + *"-type d -group install"*) + ok "sealing finds install-group directories, not just a fixed list" ;; + *) ok_or_bad=1; bad "install dirs outside the static list stay unsealed" ;; + esac + case "$sd" in + *'[ -k "$d" ] && continue'*) + ok "an already-sticky directory is skipped" ;; + *) bad "sealing re-chmods directories needlessly" ;; + esac + ba="$(sed -n '/^cmd_build_all() {/,/^}/p' "$helper_src")" + case "$ba" in + *'[ "$s" = "last-step" ] && ! _install_dirs_are_sealed'*) + ok "sealing happens immediately before the final step" ;; + *) bad "sealing is not tied to the step before last-step" ;; + esac +fi + +# ---- adoption is for chapter 5-6 packages ONLY ------------------------------ # +# Adoption exists for one thing: packages built outside the chroot, before any +# package user existed, whose files are still root-owned. Anything in this +# chroot's step list is built by lfs-helper, which assigns ownership as part of +# the build -- walking its manifest is pure waste, and it made the pass appear +# on every single build-all with no explanation of why. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + na="$(sed -n '/^_needs_adoption() {/,/^}/p' "$helper_src")" + case "$na" in + *'steps | grep -qxF "$name"'*) + ok "packages built in the chroot are never adopted" ;; + *) bad "chroot-built packages are still walked by the adoption pass" ;; + esac + # the cheap check must come BEFORE statting thousands of files + i_step="$(printf '%s' "$na" | grep -n 'steps | grep -qxF' | head -1 | cut -d: -f1)" + i_stat="$(printf '%s' "$na" | grep -n '_manifest_size' | head -1 | cut -d: -f1)" + if [ -n "$i_step" ] && [ -n "$i_stat" ] && [ "$i_step" -lt "$i_stat" ]; then + ok "the cheap check runs before reading manifests" + else + bad "manifests are read before the step-list check" + fi + ae="$(sed -n '/^cmd_adopt_existing() {/,/^}/p' "$helper_src")" + case "$ae" in + *"CHAPTER 5-6"*) + ok "the pass says which packages it is for" ;; + *) bad "the pass does not explain what it is adopting" ;; + esac +fi + +# ---- files from outside the chroot need an owner that exists in it ---------- # +# The lfs user exists on the HOST, not inside, so everything it created -- +# /sources, the tree handed over at 7.2 -- shows as a bare uid and every +# ownership check calls it UNKNOWN. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q "^LFS_BUILD_UID=9998" "$helper_src" \ + && ok "the build user has a fixed id, clear of the package users" \ + || bad "no reserved id for the outside build user" + grep -q "^cmd_fix_orphans()" "$helper_src" \ + && ok "files with no owner can be repaired" \ + || bad "orphaned files stay owned by nobody" + fo="$(sed -n '/^cmd_fix_orphans() {/,/^}/p' "$helper_src")" + case "$fo" in + *"-nouser"*) ok "it finds files by missing passwd entry, not by guessing" ;; + *) bad "orphan detection does not use -nouser" ;; + esac + # the virtual filesystems are the host's -- never touch them + for d in "dev" "proc" "sys" "run"; do + case "$fo" in + *"-not -path \"\$r/$d/*\""*) ;; + *) bad "orphan repair could reach /$d" ;; + esac + done + ok "the virtual filesystems are excluded from orphan repair" + ba="$(sed -n '/^cmd_build_all() {/,/^}/p' "$helper_src")" + case "$ba" in + *"cmd_fix_orphans --run"*) + ok "the build recreates the outside build user by itself" ;; + *) bad "the user has to know to run fix-orphans" ;; + esac + case "$ba" in + *"cmd_sort_users --run"*) + ok "passwd and group end up sorted by id" ;; + *) bad "passwd stays in build order" ;; + esac + # sorting must be the LAST thing, once every package user exists + i_sort="$(printf '%s' "$ba" | grep -n 'cmd_sort_users --run' | head -1 | cut -d: -f1)" + i_seal="$(printf '%s' "$ba" | grep -n 'cmd_seal_install_dirs --run' | tail -1 | cut -d: -f1)" + if [ -n "$i_sort" ] && [ -n "$i_seal" ] && [ "$i_sort" -gt "$i_seal" ]; then + ok "sorting happens after every user has been created" + else + bad "passwd is sorted before all the users exist" + fi +fi + +# ---- what 4.3 hands over, 7.2 must hand back -------------------------------- # +# The two lists were written separately and drifted: 4.3 included `lib` and 7.2 +# did not, so /lib stayed owned by the lfs user for the rest of the build while +# /bin and /sbin -- never handed over -- stayed root. One list serves both now. +python3 - "$LFS_TOOL" <<'PY53' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +src = open(sys.argv[1]).read() +problems = [] +if not hasattr(lfs, "LFS_HANDOVER_DIRS"): + print(" FAIL the handover set is not defined in one place"); sys.exit(1) +for d in ("usr", "lib", "lib64", "var", "etc", "tools", "bin", "sbin"): + if d not in lfs.LFS_HANDOVER_DIRS: + problems.append("%s is missing from the handover set" % d) +# /sources belongs to the lfs user by design and must NOT be handed back +if "sources" in lfs.LFS_HANDOVER_DIRS: + problems.append("/sources would be taken from the lfs user") +# both sites must use the shared list, not a literal +if src.count("LFS_HANDOVER_DIRS") < 3: + problems.append("one of the two chowns still uses its own list") +# symlinks: /lib is a symlink, so the give-back needs -h +i = src.index('"give the tree to root"') +if "-h -R root:root" not in src[i:i + 400]: + problems.append("the give-back does not touch symlinks") +if problems: + for p in problems: print(" FAIL %s" % p) + sys.exit(1) +print(" PASS the tree is handed over and handed back symmetrically") +PY53 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- scratch and human directories are never owned by a package ------------- # +# man-pages ended up owning /tmp, which would break every other user of it. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + ( SNAP_ROOT=/; INSTALLDIRS="" + eval "$(sed -n '/^install_dirs_list() {/,/^}/p' "$helper_src")" + eval "$(sed -n '/^install_dir_patterns() {/,/^}/p' "$helper_src")" + eval "$(sed -n '/^matches_install_pattern() {/,/^}/p' "$helper_src")" + eval "$(sed -n '/^is_install_dir() {/,/^}/p' "$helper_src")" + for d in /tmp /var/tmp /root /home; do is_install_dir "$d" || exit 1; done + exit 0 ) \ + && ok "/tmp, /var/tmp, /root and /home are never adopted" \ + || bad "a package could come to own /tmp or a home directory" + + # orphan repair must fix everything it found, not a sample of it + fo="$(sed -n '/^cmd_fix_orphans() {/,/^}/p' "$helper_src")" + case "$fo" in + *"head -n 2000"*) bad "orphan repair still only fixes the first 2000" ;; + *) ok "orphan repair fixes every file it found" ;; + esac + case "$fo" in + *'gave $fixed of $n path'*) + ok "it reports how many it actually changed" ;; + *) bad "orphan repair claims success without counting" ;; + esac +fi + +# ---- "not found in book" when there IS no book ------------------------------ # +# A fresh chroot got the LFS book but not the BLFS one, so every lookup failed +# with "not found in book -- skipping" and then "Nothing to do -- everything is +# installed and up to date" -- which is the opposite of the truth, and exit 0. +pmt="$(dirname "$LFS_TOOL")/packagemanager" +if [ -f "$pmt" ]; then + grep -q "^def _blfs_book_available" "$pmt" \ + && ok "a missing book is distinguished from a missing package" \ + || bad "no book and no package give the same message" + bb="$(sed -n '/^def _blfs_book_available/,/^def /p' "$pmt")" + case "$bb" in + *'run_blfs(\["books"\])'*) + bad "book availability is asked of `blfs books`, which lists the site too" ;; + *) ok "book availability is checked on disk, not from a listing" ;; + esac + grep -q 'printError("Could not install: "' "$pmt" \ + && ok "a failed lookup is an error, not a success message" \ + || bad "a failed lookup still reports everything up to date" + # and it must exit non-zero, or a script cannot tell + ci="$(sed -n '/plan, missing, why, not_found = build_install_plan/,/printSuccess("Nothing to do/p' "$pmt")" + case "$ci" in + *"sys.exit(1)"*) ok "it exits non-zero when nothing could be looked up" ;; + *) bad "a failed install still exits 0" ;; + esac + # the status line must be cleared, or it runs into the message + case "$(sed -n '/nv = book_version(target)/,/continue/p' "$pmt")" in + *clear_status*) ok "the progress line is cleared before the message" ;; + *) bad "the status line runs into the error text" ;; + esac +fi + +# the BLFS book must travel into the chroot with the tools +python3 - "$LFS_TOOL" <<'PY54' +import sys +src = open(sys.argv[1]).read() +if "/usr/share/blfs" not in src: + print(" FAIL install-tools never copies the BLFS book in"); sys.exit(1) +if "BLFS book" not in src: + print(" FAIL install-tools does not report the BLFS book"); sys.exit(1) +print(" PASS the BLFS book is installed into the chroot with the tools") +PY54 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- a cached book must actually be usable ---------------------------------- # +# A book imported by hand, or copied in by install-tools, has no metadata +# saying which selector it is. resolve_book only looked at the site, so with +# the book sitting right there every lookup came back +# wget: not in the BLFS book +# while `blfs books` listed it as cached. +bt="$(dirname "$LFS_TOOL")/blfs" +if [ -f "$bt" ]; then + grep -q "^def _cached_book_files" "$bt" \ + && ok "a cached book is found when the selector does not resolve" \ + || bad "a cached book is ignored unless the site confirms it" + grep -q "^def _best_cached_for" "$bt" \ + && ok "the closest cached book is chosen, not just the first" \ + || bad "any cached book would be picked at random" + # urlsplit(None) yields bytes and fails deep inside os.path.join + case "$(sed -n '/is_svn = "svn" in selector.lower()/,/return url, os.path.join/p' "$bt")" in + *"if url is None:"*) ok "a cached book with no URL does not crash" ;; + *) bad "a cached book would fail in urlsplit" ;; + esac + # and the error, when there really is no book, must say what to do + case "$(sed -n '/no book for/,/blfs import/p' "$bt")" in + *"blfs fetch"*"blfs import"*) + ok "with no book at all it says how to get one" ;; + *) bad "the no-book error gives no way forward" ;; + esac +fi + +# ---- missing settings are asked for, not silently defaulted ----------------- # +pmt="$(dirname "$LFS_TOOL")/packagemanager" +if [ -f "$pmt" ]; then + grep -q "^_CONFIG_PROMPTS" "$pmt" \ + && ok "packagemanager asks for settings it does not have" \ + || bad "a missing setting is silently left blank" + grep -q '"--show"' "$pmt" \ + && ok "--show prints the settings without asking" \ + || bad "config always prompts, even when only reading" + grep -q "^def _book_in_use" "$pmt" \ + && ok "config shows which book each tool would use" \ + || bad "there is no way to see which book is in use" + # only genuinely unset values -- never re-ask for something already chosen + case "$(sed -n '/for key, prompt, default in _CONFIG_PROMPTS/,/changed = True/p' "$pmt")" in + *'if cfg.get(key):'*) ok "a setting already chosen is not asked again" ;; + *) bad "config would re-ask for values that are already set" ;; + esac +fi + +# ---- the tools must agree on the collector prefix --------------------------- # +# The prefix decides which groups shared directories carry. install-tools +# wrote packagemanager.conf as JSON while packagemanager parses key=value, so +# the prefix never arrived: every package installed after the build joined +# `sysgroup_*` groups while the built system used `nimgnu_*` -- two parallel +# sets of groups over the same directories, and nothing to say so. +python3 - "$LFS_TOOL" <<'PY55' +import sys +src = open(sys.argv[1]).read() +seg = src[src.index("pmconf = os.path.join(cfg_dir"):] +seg = seg[:seg.index("_install_pkgusr_skel")] +problems = [] +if "json.dump" in seg: + problems.append("packagemanager.conf is still written as JSON") +if "collector_prefix=%s" not in seg: + problems.append("the prefix is not written in key=value form") +print(" FAIL " + problems[0] if problems else + " PASS install-tools writes a config packagemanager can read") +sys.exit(1 if problems else 0) +PY55 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +pmt="$(dirname "$LFS_TOOL")/packagemanager" +if [ -f "$pmt" ]; then + grep -q "^_SHARED_WITH_LFS" "$pmt" \ + && ok "settings shared with lfs have one owner" \ + || bad "packagemanager keeps its own copy of shared settings" + grep -q "which is what the system was built" "$pmt" \ + && ok "a disagreeing prefix is reported, not silently preferred" \ + || bad "a prefix mismatch would pass unnoticed" + # the book must be read from the config, not by shelling out to `lfs` + bu="$(sed -n '/^def _book_in_use/,/^def /p' "$pmt")" + case "$bu" in + *_shared_lfs_config*) ok "the LFS book is read from the config file" ;; + *) bad "the LFS book needs a mounted tree to report" ;; + esac +fi +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q "^_read_shared_prefix()" "$helper_src" \ + && ok "lfs-helper reads the same prefix as the others" \ + || bad "lfs-helper could use a different prefix" +fi + +# ---- packages re-creating directories that already exist -------------------- # +# make-ca's Makefile runs `install -vdm755 /usr/sbin`. The directory is +# already there with the right mode, but install still tries to chmod it, and +# under the package-user model that is a hard error: +# install: cannot change permissions of '/usr/sbin': Operation not permitted +# lfs-helper wraps install/chmod/chown during the chroot build for exactly +# this; packagemanager_install did not, so the same package failed after boot. +pmi="$(dirname "$LFS_TOOL")/packagemanager_install" +if [ -f "$pmi" ]; then + grep -q "^_make_build_wrappers()" "$pmi" \ + && ok "packagemanager wraps install/chmod for the build" \ + || bad "a package re-moding an existing directory still fails" + grep -q "PATH='\$_wrapdir'" "$pmi" \ + && ok "the wrappers are ahead of the real tools during the build" \ + || bad "the wrappers are created but never used" + + # behaviour, not just presence + wd="$T/wrapmk"; mkdir -p "$wd/wrap" "$wd/exists" + ( eval "$(sed -n '/^_make_build_wrappers() {/,/^}/p' "$pmi")" + _make_build_wrappers "$wd/wrap" ) >/dev/null 2>&1 + if PATH="$wd/wrap:$PATH" install -vdm755 "$wd/exists" >/dev/null 2>&1; then + ok "install -d on an existing directory succeeds" + else + bad "install -d on an existing directory still errors" + fi + if PATH="$wd/wrap:$PATH" install -vdm755 "$wd/fresh" >/dev/null 2>&1 \ + && [ -d "$wd/fresh" ]; then + ok "install -d still creates directories that are missing" + else + bad "the wrapper broke directory creation" + fi + echo data > "$wd/src" + if PATH="$wd/wrap:$PATH" install -m644 "$wd/src" "$wd/exists/f" >/dev/null 2>&1 \ + && [ -f "$wd/exists/f" ]; then + ok "installing a file is passed through untouched" + else + bad "the wrapper broke ordinary file installs" + fi + # a real failure must still be a failure + if PATH="$wd/wrap:$PATH" install -d "$wd/nonexistent-parent/sub/deeper/x" \ + 2>/dev/null && [ ! -d "$wd/nonexistent-parent/sub/deeper/x" ]; then + bad "the wrapper reports success for a directory it did not create" + else + ok "a genuine failure is not masked" + fi +fi + +# ---- a step's manifest lists what it owns, not what happened while it ran --- # +# last-step builds wget as the package user 'wget'. Everything written during +# the step was recorded as last-step's, so /usr/bin/wget landed in last-step's +# manifest -- and the next thing to act on it took the file from wget: +# -rwxr-xr-x 1 last-step last-step ... /usr/bin/wget +# after which wget could not reinstall itself. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q "^_drop_other_packages_files()" "$helper_src" \ + && ok "another package's files stay out of a step's manifest" \ + || bad "a step still records files another package installed" + # it must be applied where the manifest is written + case "$(sed -n '/local man="\$MANIFESTS\/\$name.files"/,/tracked \$/p' "$helper_src")" in + *_drop_other_packages_files*) + ok "the filter is applied when the manifest is written" ;; + *) bad "the filter exists but is not used" ;; + esac + # root-owned files must still be recorded -- that is the normal case for a + # root step, and dropping them would empty every chapter-7 manifest + df="$(sed -n '/^_drop_other_packages_files() {/,/^}/p' "$helper_src")" + case "$df" in + *'"$cur" != root'*) ok "root-owned files are still recorded" ;; + *) bad "a root step would record nothing at all" ;; + esac +fi + +# ---- build steps are not packages ------------------------------------------- # +# last-step, init-*, cfg-* and refind are actions, not software. A `last-step` +# package user was created because a manifest existed under that name, and +# everything the step touched -- including wget's binary, installed as the wget +# user -- was then handed to it. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q "^_is_not_a_package()" "$helper_src" \ + && ok "steps that are not packages are recognised" \ + || bad "any step name could become a package user" + au="$(sed -n '/^cmd_add_user() {/,/^}/p' "$helper_src")" + case "$au" in + *_is_not_a_package*) ok "add-user refuses to create one for a step" ;; + *) bad "add-user would still create a last-step user" ;; + esac + na="$(sed -n '/^_needs_adoption() {/,/^}/p' "$helper_src")" + case "$na" in + *_is_not_a_package*) ok "adoption skips step names too" ;; + *) bad "adoption could recreate the user add-user refused" ;; + esac +fi + +# the last-step script installs each package as its OWN user, from wheels +python3 - "$LFS_TOOL" <<'PY56' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +t = lfs.LAST_STEP_DEFAULT +sources = t.split("SOURCES=(")[1].split(")")[0] +problems = [] +if "install_python_module" not in t: + problems.append("the python modules the tools need are not installed") +if "build_as_package_user wget" not in t: + problems.append("wget is not built as the wget package user") +# A modern sdist needs its build backend, and with no index reachable pip +# cannot fetch one: "BackendUnavailable: Cannot import 'hatchling.build'". +# A wheel is already built, so pip just unpacks it. +for line in sources.splitlines(): + line = line.strip() + if "files.pythonhosted.org" not in line: + continue + if not line.rstrip('"').endswith(".whl"): + problems.append("a python module is fetched as an sdist: %s" + % line.split("/")[-1][:40]) +for mod in ("requests", "beautifulsoup4"): + if mod not in sources: + problems.append("%s is not in SOURCES" % mod) +if "--no-build-isolation" in t: + problems.append("--no-build-isolation is still used") +# pip is told --no-deps, so every runtime dependency must be listed itself +for dep in ("urllib3", "idna", "certifi", "soupsieve"): + if dep not in t: + problems.append("%s (a runtime dependency) is never installed" % dep) +if t.index("install_python_module urllib3") > t.index("install_python_module requests"): + problems.append("requests is installed before its dependencies") +if problems: + for p in problems: print(" FAIL %s" % p) + sys.exit(1) +print(" PASS the last step installs each package as its own user, from wheels") +PY56 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# a hint that installs a package must pull its dependencies too +python3 - "$LFS_TOOL" <<'PY58' +import sys, re, importlib.machinery as m +src = open(sys.argv[1]).read() +bad = [l.strip() for l in src.splitlines() + if re.search(r"packagemanager install [a-z][a-z0-9-]*\s*\"?$", l) + and "--recursive" not in l] +if bad: + for b in bad[:3]: + print(" FAIL hint installs without dependencies: %s" % b[:60]) + sys.exit(1) +print(" PASS install hints pull dependencies with --recursive") +PY58 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- site-packages is python's, not shared infrastructure ------------------- # +# A module user cannot write into site-packages until it joins the collector +# group that owns it. The fix must be the SAME mechanism every other package +# uses -- join the existing collector group, or ask which group should own the +# directory. Handing site-packages to the `install` group would mean "anyone +# may install here", which is exactly what the collector groups exist to avoid. +python3 - "$LFS_TOOL" <<'PY59' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +t = lfs.LAST_STEP_DEFAULT +problems = [] +if "_grant_site_packages" not in t: + problems.append("nothing gives the module user access to site-packages") +if "lfs-helper grant-dir" not in t: + problems.append("it does not use the normal grant mechanism") +# the install group must never be applied to python's tree +if "grp=install" in t or "chgrp install" in t: + problems.append("site-packages would be handed to the install group") +# and it must happen before pip, not after it fails +if "_grant_site_packages" in t and \ + t.index('_grant_site_packages "$user"') > t.index("python3 -m pip install"): + problems.append("access is granted after pip has already failed") +if problems: + for p in problems: print(" FAIL %s" % p) + sys.exit(1) +print(" PASS site-packages access uses the normal collector-group rules") +PY59 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q "^cmd_grant_dir()" "$helper_src" \ + && ok "access to one directory can be asked for directly" \ + || bad "a script cannot ask for access before failing" + gd="$(sed -n '/^cmd_grant_dir() {/,/^}/p' "$helper_src")" + case "$gd" in + *grant_dir_access*) ok "it reuses the same rules as a failed build" ;; + *) bad "grant-dir implements its own logic" ;; + esac + case "$gd" in + *'test -w'*) ok "it does nothing when access already exists" ;; + *) bad "grant-dir would regroup a directory needlessly" ;; + esac +fi + + +# ---- helper functions must be defined before they are used ------------------ # +# bash parses top to bottom: a function called at line 437 but defined at 466 +# does not exist yet, and the build ran without its wrappers: +# packagemanager_install: line 437: _make_build_wrappers: command not found +pmi="$(dirname "$LFS_TOOL")/packagemanager_install" +if [ -f "$pmi" ]; then + python3 - "$pmi" <<'PY60' +import re, sys +lines = open(sys.argv[1]).read().split("\n") +defs, bad = {}, [] +for i, l in enumerate(lines): + m = re.match(r"^([a-zA-Z_][a-zA-Z0-9_]*)\(\) \{", l) + if m: + defs.setdefault(m.group(1), i) +for name, dline in defs.items(): + for i, l in enumerate(lines[:dline]): + if re.search(r"^\s+%s\b" % re.escape(name), l) and not l.strip().startswith("#"): + enclosing = max((d for d in defs.values() if d < i), default=-1) + if enclosing == -1: + bad.append("%s used at line %d, defined at %d" % (name, i + 1, dline + 1)) + break +if bad: + for b in bad: print(" FAIL %s" % b) + sys.exit(1) +print(" PASS every helper is defined before it is used") +PY60 + if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + + # ---- one implementation of the collector-group rules --------------------- # + # A package installed after boot must behave exactly like one built during + # the chroot stage. Two copies of the rules drift; lfs-helper owns them. + gd="$(sed -n '/^grant_dir_to_user() {/,/^}/p' "$pmi")" + case "$gd" in + *"lfs-helper grant-dir"*) + ok "packagemanager uses lfs-helper's collector-group rules" ;; + *) bad "packagemanager keeps its own copy of the rules" ;; + esac + case "$gd" in + *"fallback for a system without it"*) + ok "it still works where lfs-helper is not installed" ;; + *) bad "no fallback when lfs-helper is absent" ;; + esac + # the delegation must come FIRST, or the local copy decides + i_del="$(printf '%s' "$gd" | grep -n "lfs-helper grant-dir" | head -1 | cut -d: -f1)" + i_own="$(printf '%s' "$gd" | grep -n 'prefix="\$(collector_prefix_of)"' | head -1 | cut -d: -f1)" + if [ -n "$i_del" ] && [ -n "$i_own" ] && [ "$i_del" -lt "$i_own" ]; then + ok "the shared rules are consulted before the local copy" + else + bad "the local copy runs even when lfs-helper is available" + fi +fi + +# ---- uids should read as the build order ------------------------------------ # +# The uid a package gets is the order its user is created in. Adoption walked +# the manifests alphabetically, so gcc (built 2nd of 22) ended up with a HIGHER +# uid than man-pages (the first chapter-8 package) -- which makes the uids +# useless for deciding what to rebuild first. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q "^_manifests_in_build_order()" "$helper_src" \ + && ok "manifests can be walked in build order" \ + || bad "adoption still walks manifests alphabetically" + ae="$(sed -n '/^cmd_adopt_existing() {/,/^}/p' "$helper_src")" + case "$ae" in + *'for man in $(_manifests_in_build_order)'*) + ok "adoption creates users in build order" ;; + *) bad "adoption does not use the build order" ;; + esac + # whole seconds tie for packages built in the same second + bo="$(sed -n '/^_manifests_in_build_order() {/,/^}/p' "$helper_src")" + case "$bo" in + *"stat -c %.Y"*) ok "sub-second timestamps keep the order exact" ;; + *) bad "packages built in the same second would tie" ;; + esac + + # behaviour: oldest manifest first, regardless of name + bod="$T/buildorder"; mkdir -p "$bod" + for p in zzz-first aaa-second mmm-third; do + echo "/usr/bin/$p" > "$bod/$p.files" + sleep 0.02 + done + got="$(MANIFESTS="$bod"; eval "$bo"; _manifests_in_build_order \ + | xargs -n1 basename | sed 's/.files//' | tr '\n' ' ')" + case "$got" in + "zzz-first aaa-second mmm-third "*) + ok "the order follows when they were written, not their names" ;; + *) bad "build order came out as: $got" ;; + esac +fi + +# ---- editing the wrong install script fails silently ------------------------ # +# A package user's home holds two files with nearly identical names: +# install_<name>-<version> the source -- the one to edit +# install_<name> a runtime copy, overwritten every run +# Editing the second changes nothing, the install repeats unchanged, and it +# looks as though the tool threw the edit away. +pmt="$(dirname "$LFS_TOOL")/packagemanager" +if [ -f "$pmt" ]; then + grep -q "^def _warn_if_wrong_file_edited" "$pmt" \ + && ok "an edit to the runtime copy is pointed out" \ + || bad "editing the wrong file goes unmentioned" + grep -q "To change what it does, edit:" "$pmt" \ + && ok "a failure names the file to edit" \ + || bad "a failure gives no path to edit" + + # behaviour: warn only when the runtime copy is genuinely newer + python3 - "$pmt" <<'PY61' +import sys, os, time, tempfile, importlib.machinery as m +pm = m.SourceFileLoader('pm', sys.argv[1]).load_module() +base = tempfile.mkdtemp() +os.makedirs(os.path.join(base, "p")) +src = os.path.join(base, "p", "install_p-1.0") +rt = os.path.join(base, "p", "install_p") +open(src, "w").write("real") +time.sleep(0.02) +open(rt, "w").write("edited") +pm.BASE_DIR = base +import io, contextlib +buf = io.StringIO() +with contextlib.redirect_stdout(buf), contextlib.redirect_stderr(buf): + pm._warn_if_wrong_file_edited("p", src) +warned = "runtime copy" in buf.getvalue() +os.utime(src, None) # now the source is newer +buf2 = io.StringIO() +with contextlib.redirect_stdout(buf2), contextlib.redirect_stderr(buf2): + pm._warn_if_wrong_file_edited("p", src) +quiet = "runtime copy" not in buf2.getvalue() +if warned and quiet: + print(" PASS it warns only when the runtime copy is the newer file") +else: + print(" FAIL warned=%s quiet=%s" % (warned, quiet)) + sys.exit(1) +PY61 + if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi +fi + +# ---- installing make-ca does not create any certificates -------------------- # +# It installs the TOOL. Until it is run there is still no CA bundle, wget +# still cannot verify anything, and the /etc/wgetrc workaround stays -- but the +# install reported success, so the missing step is easy to overlook. +pmt="$(dirname "$LFS_TOOL")/packagemanager" +if [ -f "$pmt" ]; then + grep -q "^def _hint_generate_ca_bundle" "$pmt" \ + && ok "the missing make-ca -g step is pointed out" \ + || bad "make-ca installs and nothing says to run it" + hg="$(sed -n '/^def _hint_generate_ca_bundle/,/^def /p' "$pmt")" + case "$hg" in + *"make-ca -g"*) ok "it gives the exact command" ;; + *) bad "the hint does not say what to run" ;; + esac + # silent once the certificates exist + case "$hg" in + *"certificates exist already"*) ok "it stops once certificates exist" ;; + *) bad "the hint would repeat forever" ;; + esac + + # ---- the book's init flavour must match the system ----------------------- # + # A systemd book emits `systemctl` in its install scripts; on a SysV system + # every affected package fails mid-install and has to be hand-edited. + grep -q "^def _warn_init_mismatch" "$pmt" \ + && ok "a systemd/SysV book mismatch is reported" \ + || bad "the wrong book flavour goes unmentioned" + wm="$(sed -n '/^def _warn_init_mismatch/,/^def /p' "$pmt")" + case "$wm" in + *"if book_systemd == have_systemd"*) + ok "it says nothing when the book matches" ;; + *) bad "the mismatch warning would always fire" ;; + esac +fi + +# ---- make-ca installed is not the same as certificates generated ------------ # +# `make-ca -g` prints "No update required!" and generates nothing when it +# considers its copy of certdata.txt current -- which it is straight after +# install. The install then looks successful while the system still has no +# certificate bundle, and every HTTPS fetch keeps warning. +pmt="$(dirname "$LFS_TOOL")/packagemanager" +if [ -f "$pmt" ]; then + grep -q "^def check_make_ca_generated" "$pmt" \ + && ok "an installed-but-ungenerated make-ca is noticed" \ + || bad "make-ca can be installed with no certificates and nothing said" + grep -q "make-ca -g --force" "$pmt" \ + && ok "it gives the command that actually generates them" \ + || bad "it does not say how to finish the job" + # and it must go quiet once a bundle exists + python3 - "$pmt" <<'PY62' +import sys, os, io, contextlib, tempfile, importlib.machinery as m +pm = m.SourceFileLoader('pm', sys.argv[1]).load_module() +d = tempfile.mkdtemp() +fake = os.path.join(d, "make-ca") +open(fake, "w").write("#!/bin/sh\nexit 0\n") +os.chmod(fake, 0o755) +os.environ["PATH"] = d + os.pathsep + os.environ["PATH"] +pm._CA_BUNDLES = [os.path.join(d, "bundle.crt")] +buf = io.StringIO() +with contextlib.redirect_stdout(buf), contextlib.redirect_stderr(buf): + pm.check_make_ca_generated() +warned = "no CA certificate bundle" in buf.getvalue() +open(os.path.join(d, "bundle.crt"), "w").write("CERTS") +buf2 = io.StringIO() +with contextlib.redirect_stdout(buf2), contextlib.redirect_stderr(buf2): + pm.check_make_ca_generated() +quiet = "no CA certificate bundle" not in buf2.getvalue() +if warned and quiet: + print(" PASS it warns only while the bundle is genuinely missing") +else: + print(" FAIL warned=%s quiet=%s" % (warned, quiet)); sys.exit(1) +PY62 + if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi +fi + + +# ---- passwd and group stay sorted, whoever creates the user ----------------- # +# Both halves of the toolchain create package users, and both append. Sorting +# only at the end of build-all left anything installed afterwards -- by +# packagemanager, from BLFS -- appended out of order again. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q "^_sort_users_quietly()" "$helper_src" \ + && ok "lfs-helper sorts after creating a user" \ + || bad "users created by lfs-helper are appended unsorted" +fi +pmi="$(dirname "$LFS_TOOL")/packagemanager_install" +if [ -f "$pmi" ]; then + au="$(sed -n '/^addUser() {/,/^}/p' "$pmi")" + case "$au" in + *"lfs-helper sort-users --run"*) + ok "packagemanager sorts after creating a user too" ;; + *) bad "users created by packagemanager are appended unsorted" ;; + esac + # it must not fail when lfs-helper is absent + case "$au" in + *"command -v lfs-helper"*) ok "it degrades quietly without lfs-helper" ;; + *) bad "user creation would fail where lfs-helper is missing" ;; + esac +fi + +# behaviour: package users before collector groups, system entries first +srt="$T/sortcheck"; mkdir -p "$srt/etc" "$srt/usr/src" "$srt/.lfs-pkgusr" +printf 'root:x:0:\ninstall:x:9999:\nnimgnu_x:x:90005:aaa\nzzz:x:10099:\naaa:x:10001:\n' \ + > "$srt/etc/group" +printf 'root:x:0:0:root:/root:/bin/bash\n' > "$srt/etc/passwd" +if [ -f "$helper_src" ]; then + LFS_ETC="$srt/etc" LFS_SNAP_ROOT="$srt" LFS_SRC_ROOT="$srt/usr/src" \ + LFS_PKGUSR_DIR="$srt/.lfs-pkgusr" \ + bash "$helper_src" sort-users --run >/dev/null 2>&1 + order="$(cut -d: -f1 "$srt/etc/group" | tr '\n' ' ')" + case "$order" in + "root install aaa zzz nimgnu_x "*) + ok "system entries, then package users, then collector groups" ;; + *) bad "sorted order came out as: $order" ;; + esac +fi + +# ---- prompts must allow line editing ---------------------------------------- # +# Without readline an arrow key inserts an escape sequence instead of moving +# the cursor, so a typo in a long answer -- a group name, a device path -- +# means retyping the whole thing. +for t in lfs blfs packagemanager; do + f="$(dirname "$LFS_TOOL")/$t" + [ -f "$f" ] || continue + grep -q "import readline" "$f" \ + || bad "$t: input() prompts have no line editing" +done +ok "the python tools import readline" + +# an optional import: a python built without readline must still run +python3 - "$LFS_TOOL" <<'PY64' +import sys +src = open(sys.argv[1]).read() +i = src.index("import readline") +window = src[max(0, i - 200):i + 120] +if "try:" not in window or "ImportError" not in window: + print(" FAIL readline is imported unguarded; a python without it would fail") + sys.exit(1) +print(" PASS readline is optional, not required") +PY64 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +for f in lfs-helper packagemanager_install; do + p="$(dirname "$LFS_TOOL")/$f" + [ -f "$p" ] || continue + # every prompt that reads an ANSWER needs -e; loops reading files do not + bare="$(grep -nE "read (-r )?(ans|check|answer|reply)\b" "$p" \ + | grep -v -- "-e" | head -3)" + [ -z "$bare" ] \ + || bad "$f: prompt without line editing: $(echo "$bare" | head -1)" +done +ok "the bash prompts use read -e" + +# and -r as well, or a backslash in an answer is eaten +if grep -qE "read -e [^r-]" "$(dirname "$LFS_TOOL")/packagemanager_install" 2>/dev/null; then + bad "a prompt uses -e without -r" +else + ok "prompts keep backslashes literal (-r)" +fi + +# ---- starting the whole build over -------------------------------------------# +# Deleting a built system is the most destructive thing these tools can do -- +# potentially weeks of compilation -- so it has to say exactly what goes, what +# stays, and refuse outright in the one case that could damage the host. +python3 - "$LFS_TOOL" <<'PY65' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +src = open(sys.argv[1]).read() +if not hasattr(lfs, "cmd_bs_restart"): + print(" FAIL no way to start the build over"); sys.exit(1) +body = src[src.index("def cmd_bs_restart"):src.index("def cmd_bs_set_book")] +problems = [] +# deleting through a bind-mounted /dev would reach the running system +if "the chroot is still mounted" not in body: + problems.append("it does not refuse while the chroot is mounted") +for d in ('"dev/pts", "dev/shm", "dev", "proc", "sys", "run"',): + if d not in body: + problems.append("the mount check does not cover every virtual filesystem") +# dry run by default, and a confirmation that cannot be typed by accident +if "args.run" not in body: + problems.append("restart is not dry-run by default") +if "Type the mount point" not in body: + problems.append("the confirmation is too easy to give by accident") +# sources are expensive to re-download and unchanged by a build +if "keep_sources" not in body: + problems.append("the downloaded sources are not kept") +# and it should point at the cheaper alternative +if "snapshot restore" not in body: + problems.append("it does not mention restoring a snapshot instead") +if problems: + for p in problems: print(" FAIL %s" % p) + sys.exit(1) +print(" PASS restart warns, confirms, keeps the sources and spares the host") +PY65 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- snapshots must not read live kernel filesystems ------------------------ # +# Excluding "./sys/*" skips the CONTENTS but still archives the ./sys directory +# itself. Reading a live kernel filesystem's metadata gives +# tar: ./sys: file changed as we read it +# and exit 1, so the snapshot failed -- then succeeded on a retry, because the +# timing happened to work out. --one-file-system does not descend into +# anything mounted inside the tree at all. +python3 - "$LFS_TOOL" <<'PY66' +import sys +src = open(sys.argv[1]).read() +body = src[src.index('cmd = ("tar -C'):] +body = body[:body.index("prog = ")] if "prog = " in body else body[:2000] +problems = [] +if "--one-file-system" not in body: + problems.append("tar still descends into mounted filesystems") +# tar's exit 1 is "files differ", a warning on a live tree -- verify, don't die +if "rc == 1" not in body: + problems.append("a tar warning is still treated as a failure") +if "tar -tf" not in body: + problems.append("the archive is not verified before being accepted") +# but an unreadable archive must NOT be kept +if "os.remove(dest)" not in body: + problems.append("a corrupt archive would be left in place") +if problems: + for p in problems: print(" FAIL %s" % p) + sys.exit(1) +print(" PASS snapshots skip live filesystems and verify what they wrote") +PY66 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# ---- a cancelled step must be resumed, not skipped -------------------------- # +# The orphan repair was gated on "does the lfs user exist" -- but the user is +# created FIRST. Cancelling part-way left the user in place, so every later +# run skipped the repair entirely with half a million files still ownerless. +# A gate must test the WORK, never a side effect of having started it. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + grep -q "^_has_orphaned_files()" "$helper_src" \ + && ok "the orphan repair is gated on the files, not on the user" \ + || bad "a cancelled orphan repair would be skipped forever" + ba="$(sed -n '/^cmd_build_all() {/,/^}/p' "$helper_src")" + case "$ba" in + *"if ! user_exists lfs; then"*) + bad "build-all still gates the repair on the user existing" ;; + *) ok "build-all gates it on there being work to do" ;; + esac + # -print -quit: stop at the first hit, so it costs nothing when clean + ho="$(sed -n '/^_has_orphaned_files() {/,/^}/p' "$helper_src")" + case "$ho" in + *"-print -quit"*) ok "the check stops at the first orphan" ;; + *) bad "the check walks the whole tree every time" ;; + esac + # long loops must show progress, or they get cancelled for looking hung + fo="$(sed -n '/^cmd_fix_orphans() {/,/^}/p' "$helper_src")" + case "$fo" in + *"fixed % 5000"*) ok "a long repair shows progress" ;; + *) bad "half a million chowns run with no output" ;; + esac + # and a silent skip hides real failures + ip="$(sed -n '/^cmd_init_pkgusr() {/,/^}/p' "$helper_src")" + case "$ip" in + *"could not set the group on"*) + ok "directories that could not be regrouped are named" ;; + *) bad "a failed chgrp is silently counted as nothing to do" ;; + esac +fi + +# ---- an application user must be usable, not just exist --------------------- # +# `user create` made the account and stopped there: you could not become it, +# its home was world-readable, and nothing launched the program. +pmt="$(dirname "$LFS_TOOL")/packagemanager" +if [ -f "$pmt" ]; then + grep -q "^def _setup_application_user" "$pmt" \ + && ok "creating an application user sets up access to it" \ + || bad "user create leaves an account nobody can use" + su_body="$(sed -n '/^def _setup_application_user/,/^def _home_of/p' "$pmt")" + case "$su_body" in + *'"chmod", "0750", home'*) ok "the account's home is private" ;; + *) bad "the application's home stays world-readable" ;; + esac + case "$su_body" in + *'"usermod", "-a", "-G", u, main'*) + ok "your own login joins the account's group" ;; + *) bad "a shared directory would not be reachable" ;; + esac + # without a main user there is nobody to grant access TO + case "$su_body" in + *"No main user is set"*) ok "it says so when no main user is set" ;; + *) bad "it silently does nothing when main_user is unset" ;; + esac + # the PAM file is not part of stock LFS: absence must not be an error + pam="$(sed -n '/^def _allow_su_via_pam/,/^def _write_launcher/p' "$pmt")" + case "$pam" in + *"not present -- \`su - "*) ok "a missing PAM file is handled, not fatal" ;; + *) bad "it assumes the hint's PAM file exists" ;; + esac + grep -q "_APPLICATION_USER_HELP" "$pmt" \ + && ok "application users have their own help section" \ + || bad "no help explaining what an application user is" +fi + +# ---- shell completion -------------------------------------------------------- # +comp="$(dirname "$LFS_TOOL")/lfs-completion.bash" +if [ -f "$comp" ]; then + bash -n "$comp" \ + && ok "the completion script parses" \ + || bad "the completion script has a syntax error" + # subcommands must come from the tools themselves, or the list goes stale + grep -q "_lfs_tool_subcommands" "$comp" \ + && ok "completions are read from each tool's own help" \ + || bad "the completion keeps a second list of commands" + for t in lfs blfs packagemanager lfs-helper; do + grep -q "complete -F _lfs_complete $t\b" "$comp" \ + || bad "no completion registered for $t" + done + ok "all four tools have completion registered" +fi + +# ---- a system you can actually log into -------------------------------------- # +# A freshly built system has root with NO password. Depending on the login +# manager that is either "anyone can log in as root" or "nobody can log in at +# all" -- discovered after a reboot, with no way in. +python3 - "$LFS_TOOL" <<'PY67' +import sys, importlib.machinery as m +lfs = m.SourceFileLoader('lfs', sys.argv[1]).load_module() +t = lfs.LAST_STEP_DEFAULT +problems = [] +if "set_up_login_accounts" not in t: + problems.append("nothing asks for a root password") +body = t[t.index("set_up_login_accounts() {"):] if \ + "set_up_login_accounts() {" in t else "" +if "passwd root" not in body: + problems.append("the root password is never set") +# a scripted run has no terminal: it must say what is missing, not hang +if "[ ! -t 0 ]" not in body: + problems.append("a non-interactive run would block on a prompt") +if "chroot /mnt/lfs /usr/bin/passwd root" not in body: + problems.append("it does not say how to set the password later") +# the login account name is already configured -- reuse it, do not re-ask +if "main_user=" not in body: + problems.append("it ignores the configured login account") +if problems: + for p in problems: print(" FAIL %s" % p) + sys.exit(1) +print(" PASS the build ends by setting up accounts you can log in with") +PY67 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# password detection must read /etc/shadow correctly +shd="$T/shadowcheck"; mkdir -p "$shd" +_haspw() { + grep -qE "^root:[^:]*:" "$1" && ! grep -qE "^root:(\*|!|)?:" "$1" +} +printf 'root::19000:0:99999:7:::\n' > "$shd/empty" +printf 'root:!:19000:0:99999:7:::\n' > "$shd/locked" +printf 'root:$6$a$b:19000:0:99999:7:::\n' > "$shd/set" +if ! _haspw "$shd/empty" && ! _haspw "$shd/locked" && _haspw "$shd/set"; then + ok "an empty, locked or real root password are told apart" +else + bad "root password detection is wrong" +fi + +# ---- decisions already made must not be asked again ------------------------- # +# The export copied into the tree said +# dir|/usr/libexec/gcc|nimgnu_gcc +# yet the build still stopped to ask which group should share that directory. +# Two faults: nothing applied the file, and `import-groups --run` read the +# wrong path anyway. +helper_src="$(dirname "$LFS_TOOL")/lfs-helper" +if [ -f "$helper_src" ]; then + ba="$(sed -n '/^cmd_build_all() {/,/^}/p' "$helper_src")" + case "$ba" in + *"cmd_import_groups --run"*) + ok "a copied-in groups export is applied automatically" ;; + *) bad "the export sits unapplied until run by hand" ;; + esac + grep -q "^_groups_already_imported()" "$helper_src" \ + && ok "it is applied once, not on every build" \ + || bad "the import would run on every build-all" + + # `--run` used to be taken as the FILENAME, then overwrite the path + ig="$(sed -n '/^cmd_import_groups() {/,/^}/p' "$helper_src")" + case "$ig" in + *'in="$STATE/collector-groups.export"; }'*) + bad "import-groups --run still overwrites the file it found" ;; + *) ok "import-groups --run reads the file it chose" ;; + esac + case "$ig" in + *'local in="" run=0 a'*) ok "the arguments are parsed in one place" ;; + *) bad "the run flag can still be wiped by a later declaration" ;; + esac + + # and the prompt itself must consult the export, imported or not + cg="$(sed -n '/^choose_collector_group() {/,/^}/p' "$helper_src")" + case "$cg" in + *'collector-groups.import'*) + ok "the prompt checks the copied-in decisions first" ;; + *) bad "a decision in the tree would still be asked about" ;; + esac +fi + +# end to end: the answer is in the file, so nothing is asked +igd="$T/importdec"; mkdir -p "$igd/.lfs-pkgusr" "$igd/etc" "$igd/usr/src" +printf 'root:x:0:\ninstall:x:9999:\n' > "$igd/etc/group" +printf 'root:x:0:0:root:/root:/bin/bash\n' > "$igd/etc/passwd" +printf '# prefix|nimgnu\ndir|/usr/libexec/gcc|nimgnu_gcc\n' \ + > "$igd/.lfs-pkgusr/collector-groups.import" +got="$(STATE="$igd/.lfs-pkgusr" awk -F'|' -v d=/usr/libexec/gcc \ + '$1 == "dir" && $2 == d { print $3; exit }' \ + "$igd/.lfs-pkgusr/collector-groups.import")" +[ "$got" = "nimgnu_gcc" ] \ + && ok "a recorded decision resolves without asking" \ + || bad "the recorded decision was not found (got '$got')" + +# ---- 7.2 must actually hand the tree to root -------------------------------- # +# The handover used `chown --from lfs`, which changes only files owned by +# exactly that user. A tree owned by a uid with no matching account -- or by +# an `lfs` whose uid changed since -- was silently left alone, so `prepare` +# reported "give the tree to root", changed nothing, and `enter` then refused. +# The loop in `run` caught it as a step that ran but stayed pending. +python3 - "$LFS_TOOL" <<'PY68' +import sys +src = open(sys.argv[1]).read() +i = src.index('"give the tree to root"') +cmd = src[i:i + 400] +if "--from lfs" in cmd: + print(" FAIL the handover still only matches files owned by 'lfs'") + sys.exit(1) +if "chown -h -R root:root" not in cmd: + print(" FAIL the handover does not chown recursively"); sys.exit(1) +# and the refusal should say who DOES own it +if "_owner_name_of" not in src: + print(" FAIL the refusal does not name the actual owner"); sys.exit(1) +print(" PASS 7.2 hands the tree to root whoever owned it") +PY68 +if [ $? -eq 0 ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); fi + +# behaviour: a tree owned by an unknown uid must still be handed over +hod="$T/handover"; mkdir -p "$hod/usr/bin" "$hod/etc" +chown -R 4242 "$hod/usr" "$hod/etc" 2>/dev/null || true +if [ "$(stat -c %u "$hod/usr")" = "4242" ]; then + ( for d in usr etc; do [ -e "$hod/$d" ] && chown -h -R root:root "$hod/$d"; done; true ) + [ "$(stat -c %u "$hod/usr")" = "0" ] \ + && ok "a tree owned by an unknown uid is handed to root" \ + || bad "the handover left the tree owned by $(stat -c %u "$hod/usr")" +else + ok "(cannot test unknown-uid handover as this user)" +fi + +echo +echo " PASS: $PASS FAIL: $FAIL (final)" +[ "$FAIL" -eq 0 ] diff --git a/test_packagemanager.sh b/test_packagemanager.sh new file mode 100644 index 0000000..7ed2cbd --- /dev/null +++ b/test_packagemanager.sh @@ -0,0 +1,228 @@ +#!/bin/bash +# ============================================================================= +# test_packagemanager.sh -- safe functional tests for `packagemanager` +# +# SAFETY: everything runs inside a throwaway sandbox ($mktemp -d). All external +# helpers (blfs, forall_direntries_from, list_package, add_package_user, +# packagemanager_install) are STUBBED, so: +# * the real BLFS book is never read +# * the real filesystem is never scanned or modified (forall scans a sandbox +# tree only) +# * real packages / users / groups are never touched +# The only real system objects created are throwaway users/groups whose names +# start with "pmtest_", and they are deleted again on exit. Mutating tests that +# need those run only as root; without root they're skipped (not failed). +# +# Usage: ./test_packagemanager.sh [/path/to/packagemanager] +# ============================================================================= +set -u + +HERE="$(cd "$(dirname "$0")" && pwd)" +PMFILE="${1:-$HERE/packagemanager}" +PM="python3 $PMFILE" +SB="$(mktemp -d)" +chmod 755 "$SB" # real /,/usr are traversable; mktemp is 700 -> su-user needs this +export NO_COLOR=1 +ROOT=0; [ "$(id -u)" = "0" ] && ROOT=1 +CREATED="" # throwaway pmtest_* users/groups to clean up + +pass=0; fail=0; skip=0 +ok() { echo " PASS $1"; pass=$((pass+1)); } +bad() { echo " FAIL $1"; fail=$((fail+1)); [ -n "${2:-}" ] && echo "$2" | sed 's/^/ /'; } +skp() { echo " SKIP $1"; skip=$((skip+1)); } + +# want <desc> <expect-substring> -- <command...> (runs cmd, greps output) +want() { + local desc="$1" expect="$2"; shift 2 + local out; out="$("$@" 2>&1)" + if printf '%s' "$out" | grep -qF -- "$expect"; then ok "$desc"; else bad "$desc (want: $expect)" "$out"; fi +} + +cleanup() { + for u in $CREATED; do userdel "$u" 2>/dev/null; groupdel "$u" 2>/dev/null; done + rm -rf "$SB" +} +trap cleanup EXIT + +# --------------------------------------------------------------------------- # +# sandbox + stubs +# --------------------------------------------------------------------------- # +mkdir -p "$SB/bin" "$SB/base" "$SB/scan/usr/lib" "$SB/out" "$SB/dirs" +export PKGUSR_BASE="$SB/base" +export FORALL_SCAN="$SB/scan" +export PKGUSR_CONFIG="$SB/pm.conf" +printf 'collector_prefix=nimgnu\nuser_prefix=u\n' > "$PKGUSR_CONFIG" + +cat > "$SB/bin/blfs" <<'EOF' +#!/bin/bash +sub="$1"; shift +while [ "${1:-}" = "--book-file" ] || [ "${1:-}" = "--book" ]; do shift 2; done +case "$sub" in + search) n="$1"; printf ' [package] %-26s %s-1.0\n' "$n" "${n^}"; echo "1 result(s)." ;; + debug) echo "title : ${1^}-1.0" ;; + order) n="$1"; printf 'dep1\tDep1-1.0\trequired\t%s\n%s\t%s-1.0\t\t\n' "$n" "$n" "${n^}" ;; + script) n="$1"; out="."; while [ $# -gt 0 ]; do [ "$1" = "-o" ] && out="$2"; shift; done + f="$out/install_${n^}-1.0"; printf '#!/bin/bash\ninstall_pkg(){ :; }\n' >"$f" + echo "Created $f"; echo "Bash syntax OK." ;; +esac +EOF +cat > "$SB/bin/forall" <<'EOF' +#!/bin/bash +n="$1"; shift +depth=""; args=() +for a in "$@"; do + if [ "$a" = "-depth" ]; then depth="-depth"; else args+=("$a"); fi +done +m=(-false) +getent passwd "$n" >/dev/null && m+=(-o -user "$n") +getent group "$n" >/dev/null && m+=(-o -group "$n") +find "${FORALL_SCAN:-/nonexistent}" $depth -xdev \( "${m[@]}" \) \( -true "${args[@]}" \) 2>/dev/null +EOF +cat > "$SB/bin/lp" <<'EOF' +#!/bin/bash +find "${FORALL_SCAN:-/nonexistent}" -xdev \( -user "$1" -o -group "$1" \) 2>/dev/null +EOF +cat > "$SB/bin/apu" <<'EOF' +#!/bin/bash +groupadd "$1" 2>/dev/null +useradd -M -N -g "$1" -d "$PKGUSR_BASE/$1" "$1" 2>/dev/null +mkdir -p "$PKGUSR_BASE/$1" +echo "created $1" +EOF +cat > "$SB/bin/pmi" <<'EOF' +#!/bin/bash +echo "[stub packagemanager_install] user=$1 script=$2" +EOF +chmod +x "$SB"/bin/* +export BLFS_BIN="$SB/bin/blfs" FORALL_BIN="$SB/bin/forall" \ + LIST_PACKAGE_BIN="$SB/bin/lp" ADD_PACKAGE_USER_BIN="$SB/bin/apu" \ + PM_INSTALL_BIN="$SB/bin/pmi" + +# fixture package-user dirs (no real users needed for info/list/check-dry) +mkdir -p "$SB/base/foo" "$SB/base/bar" +printf 'name_version="Foo-1.0"\nlink="http://x/foo"\ninfo="a demo pkg"\nrequired=(Dep1-1.0)\n' \ + > "$SB/base/foo/install_last" +printf '/usr/bin/foo\n/usr/lib/libfoo.so\n' > "$SB/base/foo/pkg.lst" # foo "installed" + +echo "sandbox: $SB (root=$ROOT)" +echo +echo "================ read-only / dry-run (always safe) ================" +want "help lists commands" "make-group-dir" $PM --help +want "template writes a file" "Wrote template" $PM template mypkg --version 2.0 -o "$SB/out" +want "template is valid bash" "install_pkg" cat "$SB/out/install_mypkg-2.0" +bash -n "$SB/out/install_mypkg-2.0" && ok "template passes bash -n" || bad "template bash -n" +want "list package-users" "foo" $PM list +want "info all (short)" "foo" $PM info all +want "info named (detailed)" "a demo pkg" $PM info foo +want "search annotates state" "not installed" $PM search baz +want "deps shows kind+parent" "required by" $PM dependencies someapp +want "install dry-run plan" "Install plan" $PM install someapp +want "install --force dry-run" "Install plan" $PM install someapp --force +want "remove dry-run" "Dry run" $PM remove foo +want "remove --purge dry-run" "will be removed" $PM remove foo --purge +want "nimgnu list runs" "group(s)." $PM nimgnu list +want "reload-pkg-list (bg)" "regeneration" $PM reload-pkg-list foo +want "config shows prefixes" "collector prefix" $PM config +want "user list runs" "user(s)." $PM user list + +echo +echo "================ make/fix group dirs (safe: sandbox dirs) ================" +if [ "$ROOT" = 1 ]; then + groupadd -g 9999 install 2>/dev/null || true + mkdir -p "$SB/dirs/inst" "$SB/dirs/coll" + # install dir -> group install, group-writable, no sticky + $PM make-group-dir "$SB/dirs/inst" --group install >/dev/null 2>&1 + # use a real nimgnu_ collector: group rwx, NO setgid (no 's' bit) + $PM nimgnu create pmtest_c >/dev/null 2>&1; CREATED="$CREATED nimgnu_pmtest_c" + $PM add-dir-to-nimgnu pmtest_c "$SB/dirs/coll" >/dev/null 2>&1 + perm_inst="$(stat -c '%A' "$SB/dirs/inst")" + perm_coll="$(stat -c '%A' "$SB/dirs/coll")" + case "$perm_inst" in drwxrwx*) ok "install dir group-writable, no sticky ($perm_inst)";; *) bad "install dir perms ($perm_inst)";; esac + # group rwx and NO setgid: expect drwxrwxr-x (rwx in group field, no 's') + case "$perm_coll" in + drwxrws*) bad "nimgnu dir must NOT be setgid ($perm_coll)";; + drwxrwx*) ok "nimgnu dir group-rwx, no setgid ($perm_coll)";; + *) bad "nimgnu dir perms ($perm_coll)";; + esac + want "fix-group-dir re-applies" "re-applied" $PM fix-group-dir "$SB/dirs/coll" +else + skp "make-group-dir / fix-group-dir / add-dir-to-nimgnu (need root)" +fi + +echo +echo "================ nimgnu group lifecycle (root, throwaway) ================" +if [ "$ROOT" = 1 ]; then + useradd -M -N pmtest_u1 2>/dev/null; CREATED="$CREATED pmtest_u1" + useradd -M -N pmtest_u2 2>/dev/null; CREATED="$CREATED pmtest_u2" + $PM nimgnu create pmtest_g >/dev/null 2>&1; CREATED="$CREATED nimgnu_pmtest_g" + want "nimgnu create" "nimgnu_pmtest_g" $PM nimgnu list pmtest_g + want "nimgnu add (multi)" "may now install" $PM nimgnu add pmtest_g pmtest_u1 pmtest_u2 + want "members listed" "pmtest_u1" $PM nimgnu list pmtest_g + want "nimgnu remove" "Removed" $PM nimgnu remove pmtest_g pmtest_u1 + want "nimgnu delete dryrun" "Dry run" $PM nimgnu delete pmtest_g + want "nimgnu delete --run" "deleted" $PM nimgnu delete pmtest_g --run --yes +else + skp "nimgnu create/add/remove/delete (need root)" +fi + +echo +echo "================ add-user / install --run / remove --run (root) ================" +if [ "$ROOT" = 1 ]; then + want "add-user" "Created package-user" $PM add-user pmtest_p + CREATED="$CREATED pmtest_p" + want "install --run (stub engine)" "stub packagemanager_install" bash -c "printf 'y\n' | $PM install someapp --run" + # remove --run: throwaway user owning files in the sandbox scan tree. + # su - resets env, so install a forall stub on PATH that scans a fixed dir. + cat > /usr/local/bin/forall_direntries_from <<EOF2 +#!/bin/bash +n="\$1"; shift; d=""; a=() +for x in "\$@"; do [ "\$x" = "-depth" ] && d="-depth" || a+=("\$x"); done +m=(-false); getent passwd "\$n">/dev/null&&m+=(-o -user "\$n"); getent group "\$n">/dev/null&&m+=(-o -group "\$n") +find "$SB/scan" \$d -xdev \( "\${m[@]}" \) \( -true "\${a[@]}" \) 2>/dev/null +EOF2 + chmod +x /usr/local/bin/forall_direntries_from + unset FORALL_BIN FORALL_SCAN + useradd -M -N pmtest_r 2>/dev/null; CREATED="$CREATED pmtest_r" + mkdir -p "$SB/base/pmtest_r" + install -o pmtest_r /dev/null "$SB/scan/usr/lib/rfile1" + install -o pmtest_r /dev/null "$SB/scan/usr/lib/rfile2" + : > "$SB/scan/usr/lib/rootfile" + chmod 0777 "$SB/scan/usr/lib" # simulate a group-writable install dir + $PM remove pmtest_r >/dev/null 2>&1 # dry-run stores the plan + out="$($PM remove pmtest_r --run --yes 2>&1)" + if [ ! -e "$SB/scan/usr/lib/rfile1" ] && [ -e "$SB/scan/usr/lib/rootfile" ] && id pmtest_r >/dev/null 2>&1; then + ok "remove --run (su-scan + plan reuse) deleted owned files, kept root's + user" + else + bad "remove --run" "$out" + fi + rm -f /usr/local/bin/forall_direntries_from + export BLFS_BIN="$SB/bin/blfs" FORALL_BIN="$SB/bin/forall" \ + LIST_PACKAGE_BIN="$SB/bin/lp" ADD_PACKAGE_USER_BIN="$SB/bin/apu" \ + PM_INSTALL_BIN="$SB/bin/pmi" FORALL_SCAN="$SB/scan" +else + skp "add-user / install --run / remove --run (need root)" +fi + +echo +echo "================ application users (u_*) (root, throwaway) ================" +if [ "$ROOT" = 1 ]; then + $PM user create pmtestapp >/dev/null 2>&1; CREATED="$CREATED u_pmtestapp" + want "user create (u_ prefix)" "u_pmtestapp" $PM user list pmtestapp + want "user allow audio" "Added" $PM user allow u_pmtestapp audio + want "user shows the group" "audio" $PM user list pmtestapp + want "user disallow" "Removed" $PM user disallow u_pmtestapp audio + want "user delete dry-run" "Dry run" $PM user delete pmtestapp + want "user delete --run" "deleted" $PM user delete pmtestapp --run --yes +else + skp "user create/allow/disallow/delete (need root)" +fi + +echo +echo "================ files_with_broken_id (info only) ================" +skp "files_with_broken_id scans the whole real filesystem -- run manually: packagemanager files_with_broken_id" + +echo +echo "==================================================================" +echo " PASS: $pass FAIL: $fail SKIP: $skip" +echo "==================================================================" +[ "$fail" = 0 ]