Packagemanager-LFS-PackageUser-System git · main
Package-user based package manager for LFS/BLFS
Shell 63.7% Markdown 35.8%git clone https://git.christianimmanuel.de/linux-from-scratch/Packagemanager-LFS-PackageUser-System.gitwget https://git.christianimmanuel.de/linux-from-scratch/Packagemanager-LFS-PackageUser-System/archive/Packagemanager-LFS-PackageUser-System.tar.gzlfs raw
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# lfs -- Linux From Scratch book tool (follow one cached book)
#
# The LFS counterpart to the `blfs` tool. You cache one or more LFS "nochunks"
# HTML books, pick a default (main) book, and query it:
# * list chapter-8 build packages and their versions,
# * dump the shell commands for any section, and
# * print the package source URLs (from the book's wget-list).
#
# Building (chroot / cross-compile / package-user aware install + update) comes
# in later steps on top of this.
#
# Usage:
# lfs fetch VERSION download + cache a book (+ wget-list)
# lfs import FILE --version V cache a LOCAL book file (offline)
# lfs books [--remote] list cached books + the default
# lfs set-default VERSION choose the main book
# lfs packages chapter-8 packages (name version id)
# lfs sections [BOOK] list section ids + titles
# lfs commands SECTION-ID [BOOK] a section's shell commands
# lfs sources package source URLs (wget-list)
#
# Global: --book VERSION (use a cached version) --book-file FILE (local html)
# Store: $LFS_STORE (default /usr/share/lfs, then ~/.cache/lfs).
#
# GPLv2-or-later. Part of the pkgusr toolchain.
import argparse
import json
import os
import re
import shlex
try:
# arrow keys, backspace and history inside input() prompts. Importing it
# is enough -- Python wires it into input() automatically. Optional: a
# Python built without readline still runs, just without line editing.
import readline # noqa: F401
except ImportError:
pass
import stat
import subprocess
import sys
import time
try:
from bs4 import BeautifulSoup
try:
from bs4 import XMLParsedAsHTMLWarning
import warnings
warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning)
except Exception:
pass
except ImportError:
sys.stderr.write(
"error: this needs beautifulsoup4 to parse the book.\n"
" Install it as a package user (not with a bare pip install, which\n"
" would leave the files owned by root):\n"
" packagemanager pip install beautifulsoup4\n")
sys.exit(2)
LFS_VERSION = "1.14.371"
def _build_id():
"""A short fingerprint of this file's own contents.
A hand-maintained version number goes stale the moment someone forgets to
bump it -- and the whole reason for printing a version is to answer "am I
running the code that was just fixed?". This cannot go stale: it changes
whenever the file does.
"""
import hashlib
try:
with open(os.path.abspath(__file__), "rb") as fh:
return hashlib.md5(fh.read()).hexdigest()[:7]
except OSError:
return "unknown"
LFS_SITE = os.environ.get("LFS_SITE", "https://linuxfromscratch.org")
# --------------------------------------------------------------------------- #
# book store: cache books + a default selection, just like blfs
# --------------------------------------------------------------------------- #
# ---- output helpers ------------------------------------------------------- #
def _tty():
return sys.stderr.isatty() and os.environ.get("NO_COLOR") is None
def _tty_out():
return sys.stdout.isatty() and os.environ.get("NO_COLOR") is None
# Four colours, one meaning each, matching lfs-helper exactly -- the two tools
# print into the same terminal one after the other, and a build log where the
# same colour means two different things is worse than no colour at all.
#
# green something finished, and finished correctly
# yellow something needs your attention -- including "nothing happened"
# red something failed
# dim detail you can skip: provenance, and what to do next
#
# Prose gets none. Colour that appears on every other line stops carrying
# information.
_C_WARN = "\033[0;33m"
_C_ERR = "\033[0;31m"
_C_OK = "\033[0;32m"
_C_DIM = "\033[2m"
_C_OFF = "\033[0m"
def _paint(msg, colour, stream):
if (stream is sys.stderr and _tty()) or (stream is sys.stdout and _tty_out()):
stream.write("%s%s%s\n" % (colour, msg, _C_OFF))
else:
stream.write("%s\n" % msg)
def warn(msg):
"""Yellow, on stderr. A dry run, a skip, a refusal -- anything the reader
might otherwise take for 'done'. Coloured because the point is that
nothing happened."""
_paint(msg, _C_WARN, sys.stderr)
def note(msg):
"""Yellow, on stdout. Needs attention, but belongs in sequence with the
build output rather than beside it."""
_paint(msg, _C_WARN, sys.stdout)
def ok(msg):
"""Green. Something finished, and finished correctly."""
_paint(msg, _C_OK, sys.stdout)
def fail(msg):
"""Red, on stderr. Something failed."""
_paint(msg, _C_ERR, sys.stderr)
def hint(msg):
"""Dim. What to do next, or provenance -- never the finding itself."""
_paint(msg, _C_DIM, sys.stdout)
def dry_run_note(what="apply"):
warn("\n(dry run -- nothing was changed. Re-run with --run to %s.)" % what)
def next_step(*cmds, label="next run:"):
"""Print a follow-up command on its OWN line, unindented and unprefixed,
so it can be copied straight out of the terminal."""
print("# " + label)
for c in cmds:
print(c)
def _colour_epilog(text):
"""Colour the group headings in the help.
The command list is what people scan to find a command, and an unbroken
wall of text is hard to search. Headings are the lines that are not
indented; commands are indented. Colour is dropped when the output is not
a terminal, so piping help into a file or a pager stays clean."""
if not _tty_out():
return text
out = []
for line in text.split("\n"):
if line and not line.startswith(" ") and line.endswith(":"):
out.append("%s%s%s" % (_C_HEAD, line, _C_OFF)) # "commands, by..."
elif line.startswith(" ") and not line.startswith(" ") and line.strip():
out.append("%s%s%s" % (_C_GROUP, line, _C_OFF)) # a group heading
elif line.startswith(" ") and line.strip():
# the command itself, up to the first run of two spaces
stripped = line.rstrip()
m = re.match(r"^(\s+)(\S+(?: \S+)*?)(\s{2,}.*)$", stripped)
if m:
out.append("%s%s%s%s%s" % (m.group(1), _C_CMD, m.group(2),
_C_OFF, m.group(3)))
else:
out.append(line)
else:
out.append(line)
return "\n".join(out)
_C_HEAD = "\033[1m" # bold
_C_GROUP = "\033[1;36m" # bold cyan -- the group you are scanning for
_C_CMD = "\033[0;32m" # green -- the thing you actually type
def _wrap(text, width):
out, line = [], ""
for w in text.split():
if len(line) + len(w) + 1 > width:
out.append(line); line = w
else:
line = (line + " " + w).strip()
if line:
out.append(line)
return out
# The directories handed to the lfs user at book 4.3 and handed BACK to root
# at 7.2. One list, used by both -- they were written separately and drifted:
# 4.3 included `lib` and 7.2 did not, so /lib stayed owned by the lfs user for
# the rest of the build while /bin and /sbin (never handed over) stayed root.
# /sources is deliberately absent: the book leaves it to the lfs user.
LFS_HANDOVER_DIRS = ("usr", "lib", "lib64", "var", "etc", "tools", "bin", "sbin")
# Where package users and application users live.
#
# TWO SETS, deliberately. The `target_*` values describe the system being
# BUILT and are written into its config; the plain ones describe THIS machine,
# if these tools also manage it. Without the split, configuring your own
# host's layout would silently change the layout of every system you build,
# which is exactly backwards -- the point of the build is that the new system
# is not this one.
#
# Defaults are the current layout, so nothing moves on an existing tree. Set
# them before a fresh build to get the tidier arrangement.
# Accounts live in subdirectories by kind, so `ls /usr/src` says what a thing
# IS rather than listing a hundred packages and a few config steps as one flat
# set:
# /usr/src/pkgusr/p_gcc a package
# /usr/src/cfg/cfg_bootscripts a config step that installs files
# /usr/src/u_firefox an application user
LAYOUT_DEFAULTS = {
# the system being built
"target_pkgusr_home": "/usr/src/pkgusr",
"target_cfguser_home": "/usr/src/cfg",
"target_appuser_home": "/home/shared_user", # one subdir, not /home itself
# this machine
"pkgusr_home": "/usr/src/pkgusr",
"cfguser_home": "/usr/src/cfg",
"appuser_home": "/home/shared_user",
}
def _accounts_root(kind):
"""Where the homes of one kind of account actually live.
The config has TWO keys that both claim to answer this: `pkgusr_home`,
whose default is /usr/src/pkgusr, and `pkgusr_subdir`, "folder under the
base for package homes (pkgusr)". Read one way the base already includes
the folder; read the other it does not -- and pkgusr_home's own help text
says "(/usr/src)", which invites the second reading. A config written
that way exported LFS_PKGUSR_ROOT=/usr/src, so every account's home became
/usr/src/p_<name>, `add_package_user` made it there, and every tool that
computes /usr/src/pkgusr/p_<name> disagreed with the passwd file -- for
the whole life of that tree.
Accept both readings: append the subdir when the base does not already
end in it.
"""
# (packagemanager already composes it this way -- see PKGUSR_SUBDIR
# there -- which is exactly why the two disagreed: one appended the
# subdir and the other exported the bare base.)
base = layout("pkgusr_home" if kind == "pkgusr" else "cfguser_home",
target=True).rstrip("/")
_c = {}
try:
# load_config, the function that exists: `read_config` did not, and
# the NameError was swallowed below -- a configured pkgusr_subdir /
# cfguser_subdir was silently ignored in favour of the default
_c = load_config() or {}
except Exception:
_c = {}
sub = str(_c.get("pkgusr_subdir" if kind == "pkgusr" else "cfguser_subdir")
or ("pkgusr" if kind == "pkgusr" else "cfg")).strip("/")
if not sub or os.path.basename(base) == sub:
return base
return base + "/" + sub
def layout(key, target=False):
"""One layout setting, target or host.
`target=True` asks about the system being built, and falls back to the
host value only when no target value is set -- so a half-configured setup
behaves like the old one rather than mixing the two."""
cfg = load_config()
if target:
v = cfg.get("target_" + key)
if v:
return v
v = cfg.get(key)
if v:
return v
return LAYOUT_DEFAULTS["target_" + key if target else key]
def store_dir():
"""One shared location so root and the lfs user see the SAME books.
Override with $LFS_STORE."""
return os.environ.get("LFS_STORE") or "/usr/share/lfs"
def _ensure_store():
"""Create the shared store (needs root/owner) and keep it world-readable so
every user can READ the cached books."""
d = store_dir()
os.makedirs(os.path.join(d, "books"), exist_ok=True)
for p in (d, os.path.join(d, "books")):
try:
os.chmod(p, 0o755)
except OSError:
pass
return d
def _make_world_readable(path):
try:
os.chmod(path, 0o644)
except OSError:
pass
def books_dir():
return os.path.join(store_dir(), "books")
def config_path():
return os.path.join(store_dir(), "config.json")
_RENAMED_KEYS = {
# old name -> new. "collector_groups" read like "the groups themselves";
# it is the file the group decisions are imported FROM.
"collector_groups": "collector_import_file",
}
def load_config():
try:
with open(config_path()) as f:
cfg = json.load(f)
except Exception:
return {}
# migrate renamed keys in memory, so an existing config keeps working
for old_k, new_k in _RENAMED_KEYS.items():
if old_k in cfg and not cfg.get(new_k):
cfg[new_k] = cfg.pop(old_k)
return cfg
def save_config(cfg):
_ensure_store()
with open(config_path(), "w") as f:
json.dump(cfg, f, indent=2)
_make_world_readable(config_path())
def default_version():
"""The book used for installing packages on a running system."""
return load_config().get("default")
def build_version():
"""The book used to BUILD the system (chapters 5-9).
Deliberately separate from `default`: you might build the system from 12.4
but then install packages on it from a newer book, or pin the build to the
exact version you started with while the default moves on. Falls back to
`default` when unset, so nothing changes for anyone who doesn't care."""
cfg = load_config()
return cfg.get("build_book") or cfg.get("default")
def book_path(ver):
return os.path.join(books_dir(), f"LFS-BOOK-{ver}-NOCHUNKS.html")
def wget_list_path(ver):
return os.path.join(books_dir(), f"wget-list-{ver}.txt")
def cached_versions():
out = []
try:
for f in sorted(os.listdir(books_dir())):
m = re.match(r"LFS-BOOK-(.+)-NOCHUNKS\.html$", f)
if m:
out.append(m.group(1))
except OSError:
pass
return out
def _http_get(url, binary=False):
import urllib.request
timeout = float(os.environ.get("LFS_HTTP_TIMEOUT", "30"))
req = urllib.request.Request(url, headers={"User-Agent": "lfs-tool"})
with urllib.request.urlopen(req, timeout=timeout) as r:
data = r.read()
return data if binary else data.decode("utf-8", "replace")
def _download_to(url, dest):
_ensure_store()
data = _http_get(url, binary=True)
with open(dest, "wb") as f:
f.write(data)
_make_world_readable(dest)
return len(data)
_IPV4_ONLY = False
def _force_ipv4():
"""From now on, resolve names to IPv4 only (this process)."""
global _IPV4_ONLY
if _IPV4_ONLY:
return
import socket
_orig = socket.getaddrinfo
def _v4(host, port, family=0, *a, **k):
return _orig(host, port, socket.AF_INET, *a, **k)
socket.getaddrinfo = _v4
_IPV4_ONLY = True
print("\n (IPv6 is not usable from here -- Errno 99 -- downloading over"
" IPv4 from now on)", flush=True)
def _alt_urls(url):
"""Other places that serve the same file.
ftp.gnu.org is overloaded and refuses often; ftpmirror.gnu.org is GNU's
own redirector to a mirror near you, with the same paths below /gnu/.
download.savannah.gnu.org (acl, attr) has a mirror host of its own."""
out = []
for pre, alt in (("https://ftp.gnu.org/gnu/", "https://ftpmirror.gnu.org/"),
("http://ftp.gnu.org/gnu/", "https://ftpmirror.gnu.org/"),
("https://download.savannah.gnu.org/releases/",
"https://download-mirror.savannah.gnu.org/releases/"),
("https://download.savannah.nongnu.org/releases/",
"https://download-mirror.savannah.gnu.org/releases/")):
if url.startswith(pre):
out.append(alt + url[len(pre):])
return out
def _wget_to(url, dest):
"""The host's wget, when Python's own fetch keeps failing: it has its own
address fallback and retry logic, and it is what got through by hand."""
import shutil as _sh
w = _sh.which("wget")
if not w:
return None
part = dest + ".part"
r = subprocess.run([w, "-q", "--tries=3", "--timeout=30", "-O", part, url])
if r.returncode == 0 and os.path.getsize(part) > 0:
os.replace(part, dest)
_make_world_readable(dest)
return os.path.getsize(dest)
try:
os.remove(part)
except OSError:
pass
return None
def _fetch_first(sources, dest, tries=3):
"""Download from the first source that has the file: (bytes, src, err).
ONE TRY WAS NOT ENOUGH. The GNU servers drop connections often enough
that a whole run came back with
acl-2.4.0.tar.xz: FAILED: <urlopen error [Errno 99] Cannot assign
requested address>
for half the GNU tarballs, while the next file from the same host came
through. A network error is retried with a pause; an HTTP answer (404:
this mirror does not carry the file) is final for THAT source and the
next one is asked at once -- the mirror miss for a self-made book label
like 13.1-sysv costs nothing."""
import urllib.error
import errno as _errno
err = None
srcs = list(sources)
for s_ in list(srcs):
for alt in _alt_urls(s_):
if alt not in srcs:
srcs.append(alt)
for src in srcs:
attempt = 0
while attempt < tries:
try:
return _download_to(src, dest), src, None
except urllib.error.HTTPError as e:
err = e
break
except Exception as e: # URLError, timeout, reset
err = e
# IPv6 WITHOUT A USABLE ADDRESS. Errno 99 on connect means
# the resolver handed out an IPv6 address this machine cannot
# send from; Python reports that FIRST error and hides what
# IPv4 did. Switch to IPv4 and retry at once, uncounted.
_r = getattr(e, "reason", e)
if getattr(_r, "errno", None) in (_errno.EADDRNOTAVAIL,
_errno.ENETUNREACH) \
and not _IPV4_ONLY:
_force_ipv4()
continue
attempt += 1
if attempt < tries:
time.sleep(3 if attempt == 1 else 8)
for src in srcs: # last resort: wget
n = _wget_to(src, dest)
if n:
return n, src, None
return None, None, err
DEFAULT_MIRROR = "https://lfs.gnlug.org/pub/lfs/lfs-packages/{book}"
def mirror_base(ver):
"""Base URL of a full package mirror, tried BEFORE the wget-list URL.
Upstream links rot (ncurses' snapshot vanishes from invisible-mirror), while
the LFS package mirrors keep the exact versions a book release needs."""
cfg = load_config()
if cfg.get("mirror") == "off":
return None
tmpl = cfg.get("mirror") or DEFAULT_MIRROR
return tmpl.replace("{book}", ver)
def remote_versions():
"""All book versions listed on the LFS downloads index (10.0, 10.0-systemd,
11.0-rc2, ...)."""
html = _http_get(f"{LFS_SITE}/lfs/downloads/")
vers = re.findall(r'href="([^"/]+)/"', html)
return [v for v in vers if re.match(r"\d", v)]
def _find_nochunks_link(page_html):
m = re.search(r'href="([^"]*NOCHUNKS\.html)"', page_html)
return m.group(1) if m else None
def md5sums_path(ver):
return os.path.join(books_dir(), f"md5sums-{ver}.txt")
def _fetch_version(ver, set_default=False, quiet=False):
"""Download a book's NOCHUNKS html + plain wget-list + md5sums + the
lfs-bootscripts tarball into the store."""
base = f"{LFS_SITE}/lfs/downloads/{ver}/"
page = _http_get(base)
link = _find_nochunks_link(page)
if not link:
raise RuntimeError(f"no NOCHUNKS.html at {base}")
book_url = link if link.startswith("http") else base + link.lstrip("/")
n = _download_to(book_url, book_path(ver))
if not quiet:
print(f" book: {n} bytes -> {book_path(ver)}")
# always the plain wget-list (not -sysv / .old)
for name, dest, why in (
("wget-list", wget_list_path(ver), "wget-list"),
("md5sums", md5sums_path(ver), "md5sums")):
try:
b = _download_to(base + name, dest)
if not quiet:
print(f" {why}: {b} bytes -> {dest}")
except Exception as e:
sys.stderr.write(f" ({why} not fetched: {e})\n")
# the lfs-bootscripts tarball (filename carries a date; read it off the page)
m = re.search(r'href="(lfs-bootscripts-[^"]+\.tar\.[a-z0-9]+)"', page)
if m:
fn = m.group(1)
try:
b = _download_to(base + fn, os.path.join(books_dir(), fn))
if not quiet:
print(f" bootscripts: {b} bytes -> {os.path.join(books_dir(), fn)}")
except Exception as e:
sys.stderr.write(f" (bootscripts not fetched: {e})\n")
if set_default or not default_version():
cfg = load_config()
cfg["default"] = ver
save_config(cfg)
if not quiet:
print(f" default book set to {ver}")
def resolve_book(args):
"""Return (label, html_path) for the book this run should use:
--book-file wins; else --book <version>; else the cached default. A named
version that isn't cached is fetched automatically."""
if getattr(args, "book_file", None):
return (os.path.basename(args.book_file), args.book_file)
# `--book` always wins; otherwise which book depends on what is being
# done: building the system, or installing onto a running one.
ver = (getattr(args, "book", None)
or (build_version() if getattr(args, "_use_build_book", False)
else default_version()))
if not ver:
sys.stderr.write(
"no book selected. See what's available and pick one:\n"
" lfs books # list cached + all site versions\n"
" lfs fetch 12.4 # download + cache (becomes default)\n"
"or import a local file:\n"
" lfs import LFS-BOOK-12.4-NOCHUNKS.html --version 12.4\n")
sys.exit(2)
p = book_path(ver)
if not os.path.isfile(p):
sys.stderr.write(f"book {ver} not cached -- fetching from {LFS_SITE}...\n")
try:
_fetch_version(ver, quiet=True)
except Exception as e:
sys.stderr.write(f"could not fetch {ver}: {e}\n"
f" see available versions: lfs books\n")
sys.exit(2)
return (ver, p)
def resolve_book_arg(val):
"""An argument that is either a local file path or a cached version."""
if os.path.isfile(val):
return val
if os.path.isfile(book_path(val)):
return book_path(val)
sys.stderr.write(f"not a file or cached version: {val}\n")
sys.exit(2)
# --------------------------------------------------------------------------- #
# book loading + section walking
# --------------------------------------------------------------------------- #
_SOUP_CACHE = {}
def load_soup(path):
"""Parse a book, once per path per process.
Without the cache `_find_lfs_package` re-parsed the whole book for EVERY
name it was asked about -- `lfs update` with 77 packages parsed a 10 MB
HTML file 77 times, which is minutes of work and long enough for a caller
with a timeout to give up on it."""
key = os.path.abspath(path)
if key in _SOUP_CACHE:
return _SOUP_CACHE[key]
with open(path, "rb") as f:
html = f.read()
soup = None
for parser in ("lxml", "html.parser"):
try:
soup = BeautifulSoup(html, parser)
break
except Exception:
continue
if soup is None:
soup = BeautifulSoup(html, "html.parser")
_SOUP_CACHE[key] = soup
return soup
def _is_admonition(tag):
"""A note/tip/caution/warning box -- its commands are illustrative."""
cls = getattr(tag, "get", lambda _k: None)("class") or []
return bool({"admon", "note", "tip", "caution", "warning",
"important"} & set(cls))
def _heading_anchor_id(tag):
"""The section id of a heading (hN) is the id of the <a id> it contains."""
a = tag.find("a", id=True)
return a["id"] if a else None
_HEADING_RE = re.compile(r"^h[1-6]$")
def parse_sections(soup):
"""Linear walk over the flat/stitched book: a heading (h1-h6) carrying an
<a id> starts a section; every <pre class="userinput"> until the next such
heading is one of its command blocks. Returns an ordered list of dicts:
{id, title, level, commands:[str], heading:<tag>}."""
sections, cur = [], None
for tag in soup.find_all([_HEADING_RE, "pre"]):
if tag.name != "pre":
aid = _heading_anchor_id(tag)
if aid:
cur = {"id": aid,
"title": re.sub(r"\s+", " ", tag.get_text(" ", strip=True)),
"level": int(tag.name[1]), "commands": [], "heading": tag}
sections.append(cur)
continue
if cur is None:
continue
classes = tag.get("class") or []
# `userinput` is the usual class, but not the only one the book uses
# for commands. 9.9's `cat > /etc/shells` block is class "root", so
# cfg_shells resolved to ZERO commands and /etc/shells was never
# written -- silently, because an empty step looked like a finished
# one. "install" covers 9.4.1.2's optional udev rules.
# NOT "screen": that class is output samples (65 of them here).
if any(c in classes for c in ("userinput", "root", "install")):
# Skip commands inside admonitions (note/tip/caution/warning).
# Those are conditional examples, not part of the build -- GMP's
# note shows "ABI=32 ./configure ..." for 32-bit hosts, and running
# that literally passes an ellipsis to configure:
# Invalid configuration '...': machine '...-unknown' not recognized
if any(_is_admonition(p) for p in tag.parents):
continue
# the whole <pre class="userinput"> is the command block as the reader
# would copy it; taking per-<kbd> text splits inline tokens wrongly.
text = tag.get_text("", strip=False).replace("\u00a0", " ").rstrip()
if text.strip():
cur["commands"].append(text)
return sections
def iter_sections(soup):
"""Yield (id, title, section_dict) in book order."""
for sec in parse_sections(soup):
yield sec["id"], sec["title"], sec
def section_commands(sec):
return sec["commands"]
# --------------------------------------------------------------------------- #
# commands: subcommands
# --------------------------------------------------------------------------- #
def _explain_fetch_failure(e):
"""Turn a network failure into something worth reading.
Inside a fresh chroot the usual cause is that no CA certificates exist yet
-- they come from BLFS' make-ca, not from LFS -- and the raw exception
("CERTIFICATE_VERIFY_FAILED ... unable to get local issuer certificate")
reads like something is broken when nothing is."""
msg = str(e)
if "CERTIFICATE_VERIFY_FAILED" in msg or "SSLCertVerification" in msg:
return ("(cannot verify HTTPS certificates: this system has no CA\n"
" certificates yet -- they come from BLFS' make-ca, not from\n"
" LFS. The cached book still works: lfs books --local)")
if "Name or service not known" in msg or "Temporary failure in name" in msg:
return ("(no DNS: nothing can be resolved from here.\n"
" In the chroot: lfs build-system chroot resolv\n"
" Or work offline: lfs books --local)")
if "Network is unreachable" in msg or "Connection refused" in msg:
return ("(no network from here -- work offline with: lfs books --local)")
return "(could not reach %s: %s -- use --local to skip)" % (LFS_SITE, e)
def cmd_books(args):
cfg = load_config()
default = cfg.get("default")
cached = set(cached_versions())
print(f"Cached books (in {books_dir()}):")
if not cached:
print(" (none)")
for v in sorted(cached):
mark = " *" if v == default else " "
wl = " [wget-list]" if os.path.isfile(wget_list_path(v)) else ""
print(f" {mark} {v}{wl}")
if default:
print(f"default: {default}")
if args.local:
return
try:
vers = remote_versions()
except Exception as e:
print("\n" + _explain_fetch_failure(e))
return
sysv = [v for v in vers if "-systemd" not in v]
systemd = [v for v in vers if "-systemd" in v]
print(f"\nAvailable on the site ({len(vers)}): * = cached")
print(" SysV:")
for v in sysv:
print(f" {'*' if v in cached else ' '} {v}")
print(" systemd:")
for v in systemd:
print(f" {'*' if v in cached else ' '} {v}")
print("\nFetch: lfs fetch <version> One-off: lfs --book <version> ... "
"(auto-fetches)")
# --------------------------------------------------------------------------- #
# snapshots: save the build so a bad step can be undone
# --------------------------------------------------------------------------- #
# Building LFS is a long sequence of irreversible steps. One bad package, one
# mistaken chown, and the choice is "start over" or "hope". A snapshot makes
# it "go back to before that".
#
# What is saved: the whole $LFS tree EXCEPT /sources (large, re-downloadable,
# and unchanged by a build) and the virtual filesystems (/dev /proc /sys /run,
# which are the host's and must never be captured or restored). Ownership,
# permissions, symlinks, hardlinks and sparse files are preserved exactly --
# the package-user model IS ownership, so a backup that loses it is worthless.
def snapshots_dir():
cfg = load_config()
d = cfg.get("snapshot_dir") or os.path.join(store_dir(), "snapshots")
os.makedirs(d, exist_ok=True)
return d
_SNAP_EXCLUDE = ["./dev/*", "./proc/*", "./sys/*", "./run/*", "./sources/*",
"./tmp/*", "./var/tmp/*"]
def _snapshot_path(name):
return os.path.join(snapshots_dir(), "%s.tar" % name)
def _list_snapshots():
out = []
for f in sorted(os.listdir(snapshots_dir())):
if not f.endswith((".tar", ".tar.zst", ".tar.gz")):
continue
p = os.path.join(snapshots_dir(), f)
meta = p.rsplit(".tar", 1)[0] + ".json"
info = {}
if os.path.isfile(meta):
try:
info = json.load(open(meta))
except ValueError:
pass
out.append((f.rsplit(".tar", 1)[0], p, os.path.getsize(p), info))
return out
def _tar_compressor():
"""zstd if available (fast, and these trees are big), else gzip."""
import shutil as _sh
if _sh.which("zstd"):
return "--zstd", ".tar.zst"
return "--gzip", ".tar.gz"
def _build_is_finished(lfs):
"""Has the build run to the end of its own step order?
The same question lfs-sanity.sh asks, answered the same way: every step in
progress/steporder is in progress/steps-built. Nothing else in the tree
distinguishes "chapter 6, and something went wrong" from "finished", and
several checks mean opposite things on either side of that line.
"""
prog = pkgusr_state(lfs, "progress")
try:
order = [l for l in open(os.path.join(prog, "steporder")) if l.strip()]
built = [l for l in open(os.path.join(prog, "steps-built")) if l.strip()]
except OSError:
return False
return bool(order) and len(built) >= len(order)
def _toolchain_state(lfs):
"""What state is the toolchain in, judged from the tree alone?
Checkable from the host, without entering the chroot: the C++ headers must
exist under the target the tree was built for.
Returns a short phrase for the snapshot metadata.
"""
import glob as _glob
tgt = lfs_tgt()
finished = _build_is_finished(lfs)
inc = _glob.glob(os.path.join(lfs, "usr", "include", "c++", "*"))
if not inc:
return "chapter 6 unfinished (no C++ headers)"
have = [os.path.basename(d) for p in inc
for d in _glob.glob(os.path.join(p, "*"))
if os.path.isfile(os.path.join(d, "bits", "c++config.h"))]
native = _glob.glob(os.path.join(lfs, "usr", "lib", "gcc",
"*-pc-linux-gnu"))
# A NATIVE GCC MEANS OPPOSITE THINGS ON EITHER SIDE OF THE BUILD.
#
# Chapter 6 installs only under $LFS_TGT, so during the build a native gcc
# on top of the cross one is the toolchain being clobbered. Chapter 8
# REPLACES the cross toolchain with exactly that native gcc -- it is the
# finished state, the thing the build is for. Reporting one rule for both
# told someone snapshotting a complete, working system:
# toolchain: BROKEN: a native gcc (x86_64-pc-linux-gnu) is installed
# ! this snapshot is of a tree whose toolchain is already broken.
# about a tree with nothing wrong with it. Same species as the sticky-bit
# check in lfs-sanity.sh, which had to learn the same distinction.
if native and tgt not in [os.path.basename(n) for n in native]:
if finished:
return ("ok (native %s -- chapter 8 replaced the cross toolchain)"
% os.path.basename(native[0]))
return ("BROKEN: a native gcc (%s) is installed over the cross one, "
"and the build has not finished"
% os.path.basename(native[0]))
if not have:
return "chapter 6 unfinished (no c++config.h)"
if tgt not in have:
# Finished, and the headers moved to the native triplet: chapter 8
# again, not damage.
if finished and native:
return "ok (C++ headers under %s -- finished system)" % ", ".join(have)
return "BROKEN: headers are under %s, but LFS_TGT is %s" % (
", ".join(have), tgt)
return "ok (C++ headers under %s)" % tgt
def cmd_snapshot(args):
action = args.action or "list"
if action == "list":
snaps = _list_snapshots()
if not snaps:
print("No snapshots yet. Take one before a risky step:")
print(" %slfs snapshot save before-gcc --run" % sudo_prefix())
return
print("Snapshots in %s:\n" % snapshots_dir())
for name, path, size, info in snaps:
print(" %-24s %8s %s" % (name, _human_size(size),
info.get("when", "")))
if info.get("note"):
print(" %-24s %s" % ("", info["note"]))
if info.get("progress"):
print(" %-24s %s" % ("", info["progress"]))
if info.get("toolchain"):
print(" %-24s %s" % ("", info["toolchain"]))
print("\nRestore one with: %slfs snapshot restore <name> --run"
% sudo_prefix())
return
lfs = require_mounted_lfs()
if action == "save":
name = args.name or time.strftime("%Y%m%d-%H%M%S")
comp, ext = _tar_compressor()
dest = os.path.join(snapshots_dir(), name + ext)
if os.path.exists(dest) and not args.force:
sys.stderr.write("a snapshot named '%s' already exists "
"(use --force to replace it)\n" % name)
sys.exit(1)
excl = " ".join("--exclude='%s'" % e for e in _SNAP_EXCLUDE)
print("Save %s -> %s" % (lfs, dest))
print(" excluded: /sources (re-downloadable) and the virtual "
"filesystems")
if not args.run:
dry_run_note("save it")
return
# --numeric-owner: inside the chroot the uids have no names yet, and
# names would be resolved against the HOST's /etc/passwd -- restoring
# that would silently reassign every package user's files.
# --one-file-system: do not descend into anything mounted inside the
# tree. Excluding "./sys/*" skipped the CONTENTS but still archived
# the ./sys directory itself, and reading a live kernel filesystem's
# metadata gave
# tar: ./sys: file changed as we read it
# and exit 1. The mount points are still recorded as empty
# directories, which is exactly what a restore needs.
cmd = ("tar -C '%s' %s --numeric-owner --acls --xattrs --sparse "
"--one-file-system %s -cf '%s' ." % (lfs, comp, excl, dest))
rc = _run_bash(cmd)
if rc == 1:
# tar's exit 1 is "some files differ from what was archived" -- on
# a live tree that is a warning, not a failure. Verify the archive
# rather than throwing away a snapshot that is probably fine.
print("# tar reported changes during the read -- checking the "
"archive ...")
if _run_bash("tar -tf '%s' >/dev/null 2>&1" % dest) == 0:
print("# the archive is readable and complete.")
rc = 0
else:
sys.stderr.write("the archive is not readable -- discarding\n")
try:
os.remove(dest)
except OSError:
pass
if rc != 0:
sys.stderr.write("snapshot failed (tar exited %d)\n" % rc)
sys.exit(rc)
prog = _load_crosschain_progress(lfs) or {}
done = len(prog.get("done", []))
state = _toolchain_state(lfs)
meta = {
"when": time.strftime("%Y-%m-%d %H:%M:%S"),
"lfs": lfs,
"book": build_version(),
"note": args.note or "",
"toolchain": state,
"state": "; ".join("%s %s" % (k, v) for k, v in _tree_state(lfs)
if k != "toolchain"),
"progress": "chapters 5-6: %d/%d steps"
% (done, len(CROSSCHAIN_STEPS)) if prog else "",
}
with open(os.path.join(snapshots_dir(), name + ".json"), "w") as f:
json.dump(meta, f, indent=2)
print("\nsaved %s (%s)" % (name, _human_size(os.path.getsize(dest))))
print(" toolchain: %s" % state)
if state.startswith("BROKEN"):
warn("\n! this snapshot is of a tree whose toolchain is already "
"broken.")
warn(" Restoring it later will restore the breakage too.")
return
if action == "restore":
if not args.name:
sys.stderr.write("which snapshot? see: lfs snapshot list\n")
sys.exit(2)
match = [s for s in _list_snapshots() if s[0] == args.name]
if not match:
sys.stderr.write("no snapshot named '%s'\n see: lfs snapshot "
"list\n" % args.name)
sys.exit(1)
name, path, size, info = match[0]
warn("!! This REPLACES the current build at %s" % lfs)
print()
print(" restore: %s (%s, %s)" % (name, _human_size(size),
info.get("when", "")))
if info.get("progress"):
print(" %s" % info["progress"])
print(" onto: %s" % lfs)
print()
print("Everything built since that snapshot is lost.")
print("/sources is left alone (it was never captured).")
if not args.run:
dry_run_note("restore it")
return
# Restoring into a tree with the virtual filesystems mounted would
# write through them onto the HOST. Refuse rather than risk it.
mounted = [d for d in ("dev", "proc", "sys", "run")
if _is_mounted(os.path.join(lfs, d))]
if mounted:
fail(
"\n! these are still mounted in the tree: %s"
" Restoring now could write through them onto this system.\n"
" Unmount first:\n"
" %slfs build-system chroot unmount --run\n"
% (", ".join(mounted), sudo_prefix()))
sys.exit(2)
if not args.yes:
try:
ans = input("Type the snapshot name to confirm: ").strip()
except (EOFError, KeyboardInterrupt):
print()
return
if ans != name:
print("Not confirmed -- nothing was changed.")
return
# Remove the current contents (except /sources) and unpack. Deleting
# first matters: tar alone would leave files the snapshot does not
# contain, giving a tree that is neither the old state nor the new one.
keep = "sources"
print("\nclearing the tree (keeping /%s) ..." % keep)
rc = _run_bash(
"shopt -s dotglob; for e in '%s'/*; do "
" b=\"$(basename \"$e\")\"; "
" case \"$b\" in %s|dev|proc|sys|run) continue ;; esac; "
" rm -rf \"$e\"; done" % (lfs, keep))
print("unpacking %s ..." % name)
rc = _run_bash("tar -C '%s' --numeric-owner --acls --xattrs "
"-xf '%s'" % (lfs, path))
if rc != 0:
sys.stderr.write("restore failed (tar exited %d)\n" % rc)
sys.exit(rc)
print("\nrestored %s" % name)
print()
next_step("%slfs build-system run" % sudo_prefix(),
label="continue with:")
return
if action == "remove":
if not args.name:
sys.stderr.write("which snapshot? see: lfs snapshot list\n")
sys.exit(2)
match = [s for s in _list_snapshots() if s[0] == args.name]
if not match:
sys.stderr.write("no snapshot named '%s'\n" % args.name)
sys.exit(1)
name, path, size, _i = match[0]
print("remove %s (%s)" % (name, _human_size(size)))
if not args.run:
dry_run_note("remove it")
return
os.remove(path)
meta = path.rsplit(".tar", 1)[0] + ".json"
if os.path.isfile(meta):
os.remove(meta)
print("removed %s" % name)
def cmd_reset(args):
"""Delete the tools' configuration so the next run starts fresh.
Deliberately narrow: this removes what `lfs` and `packagemanager` remember
-- settings, the step list, generated scripts -- and NOTHING else. It does
not touch the LFS tree, the downloaded sources, the cached books, or any
package user. Wiping a half-built system is never what "start over with
the settings" should mean."""
store = store_dir()
targets = [
(os.path.join(store, "config.json"), "settings (book, mount, device, prefix)"),
(os.path.join(store, "session.json"), "the saved session"),
("/etc/pkgusr/packagemanager.conf", "packagemanager's settings"),
]
lfs_mount = load_config().get("lfs_mount") or os.environ.get("LFS") or ""
if lfs_mount:
targets.append((os.path.join(lfs_mount, PKGUSR_DIR),
"the in-chroot step list and generated scripts"))
present = [(p, d) for p, d in targets if os.path.exists(p)]
if not present:
print("Nothing to clear -- no configuration found.")
return
warn("!! This will DELETE the configuration listed below.")
print()
for p, d in present:
kind = "dir " if os.path.isdir(p) else "file"
print(" %s %s" % (kind, p))
print(" %s" % d)
print()
print("Kept, untouched:")
print(" the LFS tree and everything built in it")
print(" downloaded sources and cached books")
print(" every package user and its files")
if not args.run:
dry_run_note("delete them")
return
if not args.yes:
warn("\nThis cannot be undone.")
try:
ans = input("Type 'reset' to confirm: ").strip()
except (EOFError, KeyboardInterrupt):
print()
return
if ans != "reset":
print("Not confirmed -- nothing was deleted.")
return
import shutil as _sh
n = 0
for p, _d in present:
try:
if os.path.isdir(p):
_sh.rmtree(p)
else:
os.remove(p)
n += 1
except OSError as e:
sys.stderr.write("could not remove %s: %s\n" % (p, e))
print("\ncleared %d item(s)." % n)
print()
next_step("%slfs build-system run" % sudo_prefix(), label="start again with:")
def cmd_bs_set_default(args):
"""Choose the book the SYSTEM is built from.
Separate from `lfs set-default`, which chooses the book packages are
installed from on a running system. One command that quietly meant two
different things was the confusing part."""
ver = args.version
if not os.path.isfile(book_path(ver)):
known = []
try:
known = list(remote_versions())
except Exception:
known = []
if known and ver not in known:
sys.stderr.write("no such book: %s\n see them all: lfs books\n" % ver)
sys.exit(1)
sys.stderr.write("note: %s isn't downloaded yet -- fetching it now\n" % ver)
try:
_fetch_version(ver, set_default=False)
except Exception as e:
sys.stderr.write("fetch failed: %s\n" % e)
sys.exit(1)
cfg = load_config()
cfg["build_book"] = ver
save_config(cfg)
print(f"book for BUILDING the system set to {ver}")
if cfg.get("default") and cfg["default"] != ver:
print(f" (packages are installed from {cfg['default']} -- "
f"lfs set-default changes that)")
def cmd_set_default(args):
"""Make a book the default -- but only if it actually exists.
This used to warn and save anyway, so a typo became the configured book and
every later command failed with a 404 far from where the mistake was made
("book ls not cached -- fetching ... 404"). A setting that cannot work
should not be accepted."""
ver = args.version
if not os.path.isfile(book_path(ver)):
# not cached: is it even a real version?
known = []
try:
known = list(remote_versions())
except Exception:
known = [] # offline: fall back to the cache
if known and ver not in known:
sys.stderr.write("no such book: %s\n" % ver)
near = [k for k in known if ver in k or k.startswith(ver)][:5]
if near:
sys.stderr.write("did you mean: %s\n" % ", ".join(near))
sys.stderr.write(" see them all: lfs books\n")
sys.exit(1)
if not known:
cached = sorted(cached_versions())
if ver not in cached:
sys.stderr.write(
"no such book: %s (and %s could not be reached to check)\n"
% (ver, LFS_SITE))
if cached:
sys.stderr.write(" downloaded already: %s\n"
% ", ".join(cached))
sys.exit(1)
sys.stderr.write("note: %s isn't downloaded yet -- fetching it now\n"
% ver)
try:
_fetch_version(ver, set_default=False)
except Exception as e:
sys.stderr.write("fetch failed: %s\n" % e)
sys.exit(1)
cfg = load_config()
cfg["default"] = ver
save_config(cfg)
print(f"book for installing packages set to {ver}")
def cmd_fetch(args):
try:
print(f"fetching {args.version} from {LFS_SITE} ...")
_fetch_version(args.version, set_default=args.set_default)
except Exception as e:
sys.stderr.write(f"fetch failed: {e}\n see: lfs books\n")
sys.exit(1)
def cmd_import(args):
import shutil
ver = args.version
try:
_ensure_store()
dest = book_path(ver)
shutil.copy(args.file, dest)
_make_world_readable(dest)
except PermissionError:
sys.stderr.write(f"cannot write to {books_dir()} -- run this as root "
f"(the store is shared for all users).\n")
sys.exit(1)
print(f"imported book -> {dest}")
if args.wget_list:
shutil.copy(args.wget_list, wget_list_path(ver))
_make_world_readable(wget_list_path(ver))
print(f"imported wget-list -> {wget_list_path(ver)}")
if args.set_default or not default_version():
cfg = load_config()
cfg["default"] = ver
save_config(cfg)
print(f"book for installing packages set to {ver}")
def cmd_sources(args):
"""Print the package source URLs (from the cached wget-list)."""
ver = args.book or default_version()
if not ver:
sys.stderr.write("no book selected (see: lfs books)\n")
sys.exit(2)
wl = wget_list_path(ver)
if not os.path.isfile(wl):
sys.stderr.write(f"no wget-list cached for {ver}. Re-fetch: lfs fetch {ver}\n")
sys.exit(1)
with open(wl) as f:
sys.stdout.write(f.read())
def cmd_sections(args):
label, path = resolve_book(args) if not args.book_pos else \
(args.book_pos, resolve_book_arg(args.book_pos))
soup = load_soup(path)
n = 0
for sid, title, _sec in iter_sections(soup):
print(f"{sid}\t{title}")
n += 1
sys.stderr.write(f"{n} section(s) [{label}].\n")
def cmd_commands(args):
label, path = (args.book_pos, resolve_book_arg(args.book_pos)) \
if args.book_pos else resolve_book(args)
soup = load_soup(path)
for sid, title, sec in iter_sections(soup):
if sid == args.section or title == args.section:
sys.stderr.write(f"# {sid} -- {title} [{label}]\n")
for b in section_commands(sec):
print(b)
print()
return
sys.stderr.write(f"section not found: {args.section}\n")
sys.exit(1)
# a chapter-8 package section: id ch-system-<name>, a versioned title, and it
# actually carries build commands (skips the intro / package-management pages).
_PKG_ID_RE = re.compile(r"^ch-system-(.+)$")
_PKG_TITLE_RE = re.compile(r"^\d+\.\d+\.\s+(.*\S-\S.*)$")
def package_name_version(sid, title):
m = _PKG_ID_RE.match(sid)
if not m:
return None
t = _PKG_TITLE_RE.match(title.strip())
if not t:
return None
return m.group(1), t.group(1).strip() # (name, "Glibc-2.42")
def cmd_packages(args):
label, path = resolve_book(args)
soup = load_soup(path)
n = 0
for sid, title, sec in iter_sections(soup):
nv = package_name_version(sid, title)
if not nv or not sec["commands"]:
continue
name, version = nv
print(f"{name}\t{version}\t{sid}")
n += 1
sys.stderr.write(f"{n} chapter-8 package(s) [{label}].\n")
# --------------------------------------------------------------------------- #
# --------------------------------------------------------------------------- #
# build-system: create an LFS system, step by step, straight from the book.
# (install / update commands live alongside this later.)
# --------------------------------------------------------------------------- #
def _section_script(args, section_id, join="\n\n"):
"""(label, title, script) for one section of the main book, or (label,None,None)."""
label, path = resolve_book(args)
soup = load_soup(path)
for sid, title, sec in iter_sections(soup):
if sid == section_id:
return label, title, join.join(sec["commands"])
return label, None, None
def _run_bash(script, cwd=None, strict=False):
import subprocess
flags = ["-e"] if strict else []
# never inherit the caller's cwd blindly -- if it's inside a dir a build
# step deleted, EVERY subprocess we spawn afterwards fails with a cryptic
# "getcwd: cannot access parent directories" error. Default to a dir that
# always exists.
return subprocess.run(["bash", *flags, "-c", script],
cwd=(cwd or "/")).returncode
def _lfs_env_prefix(lfs):
"""The exact variable set the book's ~/.bashrc (4.4) exports, embedded
directly rather than relied on via sourcing (see run_as_lfs). Missing PATH
here would be a real bug: gcc-pass1 needs binutils-pass1's tools on PATH,
and glibc/libstdc++ need the cross compiler -- not just LFS/LFS_TGT."""
if not lfs:
return ""
cfg = load_config()
tgt = cfg.get("lfs_tgt") or "$(uname -m)-lfs-linux-gnu"
mkf = cfg.get("makeflags") or "-j$(nproc)"
return (
# Book 4.4's ~/.bashrc opens with these two, and run_as_lfs embeds this
# prefix instead of sourcing that file -- so without them the build
# runs with bash's command hash on (stale paths to tools that have just
# been replaced) and with whatever umask root's shell had. On a host
# with umask 077 the toolchain installs unreadable.
f'set +h\n'
f'umask 022\n'
f'export LFS="{lfs}"\n'
f'export LC_ALL=POSIX\n'
f'LFS_TGT={tgt}\n'
f'export LFS_TGT\n'
f'PATH=/usr/bin\n'
f'if [ ! -L /bin ]; then PATH=/bin:$PATH; fi\n'
f'PATH="$LFS/tools/bin:$PATH"\n'
f'export PATH\n'
f'LFS_SRC_ROOT="{layout("pkgusr_home", target=True)}"\n'
f'export LFS_SRC_ROOT\n'
f'CONFIG_SITE="$LFS/usr/share/config.site"\n'
f'export CONFIG_SITE\n'
f'export MAKEFLAGS="{mkf}"\n'
+ (f'export PKGUSR_LIB="{lib}"\n' if (lib := _host_phases_lib()) else "")
)
def _host_phases_lib():
"""The phase runner, where the lfs user can read it. The checkout may sit
in /root, so the copy goes into the shared store, refreshed from the one
beside this tool whenever they differ."""
src = _find_helper_source("lfs-phases")
if not src:
return None
try:
import shutil
dest = os.path.join(_ensure_store(), "lfs-phases")
import hashlib
def _sum(f):
with open(f, "rb") as fh:
return hashlib.md5(fh.read()).hexdigest()
if not os.path.isfile(dest) or _sum(src) != _sum(dest):
shutil.copy(src, dest)
_make_world_readable(dest)
return dest
except OSError:
return src
def run_as_lfs(script, lfs=None):
"""Run a script as the lfs user via `su - lfs`. IMPORTANT: `su -` fully
resets the environment (that's what login simulation does), so anything the
root process exported -- $LFS included -- is gone in the lfs shell. We embed
the book's whole env block explicitly at the top of the script instead of
relying on ~/.bashrc being sourced (it usually isn't, in this non-interactive
invocation)."""
import subprocess
lfs = lfs or _lfs_dir()
prefix = _lfs_env_prefix(lfs)
return subprocess.run(["su", "-", "lfs", "-c", "bash -s"], cwd="/",
input=prefix + script, text=True).returncode
def _lfs_dir():
"""The LFS mount point: $LFS if exported, else the saved session config."""
return os.environ.get("LFS") or load_config().get("lfs_mount")
# THE ONE THING THAT COULD DESTROY THE MACHINE YOU ARE BUILDING ON.
#
# Every path in this tool is `$LFS/something`. If $LFS is the running system,
# `$LFS/usr` is the HOST's /usr -- and `build-system restart --run` deletes the
# top-level directories of the tree it is given. Nothing checked that the tree
# was not the host's own: a config holding `/` as the mount point, an
# `export LFS=/` in the wrong shell, or a typo in the interview was enough.
#
# The chroot's bind mounts are already guarded (restart refuses while /dev is
# mounted, because deleting through it would reach the host's /dev). This is
# the other half: refusing when the tree IS the host.
#
# It lives in require_lfs_for_root because that is the single door every
# command resolves $LFS through -- put it anywhere else and the next command
# added is the one that misses it.
_HOST_DIRS = ("/", "/usr", "/etc", "/boot", "/bin", "/sbin", "/lib", "/lib64",
"/var", "/home", "/root", "/opt", "/srv", "/run", "/dev",
"/proc", "/sys",
# merged-/usr: /bin is a symlink to usr/bin, so realpath("/bin")
# is "/usr/bin" and the names above never match it. A check that
# only looked at the resolved path let $LFS=/bin through.
"/usr/bin", "/usr/sbin", "/usr/lib", "/usr/lib64", "/usr/share",
"/usr/include", "/usr/local", "/usr/src")
# Trees that belong to the running system whole. Anything inside one of these
# is the host, however deep -- $LFS=/usr/src/lfs is not a build tree, it is a
# directory in the system's own /usr. /home and /srv are deliberately NOT
# here: /home/you/lfs is a perfectly ordinary place to build.
_HOST_TREES = ("/usr/", "/etc/", "/boot/", "/bin/", "/sbin/", "/lib/",
"/lib64/", "/proc/", "/sys/", "/dev/", "/run/", "/var/")
def _is_host_path(p):
"""True if this path is part of the running system. The test behind both
the interview's refusal and the hard stop below, so they cannot disagree."""
try:
real = os.path.realpath(p)
except OSError:
return False
# BOTH spellings. On a merged-/usr system /bin resolves to /usr/bin, so
# checking only the resolved path misses the name the person typed, and
# checking only the name misses where it actually points.
given = os.path.abspath(os.path.expanduser(str(p))).rstrip("/") or "/"
if real in _HOST_DIRS or given in _HOST_DIRS:
return True
for t in _HOST_TREES:
if real.startswith(t) or given.startswith(t):
return True
try:
if os.path.samefile(real, "/"):
return True
except OSError:
pass
for d in ("/usr", "/etc", "/boot"):
cand = os.path.join(real, d.lstrip("/"))
try:
if os.path.exists(cand) and os.path.samefile(cand, d):
return True
except OSError:
continue
return False
def _refuse_host_tree(lfs):
"""Hard stop if $LFS is, or contains, the running system."""
if not _is_host_path(lfs):
return lfs
real = os.path.realpath(lfs)
fail(
"\n!! REFUSING: %s is part of the running system."
" Every path this tool writes is $LFS/something, so this would\n"
" build into, and `restart` would delete, the machine you are\n"
" sitting on.\n"
"\n Set it to the mount point of the LFS partition:\n"
" lfs build-system session\n" % real)
sys.exit(2)
def require_lfs_for_root():
"""Ensure an LFS mount is known (book rule: root needs $LFS after 2.4). Falls
back to the saved session config and exports it so child shells inherit it."""
d = _lfs_dir()
if not d:
fail('! no LFS mount set (neither $LFS nor a saved session).\n Configure it once: lfs build-system session\n or for this shell: export LFS=/mnt/lfs')
sys.exit(2)
_refuse_host_tree(d)
os.environ["LFS"] = d
return d
def require_mounted_lfs():
"""Resolve $LFS AND verify the configured partition is actually mounted
there. Building against an unmounted $LFS silently operates on a plain
empty directory instead of the real filesystem (confusing 'No such file or
directory' errors deep inside a build), so this is a hard stop with a hint
to the exact command that fixes it -- we already know the device and mount
point from the saved session config. If no device was ever recorded (e.g.
a bind-mount / manual setup), we can't verify a mount, so we don't second-
guess it -- same behavior as require_lfs_for_root()."""
lfs = require_lfs_for_root()
device = load_config().get("lfs_device")
if device and not _is_mounted(lfs):
sys.stderr.write(
f"! {lfs} is not mounted -- the build session isn't active.\n"
f" {device} needs to be mounted there first. Start/rejoin the "
f"session (mounts it, sets $LFS, chowns the tree):\n"
f" lfs build-system session --run\n")
sys.exit(2)
return lfs
_SESSION_KEYS = [
("lfs_mount", "LFS mount point", "/mnt/lfs"),
("lfs_device", "root partition device (e.g. /dev/sda2)", ""),
("lfs_fstype", "root filesystem type", "ext4"),
("lfs_swap", "swap device (optional, blank to skip)", ""),
("lfs_home_device", "separate /home device (optional)", ""),
("collector_prefix", "prefix for collector groups (packages that install "
"into another package's directory join <prefix>_<owner>)",
"sysgroup"),
# Three kinds of account share one passwd file: package users, application
# users (u_) and collector groups. Without a prefix a package called
# `man` or `news` collides with a real system account, and nothing in the
# passwd file says which accounts belong to the build. Blank = no prefix.
("pkgusr_prefix", "prefix for package users (p gives p_gcc, p_zlib; "
"blank for none)", "p"),
# Config steps carry their OWN prefix, not the package one stacked on top
# of it. `p_cfg_bootscripts` wore both: `p_` claimed it was a package
# while the account lived in the config root, and every rule about
# prefixes had to special-case it. One account, one prefix.
("cfguser_prefix", "prefix for config-step users (cfg gives "
"cfg_bootscripts; blank for none)", "cfg"),
# Asked here with the rest of the session settings: a groups export from a
# previous build saves answering the same sharing questions again, and it
# has to be known BEFORE the chroot scripts are generated (that is when the
# file is copied into the tree). Blank means "start fresh".
("collector_import_file", "collector-group export to import from a previous "
"build (blank for none)", ""),
# Asked here rather than left to a default nobody sees. These end up in
# the generated chroot scripts, and a wrong locale or triplet only shows
# up much later -- a mismatched $LFS_TGT means the compiler looks for its
# headers under a target the headers were never installed under.
("lfs_tgt", "cross-compile target triplet ($LFS_TGT)",
"%s-lfs-linux-gnu" % os.uname().machine),
("makeflags", "parallel jobs for builds", "-j%s" % (os.cpu_count() or 1)),
("hostname", "hostname for the built system", "lfs"),
("target_pkgusr_home",
"where package users live on the BUILT system",
"/usr/src"),
("target_appuser_home",
"where application users live on the BUILT system",
"/home/shared_user"),
("locale", "system locale (see 'locale -a')", "en_US.UTF-8"),
("paper_size", "paper size for groff: A4 or letter", "A4"),
# Everything below is a decision the build used to discover halfway
# through, or at the very end -- the old final step blocked on `passwd
# root` after a six-hour build, and its non-interactive path left the
# system "unbootable-but-known". Asking here is what lets `lfs run` and
# `build-all` finish without a person.
("timezone", "time zone for the built system (e.g. Europe/Berlin, "
"or UTC)", "UTC"),
# Deleting a working cross-toolchain should never happen by default, so
# the default here is no. It is removed at the END of the build rather
# than at book 7.13, which keeps it available if chapter 8 has to be redone.
("remove_tools", "delete /tools when the build finishes? It is the "
"cross-toolchain that built the system: 1-3 GB, and "
"your way back if chapter 8 needs redoing (yes/no)",
"no"),
# Off by default and deliberately so: stripping REWRITES each binary, and
# in this system a file's owner is the record of which package installed
# it -- which is why `lfs-helper strip` runs each file as its owner.
("strip", "strip debug symbols at the end? Saves several GB, "
"but rewrites every binary -- slower, and harder to "
"debug a crash afterwards. Runs as `lfs-helper "
"strip --run` when the build finishes (yes/no)",
"no"),
# The account you will actually log in as. The `init-accounts` step reads
# this; without it, a freshly booted system has only root, with no password.
("main_user", "login account to create on the built system "
"(blank for none)", ""),
]
def _is_mounted(path):
try:
real = os.path.realpath(path)
with open("/proc/mounts") as f:
return any(ln.split()[1] == real for ln in f if len(ln.split()) > 1)
except OSError:
return False
def _prompt_session_config(cfg, force=False):
"""Ask for each session setting (keeping saved answers as defaults) + save."""
if not sys.stdin.isatty():
return cfg
for k, desc, default in _SESSION_KEYS:
cur = cfg.get(k, default)
if k == "collector_import_file" and not cur:
# offer the one install-tools wrote last time, if it is still there
guess = os.path.join(store_dir(), "collector-groups.export")
if os.path.isfile(guess):
cur = guess
if cfg.get(k) and not force:
continue
val = input(f" {desc} [{cur}]: ").strip()
val = val if val else cur
# Refuse a mount point that is the running system, HERE, where you can
# simply type another one. The guard in require_lfs_for_root catches
# it later too, but by then it is stored, every command refuses, and
# the fix is to work out which config key to edit.
if k == "lfs_mount" and val:
while _is_host_path(val):
fail(
" !! %s is part of the running system -- refusing."
" The LFS mount point is an empty directory the LFS\n"
" partition mounts on, e.g. /mnt/lfs.\n" % val)
val = input(f" {desc} [{cur}]: ").strip() or cur
if k == "collector_import_file" and val:
# Accepting a path that is not there means the groups silently
# never get imported and every sharing question comes back.
p = os.path.expanduser(val)
if not os.path.isfile(p):
sys.stderr.write(" no such file: %s -- leaving it unset\n" % val)
val = ""
else:
val = os.path.abspath(p)
# Say what is actually in it. The prompt echoed a path and
# nothing else, so a file that was configured, present and
# never read looked exactly like one that was working.
try:
txt = open(val).read().splitlines()
except OSError:
txt = []
n_dirs = sum(1 for l in txt if l.startswith("dir|"))
n_groups = sum(1 for l in txt if l.startswith("group|"))
if n_dirs or n_groups:
print(" %d directory decision(s), %d group(s) -- the "
"build will not ask about those again"
% (n_dirs, n_groups))
else:
sys.stderr.write(" that file holds no group decisions "
"(no dir| or group| lines)\n")
print(" it is re-read on every chroot entry, so editing it "
"here reaches the next build")
cfg[k] = val
save_config(cfg)
print(f" saved session config -> {config_path()}")
return cfg
def _heredoc_writes(script):
"""The `cat > FILE << \"TAG\" ... TAG` blocks in a script (the real file
creations), skipping example commands like a bare `make -j32`."""
out = []
for m in re.finditer(r'(cat >>? \S+ << *"?(\w+)"?\n.*?\n\2)\b', script, re.S):
out.append(m.group(1))
return out
def _emit_or_run(args, name, script, needs_lfs=False):
"""Print a step's commands; run them only with --run. Enforces $LFS (book
rule for root after 2.4) and warns if not root."""
if script is None:
sys.stderr.write(f"{name}: section not found in this book.\n")
sys.exit(1)
print(f"# === {name} ===")
print(script)
if not args.run:
if needs_lfs and not _lfs_dir():
fail('\n! no LFS mount set. Configure it: lfs build-system session')
dry_run_note("execute")
return
if needs_lfs:
require_mounted_lfs()
if os.geteuid() != 0:
fail('\n! not root -- these steps write under $LFS / add users and will likely fail.')
rc = _run_bash(script)
if rc != 0:
sys.stderr.write(f"\n{name}: exited {rc}\n")
sys.exit(rc)
print(f"\n# {name}: done")
def cmd_bs_version_check(args):
"""Chapter 2.2 -- run the book's version-check.sh on the HOST (read-only)."""
label, _title, script = _section_script(args, "version-check")
if not script:
sys.stderr.write("version-check section not found in this book.\n")
sys.exit(1)
if args.show:
print(script)
return
import tempfile, shutil
d = tempfile.mkdtemp(prefix="lfs-vc-")
try:
sys.stderr.write(f"# host version check [{label}]\n")
rc = _run_bash(script, cwd=d)
finally:
shutil.rmtree(d, ignore_errors=True)
sys.exit(rc)
def cmd_bs_layout(args):
_emit_or_run(args, "4.2 Creating a Limited Directory Layout",
_section_script(args, "ch-tools-creatingminlayout")[2],
needs_lfs=True)
if args.run:
# Book 4.3's chown belongs HERE, not only in `session`: layout runs as
# root and creates these directories as root. session chowns the tree
# too, but it runs BEFORE layout, when there is nothing to chown yet --
# so without this the tree stays root-owned and chapter 5 dies with
# permission errors it blames on chapter 7 having already happened.
_chown_tree_to_lfs(require_mounted_lfs())
# and the scratch directory chapter 5 unpacks into: $LFS is root-owned,
# so the build user cannot create it itself
_ensure_build_root(require_mounted_lfs())
def _chown_tree_to_lfs(lfs, quiet=False):
"""Hand the build tree to the lfs user (book 4.3). A no-op if there is no
lfs user yet -- add-user calls this again once there is.
/sources is NOT in the set. Book 4.3 chowns the handover directories and
nothing else; /sources is made world-writable in 3.1 instead, and its
contents are explicitly put back to root:root there. Chowning it to `lfs`
was writing a host uid into the tree -- see _normalize_sources.
"""
try:
import pwd
pwd.getpwnam("lfs")
except (KeyError, ImportError):
return False
dirs = [d for d in LFS_HANDOVER_DIRS
if os.path.exists(os.path.join(lfs, d))]
if not dirs:
return False
# NOT on a tree that is past chapter 7.
#
# This is `chown -R lfs` over /usr, /etc, /var and the rest. Before
# chapter 5 that is book 4.3 and exactly right. On a BUILT tree it hands
# the finished system to the host's build account in one pass and every
# package's ownership is gone -- recoverable only from the manifests.
#
# It happened: `lfs build-system run` on a 104/104 tree replayed step 4.2,
# and /usr came back
# drwxrwxr-t 1 lfs install /mnt/lfs/usr
# with the host uid 10753 written through the tree, which inside the chroot
# resolves to nothing at all -- the chroot's own `lfs` is 9998.
#
# _handover_is_safe already answers the question, from the other side: it
# is True exactly while no package user owns anything.
if not _handover_is_safe(lfs):
print("# 4.3 SKIPPED: package users own files here, so this tree is")
print("# past chapter 7. `chown -R lfs` would hand the finished")
print("# system to the build user and lose every package's")
print("# ownership. Nothing was changed.")
print("# If you meant to start over: lfs build-system restart --run")
return False
_run_bash('chown -R lfs "%s"/{%s} 2>/dev/null || true'
% (lfs, ",".join(dirs)))
if not quiet:
print("# gave %s to the lfs user (book 4.3): %s"
% (lfs, ", ".join(dirs)))
return True
def _ensure_lfs_home(run):
"""Make sure the lfs user actually has a home dir (the book's `useradd -m`
should create /home/lfs, but if it didn't the env files would land in / )."""
import subprocess
try:
import pwd
pw = pwd.getpwnam("lfs")
except (KeyError, ImportError):
return
home = pw.pw_dir if pw.pw_dir and pw.pw_dir != "/" else "/home/lfs"
if os.path.isdir(home) and pw.pw_dir == home:
return
print(f"# lfs has no proper home -- creating {home}")
if not run:
return
subprocess.run(["mkdir", "-pv", home], cwd="/")
subprocess.run(["usermod", "-d", home, "lfs"], cwd="/")
subprocess.run(["chown", "lfs:lfs", home], cwd="/")
subprocess.run(["chmod", "755", home], cwd="/")
def _setup_lfs_env(args, run):
"""Chapter 4.4 -- WRITE the lfs user's ~/.bash_profile and ~/.bashrc AS the
lfs user (into ~lfs). This only SETS UP the environment; it does not start
it (the book's `source ~/.bash_profile` is intentionally left out). Only the
file-creation heredocs are used, so the book's inline examples (make -jN,
source ...) are skipped, and LFS= is pinned to the current $LFS."""
_label, _title, script = _section_script(args, "ch-preps-settingenviron")
if not script:
sys.stderr.write("4.4 environment section not found.\n")
return 1
blocks = _heredoc_writes(script)
if not blocks:
sys.stderr.write("4.4: no ~/.bash_profile / ~/.bashrc blocks found.\n")
return 1
combined = "\n".join(blocks)
lfs = os.environ.get("LFS")
if lfs: # pin LFS= to the real path
combined = re.sub(r"^LFS=\S+", f"LFS={lfs}", combined, flags=re.M)
# ...AND THE CONFIGURED TRIPLET AND JOBS. The book's text went in as
# written: LFS_TGT=$(uname -m)-lfs-linux-gnu and MAKEFLAGS=-j$(nproc), on
# a tree configured for x86_64-nimgnu-linux-gnu and -j14. The build
# itself never reads this file (run_as_lfs sets the variables), but a
# `su - lfs` to redo a step by hand would quietly cross-compile for the
# wrong target.
_cfg = load_config()
if _cfg.get("lfs_tgt"):
combined = re.sub(r"^LFS_TGT=.*$", "LFS_TGT=%s" % _cfg["lfs_tgt"],
combined, flags=re.M)
if _cfg.get("makeflags"):
combined = re.sub(r"^export MAKEFLAGS=.*$",
"export MAKEFLAGS=%s" % _cfg["makeflags"],
combined, flags=re.M)
print("# === 4.4 Setting Up the Environment (writes ~lfs config; does NOT "
"start it) ===")
print(combined)
if not run:
dry_run_note("write them")
return 0
require_mounted_lfs()
rc = run_as_lfs(combined)
print(f"\n# 4.4 environment: {'written' if rc == 0 else 'exited ' + str(rc)}")
return rc
def cmd_bs_add_user(args):
"""4.3 create the lfs user (as root), ensure its home exists, then 4.4 write
its environment (as lfs). 4.3 + 4.4 belong together."""
script = _section_script(args, "ch-preps-addinguser")[2] or ""
# Drop the book's INTERACTIVE lines:
# * `su - lfs` -- the tool switches to the lfs user itself for 4.4.
# * `passwd lfs` -- prompts for a password and would hang a non-tty --run.
# The lfs account is a build-only account; the book only sets a password
# so you can `su` to it, which the tool does as root (no password needed).
dropped = []
kept = []
for ln in script.splitlines():
if ln.strip() in ("su - lfs", "su lfs", "passwd lfs"):
dropped.append(ln.strip())
continue
kept.append(ln)
script = "\n".join(kept)
if dropped and not args.run:
print(f"# note: skipping interactive book command(s): "
f"{', '.join(dropped)}")
_emit_or_run(args, "4.3 Adding the LFS User", script, needs_lfs=True)
print()
_ensure_lfs_home(args.run)
if args.run:
# the user now exists, so the tree can finally be handed over -- this
# is a no-op when layout already did it
_chown_tree_to_lfs(require_mounted_lfs())
if args.run:
_verify_lfs_ownership()
print()
rc = _setup_lfs_env(args, args.run)
if rc:
sys.exit(rc)
def _verify_lfs_ownership():
"""The book's 4.3 chown is what makes $LFS writable by the lfs user. If it
silently didn't apply, every later build fails deep inside `make install`
with 'Permission denied' -- so verify it here, where the fix is obvious,
instead of 20 minutes into a compile."""
lfs = _lfs_dir()
if not lfs:
return
# This function runs `chown -R lfs` on $LFS/usr, $LFS/lib64, $LFS/var and
# $LFS/etc. It resolves $LFS itself rather than going through
# require_lfs_for_root, so it was the one destructive path with no check
# that the tree is not the running system -- and on a host, /usr has no
# package users, so _handover_is_safe would have said yes.
_refuse_host_tree(lfs)
import pwd as _pwd
try:
want = _pwd.getpwnam("lfs").pw_uid
except KeyError:
return
bad = []
for sub in ("usr", "lib64", "var", "etc", "tools"):
p = os.path.join(lfs, sub)
if os.path.isdir(p) and not os.path.islink(p):
try:
if os.stat(p).st_uid != want:
bad.append(p)
except OSError:
pass
if not bad:
print(f"# ownership check: {lfs} build dirs belong to lfs -- OK")
return
# Not on a tree that is past chapter 7. This runs before a chapter 5-6
# build to make sure the lfs user can write; on a BUILT tree the same
# recursive chown destroys every package's ownership.
if not _handover_is_safe(lfs):
warn("# ownership check: package users own files here, so this tree is")
warn("# past chapter 7 -- not giving it back to the build user.")
return
print(f"# ownership check: fixing {len(bad)} dir(s) still owned by root ...")
for p in bad:
_run_bash(f'chown -R lfs "{p}"')
still = [p for p in bad if os.path.isdir(p) and os.stat(p).st_uid != want]
if still:
sys.stderr.write(
f"! could not give the lfs user ownership of: {', '.join(still)}\n"
f" Builds WILL fail with 'Permission denied' during make install.\n"
f" Fix manually as root: chown -R lfs {' '.join(still)}\n")
sys.exit(2)
print("# ownership check: fixed -- OK")
def cmd_bs_show(args):
"""Print the raw commands of any build-system section id (for inspection)."""
label, title, script = _section_script(args, args.section)
if script is None:
sys.stderr.write(f"section not found: {args.section}\n")
sys.exit(1)
sys.stderr.write(f"# {args.section} -- {title} [{label}]\n")
print(script)
def cmd_bs_session(args):
"""Start/rejoin an LFS build session: show or ask for the config (partition,
mount point, swap), mount it, set $LFS, and chown the build tree to lfs."""
cfg = load_config()
mount = cfg.get("lfs_mount")
active = mount and _is_mounted(mount)
if active and not args.reconfigure:
print(f"Active session: {mount} is already mounted.")
else:
# SAY WHICH FILE, AND WHOSE PREFIXES.
#
# The user set pkgusr_prefix empty on their HOST and then saw
# pkgusr_prefix = (unset)
# in the session that builds the new system -- the host's setting
# about to name the accounts of a different machine. 1.13.51 stopped
# the host's prefixes reaching the chroot's environment; this is the
# same leak one step earlier, in the session config itself.
#
# A tree that already has a config decides its own prefixes. And the
# file being used is named, because "which config is this?" was the
# question that could not be answered.
print("Session config: (%s)" % config_path())
_tree = cfg.get("lfs_mount") or "/mnt/lfs"
for k, desc, _ in _SESSION_KEYS:
_v = cfg.get(k, "")
_note = ""
if k in ("pkgusr_prefix", "cfguser_prefix", "collector_prefix"):
_tv = _tree_prefix(_tree, k, None)
if _tv is not None and _tv != (_v or "").rstrip("_"):
_v, _note = _tv, " <- from the tree's own config"
# AN EMPTY PREFIX IS A DECISION, NOT A GAP.
#
# "pkgusr_prefix = (unset)" read as "nothing chosen yet" when it
# was in fact an empty value -- and an empty prefix means every
# account in the BUILT system is named plainly (mesa, not
# p_mesa). The display could not tell the two apart, which is
# the same ambiguity that cost 1.13.26 and 1.13.69.
if not _v and k.endswith("_prefix"):
_shown = ("(none -- accounts will be named without a prefix)"
if k in cfg else "(unset -- will ask)")
else:
_shown = _v or "(unset)"
print(f" {k:<16} = {_shown}{_note}")
have_all = all(cfg.get(k) for k, _, d in _SESSION_KEYS
if d != "" or k == "lfs_device")
if sys.stdin.isatty():
if have_all and not args.reconfigure:
keep = input("\nKeep these settings? [Y/n]: ").strip().lower()
if keep in ("n", "no"):
cfg = _prompt_session_config(cfg, force=True)
else:
print("\nSome settings are missing -- let's set them:")
cfg = _prompt_session_config(cfg, force=args.reconfigure)
mount = cfg.get("lfs_mount")
if not mount:
sys.stderr.write("no lfs_mount configured.\n")
sys.exit(2)
os.environ["LFS"] = mount
if not args.run:
print("\n(dry run) would, with --run:")
print(f" mkdir -pv {mount}")
if cfg.get("lfs_device") and not active:
print(f" mount -v -t {cfg.get('lfs_fstype', 'ext4')} "
f"{cfg['lfs_device']} {mount}")
if cfg.get("lfs_swap"):
print(f" /sbin/swapon -v {cfg['lfs_swap']}")
print(f" chown lfs (build dirs under {mount})")
print(f"\nexport LFS={mount} # <-- run this in your shell")
print()
print("Next:")
print(f" {sudo_prefix()}lfs build-system session --run # apply the above")
print(f" {sudo_prefix()}lfs run # then build")
hint(f" {sudo_prefix()}lfs build-system next # just say what comes next")
return
require_lfs_for_root()
if os.geteuid() != 0:
fail('! session --run must be run as root (it mounts partitions and chowns the build tree).')
sys.exit(2)
import subprocess
subprocess.run(["mkdir", "-pv", mount], cwd="/")
if cfg.get("lfs_device") and not _is_mounted(mount):
rc = subprocess.run(["mount", "-v", "-t",
cfg.get("lfs_fstype", "ext4"),
cfg["lfs_device"], mount], cwd="/").returncode
if rc != 0:
sys.stderr.write("mount failed.\n")
sys.exit(rc)
if cfg.get("lfs_home_device"):
subprocess.run(["mkdir", "-pv", os.path.join(mount, "home")], cwd="/")
subprocess.run(["mount", "-v", cfg["lfs_home_device"],
os.path.join(mount, "home")], cwd="/")
if cfg.get("lfs_swap"):
subprocess.run(["/sbin/swapon", "-v", cfg["lfs_swap"]], cwd="/")
# Re-apply the 4.3 chown through _chown_tree_to_lfs, NOT by hand.
#
# This was a second copy of `chown -R lfs` over the build tree, and it had
# neither of the things the real one grew: the refusal on a tree that is
# past chapter 7, and the same directory list. This copy also omitted bin
# and sbin, so the two disagreed about what "the build tree" even means.
#
# `session --reconfigure --run` on a finished system would have handed it
# to the build user in one pass -- the same damage the guard exists to
# prevent, through a door the guard was not on.
_chown_tree_to_lfs(mount, quiet=True)
ok(f"\nSession ready. $LFS={mount}")
print("Put this in your shell (root needs it too):")
print(f" export LFS={mount}")
print()
print("Next:")
print(f" {sudo_prefix()}lfs run # build everything, step by step")
hint(f" {sudo_prefix()}lfs build-system next # just say what comes next")
# ---- chapter 3: sources ---------------------------------------------------- #
def _wget_list_urls(ver):
wl = wget_list_path(ver)
if not os.path.isfile(wl):
return None
urls = []
with open(wl) as f:
for ln in f:
ln = ln.strip()
if ln and not ln.startswith("#"):
urls.append(ln)
return urls
def _book_patch_urls(args):
"""Patch URLs listed in 3.3 Needed Patches (belt-and-suspenders on top of the
wget-list, which already includes patches)."""
_l, path = resolve_book(args)
soup = load_soup(path)
urls = set()
for a in soup.find_all("a", href=True):
h = a["href"]
if h.endswith(".patch") and h.startswith("http"):
urls.add(h)
return urls
def _human_size(n):
for unit in ("B", "KB", "MB", "GB"):
if n < 1024:
return f"{n:.0f}{unit}"
n /= 1024
return f"{n:.0f}TB"
def cmd_bs_get_sources(args):
"""Chapter 3 -- download every package + patch (from wget-list, union the
book's patch links) into $LFS/sources and verify against md5sums."""
ver = args.book or build_version()
urls = _wget_list_urls(ver)
if urls is None:
sys.stderr.write(f"no wget-list cached for {ver}. Run: lfs fetch {ver}\n")
sys.exit(1)
urls = list(dict.fromkeys(urls + sorted(_book_patch_urls(args))))
lfs = require_mounted_lfs() if args.run else (_lfs_dir() or "$LFS")
srcdir = os.path.join(lfs, "sources")
# Whatever the finished system needs before it can fetch anything itself.
#
# LFS ships no download tool, so a freshly booted system cannot get the
# sources for its own next package. `packagemanager setup` installs those,
# but the tarballs have to be here BEFORE the chroot is sealed -- and this
# is the only moment there is still a network.
#
# Resolved from the BLFS BOOK rather than a hardcoded list: the book already
# knows wget's URL and keeps it current, and a second list of download links
# is a second thing to go stale.
extra = bootstrap_sources()
if extra:
urls = list(urls) + [u for u in extra if u not in urls]
npatch = sum(1 for u in urls if u.endswith(".patch"))
print(f"{len(urls)} files ({npatch} patches) -> {srcdir}")
if not args.run:
# print the WHOLE list: a truncated one cannot be checked, diffed or
# fed to anything else, which is the only reason to run a dry run
for u in urls:
print(f" {u}")
dry_run_note("download")
return
os.makedirs(srcdir, exist_ok=True)
_run_bash(f'chmod -v a+wt "{srcdir}" 2>/dev/null || true')
got, skip, nfail = 0, 0, 0
_failed = []
total = len(urls)
for i, u in enumerate(urls, 1):
fname = u.split("/")[-1]
dest = os.path.join(srcdir, fname)
tag = f"[{i}/{total}]"
if os.path.isfile(dest) and not args.force:
skip += 1
print(f"{tag} {fname}: already have it (skip)", flush=True)
continue
print(f"{tag} {fname}: downloading ...", end=" ", flush=True)
t0 = time.time()
# try the mirror first (upstream links rot), then the book's own URL
sources = []
mb = mirror_base(ver)
if mb:
sources.append(mb.rstrip("/") + "/" + fname)
sources.append(u)
n, src, err = _fetch_first(sources, dest)
if n is not None:
got += 1
via = " (mirror)" if mb and src == sources[0] else ""
print(f"{_human_size(n)} in {time.time() - t0:.1f}s{via}", flush=True)
else:
_failed.append((fname, dest, sources))
nfail += 1
print(f"FAILED: {err}", flush=True)
# A SECOND ROUND for what failed: a server that refused three times in a
# row a minute ago often answers now.
if _failed:
print(f"\n{len(_failed)} file(s) failed -- one more round in 20s ...",
flush=True)
time.sleep(20)
for fname, dest, srcs in list(_failed):
print(f" {fname}: downloading ...", end=" ", flush=True)
n, src, err = _fetch_first(srcs, dest)
if n is not None:
got += 1
nfail -= 1
_failed.remove((fname, dest, srcs))
print(_human_size(n), flush=True)
else:
print(f"FAILED: {err}", flush=True)
print(f"\ndownloaded {got}, skipped {skip}, failed {nfail}")
if nfail:
print(" still missing: " + ", ".join(f for f, _d, _s in _failed))
print(" run it again -- files already here are skipped:")
print(" lfs build-system get-sources --run")
# Book 3.1, second half: `chown root:root $LFS/sources/*`. Whatever ran
# the download owns the files, and that uid means nothing in the built
# system -- worse, it can collide with a real package user. Do it now,
# while the set of files is exactly what was just fetched.
_normalize_sources(lfs, run=True)
md5 = md5sums_path(ver)
# THE COUNTER WAS CALLED `fail`, like the function below that prints and
# stops -- so with any file missing, the md5 step called an int:
# TypeError: 'int' object is not callable
# And a MISSING file is not a mismatch: the old message blamed a mirror
# serving error pages for 34 files that simply were not there.
if nfail:
fail("\n! %d source file(s) could not be downloaded (named above)."
"\n Nothing is wrong with the ones that are here. Run again:"
"\n lfs build-system get-sources --run" % nfail)
sys.exit(1)
if os.path.isfile(md5):
import shutil
shutil.copy(md5, os.path.join(srcdir, "md5sums"))
print("verifying md5sums...", flush=True)
# `md5sum -c … | grep -v ": OK"` reported success on a CORRUPTED tree:
# grep inverts the verdict and the pipe throws away md5sum's status.
# Verified both ways -- rc was 0 either way. Check the status itself.
rc = _run_bash(f'cd "{srcdir}" && md5sum -c md5sums --quiet')
if rc != 0:
fail('\n! some downloads do not match their md5sum.\n The lines above name them. A mirror returning an error\n page with status 200 looks exactly like this.\n Delete those files and run get-sources again.')
sys.exit(1)
print("all md5sums OK")
if nfail:
sys.exit(1)
# ---- chapter 5: the cross toolchain --------------------------------------- #
# (step name, section id, source-tarball package name)
CROSSCHAIN_STEPS = [
# ---- chapter 5: cross toolchain ----
("binutils-pass1", "ch-tools-binutils-pass1", "binutils"),
("gcc-pass1", "ch-tools-gcc-pass1", "gcc"),
("linux-headers", "ch-tools-linux-headers", "linux"),
("glibc", "ch-tools-glibc", "glibc"),
("libstdcpp", "ch-tools-libstdcpp", "gcc"),
# ---- chapter 6: temporary tools (built WITH the ch5 cross toolchain) ----
("m4", "ch-tools-m4", "m4"),
("ncurses", "ch-tools-ncurses", "ncurses"),
("bash", "ch-tools-bash", "bash"),
("coreutils", "ch-tools-coreutils", "coreutils"),
("diffutils", "ch-tools-diffutils", "diffutils"),
("file", "ch-tools-file", "file"),
("findutils", "ch-tools-findutils", "findutils"),
("gawk", "ch-tools-gawk", "gawk"),
("grep", "ch-tools-grep", "grep"),
("gzip", "ch-tools-gzip", "gzip"),
("make", "ch-tools-make", "make"),
("patch", "ch-tools-patch", "patch"),
("sed", "ch-tools-sed", "sed"),
("tar", "ch-tools-tar", "tar"),
("xz", "ch-tools-xz", "xz"),
("binutils-pass2", "ch-tools-binutils-pass2", "binutils"),
("gcc-pass2", "ch-tools-gcc-pass2", "gcc"),
]
CROSSCHAIN_MARK = "### lfs-crosschain-script"
def _title_pkgver(title):
"""The Name-Version token from a book section title.
Titles look like "7.12. Util-linux-2.41.1", "5.2. Binutils-2.45 - Pass 1",
"5.4. Linux-6.16.1 API Headers", "5.6. Libstdc++ from GCC-15.2.0". Scanning
for the first "<letters>-<digits>" match is WRONG for hyphenated names --
in "Util-linux-2.41.1" it skips past "Util-" and returns "linux-2.41.1",
which then looks for the wrong tarball entirely. So: drop the section
number, then take the first whole whitespace-separated token that ends in a
version."""
t = re.sub(r"^\d+(\.\d+)*\.\s*", "", (title or "").strip())
tok_re = re.compile(r"^[A-Za-z][A-Za-z0-9+_.]*" # name start
r"(?:(?:::|[-_])[A-Za-z][A-Za-z0-9+_.]*)*" # ::/-/_ parts
r"[-_][0-9][0-9.]*(?:-[0-9]+)*$") # -version
for tok in t.split():
if tok_re.match(tok):
return tok
return None
def _pkg_glob_for(pkgver, tarpfx):
"""Candidate tarball patterns for a package, most specific first.
Book titles are capitalised ("Python-3.13.7", "Util-linux-2.41.1") but the
tarball may be lowercase, and a few packages drop the dash entirely
(Expect-5.45.4 ships as expect5.45.4.tar.gz, Tcl-8.6.16 as
tcl8.6.16-src.tar.gz), so try each spelling rather than guessing."""
stems, seen = [], set()
def add(x):
if x and x not in seen:
seen.add(x)
stems.append(x)
if pkgver:
for base in (pkgver, pkgver.lower()):
add(base)
# Perl modules are written XML::Parser-2.47 but ship as XML-Parser-
add(base.replace("::", "-"))
# Python dists are titled Flit-Core-3.12.0 but ship as flit_core-
add(base.replace("-", "_", base.count("-") - 1)
if base.count("-") > 1 else base)
# dash-less spelling: Expect-5.45.4 -> expect5.45.4
add(re.sub(r"-(?=[0-9])", "", pkgver.lower(), count=1))
else:
add(f"{tarpfx}-*")
exts = ("tar.*", "tgz", "tbz2", "zip")
pats = [f"{st}.{e}" for st in stems for e in exts]
# last resort: allow a suffix (tcl8.6.16-src.tar.gz, foo-1.2-source.tar.gz)
pats += [f"{st}*.{e}" for st in stems for e in ("tar.*", "tgz")]
# very last: an INFIX between name and version -- SQLite-3510200 ships as
# sqlite-autoconf-3510200.tar.gz, a new package in 13.0 that broke unpack.
# The unpack loop's doc filter keeps this from grabbing sqlite-doc-* or
# systemd-man-pages-*. (from the SysV-book session)
m = re.match(r"(.+?)-([0-9][\w.]*)$", pkgver or "")
if m:
pats += [f"{m.group(1).lower()}-*{m.group(2)}.{e}"
for e in ("tar.*", "tgz")]
return " ".join(pats)
# The books contain PLACEHOLDERS meant for a human to replace:
# PAGE=<paper_size> ./configure --prefix=/usr
# A shell reads "<paper_size>" as an input redirect and the build dies with
# line 4: paper_size: No such file or directory
# Substitute the ones we have settings for, and refuse to generate a script that
# still contains an unresolved one rather than letting it fail at build time.
# Placeholders come in more shapes than lowercase words:
# <paper_size> <locale name> <charmap> (lowercase)
# <FQDN> <HOSTNAME> <CC> <ll> <@modifiers> (upper, short, punctuated)
# <192.168.1.2> (an example address)
# Missing the latter wrote "<ll>_<CC>.UTF-8<@modifiers>" straight into
# /etc/profile, which then breaks every login shell:
# /etc/profile: line 10: syntax error near unexpected token `newline'
# The character class deliberately excludes < and > so shell redirects and
# heredocs (<<EOF, 2>&1) are never touched.
_PLACEHOLDER_RE = re.compile(r'<([A-Za-z0-9_.@ -]{1,40})>')
def _fill_placeholders(text, name):
cfg = load_config()
loc = cfg.get("locale") or "en_US.UTF-8"
host = cfg.get("hostname") or "lfs"
# The book spells the locale out piecewise in /etc/profile:
# export LANG=<ll>_<CC>.<charmap><@modifiers>
# Substituting the parts individually leaves a broken hybrid, so replace
# the whole construct first.
text = re.sub(r'<ll>_<CC>\.<charmap><@modifiers>', loc, text)
text = re.sub(r'<ll>_<CC>\.[A-Za-z0-9-]+<@modifiers>', loc, text)
text = re.sub(r'<ll>_<CC>[^\s]*', loc, text)
# /etc/hosts' optional static-IP line is a decision, not a default: comment
# it out rather than writing a made-up address into the file.
text = re.sub(r'^(<\d+\.\d+\.\d+\.\d+>.*)$',
r'# \1 <-- uncomment and edit if this host has a static IP',
text, flags=re.M)
# 8.5.2.2 ends with
# tzselect
# ln -sfv /usr/share/zoneinfo/<xxx> /etc/localtime
# tzselect is an INTERACTIVE menu whose only output is the name to put in
# <xxx>. We asked for the timezone in `session`, so both the menu and the
# placeholder are already answered -- running the menu inside a scripted
# build just blocks forever.
tz = cfg.get("timezone") or "UTC"
if re.search(r'^\s*tzselect\s*$', text, flags=re.M):
text = re.sub(r'^\s*tzselect\s*$',
'# tzselect -- answered by `lfs config timezone %s`' % tz,
text, flags=re.M)
text = text.replace('/usr/share/zoneinfo/<xxx>', '/usr/share/zoneinfo/%s' % tz)
# The same section unpacks tzdata with a path relative to GLIBC's build
# directory:
# tar -xf ../../tzdata2025b.tar.gz
# Lifted out of glibc and run as its own step, ".." is somewhere else
# entirely and the tarball is simply not found. Take it from the sources
# directory by name, and unpack into scratch rather than beside it.
def _tzdata(m):
return ('_tzd="$(ls -1 "${SOURCES_DIR:-/sources}"/%s* 2>/dev/null '
'| head -n1)"\n'
'[ -n "$_tzd" ] || { echo "no %s tarball in '
'${SOURCES_DIR:-/sources}" >&2; exit 1; }\n'
'_tzw="${BUILD_ROOT:-/build}/tzdata"\n'
'rm -rf "$_tzw"; mkdir -p "$_tzw"; cd "$_tzw"\n'
'tar -xf "$_tzd"' % (m.group(1), m.group(1)))
text = re.sub(r'tar -xf \.\./(?:\.\./)?(tzdata)[^\s]*', _tzdata, text)
known = {
"paper_size": cfg.get("paper_size") or "A4",
"xxx": tz,
"lfs": cfg.get("hostname") or "lfs",
"locale name": loc,
"charmap": cfg.get("charmap") or "UTF-8",
"HOSTNAME": host,
"FQDN": "%s.localdomain" % host,
"ll": loc.split("_")[0],
"CC": (loc.split("_")[1].split(".")[0]
if "_" in loc else "US"),
"@modifiers": "",
}
def sub(m):
key = m.group(1)
if key in known:
return known[key]
return m.group(0)
out = _PLACEHOLDER_RE.sub(sub, text)
# Report only things that are really a "fill this in" instruction. The
# angle brackets also enclose perfectly ordinary text:
# sed '/unistd.h/i #include <string.h>' (a C include)
# # maintained by <roryo@roryo.dynup.net> (an email in a comment)
# and anything already commented out needs no action either.
def actionable(ph, line):
if line.lstrip().startswith("#"):
return False
if re.fullmatch(r'[A-Za-z0-9_/.+-]+\.[hc]', ph): # <string.h>
return False
if "@" in ph and "." in ph: # an address
return False
if re.fullmatch(r'[\d.]+', ph): # a bare IP
return False
return True
left = []
for line in out.splitlines():
for ph in _PLACEHOLDER_RE.findall(line):
if actionable(ph, line) and ph not in left:
left.append(ph)
return out, sorted(left)
def _crosschain_pkgver(args, sid):
"""(version-label, commands) for a crosschain step's section."""
_l, path = resolve_book(args)
soup = load_soup(path)
for s, t, sec in iter_sections(soup):
if s == sid:
return (_title_pkgver(t) or t), sec["commands"]
return None, []
def _crosschain_all_pkgvers(args):
"""(name -> (version-label, commands)) for every crosschain step, parsing
the book exactly ONCE instead of once per step (that repeated re-parse was
what made --list slow)."""
_l, path = resolve_book(args)
soup = load_soup(path)
sid_to_name = {sid: name for name, sid, _tarpfx in CROSSCHAIN_STEPS}
out = {}
for s, t, sec in iter_sections(soup):
name = sid_to_name.get(s)
if name:
out[name] = ((_title_pkgver(t) or t), sec["commands"])
return out
def crosschain_dir(ver):
"""Scripts don't hardcode $LFS (they use the shell variable), so they're
reusable across sessions -- stored once per BOOK version in the shared
store, next to the book itself, world-readable/editable."""
d = os.path.join(store_dir(), "crosschain", ver)
os.makedirs(d, exist_ok=True)
try:
os.chmod(d, 0o755)
except OSError:
pass
return d
def crosschain_script_path(ver, name):
return os.path.join(crosschain_dir(ver), f"{name}.sh")
CROSSCHAIN_SCRIPT_VERSION = 18 # v18: the runner moved out into lfs-phases
_INSTALL_BLOCK_RE = re.compile(r'\b(make|ninja)\b[^\n]*\binstall\b')
# A block that runs the package's test suite. These are pulled OUT of the build
# phase: the books expect some tests to fail ("A few failures out of over 6000
# tests can generally be ignored" for Glibc), so a non-zero `make check` must not
# abort the build the way a failed compile does.
# Test invocations take several shapes in the books:
# make check
# make -k check
# su tester -c "PATH=$PATH make -k check"
# spawn make tests (inside an expect heredoc)
# ninja test / meson test
# so match the make/ninja/meson call anywhere in the block, not just at the
# start of a line -- anchoring missed everything wrapped in `su tester -c`.
_TEST_BLOCK_RE = re.compile(
r'\b(?:make|ninja|meson)\b[^\n]*?[\s"\']'
# the target may carry a suffix: test_harness, check-TESTS, check-recursive
r'(?:-k\s+)?(?:check|test|tests)(?:[-_][A-Za-z_]+)?(?:\s|"|\'|$)')
# Diagnostics that belong WITH the test suite, not with the build. Glibc's
# grep "Timed out" $(find -name \*.out)
# inspects test output; when nothing matches -- the normal, healthy case --
# grep exits 1 and `set -e` kills the build with no message at all.
_TEST_DIAG_RE = re.compile(
r'Timed out|find\s+-name\s+\\?\*\.out|\btest-suite\.log\b'
r'|[\w.-]*check-log\b|\bcheck\.log\b|\btests?\.log\b'
# Preparation and summary for a test run that, for us, never happens.
# GCC's chapter-8 section is:
# make
# ulimit -s -H unlimited
# sed ... ../gcc/testsuite/gcc.dg/plugin/plugin.exp
# sed ... ../gcc/testsuite/gcc.target/i386/...
# chown -R tester .
# su tester -c "PATH=$PATH make -k check"
# ../contrib/test_summary
# Only the chown/su lines were recognised, so the ulimit, the testsuite
# seds and test_summary stayed in the BUILD phase. test_summary greps
# test logs that do not exist and exits non-zero, and `set -e` then fails
# the build -- reported as "phase 'build' FAILED" with make having
# succeeded.
r'|\btestsuite\b|\btest_summary\b|^\s*ulimit\b',
re.M)
# Scaffolding that exists only so the test suite can run: the books create a
# `tester` user and a throwaway group, hand the build tree to them, and tear it
# down afterwards. Coreutils does
# groupadd -g 102 dummy -U tester
# chown -R tester .
# su tester -c "... make ... check"
# groupdel dummy
# Left in the build phase, those run as the package user and die with
# groupadd: Permission denied.
# They belong with the tests -- and are skipped entirely unless tests are asked
# for, which is why the build no longer needs to touch /etc/group at all.
_TEST_SETUP_RE = re.compile(
r'^\s*(?:groupadd|groupdel|userdel|usermod)\b[^\n]*\btester\b'
r'|^\s*(?:groupadd|groupdel)\b[^\n]*\bdummy\b'
r'|^\s*chown\b[^\n]*\btester\b'
r'|^\s*(?:useradd|adduser)\b[^\n]*\btester\b',
re.M)
# files a test block writes (its log), so the block that reads them back can be
# kept with the tests instead of being left behind in the build
_TEE_TARGET_RE = re.compile(r'\|\s*tee\s+(?:-a\s+)?([\w./-]+)')
_REDIR_TARGET_RE = re.compile(r'&?>\s*([\w./-]+\.(?:log|out|txt))')
def _is_test_block(b):
return bool(_TEST_BLOCK_RE.search(b) or _TEST_DIAG_RE.search(b)
or _TEST_SETUP_RE.search(b))
# A lone `grep ...` in a build sequence is informational ("did anything go
# wrong?"), and grep exits 1 when it finds nothing. Under `set -e` that aborts
# the build silently, so such lines get an explicit `|| true`.
_LONE_GREP_RE = re.compile(r'^\s*grep\b[^\n]*$')
# Greps that are the book's TOOLCHAIN GATE, not diagnostics.
#
# In 5.5 and 8.29 these must each produce output: no match means the toolchain
# is still linked against the host, and the book says to stop. Appending
# `|| true` to them -- as _soften_diagnostics does to every other lone grep --
# turned the only real check in the build into a guaranteed pass.
_TOOLCHAIN_GATE_RE = re.compile(
r'crt\[1in\].*succeeded'
r'|\^ ?/?(usr/)?include'
r'|SEARCH.*/usr/lib'
r'|/lib.*/libc\.so\.6'
r'|^\s*grep\s+found\s+dummy\.log'
r'|grep\s+--version',
re.M)
# Re-running a phase must not fail just because it already did its job. The
# books write `ln -sv target /usr/lib` with no -f, so the second attempt dies
# with "failed to create symbolic link '/usr/lib/cpp': File exists" -- which
# makes retrying an install or configure phase impossible. Adding -f keeps the
# intent (make this symlink point here) and makes the step repeatable.
_LN_FLAGS_RE = re.compile(r'\bln\s+(-[A-Za-z]+)')
def _idempotent_links(block):
def fix(m):
flags = m.group(1)
if "s" in flags and "f" not in flags:
return f"ln {flags}f"
return m.group(0)
return _LN_FLAGS_RE.sub(fix, block)
def _soften_diagnostics(block):
out = []
for ln in block.split("\n"):
if _TOOLCHAIN_GATE_RE.search(ln):
# the book's toolchain gate: no match means the build is linked
# against the HOST, and it must fail loudly rather than pass
out.append(ln)
elif _LONE_GREP_RE.match(ln) and not ln.rstrip().endswith(("\\", "|| true")):
out.append(ln.rstrip() + " || true # informational: no match is fine")
else:
out.append(ln)
return "\n".join(out)
def _split_crosschain_phases(cmds):
"""Split a step's command BLOCKS (the book gives us discrete <pre> blocks)
into build / install / configure / test -- the same phases BLFS uses.
Everything up to the first 'make/ninja ... install' block is build; that
block is install; anything after it (sanity checks, symlink/doc fixups) is
configure. Test-suite blocks are lifted out of whichever phase they landed
in and returned separately. If no install-style block exists (e.g. Linux
API headers, which install with a plain `cp`), everything stays in build --
the same convention BLFS uses for packages that install during their build.
"""
tests = [b for b in cmds if _is_test_block(b)]
# A test block usually writes a log, and a LATER block reads it back to
# summarise the run -- GMP does
# make check 2>&1 | tee gmp-check-log
# awk '/# PASS:/{total+=$3} ; END{print total}' gmp-check-log
# If only the first moves to the test phase, the awk stays in the build and
# dies with "cannot open file `gmp-check-log'". So anything referring to a
# file the tests produced belongs with the tests.
artifacts = set()
for b in tests:
artifacts |= set(_TEE_TARGET_RE.findall(b))
artifacts |= set(_REDIR_TARGET_RE.findall(b))
artifacts = {a for a in artifacts if len(a) > 3 and not a.startswith("/")}
if artifacts:
extra = [b for b in cmds
if b not in tests and any(a in b for a in artifacts)]
tests = [b for b in cmds if b in tests or b in extra]
rest = [_idempotent_links(_soften_diagnostics(b))
for b in cmds if b not in tests]
idx = next((i for i, b in enumerate(rest) if _INSTALL_BLOCK_RE.search(b)),
None)
test_cmds = "\n\n".join(tests)
if idx is None:
return "\n\n".join(rest), "", "", test_cmds
return ("\n\n".join(rest[:idx]), rest[idx],
"\n\n".join(rest[idx + 1:]), test_cmds)
def _indent(text, tab="\t"):
return "\n".join((tab + ln if ln.strip() else ln)
for ln in text.splitlines())
def _fn_body(cmds, empty=": # (nothing to do in this phase)"):
"""A command block as a function body. NOT indented: the books are full
of heredocs (cat > file << "EOF" ... EOF), and an indented terminator is
no terminator. The commands read exactly as the book prints them."""
if not cmds or not cmds.strip():
return empty
body = cmds.strip("\n")
# a body of comments alone is a syntax error to bash; give it a command
if not any(ln.strip() and not ln.lstrip().startswith("#")
for ln in body.split("\n")):
body += "\n:"
return body
def _crosschain_script_body(name, sid, pkgver, glob, cmds):
"""A generated install script: the package's variables at the top, its
commands in one function per phase, and one line at the end that hands
over to the shared runner (lfs-phases). Everything that is the same for
every package -- finding the tarball, unpacking, running each phase as its
own process under set -e, resuming in the build subdir, the dispatcher --
lives in the runner, not in every script."""
ts = time.strftime("%Y-%m-%d %H:%M:%S")
build_cmds, install_cmds, config_cmds, test_cmds = \
_split_crosschain_phases(cmds)
book = "LFS %s" % (load_config().get("default") or "?")
L = []
w = L.append
w("#!/bin/bash")
w("############################################")
w(f"### {name} ({pkgver})")
# The SHARED stamp: both generators write this line, so a tool can ask
# what a script is instead of guessing from its dialect. `origin` says
# whose format counter the number belongs to.
w(f"{CROSSCHAIN_MARK} v{CROSSCHAIN_SCRIPT_VERSION}: {name}")
w(f"### book : {book}")
w(f"### generated : {ts}")
w(f"### script format v{CROSSCHAIN_SCRIPT_VERSION} origin=lfs")
w("### phases: all (default) | unpack | build | test | install | configure")
w("### This is YOUR editable script -- lfs reuses it as-is on every --run.")
w(f"### Fresh copy from the book (discarding edits): "
f"lfs build-system crosschain {name} --regenerate")
w("")
w(f'name_version="{pkgver}"')
w(f'pkg_glob="{glob}"')
w(f'section="{sid}"')
w('pkgusr_stage="cross"')
w("")
w("build_pkg() {")
w(_fn_body(build_cmds, ": # (nothing to build)"))
w("}")
w("")
w("install_pkg() {")
w(_fn_body(install_cmds, ": # (this step installs during the build phase)"))
w("}")
if config_cmds.strip():
w("")
w("configure_pkg() {")
w(_fn_body(config_cmds))
w("}")
w("")
w("test_pkg() {")
w(_fn_body(test_cmds, ": # the book defines no test suite for this step"))
w("}")
w("")
w('. "${PKGUSR_LIB:-lfs-phases}" && pkgusr_run "$@"')
return "\n".join(L) + "\n"
def ensure_crosschain_script(args, ver, name, sid, tarpfx, force_regen=False,
book_info=None):
"""Return the on-disk script path for a step, writing/refreshing it from the
book if it doesn't exist yet or --regenerate was asked for. Prints which one
it's using so it's never ambiguous. `book_info` (name -> (pkgver, cmds)),
if given, avoids re-parsing the book per step (see _crosschain_all_pkgvers)."""
path = crosschain_script_path(ver, name)
if os.path.isfile(path) and not force_regen:
print(f"# using SAVED script (edit it freely): {path}")
return path
if book_info is not None and name in book_info:
pkgver, cmds = book_info[name]
else:
pkgver, cmds = _crosschain_pkgver(args, sid)
glob = _pkg_glob_for(pkgver, tarpfx)
text = _crosschain_script_body(name, sid, pkgver, glob, cmds)
if os.path.isfile(path):
import shutil
shutil.copy(path, path + ".bak")
print(f"# regenerated from the book (previous version saved as "
f"{path}.bak)")
with open(path, "w") as f:
f.write(text)
_make_world_readable(path)
print(f"# wrote new script -> {path}")
return path
# ---- crosschain resume: remember progress across steps so a cancelled run can
# continue where it left off instead of restarting from step 1 ----------------
def _crosschain_progress_path(lfs):
d = pkgusr_state(lfs, "progress")
os.makedirs(d, exist_ok=True)
return os.path.join(d, "crosschain-progress.json")
def _chapter56_has_output(lfs):
"""Is there real chapter 5-6 output in this tree?
The cross toolchain leaves unmistakable traces: a cross-gcc in tools/bin,
or a populated $LFS/usr/bin. An empty tree that root happens to own is
just `layout` having run."""
tools_bin = os.path.join(lfs, "tools", "bin")
if os.path.isdir(tools_bin):
for f in os.listdir(tools_bin):
if f.endswith(("-gcc", "-ld", "-as")):
return True
usr_bin = os.path.join(lfs, "usr", "bin")
try:
if len(os.listdir(usr_bin)) > 20:
return True
except OSError:
pass
return os.path.isfile(_crosschain_progress_path(lfs))
def _load_crosschain_progress(lfs):
try:
with open(_crosschain_progress_path(lfs)) as f:
return json.load(f)
except Exception:
return None
def _mark_crosschain_done(lfs, ver, name):
"""Record ONE completed step. Progress is a global set of finished steps,
not a snapshot of whatever list the last command happened to run -- so
building a single package in between (crosschain ncurses --run) adds to the
record instead of wiping the full run's progress."""
prog = _load_crosschain_progress(lfs) or {}
if prog.get("book") != ver:
prog = {"book": ver, "done": []}
done = [d for d in prog.get("done", []) if d != name]
done.append(name)
# Record the target triplet with the progress. gcc-pass2 installs its
# headers under $LFS_TGT; change LFS_TGT afterwards and the compiler looks
# under the new name while the headers sit under the old one, which shows
# up only much later as
# fatal error: bits/c++config.h: No such file or directory
with open(_crosschain_progress_path(lfs), "w") as f:
json.dump({"book": ver, "done": done, "lfs_tgt": lfs_tgt()},
f, indent=2)
def lfs_tgt():
cfg = load_config()
return cfg.get("lfs_tgt") or "%s-lfs-linux-gnu" % os.uname().machine
def _crosschain_tgt_mismatch(lfs):
"""Was the tree built with a different LFS_TGT than is configured now?
Returns (recorded, current) when they differ, else None."""
prog = _load_crosschain_progress(lfs) or {}
was = prog.get("lfs_tgt")
now = lfs_tgt()
if was and was != now:
return (was, now)
return None
def _clear_crosschain_progress(lfs):
try:
os.remove(_crosschain_progress_path(lfs))
except OSError:
pass
def _require_lfs_writable_by_lfs(lfs):
"""Fail fast if the lfs user can't write where the build installs. Without
this, binutils/gcc compile happily for minutes and only die inside `make
install` with a wall of 'mkdir: Permission denied' -- the real cause (the
book's 4.3 chown never applied, or a root-created subdir) being far off
screen by then."""
import pwd as _pwd
try:
want = _pwd.getpwnam("lfs").pw_uid
except KeyError:
fail("! the 'lfs' user doesn't exist yet.\n Run: lfs build-system add-user --run")
sys.exit(2)
# After book 7.2 the whole tree is handed to root on purpose, and chapters
# 5-6 are finished -- don't demand it be given back to lfs.
#
# But root owning the tree is NOT by itself evidence of that: `layout`
# runs as root and creates the tree as root, so a brand-new tree looks
# identical. Only refuse when chapter 5-6 work actually exists; otherwise
# this is a fresh tree that simply needs handing to the lfs user.
if _owner_is_root(lfs) and not _chapter56_has_output(lfs):
# a fresh tree that layout made as root: just hand it over
if _chown_tree_to_lfs(lfs):
pass
if _owner_is_root(lfs) and _chapter56_has_output(lfs):
sys.stderr.write(
f"\n! {lfs} is owned by root -- chapter 7 (chroot) has already been\n"
f" prepared, so the chapter 5-6 steps are done and shouldn't run\n"
f" as the lfs user again.\n"
f" Work inside the chroot instead:\n"
f" {sudo_prefix()}lfs build-system chroot enter\n"
f" (to deliberately go back, run: {sudo_prefix()}lfs build-system "
f"fix-ownership --run)\n")
sys.exit(2)
bad = []
for sub in ("tools", "usr", "var", "etc", "lib64"):
p = os.path.join(lfs, sub)
if os.path.isdir(p) and not os.path.islink(p):
try:
if os.stat(p).st_uid != want:
bad.append(p)
except OSError:
pass
# also catch root-owned dirs NESTED inside those trees: a single root-owned
# $LFS/var/lib is enough to break glibc's install (var/lib/nss_db), even
# though $LFS/var itself looks fine.
nested = []
for sub in ("var", "usr", "etc", "tools"):
root_p = os.path.join(lfs, sub)
if not os.path.isdir(root_p) or os.path.islink(root_p):
continue
for dirpath, dirnames, _files in os.walk(root_p):
dirnames[:] = [d for d in dirnames
if not os.path.islink(os.path.join(dirpath, d))]
try:
if os.stat(dirpath).st_uid != want and dirpath not in bad:
nested.append(dirpath)
except OSError:
pass
if len(nested) > 20:
break
problem = bad + nested
if problem:
shown = problem[:10]
sys.stderr.write(
f"\n! these build dirs are not owned by the 'lfs' user:\n"
f" {', '.join(shown)}"
+ (f" (and {len(problem) - len(shown)} more)" if len(problem) > len(shown) else "")
+ f"\n The build would fail inside 'make install' with 'Permission "
f"denied'.\n"
f" Fix all of them with:\n"
f" {sudo_prefix()}lfs build-system fix-ownership --run\n")
sys.exit(2)
def cmd_bs_fix_ownership(args):
"""Give the lfs user ownership of the whole build tree again. Useful after
something ran as root and created a root-owned dir inside $LFS (which then
blocks packages installing as lfs)."""
lfs = require_mounted_lfs()
targets = [os.path.join(lfs, s) for s in
("tools", "usr", "var", "etc", "lib64", "lib", "bin", "sbin",
"sources", PKGUSR_DIR)]
targets = [p for p in targets if os.path.exists(p) and not os.path.islink(p)]
print(f"# would give 'lfs' ownership of:\n" +
"\n".join(f" {p}" for p in targets))
if not args.run:
dry_run_note("apply")
return
if os.geteuid() != 0:
fail('! must be run as root.')
sys.exit(2)
# The third copy of this chown, and the third guard. `fix-ownership` is a
# chapter 5-6 repair: it hands the tree to the build user so packages can
# install as `lfs`. After chapter 7 the packages ARE the users, and doing
# it would strip all of them in one pass.
if not _handover_is_safe(lfs):
fail("! not doing that: package users own files here, so this tree is\n past chapter 7. chown -R lfs would strip every package's\n ownership, and only the manifests could rebuild it.\n To repair ownership on a built tree, inside the chroot:\n lfs-helper verify --fix")
sys.exit(2)
for p in targets:
_run_bash(f'chown -R lfs "{p}"')
ok("# ownership restored to lfs.")
# Every user-settable key, with a description and how the default is derived.
# Which settings belong to which job. Two different things share this file:
# building a system (a one-off, with a partition and a target triplet), and
# installing packages onto one (ongoing). Showing them in one flat list made
# it impossible to tell which was which -- particularly for the two book
# settings, where "which book?" has two different answers.
CONFIG_SECTIONS = [
("installing packages on this system", [
"default", "mirror", "editor", "sudo",
]),
("building a system: the disk", [
"lfs_mount", "lfs_device", "lfs_fstype", "lfs_swap", "lfs_home_device",
"esp", "bootloader",
]),
("building a system: how it is built", [
"build_book", "lfs_tgt", "makeflags", "collector_prefix",
"collector_import_file",
]),
("the system being built: how it is set up", [
"hostname", "locale", "charmap", "paper_size", "locales", "network",
"timezone", "keymap", "console_font",
]),
]
# Which part of the tool each setting belongs to. Two different jobs share
# one config file: building a system from the book, and installing packages on
# a machine afterwards. Printing them in one flat list made it impossible to
# tell which setting affected which -- and "set-default" ambiguous.
CONFIG_SECTIONS = [
("layout of the system being built", [
"target_pkgusr_home", "target_appuser_home",
]),
("layout of this machine", [
"pkgusr_home", "appuser_home",
]),
("building a system", [
"build_book", "lfs_mount", "lfs_device", "lfs_fstype", "lfs_swap",
"lfs_home_device", "lfs_tgt", "makeflags", "bootloader", "esp",
"snapshot_dir",
]),
("the system being built (its settings, not yours)", [
"locale", "charmap", "paper_size", "hostname", "network",
"static_ip", "gateway", "dns", "domain", "fqdn", "timezone",
]),
("installing packages", [
"default", "mirror", "collector_prefix", "collector_import_file",
]),
("this tool", [
"editor", "sudo",
]),
]
CONFIG_KEYS = [
("default", "book used for installing packages (when --book isn't given)"),
("build_book", "book used to BUILD the system; falls back to 'default'"),
("lfs_mount", "where the LFS partition is mounted ($LFS)"),
("lfs_device", "the LFS root partition device"),
("lfs_fstype", "filesystem type of that device"),
("lfs_swap", "swap device (optional)"),
("lfs_home_device", "separate /home device (optional)"),
("lfs_tgt", "cross-compile target triplet ($LFS_TGT); "
"default: $(uname -m)-lfs-linux-gnu"),
("makeflags", "parallelism for builds; default: -j$(nproc)"),
("mirror", "package mirror base URL ('off' to disable); "
"{book} is replaced by the version"),
("editor", "editor for --edit; default: $EDITOR or vim"),
("sudo", "prefix suggested root commands with 'sudo' "
"(on/off; default off)"),
("bootloader", "which bootloader to set up: none (DEFAULT -- nothing "
"is written to any ESP, boot sector or partition "
"table, and the machine keeps booting exactly as it "
"does now), refind, or grub. GRUB's book instructions "
"run 'grub-install /dev/sda', which writes to a disk's "
"boot sector -- with 'refind' that section is skipped "
"entirely and a rEFInd script is generated instead, "
"which only ADDS an entry beside your existing "
"bootloader"),
("esp", "path of the EFI System Partition inside the built "
"system (e.g. /boot/efi). rEFInd installs there; it is "
"never formatted, and an existing installation is left "
"alone unless you ask otherwise"),
("locale", "system locale, e.g. en_US.UTF-8 or de_DE.UTF-8 "
"(default en_US.UTF-8). The book writes "
"'LC_ALL=<locale name>', which a shell would read as an "
"input redirect. Check what is available with 'locale -a'"),
("charmap", "character map for that locale (default UTF-8)"),
("paper_size", "paper size for groff and friends: A4 (default) or "
"letter. The book writes 'PAGE=<paper_size>', which a "
"shell would read as an input redirect"),
("hostname", "hostname for the built system (default: lfs)"),
("locales", "languages to keep for translated man pages and "
"message catalogues (space separated; English is "
"always kept). Used by 'lfs-helper prune-locales'. "
"Default: en de"),
("network", "how the built system gets its network: dhcp "
"(default) or static. With dhcp no static interface or "
"resolv.conf files are generated -- you install a DHCP "
"client (dhcpcd) from BLFS afterwards"),
("collector_import_file", "path to a collector-group export (from "
"'lfs-helper export-groups'). When set, it is copied "
"into the chroot and imported automatically, so a "
"rebuild reuses the group names you already chose "
"instead of asking again"),
("collector_prefix", "prefix for COLLECTOR groups. Packages sometimes "
"install into a directory another package owns (gcc "
"writes gcc.mo into glibc's /usr/share/locale/*/"
"LC_MESSAGES). Such a directory keeps its owner but is "
"handed to a group named <prefix>_<owner> and made "
"group-writable, and both packages join it -- so the "
"installer gets exactly the access it needs and nothing "
"more. Default: sysgroup"),
]
REQUIRED_CONFIG = [
("default", "which book to build", "lfs fetch <version>"),
("lfs_mount", "where $LFS is mounted", "lfs config lfs_mount /mnt/lfs"),
("collector_prefix", "prefix for collector groups",
"lfs config collector_prefix sysgroup"),
]
def require_complete_config():
"""Refuse to go further with a half-configured session. A missing
collector_prefix in particular only shows up much later, as a failed
install deep inside the chroot, so it's checked up front."""
cfg = load_config()
missing = [(k, why, how) for k, why, how in REQUIRED_CONFIG if not cfg.get(k)]
if not missing:
return
fail('! the build session is not fully configured:')
for k, why, how in missing:
sys.stderr.write(f" {k:<18} -- {why}\n"
f" {'':<18} set it with: {sudo_prefix()}{how}\n")
sys.stderr.write("\n See every setting with: lfs config\n")
sys.exit(2)
def extra_packages():
v = load_config().get("extra_packages")
if v is None:
v = "wget"
return [p for p in v.split() if p]
def network_mode():
return (load_config().get("network") or "dhcp").lower()
def bootloader():
"""Which bootloader step to generate: none (default), refind, or grub.
NONE by default, and that is the important part. Most people running this
already have a working EFI partition and a bootloader that finds it -- the
machine booted in order to build LFS at all. Generating a rEFInd step for
them turns the last stretch of a six-hour build into a stop:
== rEFInd needs you to do something first ==
/boot/efi does not exist, so there is nowhere to install to.
which is a question nobody asked to be asked. A bootloader is a decision
about the whole machine, not a package, so it is opt-in: say so in
`session`, or `lfs config bootloader refind` later.
"""
return (load_config().get("bootloader") or "none").lower()
def collector_prefix():
return (load_config().get("collector_prefix") or "sysgroup").rstrip("_")
def pkgusr_prefix():
"""Prefix on package users: `p_` gives p_gcc. Blank means none.
`lfs-helper` has always READ this (falling back to `p`), but until now
nothing ever wrote it, so the default applied by accident and
`packagemanager` on the built system did not implement it at all -- the
build would create p_gcc and the built system would look for gcc.
Writing it here is what makes the three tools agree.
An explicitly empty value means "no prefix" and is not the same as unset.
"""
cfg = load_config()
v = cfg.get("pkgusr_prefix", "p") if "pkgusr_prefix" in cfg else "p"
v = (v or "").rstrip("_")
return (v + "_") if v else ""
def cfguser_prefix():
"""Prefix on config-step users: `cfg_` gives cfg_bootscripts.
Its own prefix, not the package one on top of it. Written into the chroot
env for the same reason as pkgusr_prefix: three tools deciding an account's
name separately is how they end up building two parallel sets of accounts.
An explicitly empty value means "no prefix" and is not the same as unset.
"""
cfg = load_config()
v = cfg.get("cfguser_prefix", "cfg") if "cfguser_prefix" in cfg else "cfg"
v = (v or "").rstrip("_")
return (v + "_") if v else ""
def sudo_prefix():
"""'sudo ' or '' -- some systems (and root shells) have no sudo at all, so
suggested commands are shown bare unless you turn this on."""
v = str(load_config().get("sudo", "")).lower()
return "sudo " if v in ("1", "on", "true", "yes") else ""
def remove_tools():
"""Delete /tools at the end of the build (book 7.13)?
Default off. The book removes it as soon as chapter 7 is over, but /tools
is the cross-toolchain that BUILT chapter 8 -- keeping it costs 1-3 GB and
buys a way back if chapter 8 has to be redone. Deleting it at the END
rather than at 7.13 keeps that fallback for the part of the build most
likely to need it.
"""
v = str(load_config().get("remove_tools", "")).lower()
return v in ("1", "on", "true", "yes")
def strip_binaries():
"""Strip debug symbols at the end of the build (book 8.85)?
Default off, and deliberately so. Stripping REWRITES each binary, so in a
package-user system it has to run as the file's owner or ownership drifts
away from the package that installed it -- which is the one thing this
whole scheme exists to keep true.
"""
v = str(load_config().get("strip", "")).lower()
return v in ("1", "on", "true", "yes")
# The keys worth asking about, with what they mean. Anything else can still
# be set with `lfs config --target <key> <value>`.
_TARGET_KEYS = (
("pkgusr_prefix",
"prefix for package users in the new system (p -> p_mesa; empty -> mesa)"),
("cfguser_prefix", "prefix for its config users (cfg -> cfg_bootscripts)"),
("collector_prefix", "prefix for its shared install-dir groups"),
("pkgusr_subdir", "folder under the base for package homes (pkgusr)"),
("cfguser_subdir", "folder under the base for config-user homes (cfg)"),
("pkgusr_home", "where package homes live (/usr/src/pkgusr); the "
"subdir is appended if you give just the base"),
)
def _target_config_path():
return os.path.join(store_dir(), "target-config.json")
def cmd_config_target(args):
"""Show or change the config the BUILT system will get.
Laid out like `packagemanager config --show`: value, then where it comes
from. The user asked for the config commands to look and behave the same
across the tools, and this is the first of them to follow that shape.
"""
path = _target_config_path()
try:
with open(path) as f:
cfg = json.load(f)
except (OSError, ValueError):
cfg = {}
# INTERACTIVE, LIKE THE OTHERS.
#
# `packagemanager config` asks for what is missing and `packagemanager env
# --ask` walks the build variables; this took only a key and a value -- so
# the config a person most needs to set deliberately, the NEW SYSTEM'S,
# was the one with no guided way to set it.
if getattr(args, "ask", False):
print()
print("# === Config the BUILT system will get ===")
print(" file: %s" % path)
print(" Enter keeps the current value; '-' clears it (means 'none').")
print()
for key, what in _TARGET_KEYS:
cur = cfg.get(key)
shown = ("(none)" if cur == "" else
cur if cur is not None else "this host's value")
try:
ans = input(" %s\n %-16s [%s]: " % (what, key, shown)).strip()
except EOFError:
print()
break
if ans == "":
continue
cfg[key] = "" if ans == "-" else ans
os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
with open(path, "w") as f:
json.dump(cfg, f, indent=2, sort_keys=True)
f.write("\n")
print()
print(" saved %s" % path)
print(" it reaches the build system on the next `lfs run`.")
return
if args.key and args.value is not None:
cfg[args.key] = args.value
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w") as f:
json.dump(cfg, f, indent=2, sort_keys=True)
f.write("\n")
print(" %s = %r -> %s" % (args.key, args.value, path))
print(" it reaches the build system on the next `lfs run`.")
return
print()
print("# === Config the BUILT system will get ===")
print(" file: %s%s" % (path, "" if os.path.isfile(path)
else " (not created yet)"))
print()
if not cfg:
print(" nothing set -- the build system will use this host's values")
print(" for the shared keys (prefixes, layout, books).")
print()
print(" set one with, for example:")
print(" lfs config --target pkgusr_prefix p")
print(" lfs config --target pkgusr_subdir packages")
return
for k in sorted(cfg):
print(" %-22s = %r" % (k, cfg[k]))
print()
print(" these override this host's values when the tree is built;")
print(" a config the tree already has wins over both.")
def cmd_config(args):
if getattr(args, "target", False):
return cmd_config_target(args)
"""Show or change every setting, and show where things live on disk."""
cfg = load_config()
if args.edit:
ed = cfg.get("editor") or os.environ.get("EDITOR") or "vim"
_ensure_store()
if not os.path.isfile(config_path()):
save_config(cfg)
import subprocess
subprocess.run([ed, config_path()], cwd="/")
return
if args.unset:
for k in args.unset:
cfg.pop(k, None)
save_config(cfg)
print(f"unset: {', '.join(args.unset)}")
return
if args.key and args.value is not None:
known = [k for k, _d in CONFIG_KEYS]
if args.key not in known:
sys.stderr.write(f"unknown key '{args.key}'. Known keys:\n "
+ "\n ".join(known) + "\n")
sys.exit(2)
cfg[args.key] = args.value
save_config(cfg)
print(f"{args.key} = {args.value}")
return
if args.key:
print(cfg.get(args.key, ""))
return
print(f"Settings ({config_path()})\n")
shown = set()
for title, keys in CONFIG_SECTIONS:
rows = [(k, d) for k, d in CONFIG_KEYS if k in keys]
if not rows:
continue
print(" %s%s%s" % (_C_GROUP if _tty_out() else "", title,
_C_OFF if _tty_out() else ""))
for k, desc in rows:
shown.add(k)
val = cfg.get(k)
# one line per setting: the value is what people are looking for,
# so it goes first and the explanation is only shown with --long
print(" %-22s %s" % (k, val if val not in (None, "") else "-"))
if getattr(args, "long", False):
for line in _wrap(desc, 60):
print(" %-22s %s" % ("", line))
print()
rest = [(k, d) for k, d in CONFIG_KEYS if k not in shown]
if rest:
print(" other")
for k, desc in rest:
val = cfg.get(k)
print(" %-22s %s" % (k, val if val not in (None, "") else "-"))
print()
b, d = build_version(), default_version()
if b or d:
print("\nWhich book is used where:")
print(" building the system : %s%s"
% (b or "(none chosen)",
" (from 'default')" if b and not cfg.get("build_book") else ""))
print(" installing packages : %s" % (d or "(none chosen)"))
print("\nEffective build environment (what the lfs user gets):")
lfs = _lfs_dir() or "(unset -- run: lfs build-system session)"
tgt = cfg.get("lfs_tgt") or f"{os.uname().machine}-lfs-linux-gnu"
mk = cfg.get("makeflags") or "-j$(nproc)"
print(f" LFS = {lfs}")
print(f" LFS_TGT = {tgt}")
print(f" MAKEFLAGS = {mk}")
print(f" PATH = $LFS/tools/bin:/usr/bin[:/bin]")
print(f" CONFIG_SITE = $LFS/usr/share/config.site")
print("\nPaths:")
print(f" store {store_dir()}")
print(f" books {books_dir()}")
# the paths below belong to the book the SYSTEM is built from
ver = build_version()
if ver:
print(f" book file {book_path(ver)}")
print(f" wget-list {wget_list_path(ver)}")
print(f" md5sums {md5sums_path(ver)}")
print(f" crosschain dirs {crosschain_dir(ver)}")
if mirror_base(ver):
print(f" mirror {mirror_base(ver)}")
if _lfs_dir():
d = _lfs_dir()
print(f" sources {os.path.join(d, 'sources')}")
print(f" manifests {os.path.join(d, PKGUSR_DIR, 'manifests')}")
print(f" build progress {_crosschain_progress_path(d)}")
print("\nChange a value: lfs config <key> <value>"
"\nEdit everything: lfs config --edit"
"\nScript paths: lfs build-system crosschain --list")
# ---- chapter 7: the chroot environment ------------------------------------ #
# Virtual kernel filesystems, in mount order. Teardown reverses this.
_KERNFS = ["dev/pts", "proc", "sys", "run", "dev/shm", "dev"]
def _chroot_mounts(lfs):
"""Which of our virtual filesystems are currently mounted, in mount order."""
live = []
try:
with open("/proc/mounts") as f:
mounted = {ln.split()[1] for ln in f if len(ln.split()) > 1}
except OSError:
return live
for sub in _KERNFS:
p = os.path.realpath(os.path.join(lfs, sub))
if p in mounted:
live.append(sub)
return live
def _owner_is_root(lfs):
try:
return os.stat(os.path.join(lfs, "usr")).st_uid == 0
except OSError:
return False
# Package users start here; anything below is the host's own accounts.
PKG_UID_MIN = 10000
def _find_uid(lfs, expr, limit=1):
"""First path under the handover trees matching a uid expression.
`find ... -print -quit` stops at the first hit, so this costs almost
nothing even on a full /usr.
"""
import subprocess
dirs = [os.path.join(lfs, d) for d in LFS_HANDOVER_DIRS
if os.path.exists(os.path.join(lfs, d))]
if not dirs:
return None
try:
out = subprocess.run(["find"] + dirs + ["-xdev"] + expr
+ ["-print", "-quit"],
capture_output=True, text=True, timeout=120)
return (out.stdout or "").strip() or None
except (OSError, subprocess.SubprocessError):
return None
def _build_user_uid():
"""The HOST's lfs account, by number.
Book 4.3's useradd pins no uid, so this is whatever the host had free --
10753 on one Debian machine. The chroot has its own `lfs` at 9998, so the
same file reads as two different owners depending on which side you are on,
and inside the chroot the host's uid resolves to no name at all. Only the
number is meaningful across that boundary.
"""
try:
import pwd
return pwd.getpwnam("lfs").pw_uid
except (KeyError, ImportError):
return None
def _handover_build_user_only(lfs, run=True):
"""Give back exactly what the BUILD USER holds, and nothing else.
Book 7.2 is `chown -R root:root` over eight trees, and on a chapter-8 tree
that is destructive -- it strips every package user in one pass. So the
recursive form is refused there, which left a tree with no way forward at
all: 4.3 had taken /usr, /etc, /var and the root symlinks, 7.2 would not
give them back, and `chroot enter` refused because /usr was not root's.
This is the precise version. Selecting on the build user's uid touches
what 4.3 took and leaves every package user's files alone, so it is safe at
any stage of the build.
`chown -h root`, user only: 4.3 chowned the user only, so the inverse does
too. Taking the group as well would strip `install` from every shared
directory and undo the ownership epoch's work. And `-h`, because /bin,
/lib and /sbin are symlinks -- following them would retarget /usr/bin.
"""
uid = _build_user_uid()
if uid is None:
return 0
# THE HOST'S lfs UID CAN BE A PACKAGE USER'S UID IN THE TREE.
#
# This selects files by NUMBER, because that is the only thing that means
# the same on both sides of the chroot. Book 4.3 pins no uid for `lfs`,
# so the host's is whatever was free -- 10753 on one machine -- and
# package users here are created in the 10000-20000 range. If those
# numbers ever meet, this hands a package user's files to root and the
# package silently loses every one of them.
#
# Ask the tree's own passwd before touching anything.
_clash = []
_pw = os.path.join(lfs, "etc", "passwd")
try:
with open(_pw, encoding="utf-8", errors="replace") as f:
for line in f:
parts = line.split(":")
if len(parts) > 2 and parts[2].strip() == str(uid):
_clash.append(parts[0])
except OSError:
pass
if _clash:
warn("REFUSING to hand files back: the host's build user is uid %d,"
% uid)
warn("and that is also %s in %s." % (", ".join(_clash), _pw))
note(" Selecting by uid cannot tell them apart, so this would give")
note(" that account's files to root and the package would lose them.")
note(" See what it owns:")
note(" find %s -xdev -uid %d | head" % (lfs, uid))
note(" Then renumber the HOST's lfs account -- it exists only to")
note(" build, so nothing of yours depends on its number:")
note(" usermod -u <free-uid> lfs # on the host, then retry")
return 0
dirs = [os.path.join(lfs, d) for d in LFS_HANDOVER_DIRS
if os.path.exists(os.path.join(lfs, d))]
if not dirs:
return 0
import subprocess
try:
out = subprocess.run(["find"] + dirs + ["-xdev", "-uid", str(uid)],
capture_output=True, text=True, timeout=600)
paths = [p for p in (out.stdout or "").splitlines() if p]
except (OSError, subprocess.SubprocessError):
return 0
if not paths:
return 0
if run:
_run_bash('find %s -xdev -uid %d -exec chown -h root {} + 2>/dev/null'
% (" ".join('"%s"' % d for d in dirs), uid))
return len(paths)
def _handover_needed(lfs):
"""Is anything in the handover trees still owned by the BUILD user?
`_owner_is_root` alone decided this, and it looks at one inode --
$LFS/usr. The pass it guards is `chown -h -R root:root` over eight trees.
So a tree whose /usr was already root-owned got no handover at all, while
everything chapters 5-6 created below it kept the host uid:
drwxr-xr-x 1 lfs lfs /usr/share/gcc-15.2.0/python/libstdcxx
lfs:install drwxrwxr-x /usr/share
lfs:install drwxrwxr-x /lib64
Ask the tree, not one directory in it.
"""
if not _owner_is_root(lfs):
return True
# Anything not owned by root. NOT "a uid outside the package range" --
# that was the first attempt and it was wrong for the oldest reason in this
# project: book 4.3's useradd pins no uid, so the host's `lfs` account lands
# wherever the host had a gap. On a Debian host that was 10753, which is
# inside the package-user range, so the filter excluded the very user it was
# looking for. The tree then reported "7.2 already done" while
# lrwxrwxrwx 1 lfs lfs /bin -> usr/bin
# was still sitting there. _handover_is_safe already establishes that no
# package owns anything, so here anything non-root is the build user's.
return _find_uid(lfs, ["!", "-uid", "0"]) is not None
def _handover_is_safe(lfs):
"""True while no package user owns anything.
`chown -R root:root` over $LFS/usr is exactly right before chapter 8 and
catastrophic after it: it would strip every package user's ownership in one
pass and there is no record left to rebuild it from.
Asked of the TREE's own passwd, not of a uid range. A uid range cannot tell
a package user from the host's build account -- book 4.3 pins no uid for
`lfs`, so on one host it came out 10753, inside the package range. The
tree's passwd is unambiguous: these are the accounts that exist in there.
"""
users = _pkg_users_in_tree(lfs)
if not users:
return True # no package user exists; nothing to strip
for uid in users:
if _find_uid(lfs, ["-uid", str(uid)]):
return False
return True
def _pkg_users_in_tree(lfs):
"""uids of the package users in the TREE's passwd, by name prefix."""
pfx = pkgusr_prefix()
out = []
try:
with open(os.path.join(lfs, "etc", "passwd")) as f:
for line in f:
parts = line.split(":")
if len(parts) > 2 and (not pfx or parts[0].startswith(pfx)):
try:
out.append(int(parts[2]))
except ValueError:
pass
except OSError:
return []
return out
def _owner_name_of(path):
"""Who owns this, by name where possible.
`chown --from lfs` matched only files owned by exactly that user, so a
tree owned by a uid with no matching name -- or by an `lfs` account whose
uid changed since -- was silently left alone, and 7.2 appeared to run
while changing nothing. Book 7.2 chowns unconditionally; so do we now."""
try:
st = os.stat(path)
except OSError:
return "?"
try:
import pwd
return pwd.getpwuid(st.st_uid).pw_name
except (KeyError, ImportError):
return "uid %d (no such user here)" % st.st_uid
# ---- chapter 7 (in chroot) + chapter 8: scripts for lfs-helper ------------ #
# Chapter 7's in-chroot packages, in book order. Chapter 8 is appended from the
# book's own package list, so this tracks whatever release you're building.
# These are TEMPORARY tools -- chapter 8 rebuilds every one of them for real.
# They get a -tmp suffix so they don't collide with the chapter 8 package of the
# same name, and the book builds them as root (the package-user system only
# takes over for the real chapter 8 system).
# Chapter 7 housekeeping that must happen INSIDE the chroot before any package
# is built: 7.5 creates the directory tree, 7.6 creates /etc/passwd, /etc/group
# and friends (until 7.6 runs the shell shows "I have no name!", because there
# is no passwd file yet). These are plain root command blocks, not package
# builds, so they're generated as simple scripts.
CHROOT_INIT_STEPS = [
("init-dirs", "ch-tools-creatingdirs"),
("init-files", "ch-tools-createfiles"),
]
def _auto_file_block(soup_or_raw, sid, dest_path):
"""Some sections ship a FILE, not commands.
Book 9.6.8 (The rc.site File) has no `<kbd class="command">` at all -- it
prints /etc/sysconfig/rc.site as a `<pre class="auto">` block with every
line commented out. A reader is meant to copy it and uncomment what they
want. Because the parser only harvests command blocks, cfg_rcsite
resolved to zero blocks and the file was simply never created.
An all-commented file is still worth installing: it is the documentation
of every knob the boot scripts have, in the place the boot scripts look.
Without it you have to go back to the book to find out what exists.
Returns a single shell command that writes the file, or None.
"""
import re as _re
raw = soup_or_raw if isinstance(soup_or_raw, str) else str(soup_or_raw)
i = raw.find('id="%s"' % sid)
if i < 0:
return None
# stop at the next section so a later <pre> cannot be picked up by mistake
j = _re.search(r'<div class="(?:sect1|sect2)\b', raw[i + 1:])
seg = raw[i:i + 1 + j.start()] if j else raw[i:i + 40000]
m = _re.search(r'<pre[^>]*class="auto"[^>]*>(.*?)</pre>', seg, _re.S)
if not m:
return None
import html as _html
body = _html.unescape(_re.sub(r'<[^>]+>', '', m.group(1))).strip("\n")
if not body.strip():
return None
# A quoted heredoc: the body is full of #comments and shell metacharacters
# that must land in the file exactly as the book prints them.
return ('mkdir -pv %s\ncat > %s << "EOF"\n%s\nEOF'
% (os.path.dirname(dest_path), dest_path, body))
# Sections that ship a file body instead of commands: name -> (section, path).
CONFIG_FILE_SECTIONS = {
"cfg_rcsite": ("ch-config-site", "/etc/sysconfig/rc.site"),
}
def _resolve_section(secs, sid):
"""Find a book section by id, falling back to its title.
Most sections carry a stable id (`ch-config-hostname`), but some carry an
id the toolchain GENERATED when the book was built:
idm139921653990176 8.5.2.1. Adding nsswitch.conf
idm139921653984688 8.5.2.2. Adding Time Zone Data
Those change every time the book is regenerated -- including between point
releases of the same edition -- so an entry that names one works exactly
once and then silently resolves to nothing. Silently is the problem:
`/etc/shells` went missing from finished systems because cfg_shells
resolved to zero blocks and was skipped without a word.
So a step may name either an id or a title fragment. A fragment is matched
case-insensitively against section titles, and must match exactly ONE
section -- an ambiguous fragment is a mistake in the table, not something
to resolve by picking the first.
"""
if sid in secs:
return secs[sid]
want = sid.lower()
hits = [(k, v) for k, v in secs.items() if want in v[0].lower()]
if len(hits) == 1:
return hits[0][1]
if len(hits) > 1:
warn(" '%s' matches %d sections (%s) -- make it more specific"
% (sid, len(hits), ", ".join(h[1][0][:28] for h in hits[:3])))
return None
# Chapter 8 configuration the book performs mid-chapter, between packages.
#
# These are files, not software: nothing built in chapter 8 reads them, so
# running them in the configuration phase with the rest is equivalent and
# keeps all configuration in one place.
#
# Two of the three sit in sub-sections whose ids are generated, so they are
# named by TITLE -- see _resolve_section.
CHROOT_CONFIG_STEPS_8 = [
# glibc leaves /etc/nsswitch.conf absent; without it name resolution falls
# back to compiled-in defaults and getent behaves inconsistently.
("cfg_nsswitch", "Adding nsswitch.conf"),
# /etc/localtime -- without it the system runs UTC and every log line,
# timestamp and `date` is offset.
("cfg_timezone", "Adding Time Zone Data"),
# The glibc step already does `touch /etc/ld.so.conf`; this is the rest of
# 8.5.2.3, which matters as soon as anything installs outside /usr/lib.
("cfg_ld", "conf-ld"),
]
# Chapter 9: configuring the system. These are root housekeeping steps that
# write config files -- no package is built -- so they are generated as plain
# root scripts, like 7.5 and 7.6. Several ask for a decision (hostname, network,
# locale), so they are placed after the packages and flagged where the book
# leaves a placeholder for you to fill in.
CHROOT_CONFIG_STEPS_9 = [
("cfg_bootscripts", "ch-config-bootscripts"),
("cfg_network", "ch-config-network"),
("cfg_resolv", "resolv.conf"),
("cfg_hostname", "ch-config-hostname"),
("cfg_hosts", "ch-config-hosts"),
("cfg_sysvinit", "conf-sysvinit"),
("cfg_clock", "ch-config-clock"),
("cfg_console", "ch-config-console"),
("cfg_sysklogd", "ch-config-sysklogd"),
("cfg_rcsite", "ch-config-site"),
("cfg_locale", "ch-config-locale"),
("cfg_inputrc", "ch-config-inputrc"),
("cfg_shells", "ch-config-shells"),
]
CHROOT_STEPS_7 = [
("gettext-tmp", "ch-tools-gettext"),
("bison-tmp", "ch-tools-bison"),
("perl-tmp", "ch-tools-perl"),
("python-tmp", "ch-tools-Python"),
("texinfo-tmp", "ch-tools-texinfo"),
("util-linux-tmp", "ch-tools-util-linux"),
]
def write_chroot_env(lfs, quiet=False):
"""Persist the configured build variables INSIDE the tree, so lfs-helper
(which runs in the chroot, where our config file isn't reachable and the
environment has been reset) uses the same LFS_TGT/MAKEFLAGS you set here.
Without this a custom triplet silently reverts to the default."""
cfg = load_config()
tgt = cfg.get("lfs_tgt") or f"{os.uname().machine}-lfs-linux-gnu"
mkf = cfg.get("makeflags") or "-j$(nproc)"
# config/, because it holds settings; lfs-helper sources it from there.
d = pkgusr_state(lfs, "config")
os.makedirs(d, exist_ok=True)
path = os.path.join(d, "env")
with open(path, "w") as f:
f.write("# written by `lfs` -- sourced by lfs-helper inside the chroot\n"
f'export LFS_TGT="{tgt}"\n'
f'export MAKEFLAGS="{mkf}"\n'
f'# where this tree is mounted OUTSIDE the chroot. Manifests\n'
f'# written before chrooting hold host paths ({lfs}/usr/bin/...)\n'
f'# which are simply /usr/bin/... in here, so lfs-helper strips\n'
f'# this prefix when it reads them.\n'
f'export LFS_HOST_MOUNT="{lfs}"\n'
f'# prefix for collector groups -- see `lfs config`\n'
f'export LFS_COLLECTOR_PREFIX="{_tree_prefix(lfs, "collector_prefix", collector_prefix())}"\n'
f'# prefix for package users. lfs-helper has always READ this;\n'
f'# nothing wrote it, so the default applied by accident.\n'
f'export LFS_PKGUSR_PREFIX="{_tree_prefix(lfs, "pkgusr_prefix", pkgusr_prefix())}"\n'
f'# prefix for config-step users. Its own, NOT the package\n'
f'# prefix stacked on top -- one account, one prefix.\n'
f'export LFS_CFGUSR_PREFIX="{_tree_prefix(lfs, "cfguser_prefix", cfguser_prefix())}"\n'
f'# where accounts live, by kind -- see LAYOUT_DEFAULTS\n'
f'export LFS_PKGUSR_ROOT="{_accounts_root("pkgusr")}"\n'
f'export LFS_CFGUSR_ROOT="{_accounts_root("cfg")}"\n'
f'# languages to keep -- see `lfs-helper prune-locales`\n'
f'export LFS_LOCALES="{cfg.get("locales") or "en de"}"\n'
f'# answers collected by `lfs build-system session`, so the\n'
f'# chroot side never has to stop and ask. `init-accounts`\n'
f'# reads LFS_MAIN_USER instead of blocking on a prompt after\n'
f'# a six-hour build.\n'
f'export LFS_TIMEZONE="{cfg.get("timezone") or "UTC"}"\n'
f'export LFS_MAIN_USER="{cfg.get("main_user") or ""}"\n'
f'export LFS_STRIP="{"1" if strip_binaries() else "0"}"\n'
f'export LFS_REMOVE_TOOLS="{"1" if remove_tools() else "0"}"\n')
_make_world_readable(path)
if not quiet:
print(f"# build env -> {path} (LFS_TGT={tgt}, MAKEFLAGS={mkf})")
return path
def _find_helper_source(name="lfs-helper"):
"""One of our own scripts, shipped next to this tool."""
here = os.path.dirname(os.path.abspath(__file__))
for cand in (os.path.join(here, name),
os.path.join(here, "..", name),
"/usr/bin/" + name, "/usr/share/lfs/" + name):
if os.path.isfile(cand):
return os.path.abspath(cand)
return None
# Our own scripts, copied into the tree beside lfs-helper.
#
# Only lfs-helper used to go in, because only lfs-helper runs during the build.
# But the built system needs the rest -- `packagemanager` cannot be installed
# by `packagemanager` -- and putting them in at build time is free: they are a
# few files, they refresh on every `lfs run`, and a system that boots without
# them has no way to install anything at all.
#
# They cannot be USED until the last build step provides Python and the two
# book-parsing modules. That is fine: they are there, and `packagemanager
# setup` completes them.
# packagemanager_install is retired (1.12.0): folded into `lfs-helper
# pm-install`. Four tools is the endpoint item 6 named.
# Every tool `make install` places, so a new one is refreshed into the chroot
# without anyone remembering this list. lfs-kernel was added to the Makefile
# and not here, so `lfs run` quietly refreshed five of six tools -- and the
# sixth was the one just written. The Makefile's TOOLS line is the source of
# truth; the literal tuple below is the fallback for a tree with no Makefile
# beside the tools.
def _toolchain_scripts():
here = os.path.dirname(os.path.abspath(__file__))
mk = os.path.join(here, "Makefile")
try:
with open(mk) as fh:
for line in fh:
if re.match(r"^TOOLS\s*:?=", line):
names = line.split("=", 1)[1].split()
if names:
return tuple(names)
except OSError:
pass
return ("lfs-helper", "packagemanager", "blfs", "lfs", "lfs-phases",
"lfs-kernel", "lfs-sysvbook")
TOOLCHAIN_SCRIPTS = _toolchain_scripts()
# The package user that owns them.
#
# They are software installed into /usr/bin like anything else, so they get an
# owner like anything else -- otherwise they show up forever as
# 1516 root-owned file(s) that NO manifest claims:
# /usr/bin/lfs-helper
# /usr/bin/list_package
# which is true, unhelpful, and buries the cases that matter. One user for the
# whole repo: they are one thing, versioned and installed together.
PKGUSR_TOOLS_USER = "pkgusr"
def install_helper(lfs, quiet=False):
"""Put our scripts inside the LFS tree so they are on PATH in the chroot.
Called on every `lfs run` and before entering the chroot, so the copies
track the host's -- a stale chroot copy running old code has cost more
debugging time here than any other single thing, which is why each tool
prints a build id.
"""
import shutil
dest_dir = os.path.join(lfs, "usr", "bin")
os.makedirs(dest_dir, exist_ok=True)
helper_dest, copied, missing = None, [], []
for name in TOOLCHAIN_SCRIPTS:
src = _find_helper_source(name)
if not src:
missing.append(name)
continue
dest = os.path.join(dest_dir, name)
shutil.copy(src, dest)
os.chmod(dest, 0o755)
copied.append(name)
if name == "lfs-helper":
helper_dest = dest
# Sweep retired tools OUT of the tree. packagemanager_install was folded
# into `lfs-helper pm-install` (1.12.0); a stale copy left on the chroot's
# PATH is exactly the "old code running after the fix" this sync exists to
# prevent.
for retired in ("packagemanager_install",):
r = os.path.join(dest_dir, retired)
if os.path.lexists(r):
try:
os.unlink(r)
if not quiet:
print("# removed retired %s from the tree" % retired)
except OSError as e:
fail("! could not remove retired %s: %s" % (retired, e))
# Give them an owner. The account may not exist yet -- chapter 7 creates
# /etc/passwd -- so this is best-effort now and repeated on the next run,
# by which time it will exist. Failing to chown is not worth stopping a
# build over, but silence is: that is how a whole tree came out root-owned.
_chown_tools(lfs, dest_dir, copied, quiet=quiet)
if not quiet:
if copied:
print("# installed %s -> %s (on PATH in the chroot)"
% (", ".join(copied), dest_dir))
for name in missing:
fail("! %s not found next to this tool -- the chroot "
"won't have it." % name)
if helper_dest is None and not quiet:
fail('! lfs-helper is missing: the chroot cannot build.')
return helper_dest
def _chown_tools(lfs, dest_dir, names, quiet=False):
"""Hand our scripts to their package user inside the tree."""
if not names:
return
user = pkgusr_prefix() + PKGUSR_TOOLS_USER
uid = _uid_in_tree(lfs, user)
if uid is None:
if not quiet:
print("# (%s does not exist yet -- ownership set on a later run)"
% user)
return
n = 0
for name in names:
p = os.path.join(dest_dir, name)
try:
os.chown(p, uid, uid)
n += 1
except OSError as e:
fail("! could not give %s to %s: %s" % (p, user, e))
if n and not quiet:
print("# gave %d tool(s) to '%s'" % (n, user))
def _uid_in_tree(lfs, user):
"""Look a user up in the TREE's passwd file, not this machine's.
The host has its own accounts and they mean nothing inside the chroot;
reading the host's would write a host uid into the tree, which is exactly
the mistake book 3.1 warns about for /sources.
"""
pw = os.path.join(lfs, "etc", "passwd")
try:
with open(pw) as f:
for line in f:
parts = line.split(":")
if len(parts) > 2 and parts[0] == user:
return int(parts[2])
except (OSError, ValueError):
pass
return None
def _chrootify(text):
"""Turn a cross-compile script into an IN-CHROOT one. Inside the chroot
the final system is simply '/', so $LFS is not required, sources live at
/sources and there is no cross-compiler prefix to put on PATH. The runner
(lfs-phases) decides all of that from one variable."""
return text.replace('pkgusr_stage="cross"', 'pkgusr_stage="chroot"', 1)
def _plain_root_script(name, sid, title, cmds):
"""A straight root command block (book 7.5 / 7.6) -- no unpack/build/install
phases, because nothing is being compiled."""
ts = time.strftime("%Y-%m-%d %H:%M:%S")
# same treatment as the package phases: re-runnable symlinks, and a bare
# informational grep must not abort the step
body = "\n\n".join(_idempotent_links(_soften_diagnostics(c)) for c in cmds)
# `exec /usr/bin/bash --login` in 7.6 restarts the reader's shell so the
# prompt picks up the new /etc/passwd. Running it here would replace this
# script's shell and swallow the rest of the step, so drop it and tell the
# user instead.
body = re.sub(r'^\s*exec /usr/bin/bash --login\s*$', '', body, flags=re.M)
# Substitute FIRST, then look for what is left. Checking the raw text
# flags placeholders we can perfectly well fill in from config.
body, left = _fill_placeholders(body, name)
guard = ""
if left:
# Refuse to run rather than misfire. "LC_ALL=<locale name>" is an input
# redirect to bash, and the failure it produces
# line 13: locale: No such file or directory
# says nothing about the real problem.
names = ", ".join("<%s>" % x for x in left)
guard = (
'cat >&2 <<"__PH__"\n'
'\n== this step needs a decision from you ==\n'
'The book leaves a placeholder here for you to fill in: %s\n'
'\nEither set it once and regenerate:\n'
' lfs config locale en_US.UTF-8 # outside the chroot\n'
' lfs build-system gen-chroot-scripts --run --overwrite\n'
'\nor edit this script directly, then:\n'
' lfs-helper build %s --force\n'
'__PH__\nexit 3\n' % (names, name))
book = "LFS %s" % (load_config().get("default") or "?")
return f"""#!/bin/bash
############################################
### {name} -- {title}
{CROSSCHAIN_MARK} v{CROSSCHAIN_SCRIPT_VERSION}: {name}
### script format v{CROSSCHAIN_SCRIPT_VERSION} origin=lfs
### section : {sid}
### book : {book}
### generated : {ts}
###
### Book housekeeping run as root inside the chroot (no package is built).
### Editable -- lfs-helper runs this file as-is.
set -e
{guard}
{body}
"""
REFIND_SCRIPT = r'''#!/bin/bash
############################################
### refind -- install the rEFInd boot manager
{MARK} v{VER}: refind
### section : (not from the LFS book -- generated by `lfs`)
### generated : {TS}
###
### rEFInd instead of GRUB. The book's chapter 10 runs
### grub-install /dev/sda
### which writes a disk's boot sector. This script never does anything of the
### sort. It only ever ADDS files to an existing, already-formatted EFI System
### Partition:
###
### * it refuses to run if the ESP is not mounted;
### * it never formats, partitions, or writes to a raw device;
### * it never deletes an existing bootloader -- your current one keeps
### working, and rEFInd is added alongside it;
### * it refuses to overwrite an existing rEFInd unless REFIND_OVERWRITE=1.
###
### Set the ESP with: lfs config esp /boot/efi
### Homepage: https://www.rodsbooks.com/refind/
set -e
ESP="${{ESP:-{ESP}}}"
KERNEL_VERSION="${{KERNEL_VERSION:-}}"
# exit 3 = "this needs YOU", not a build error: the tooling must not try to
# fix it by handing out permissions.
need_you() {{ echo "" >&2; echo "== rEFInd needs you to do something first ==" >&2
echo "$*" >&2; echo "" >&2; exit 3; }}
die() {{ echo "!! $*" >&2; exit 1; }}
# ---------------------------------------------------------------- safety ----
[ -n "$ESP" ] || need_you "No ESP configured. Set it with:
lfs config esp /boot/efi"
# Show the user their actual candidates instead of a generic instruction.
suggest_esp() {{
local found=""
# an ESP is a FAT partition, usually flagged as "EFI System"
if command -v lsblk >/dev/null 2>&1; then
found="$(lsblk -rno NAME,FSTYPE,SIZE,PARTTYPENAME,LABEL 2>/dev/null \
| awk '$2 ~ /^(vfat|fat32|fat16|msdos)$/ {{ print }}' || true)"
fi
if [ -z "$found" ] && command -v blkid >/dev/null 2>&1; then
found="$(blkid -t TYPE=vfat -o device 2>/dev/null || true)"
fi
if [ -n "$found" ]; then
echo " FAT partitions on this machine (your ESP is almost certainly one):" >&2
# The two sources disagree about the prefix: `lsblk -rno NAME` gives a
# bare name (nvme0n1p1) while `blkid -o device` gives a full path
# (/dev/nvme0n1p1). Prefixing both produced
# /dev//dev/nvme0n1p1
# in the one message whose whole job is to tell you what to type.
# Add /dev/ only where it is missing.
printf '%s\n' "$found" \
| sed -e 's|^\([^/]\)|/dev/\1|' -e 's|^| |' >&2
else
echo " Could not list partitions from in here. Run this OUTSIDE the chroot:" >&2
echo " lsblk -o NAME,SIZE,FSTYPE,PARTTYPENAME,MOUNTPOINT" >&2
fi
}}
if [ ! -d "$ESP" ]; then
{{
echo ""
echo "== rEFInd needs you to do something first =="
echo "$ESP does not exist, so there is nowhere to install to."
echo ""
echo "rEFInd goes on your EFI System Partition -- the one your firmware"
echo "already boots from. It is ADDED alongside what is there; nothing is"
echo "formatted and no existing bootloader is removed."
echo ""
suggest_esp
echo ""
echo " Then, inside the chroot:"
echo " mkdir -pv $ESP"
echo " mount -v -t vfat /dev/<esp> $ESP"
echo " lfs-helper build refind --force"
echo ""
echo " Not ready to deal with the bootloader yet? Skip it:"
echo " lfs-helper done refind"
}} >&2
exit 3
fi
if ! mountpoint -q "$ESP" 2>/dev/null && ! grep -qs " ${{ESP}} " /proc/mounts; then
need_you "$ESP exists but nothing is mounted there.
Writing into an unmounted directory would put the bootloader on the wrong
filesystem, so this stops here.
mount -v /dev/<your-esp-partition> $ESP"
fi
# An ESP is FAT; if this looks like anything else, we are in the wrong place.
esp_fs="$(findmnt -no FSTYPE --target "$ESP" 2>/dev/null || true)"
case "$esp_fs" in
vfat|msdos|fat|fat32|"") : ;;
*) need_you "$ESP is a $esp_fs filesystem, not FAT -- that is not an ESP.
Refusing to touch it. Check which partition is your ESP:
lsblk -o NAME,SIZE,FSTYPE,PARTTYPENAME,MOUNTPOINT" ;;
esac
echo "# ESP: $ESP ($esp_fs)"
echo "# existing EFI entries (these are NOT touched):"
ls -1 "$ESP/EFI" 2>/dev/null | sed 's/^/# /' || echo "# (none)"
dest="$ESP/EFI/refind"
# Add a menu entry for this system to an existing rEFInd.
#
# If rEFInd is already installed -- as it will be if this machine already boots
# with it -- reinstalling is the wrong move. What is actually needed is an
# entry so the firmware menu can boot LFS. That is additive: the config is
# backed up first, nothing else in it is changed, and other bootloaders on the
# ESP are untouched.
add_lfs_entry() {{
local conf="$1"
[ -f "$conf" ] || return 1
if grep -q '^menuentry "LFS"' "$conf" 2>/dev/null; then
return 2 # already there
fi
local bak="$conf.bak-$(date +%Y%m%d%H%M%S)"
cp -p "$conf" "$bak"
local root_dev root_uuid root_spec kern
root_dev="$(findmnt -no SOURCE / 2>/dev/null || echo /dev/CHANGE-ME)"
root_uuid="$(blkid -s UUID -o value "$root_dev" 2>/dev/null || true)"
if [ -n "$root_uuid" ]; then root_spec="UUID=$root_uuid"; else root_spec="$root_dev"; fi
kern="$(ls -1 /boot/vmlinuz-* 2>/dev/null | sort -V | tail -n1 || true)"
kern="${{kern#/boot/}}"
cat >> "$conf" <<RENTRY
# --- added by lfs on $(date +%Y-%m-%d) ---------------------------------
menuentry "LFS" {{
icon /EFI/refind/icons/os_linux.png
loader /boot/${{kern:-vmlinuz}}
options "root=$root_spec ro"
}}
RENTRY
echo "# added an \"LFS\" entry to $conf"
echo "# root: $root_spec"
echo "# kernel: /boot/${{kern:-vmlinuz (NO KERNEL BUILT YET -- edit this)}}"
echo "# backup: $bak"
return 0
}}
if [ -d "$dest" ] && [ "${{REFIND_OVERWRITE:-0}}" != "1" ]; then
echo "# rEFInd is already installed at $dest -- not reinstalling it."
conf="$dest/refind.conf"
# `set -e` aborts on a non-zero return, so capture the status rather than
# letting the function's "2 = already there" kill the script
rc_entry=0
add_lfs_entry "$conf" || rc_entry=$?
case $rc_entry in
0) exit 4 ;; # entry added; nothing lands in the LFS tree
2) echo "# an \"LFS\" entry is already in $conf -- nothing to do."
exit 4 ;; # 4 = deliberately nothing to do
*) echo "# no $conf found; rEFInd is installed but unconfigured."
echo "# To install this rEFInd build over it: REFIND_OVERWRITE=1 $0"
exit 4 ;;
esac
fi
# ------------------------------------------------------------- the files ----
# rEFInd ships as a binary zip. Point REFIND_SRC at it, or drop it in /sources.
src="${{REFIND_SRC:-}}"
if [ -z "$src" ]; then
src="$(ls -1 /sources/refind-bin-*.zip /sources/refind-bin-*.tar.gz 2>/dev/null | head -n1 || true)"
fi
if [ -z "$src" ]; then
src="$(ls -1 /sources/refind-src-*.tar.gz 2>/dev/null | head -n1 || true)"
fi
[ -n "$src" ] && [ -e "$src" ] || need_you "No rEFInd archive in /sources.
Binary build (simplest -- no compiler or gnu-efi needed):
https://www.rodsbooks.com/refind/getting.html
-> refind-bin-<version>.zip
Or the source:
wget -O /sources/refind-src-0.14.2.tar.gz \\
'https://sourceforge.net/projects/refind/files/0.14.2/refind-src-0.14.2.tar.gz/download'
Then: lfs-helper build refind --force"
work="$(mktemp -d)"
trap 'rm -rf "$work"' EXIT
case "$src" in
*.zip) unzip -q "$src" -d "$work" ;;
*.tar.gz) tar -xf "$src" -C "$work" ;;
*) die "don't know how to unpack $src" ;;
esac
root="$(find "$work" -maxdepth 1 -type d -name 'refind-bin-*' | head -n1)"
if [ -z "$root" ]; then
# a source tree: build it (needs gnu-efi)
srcroot="$(find "$work" -maxdepth 1 -type d -name 'refind-*' | head -n1)"
if [ -n "$srcroot" ] && [ -f "$srcroot/Makefile" ]; then
echo "# source archive -- building rEFInd (needs gnu-efi)"
if ! make -C "$srcroot" gnuefi >/dev/null 2>&1; then
need_you "Could not build rEFInd from source.
It needs gnu-efi, which LFS does not install. The binary build avoids this
entirely -- download refind-bin-<version>.zip into /sources instead:
https://www.rodsbooks.com/refind/getting.html"
fi
root="$srcroot"
fi
fi
[ -n "$root" ] || root="$work"
arch=x64
case "$(uname -m)" in
x86_64) arch=x64 ;;
aarch64) arch=aa64 ;;
i?86) arch=ia32 ;;
esac
mkdir -p "$dest/drivers_$arch" "$dest/tools_$arch"
install -m 644 "$root/refind/refind_$arch.efi" "$dest/"
[ -d "$root/refind/icons" ] && cp -r "$root/refind/icons" "$dest/"
for d in "$root/refind/drivers_$arch"/*.efi; do
[ -e "$d" ] && install -m 644 "$d" "$dest/drivers_$arch/"
done
# ------------------------------------------------------------ refind.conf ---
if [ ! -f "$dest/refind.conf" ]; then
if [ -f "$root/refind/refind.conf-sample" ]; then
install -m 644 "$root/refind/refind.conf-sample" "$dest/refind.conf"
else
: > "$dest/refind.conf"
fi
cat >> "$dest/refind.conf" <<'RCONF'
# --- added by lfs -------------------------------------------------------
# rEFInd scans for other bootloaders by default, so anything already on this
# ESP keeps working. The entry below boots this LFS system explicitly.
timeout 10
RCONF
fi
# a manual stanza for this system, appended only once
if ! grep -q "^menuentry \"LFS\"" "$dest/refind.conf" 2>/dev/null; then
root_dev="$(findmnt -no SOURCE / 2>/dev/null || echo /dev/CHANGE-ME)"
root_uuid="$(blkid -s UUID -o value "$root_dev" 2>/dev/null || true)"
if [ -n "$root_uuid" ]; then root_spec="UUID=$root_uuid"; else root_spec="$root_dev"; fi
kern="$(ls -1 /boot/vmlinuz-* 2>/dev/null | sort -V | tail -n1 || true)"
kern="${{kern#/boot/}}"
cat >> "$dest/refind.conf" <<RENTRY
menuentry "LFS" {{
icon /EFI/refind/icons/os_linux.png
volume LFS
loader /boot/${{kern:-vmlinuz}}
options "root=$root_spec ro"
}}
RENTRY
fi
echo "# rEFInd installed to $dest"
echo "#"
echo "# NOT done automatically (they change your firmware's boot order):"
echo "# efibootmgr --create --disk /dev/<esp-disk> --part <n> \\"
echo "# --loader '\\EFI\\refind\\refind_$arch.efi' --label rEFInd"
echo "# Run that yourself once you have checked the entry above is correct."
# Everything rEFInd writes goes to the ESP, never into the LFS filesystem, so
# there is nothing for the file tracker to see. Exit 4 says "success, but by
# design nothing was installed into the system tree".
exit 4
'''
def _refind_script():
import time as _t
return REFIND_SCRIPT.format(
MARK=CROSSCHAIN_MARK, VER=CROSSCHAIN_SCRIPT_VERSION,
TS=_t.strftime("%Y-%m-%d %H:%M:%S"),
ESP=(load_config().get("esp") or "/boot/efi"))
def _install_collector_groups(lfs):
"""Copy a collector-group export into the tree so lfs-helper can import it.
Without this a rebuild asks the same "which group should share this
directory?" question for every package all over again."""
src = load_config().get("collector_import_file")
if not src:
# fall back to one sitting in the store, so a rebuild picks up the
# groups from the previous system without any configuration at all
default = os.path.join(store_dir(), "collector-groups.export")
if os.path.isfile(default):
src = default
else:
return
src = os.path.expanduser(src)
if not os.path.isfile(src):
fail("! collector_import_file is set but %s does not "
"exist" % src)
return
import shutil
# groups/, where lfs-helper looks for it. These two paths are the whole
# mechanism: `lfs` puts the file where `lfs-helper` reads it, and nothing
# else connects them. When the state directory was sorted in 1.9.0 this
# one kept writing to the top level while the reader moved into groups/,
# so a rebuild with a perfectly good export asked
# 'p_gcc' needs to install into: /usr/lib/bfd-plugins
# Which group should share it?
# for a directory the file already answered.
dst_dir = pkgusr_state(lfs, "groups")
os.makedirs(dst_dir, exist_ok=True)
dst = os.path.join(dst_dir, "collector-groups.import")
# Only say anything when it CHANGES. This runs on every chroot entry, so
# that an edit on the host reaches the build without regenerating the
# scripts -- but three unchanging lines on every entry is noise, and noise
# is what stopped anyone noticing the file was going to the wrong place.
same = (os.path.isfile(dst)
and open(dst, "rb").read() == open(src, "rb").read())
shutil.copy(src, dst)
_make_world_readable(dst)
if same:
return
n_dirs = n_groups = 0
for line in open(dst):
if line.startswith("dir|"):
n_dirs += 1
elif line.startswith("group|"):
n_groups += 1
ok("# collector groups: %d directory decision(s), %d group(s) from %s"
% (n_dirs, n_groups, src))
hint("# the build will not ask about those directories again")
hint("# apply them now with: lfs-helper import-groups --run")
def _gen_blfs_extra(outdir, order, unresolved):
"""Generate install scripts for the BLFS packages we want during the build.
LFS ships no download tool at all, so a freshly booted system cannot fetch
anything -- not even the sources for its own next package. wget is the
smallest fix, and BLFS already has the script for it; there is no reason to
write a second one here."""
extras = extra_packages()
if not extras:
return 0
blfs = _find_blfs()
if not blfs:
fail("! extra_packages is set (%s) but the 'blfs' tool is "
"not next to this one -- skipping" % " ".join(extras))
return 0
import subprocess
written = 0
for pkg in extras:
# `blfs script` WRITES install_<Name-Version> into the current
# directory and prints only a summary, so run it in a scratch dir and
# pick up the file rather than trying to read it from stdout.
import tempfile
tmp = tempfile.mkdtemp()
try:
# honour the same book selectors blfs takes, so a pinned book
# file or version is used here too
sel = []
if os.environ.get("BLFS_BOOK_FILE"):
sel = ["--book-file", os.environ["BLFS_BOOK_FILE"]]
elif os.environ.get("BLFS_BOOK"):
sel = ["--book", os.environ["BLFS_BOOK"]]
r = subprocess.run(blfs + sel + ["script", pkg],
capture_output=True, text=True,
timeout=600, cwd=tmp)
except (OSError, subprocess.SubprocessError) as e:
fail("! blfs script %s: %s" % (pkg, e))
continue
made = [f for f in os.listdir(tmp) if f.startswith("install_")]
if r.returncode != 0 or not made:
detail = ""
for line in (r.stderr or r.stdout or "").strip().splitlines()[::-1]:
if line.strip():
detail = ": " + line.strip()
break
fail("! blfs has no script for '%s'%s" % (pkg, detail))
continue
with open(os.path.join(tmp, made[0])) as f:
raw = f.read()
import shutil as _sh
_sh.rmtree(tmp, ignore_errors=True)
text = _chrootify(raw)
text, left = _fill_placeholders(text, pkg)
if left:
unresolved.append((pkg, left))
dest = os.path.join(outdir, "%s.sh" % pkg)
with open(dest, "w") as f:
f.write(text)
os.chmod(dest, 0o755)
if pkg not in order:
order.append(pkg)
written += 1
print("# %s: install script from the BLFS book" % pkg)
return written
def bootstrap_sources():
"""What the built system needs downloaded before it can fetch anything.
ASKED OF `packagemanager setup`, which owns that decision -- wget and the
Python modules the tools need are its job to install, so the list of what
to download is its to declare. This command only has to make sure they are
on disk before the chroot is sealed, because get-sources is the last moment
there is still a network.
Two tools, one list. A copy here would be a second thing to go stale, and
the first version of this did exactly that: `bootstrap_packages` in lfs's
own config, resolving wget separately from the tool that installs it.
"""
import subprocess
try:
# `bootstrap` since 1.11.7; `setup` is still accepted as an alias, so
# this keeps working against an older packagemanager on the tree.
out = subprocess.run(["packagemanager", "bootstrap", "--sources"],
capture_output=True, text=True, timeout=180)
if out.returncode != 0 and not (out.stdout or "").strip():
out = subprocess.run(["packagemanager", "setup", "--sources"],
capture_output=True, text=True, timeout=180)
except (OSError, subprocess.SubprocessError):
warn("! could not run `packagemanager bootstrap --sources`.")
warn(" The built system will have no way to fetch anything until you")
warn(" put a download tool on it by hand.")
return []
urls = [l.strip() for l in (out.stdout or "").splitlines()
if l.strip().startswith(("http://", "https://", "ftp://"))]
if not urls:
if not _blfs_book_on_host():
# No package named here, deliberately: which packages the built
# system needs is packagemanager's list, and a copy of it in this
# file is the thing a test forbids. What lfs knows is that the
# answer comes from a book, and that there is no book.
warn("! No BLFS book on this machine, so nothing can say where")
warn(" those sources are -- and the built system cannot fetch the")
warn(" book until it has a download tool, which is itself looked")
warn(" up in the book. Break the circle here:")
warn(" blfs fetch")
warn(" then run get-sources again.")
warn("! `packagemanager bootstrap --sources` listed nothing.")
for line in (out.stderr or "").splitlines():
warn(" %s" % line)
return urls
def _write_tool_stamps(lfs, tools):
"""Record the md5 of every tool we copied in.
The chroot keeps running whatever was copied last; without a record, a fix
made on the host silently does not apply in there and the same bug appears
to come back."""
import hashlib
stamps = {}
for name, src in tools:
try:
with open(src, "rb") as f:
stamps[name] = hashlib.md5(f.read()).hexdigest()
except OSError:
continue
path = os.path.join(lfs, "usr", "share", "lfs", "tool-stamps.json")
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w") as f:
json.dump(stamps, f, indent=2)
def _sync_report(title, items, note=None):
"""One consistent shape for 'here is what I copied'.
The old output mixed three different layouts, so it was easy to miss that
lfs-helper was being copied at all."""
print(" %s" % title)
for i in items:
print(" %s" % i)
if note:
print(" (%s)" % note)
def _run_choose_book():
"""Ask which book to build, and make it the default.
The checklist step is "choose and fetch a book" -- a choice, so ask for it
instead of printing sixty versions and stopping."""
cfg = load_config()
cached = sorted(cached_versions())
if cached:
print("\nAlready downloaded:")
for i, v in enumerate(cached, 1):
print(" %2d) %s" % (i, v))
print("\nEnter a number, or any version name to download it")
print("(`lfs books` lists everything available).")
else:
print("\nNothing downloaded yet. Enter a version, e.g. 12.4")
print("(`lfs books` lists everything available).")
try:
ans = input("\nbook> ").strip()
except (EOFError, KeyboardInterrupt):
print()
return False
if not ans:
return False
if ans.isdigit() and cached:
idx = int(ans) - 1
if not (0 <= idx < len(cached)):
sys.stderr.write("no such number\n")
return False
ans = cached[idx]
argv = [sys.executable, os.path.abspath(__file__), "set-default", ans]
if subprocess.call(argv) != 0:
# not cached yet -- fetch it, then set it
if subprocess.call([sys.executable, os.path.abspath(__file__),
"fetch", ans]) != 0:
return False
if subprocess.call(argv) != 0:
return False
return True
def _blfs_book_on_host():
"""Is a BLFS book cached where install-tools will find it?
Its own store, not lfs's -- /usr/share/blfs/books. This is the same list
install-tools copies into the tree, so asking here answers exactly the
question the built system will have.
When $BLFS_STORE is set it is the ONLY place looked: an override that
half-applies is this project's oldest bug shape -- an empty overridden
store fell through to /usr/share/blfs and reported the system's books as
the override's.
"""
if os.environ.get("BLFS_STORE"):
cands = (os.environ["BLFS_STORE"],)
else:
cands = ("/usr/share/blfs", os.path.expanduser("~/.cache/blfs"))
for cand in cands:
d = os.path.join(cand, "books")
try:
if any(f.endswith((".html", ".html.gz")) for f in os.listdir(d)):
return True
except OSError:
continue
return False
def _run_choose_blfs_book():
"""Offer to fetch the BLFS book, HERE, where there is still a network.
The chicken and egg this closes: on the built system `packagemanager
bootstrap` installs wget by looking it up in the BLFS book -- and without
a book there is nothing to look wget up in, while fetching the book needs
the wget that cannot be installed:
wget: there is no BLFS book on this system to look it up in.
Could not install: wget
The host has a network and the copy machinery already exists; all that was
missing was asking, at a moment when the answer can still be acted on.
Declining is recorded, so the checklist does not ask again on every resume.
"""
print("\nThe built system needs the BLFS book to install anything beyond")
print("LFS itself -- including wget, which is how it would otherwise")
print("fetch the book. This machine has a network; that one will not")
print("until wget exists.")
print("\nFetching it now copies it into the tree with the tools.")
try:
ans = input("\nFetch the BLFS book now? [Y/n] ").strip().lower()
except (EOFError, KeyboardInterrupt):
print()
return False
if ans in ("n", "no", "skip"):
cfg = load_config()
cfg["blfs_book"] = "skip"
save_config(cfg)
print("\nSkipped. Get it later with `blfs fetch` on this machine, or")
print("inside the chroot once it has a download tool:")
print(" blfs import /path/to/BLFS-BOOK-...-nochunks.html")
return True
blfs = _find_blfs()
if not blfs:
fail('! blfs is not on this machine, so it cannot fetch.\n Install the tools first: make install')
return False
rc = subprocess.call(blfs + ["fetch"])
if rc != 0:
fail('\n! `blfs fetch` failed. It needs a working network here.\n Fetch it by hand, or skip and import it later:\n blfs books --discover\n blfs fetch <version>')
return False
if not _blfs_book_on_host():
fail('! blfs fetch reported success but cached nothing.')
return False
cfg = load_config()
cfg["blfs_book"] = "fetched"
save_config(cfg)
return True
def _run_banner(step, steps, cmd=None):
"""One consistent header before every step, so it is always clear what is
happening now, how far along it is, and what comes next. Long steps print
nothing for minutes; without this you cannot tell work from a hang."""
done = sum(1 for s in steps if s["done"])
total = len(steps)
pending = [s for s in steps if not s["done"]]
nxt = pending[1]["title"] if len(pending) > 1 else "enter the chroot"
bar_w = 20
filled = int(bar_w * done / total) if total else 0
bar = "#" * filled + "." * (bar_w - filled)
W = 66
def row(text):
print("| %-*s |" % (W - 4, text[:W - 4]))
print()
print("+" + "-" * (W - 2) + "+")
row("step %d of %d [%s]" % (done + 1, total, bar))
row("")
row("NOW: " + step["title"])
row("NEXT: " + nxt)
if cmd:
row(" $ " + cmd)
print("+" + "-" * (W - 2) + "+")
_STACK_VERSION_RE = re.compile(r"^#\s*stack-version:\s*(\S+)", re.M)
def _stack_version(path):
"""The version a stack file declares, if it declares one."""
try:
with open(path, encoding="utf-8", errors="replace") as f:
m = _STACK_VERSION_RE.search(f.read(2000))
except OSError:
return None
return m.group(1) if m else None
def _version_tuple(v):
out = []
for part in str(v).split("."):
digits = "".join(c for c in part if c.isdigit())
out.append(int(digits) if digits else 0)
return tuple(out)
def _tree_prefix(lfs, key, fallback):
"""The BUILT SYSTEM'S own prefix, not this host's.
The chroot scripts export LFS_*_PREFIX, and the values were read from the
HOST -- so a prefix the user changed on their own machine was baked into
the build system's environment, where it overrode the tree's config.
That is how an empty LFS_PKGUSR_PREFIX reached a chroot whose config says
"p".
The tree's config decides for the tree. The host's value is only a
fallback for a tree that has not been configured yet.
"""
for cand in ("usr/share/lfs/config.json", "root/.local/share/lfs/config.json"):
path = os.path.join(lfs, cand)
try:
with open(path) as f:
v = json.load(f).get(key)
if v is not None:
return str(v).rstrip("_")
except (OSError, ValueError):
continue
return fallback
def _refresh_chroot_tools(lfs, quiet=False):
"""Copy the TOOLS into the tree if what is in there is out of date.
Compares content, so an unchanged tool is not touched and there is no
output in the common case.
All five, not just lfs-helper. Refreshing only the bash driver left the
python tools inside the chroot older than the host's, and every
packagemanager run opened with "These tools differ from the copies on the
host" plus advice to sync -- from a tool that had just been given the
chance to sync and had refreshed one file.
Nothing here touches the tree's config or its books: `lfs set-default`
inside the chroot is the system's own decision, and entering must not
quietly undo it."""
import hashlib, shutil
def _md5(p):
try:
with open(p, "rb") as f:
return hashlib.md5(f.read()).hexdigest()
except OSError:
return None
here = os.path.dirname(os.path.abspath(__file__))
updated, failed, pairs = [], [], []
# Sweep retired tools out of the tree first -- a stale
# packagemanager_install on the chroot's PATH is old code waiting to run.
for retired in ("packagemanager_install",):
r = os.path.join(lfs, "usr", "bin", retired)
if os.path.lexists(r):
try:
os.unlink(r)
updated.append(retired + " (removed, retired)")
except OSError as e:
failed.append("%s (%s)" % (retired, e))
# THE STACKS TRAVEL WITH THE TOOLS.
#
# This refreshed the four binaries and nothing else, so a chroot kept the
# sway.stack it was first given -- the user updated the tools four times
# and kept reading a 39-entry file while the source had 47, wondering why
# the package count never changed. machine.conf is the exception: it is
# the user's own choice of build options, so it is only placed when it
# does not exist yet.
refreshed_stacks = []
# WHERE THE STACKS ACTUALLY ARE.
#
# This looked only beside the running tool -- and the running tool is
# /usr/bin/lfs, so it looked in /usr/bin/stacks and found nothing, in
# silence. The host had 31 book entries and the chroot kept 23. A
# checkout keeps them beside the tool; an installed system keeps them in
# /etc/pkgusr/stacks. Look in both, nearest first.
_stack_src = ""
for _cand in (os.path.join(here, "stacks"), "/etc/pkgusr/stacks"):
if os.path.isdir(_cand) and os.path.abspath(_cand) != os.path.abspath(
os.path.join(lfs, "etc", "pkgusr", "stacks")):
_stack_src = _cand
break
if not _stack_src and not quiet:
print("# no stack files found to refresh (looked beside the tool and"
" in /etc/pkgusr/stacks)")
if _stack_src and os.path.isdir(_stack_src):
# kernel.conf lives beside the stacks, not in them, and lfs-kernel
# refuses to run without it -- so refreshing the tool and not the
# config would put a working command in the chroot that cannot start.
# An edited copy in the chroot is kept, as everywhere else.
# ...found either beside the tool (a checkout) or in /etc/pkgusr
# (an installed system), the same two places as the stacks.
for _cf in ("kernel.conf",):
_cs = ""
for _c2 in (os.path.join(here, _cf),
os.path.join("/etc/pkgusr", _cf)):
if os.path.isfile(_c2) and os.path.abspath(_c2) != os.path.abspath(
os.path.join(lfs, "etc", "pkgusr", _cf)):
_cs = _c2
break
if not _cs:
continue
_cd = os.path.join(lfs, "etc", "pkgusr", _cf)
try:
os.makedirs(os.path.dirname(_cd), exist_ok=True)
if not os.path.exists(_cd):
shutil.copy2(_cs, _cd)
refreshed_stacks.append(_cf)
elif _md5(_cs) != _md5(_cd):
shutil.copy2(_cs, _cd + ".new")
refreshed_stacks.append(_cf + ".new (yours kept)")
except OSError as e:
fail("! could not refresh %s: %s" % (_cf, e))
_stack_dst = os.path.join(lfs, "etc", "pkgusr", "stacks")
try:
os.makedirs(_stack_dst, exist_ok=True)
for fn in sorted(os.listdir(_stack_src)):
# NOT EVERYTHING IN THAT DIRECTORY IS A STACK.
#
# .shipped/ is make install's record of what it last shipped
# (1.14.98), and .new/.bak are the merge leftovers. The
# refresh copied all of them into the chroot -- ".shipped/
# desktop-user.sh" and a services.stack.new that had nothing
# to do with the chroot's own file. Bookkeeping does not
# travel.
if fn.startswith(".") or fn.endswith((".new", ".bak")):
continue
src_f = os.path.join(_stack_src, fn)
dst_f = os.path.join(_stack_dst, fn)
# A STACK ENTRY MAY HAVE A DIRECTORY.
#
# stacks/mytools/install_mytools is the package-user script
# the mytools step runs, and this loop took files only -- so
# the directory never crossed into the chroot and the step
# died with "missing .../mytools/install_mytools". The
# Makefile had the same fault (1.14.102) and was fixed there;
# this is the second copy of the same walk.
if os.path.isdir(src_f):
try:
os.makedirs(dst_f, exist_ok=True)
for sub in sorted(os.listdir(src_f)):
s_sub, d_sub = os.path.join(src_f, sub), os.path.join(dst_f, sub)
if not os.path.isfile(s_sub):
continue
if os.path.exists(d_sub) and _md5(s_sub) == _md5(d_sub):
continue
shutil.copy2(s_sub, d_sub)
os.chmod(d_sub, 0o755)
refreshed_stacks.append("%s/%s" % (fn, sub))
except OSError as e:
fail("! could not refresh %s: %s" % (fn, e))
continue
if not os.path.isfile(src_f):
continue
if fn == "machine.conf" and os.path.exists(dst_f):
continue # the machine's own options: never overwritten
if os.path.exists(dst_f) and _md5(src_f) == _md5(dst_f):
continue # already current
# NEVER GO BACKWARDS. A stack file edited in the chroot, or
# one from a newer release, must not be replaced by an older
# copy from the host -- the tools are versioned, so ask.
_sv, _dv = _stack_version(src_f), _stack_version(dst_f)
if _sv and _dv and _version_tuple(_dv) > _version_tuple(_sv):
if not quiet:
print("# kept the newer %s already in the chroot (%s > %s)"
% (fn, _dv, _sv))
continue
# AND A STACK FILE THAT DIFFERS IS THE CHROOT'S OWN.
#
# Stack files carry no version, so the guard above never
# fires for them: the host's copy simply won. The user had
# just taken the 6-entry services.stack inside the chroot
# with --take-new, and this loop put the host's 3-entry file
# back over it. Same rule as `make install`: an existing
# file that differs is theirs, and the incoming one waits
# beside it.
if fn.endswith(".stack") and os.path.exists(dst_f):
shutil.copy2(src_f, dst_f + ".new")
if not quiet:
print("# %s differs in the chroot -- kept it, new one"
" is %s.new" % (fn, fn))
continue
shutil.copy2(src_f, dst_f)
if fn.endswith(".sh"):
os.chmod(dst_f, 0o755)
refreshed_stacks.append(fn)
if refreshed_stacks and not quiet:
print("# refreshed in the chroot: %s" % ", ".join(refreshed_stacks))
except OSError as e:
fail("! could not refresh the stacks: %s" % e)
for tool in TOOLCHAIN_SCRIPTS:
src = _find_helper_source() if tool == "lfs-helper" \
else os.path.join(here, tool)
if not src or not os.path.isfile(src):
continue
pairs.append((tool, src))
dst = os.path.join(lfs, "usr", "bin", tool)
if _md5(src) == _md5(dst):
continue
try:
shutil.copy(src, dst)
os.chmod(dst, 0o755)
except OSError as e:
failed.append("%s (%s)" % (tool, e))
continue
# Verify, rather than assume. A copy that does not stick -- a
# read-only mount, a busy binary, the wrong destination -- would
# otherwise show up only as this same message on every entry, which
# is exactly what it did.
if _md5(src) != _md5(dst):
failed.append("%s (copied, but the file did not change)" % tool)
continue
updated.append(tool)
# the completion travels with them; stale completion is worse than none
if updated:
try:
_install_completion(lfs, verbose=False)
except Exception:
pass
# The stamps are what `packagemanager` compares against to decide whether
# the chroot is behind. Refreshing the binaries and leaving the stamps
# alone made it announce "these tools differ from the copies on the host"
# about tools it had just been given, and tell the user to sync -- on the
# path that had synced them.
if pairs and (updated or failed):
try:
_write_tool_stamps(lfs, pairs)
except OSError:
pass
if failed and not quiet:
fail("! could not refresh in the chroot: %s" % ", ".join(failed))
if updated and not quiet:
print("# refreshed in the chroot: %s (now %s, build %s)"
% (", ".join(updated), LFS_VERSION, _build_id()))
return updated
def cmd_bs_run(args):
"""Run the whole outside-the-chroot procedure, from wherever it stands.
The checklist behind `next` already knows what is done and what comes next;
this executes it instead of printing it. After a host reboot that means:
remount, re-prepare the chroot, and drop you into it -- one command.
It stops the moment a step needs YOU (an unconfigured setting, a missing
book) or a step fails, and it never repeats a step that made no progress.
The chroot itself is the destination: once everything before it is done,
this execs `chroot enter` and you are inside."""
if inside_chroot():
_refuse_inside_chroot("build-system", "run")
# A FRESH run (mounted tree, nothing laid out yet) states the book and the
# whole saved config ONCE before touching anything. Reusing last month's
# settings -- or last month's default book -- should be a decision made
# here, not a surprise discovered nine steps in. (SysV-book session)
cfg = load_config()
ver = build_version()
lfs_dir = _lfs_dir()
mounted = bool(lfs_dir) and (not cfg.get("lfs_device")
or _is_mounted(lfs_dir))
if (ver and mounted and sys.stdin.isatty()
and not os.path.isdir(os.path.join(lfs_dir, "tools"))):
print(f"\nFresh build -- nothing is laid out under {lfs_dir} yet.")
print(f"\n book = {ver}")
for k, _desc, _d in _SESSION_KEYS:
print(f" {k:<16} = {cfg.get(k, '') or '(unset)'}")
ans = input("\nBuild with this? [Y/n/e(dit)]: ").strip().lower()
if ans in ("n", "no"):
print("stopped, nothing touched.")
print(" another book: lfs set-default <version> "
"(lfs books shows them)")
print(" other settings: lfs build-system session --reconfigure")
sys.exit(0)
if ans in ("e", "edit"):
_prompt_session_config(cfg, force=True)
def enter_chroot():
lfs = require_mounted_lfs()
# Refresh the in-chroot tools FIRST. They are copies: fixing a bug in
# lfs-helper out here changes nothing inside until it is copied in
# again. Relying on the user to remember means they keep hitting bugs
# that were fixed days ago, and every diagnosis starts from the wrong
# version of the evidence.
_refresh_chroot_tools(lfs)
print()
print("+" + "-" * 64 + "+")
print("| %-62s |" % "everything outside the chroot is done")
print("| %-62s |" % "")
print("| %-62s |" % "NOW: entering the chroot")
print("| %-62s |" % "NEXT: lfs-helper next (inside, to build the system)")
print("+" + "-" * 64 + "+")
print()
fake = argparse.Namespace(action="enter", run=False)
cmd_bs_chroot(fake)
sys.exit(0)
last_title = None
for _ in range(30):
steps, lfs = _next_steps()
pending = [st for st in steps if not st["done"]]
if not pending:
enter_chroot()
nxt = pending[0]
title = nxt["title"]
# the chroot is the destination, and everything after it is inside work
if title.startswith(("build the system inside the chroot",
"install the Python modules",
"hand over")):
enter_chroot()
if title == last_title:
fail("\n! '%s' ran but is still pending -- stopping "
"rather than looping.\n Run it by hand to see "
"why:\n %s"
% (title, nxt["cmds"][0] if nxt["cmds"] else "?"))
sys.exit(1)
last_title = title
cmd = nxt["cmds"][0] if nxt["cmds"] else ""
cmd = cmd.strip()
if cmd.startswith("sudo "):
cmd = cmd[5:]
# Steps that need a DECISION are walked through here rather than
# printed as homework. Resuming a build should not mean reading a
# list and retyping a command.
if title == "choose and fetch a book":
_run_banner(nxt, steps)
if not _run_choose_book():
sys.exit(1)
last_title = None
continue
if title == "fetch the BLFS book (for the built system)":
_run_banner(nxt, steps)
if not _run_choose_blfs_book():
sys.exit(1)
last_title = None
continue
if title == "complete the configuration":
_run_banner(nxt, steps)
argv = [sys.executable, os.path.abspath(__file__),
"build-system", "session"]
if subprocess.call(argv) != 0:
sys.exit(1)
last_title = None
continue
# Anything else must be a command that actually CHANGES something.
# `lfs books` only lists: running it can never complete a step, which
# is how this looped until the guard caught it.
applies = ("--run" in cmd or cmd.startswith("lfs build-system session"))
if not cmd.startswith("lfs ") or "<" in cmd or not applies:
print("\nThis needs you first: %s" % title)
for c in nxt["cmds"]:
print(" %s" % c)
if nxt.get("note"):
print("\n (%s)" % nxt["note"])
sys.exit(1)
_run_banner(nxt, steps, cmd)
argv = [sys.executable, os.path.abspath(__file__)] + shlex.split(cmd)[1:]
rc = subprocess.call(argv)
if rc != 0:
fail("\n! step failed (exit %d): %s"
" Fix it, then just run `lfs build-system run` "
"again -- it continues from here.\n" % (rc, title))
sys.exit(rc)
fail('! 30 steps and still not at the chroot -- something is cycling. Run `lfs build-system next --all`.')
sys.exit(1)
# --------------------------------------------------------------------------- #
# $LFS/sources: ONE convention, and it is the book's own
# --------------------------------------------------------------------------- #
# Book 3.1 says exactly two things about it:
#
# mkdir -v $LFS/sources
# chmod -v a+wt $LFS/sources # root:root, 1777
# chown root:root $LFS/sources/* # the CONTENTS too
#
# and gives the reason in full:
#
# "The file system records the owner by its UID, and the UID of a normal
# user in the host distro is not assigned in LFS. So the files will be
# left owned by an unnamed UID in the final LFS system."
#
# That is not hypothetical here, it is worse than the book's version. The host
# `lfs` account is created by book 4.3's own `useradd`, which does not pin a
# uid, so it gets whatever the host had free -- 10753 on a Debian box. Package
# users in the built system start at PKG_UID_MIN=10000. A host uid in that
# range does not merely show as a number inside the chroot: it collides with a
# REAL package user, and every tarball in /sources reports as owned by, say,
# `iana-etc`. Ownership is this scheme's source of truth, so that is a lie the
# tools will act on.
#
# Three rules had drifted apart, and the mixture is what you see on disk:
# * `_chown_tree_to_lfs` chowned /sources to `lfs` (host uid);
# * lfs-helper's install_dirs_list carried /sources, so `verify --fix` made
# it root:install 775 and the seal made it 1775 -- dropping the o+w the
# book asks for, which is what lets the unprivileged build user unpack;
# * cmd_clean_sources reset unknown owners to root:root.
#
# So: root:root, 1777, contents root:root. World-writable is what grants the
# build user access, so no group is needed and none is set. /sources is
# scratch, NOT an install directory.
def _ensure_build_root(lfs, quiet=False):
"""Create $LFS/build -- the scratch directory builds unpack into.
Same permissions as /sources and for the same reason: chapters 5-6 run on
the host as the unprivileged `lfs` user and chapters 7+ run as root or as
package users, so every one of them has to be able to create files here.
World-writable is what grants that; the sticky bit is what stops one
package user deleting another's tree. No group is involved.
$LFS itself is root-owned, so the build user cannot create this -- it has
to exist before chapter 5 starts.
"""
p = os.path.join(lfs, "build")
try:
os.makedirs(p, exist_ok=True)
os.chown(p, 0, 0)
os.chmod(p, stat.S_ISVTX | 0o777)
except OSError as e:
sys.stderr.write(" could not create %s: %s\n" % (p, e))
return None
if not quiet:
print("# build scratch: %s (root:root 1777)" % p)
return p
def _normalize_sources(lfs, run=True, quiet=False):
"""Apply book 3.1 to $LFS/sources. Returns (n_chowned, mode_fixed)."""
srcdir = os.path.join(lfs, "sources")
if not os.path.isdir(srcdir):
return 0, False
mode_fixed = False
try:
st = os.stat(srcdir)
want = stat.S_ISVTX | 0o777
if stat.S_IMODE(st.st_mode) != want or st.st_uid != 0 or st.st_gid != 0:
mode_fixed = True
if run:
os.chown(srcdir, 0, 0)
os.chmod(srcdir, want)
except OSError as e:
sys.stderr.write(" could not set %s to root:root 1777: %s\n"
% (srcdir, e))
# `chown root:root $LFS/sources/*` -- non-recursive, exactly as the book
# writes it. Directories are skipped: at book 3.1 nothing is unpacked yet,
# so the glob matches only downloaded files and this is equivalent there.
# Later it is not -- an unpacked tree belongs to whoever is building it,
# and taking it away mid-build would stop that build. Those trees are
# scratch: deleted and re-extracted, never corrected.
n = 0
try:
names = os.listdir(srcdir)
except OSError:
return n, mode_fixed
for name in names:
p = os.path.join(srcdir, name)
try:
st = os.lstat(p)
except OSError:
continue
if stat.S_ISDIR(st.st_mode):
continue
if st.st_uid == 0 and st.st_gid == 0:
continue
n += 1
if run:
try:
os.chown(p, 0, 0, follow_symlinks=False)
except OSError as e:
sys.stderr.write(" could not chown %s: %s\n" % (p, e))
n -= 1
if n and not quiet:
print("# gave %d file(s) in %s back to root:root (book 3.1)"
% (n, srcdir))
return n, mode_fixed
def _restart_source_debris(srcdir):
"""Anything in $LFS/sources that is not a downloaded file.
With the build scratch split out into $LFS/build there should be nothing
here at all: no unpacked trees, no `.cc-*` markers, no files owned by the
last build. This is the check that says so, and it is deliberately not a
cleanup pass for old layouts -- a tree built by an earlier version is
rebuilt, not repaired.
It stays because the guarantee is worth asserting: /sources holds only
what get-sources downloaded, so a rebuild cannot inherit anything from the
build before it.
Returns (markers, foreign) as sorted absolute paths.
"""
markers, foreign = [], []
if not os.path.isdir(srcdir):
return markers, foreign
try:
names = os.listdir(srcdir)
except OSError:
return markers, foreign
for n in names:
p = os.path.join(srcdir, n)
if n.startswith((".cc-", ".pkgusr-")):
markers.append(p)
continue
try:
st = os.lstat(p)
except OSError:
continue
if stat.S_ISDIR(st.st_mode):
continue
if st.st_uid != 0 or st.st_gid != 0:
foreign.append(p)
return sorted(markers), sorted(foreign)
def cmd_bs_restart(args):
"""Throw away the built system and start the build again from scratch.
Deliberately loud. This deletes everything that was built -- weeks of
compilation on a slow machine -- so it says exactly what will go, exactly
what will be kept, prints the commands it will run, and asks the user to
type the mount point before doing any of it."""
lfs = require_mounted_lfs()
# Refuse while the chroot's virtual filesystems are mounted: deleting
# through a bind-mounted /dev would take the HOST's /dev with it.
mounted = [d for d in ("dev/pts", "dev/shm", "dev", "proc", "sys", "run")
if _is_mounted(os.path.join(lfs, d))]
if mounted:
fail(
"\n! the chroot is still mounted: %s"
" Deleting through those would reach the running system.\n"
" Unmount first:\n %slfs build-system chroot unmount --run\n"
% (", ".join(mounted), sudo_prefix()))
sys.exit(2)
keep_sources = not args.wipe_sources
# Unpacked source trees are build scratch: a rebuild re-extracts every one
# of them, and leaving hundreds of thousands of stale files behind makes
# every later scan (ownership, orphans, manifests) slow and noisy. The
# TARBALLS are kept -- those are the expensive part.
# $LFS/build is scratch and is deleted whole -- it is in `doomed` below
# like any other top-level directory, so nothing here has to pick unpacked
# trees or markers out of the tarball store. That is the entire point of
# the split. /sources should hold only what get-sources downloaded; the
# survey asserts it rather than repairing an older layout.
srcdir = os.path.join(lfs, "sources")
markers, foreign = _restart_source_debris(srcdir) if keep_sources else ([], [])
entries = []
try:
entries = sorted(os.listdir(lfs))
except OSError as e:
sys.stderr.write("cannot read %s: %s\n" % (lfs, e))
sys.exit(1)
doomed = [e for e in entries
if e not in ("dev", "proc", "sys", "run")
and not (keep_sources and e == "sources")]
def _du(p):
try:
r = subprocess.run(["du", "-sh", p], capture_output=True, text=True,
timeout=120)
return r.stdout.split()[0] if r.stdout.strip() else "?"
except Exception:
return "?"
warn("!! This DELETES the system you have built.")
print()
print(" from %s:" % lfs)
for e in doomed:
print(" %-16s %s" % (e, _du(os.path.join(lfs, e))))
if not doomed:
print(" (nothing -- the tree is already empty)")
print()
if markers:
print(" from %s (build markers -- this tree predates the "
"sources/build split):" % srcdir)
for p in markers[:6]:
print(" %s" % os.path.basename(p))
if len(markers) > 6:
print(" ... and %d more" % (len(markers) - 6))
print(" Markers belong in %s now. The build rewrites them."
% os.path.join(lfs, "build"))
print()
if foreign:
print(" in %s: %d file(s) not owned by root -> root:root (book 3.1)"
% (srcdir, len(foreign)))
for p in foreign[:6]:
print(" %-40s (%s)"
% (os.path.basename(p), _owner_name_of(p)))
if len(foreign) > 6:
print(" ... and %d more" % (len(foreign) - 6))
print(" A host uid means nothing in the built system, and one")
print(" above 10000 collides with a real package user.")
print()
print(" kept:")
if keep_sources:
print(" the downloaded tarballs in %s" % srcdir)
print(" (--wipe-sources removes those too)")
print(" the books, your settings and any snapshots (outside the tree)")
print()
print(" Package users and collector groups live in %s/etc, so they go"
% lfs)
print(" with it. Users on the HOST are untouched.")
print()
print(" Then the build starts again from the beginning:")
print(" %slfs build-system run" % sudo_prefix())
print()
snaps = _list_snapshots()
if not snaps:
warn(" You have no snapshots. This cannot be undone.")
print(" Consider: %slfs snapshot save before-restart --run"
% sudo_prefix())
else:
print(" You have %d snapshot(s); the newest is '%s'."
% (len(snaps), snaps[-1][0]))
print(" Restoring one is usually faster than rebuilding:")
print(" %slfs snapshot restore %s --run"
% (sudo_prefix(), snaps[-1][0]))
print()
if not args.run:
dry_run_note("delete them")
return
if not args.yes:
warn("This cannot be undone.")
try:
ans = input("Type the mount point (%s) to confirm: " % lfs).strip()
except (EOFError, KeyboardInterrupt):
print()
return
if ans != lfs:
print("Not confirmed -- nothing was deleted.")
return
import shutil as _sh
removed = 0
for e in doomed:
p = os.path.join(lfs, e)
try:
if os.path.islink(p) or os.path.isfile(p):
os.remove(p)
else:
_sh.rmtree(p)
removed += 1
except OSError as err:
sys.stderr.write(" could not remove %s: %s\n" % (p, err))
print("\nremoved %d item(s) from %s" % (removed, lfs))
if keep_sources:
markers, _ = _restart_source_debris(srcdir)
n = 0
for p in markers:
try:
os.remove(p)
n += 1
except OSError as err:
sys.stderr.write(" could not remove %s: %s\n" % (p, err))
if n:
print("removed %d build marker(s); the next build writes its own"
% n)
# Book 3.1: /sources is root:root 1777 and so are its contents. The
# tarballs survived the wipe, so their ownership must be reset here or
# the fresh build inherits the previous system's uids.
n, mode_fixed = _normalize_sources(lfs, run=True, quiet=True)
if n:
print("gave %d source file(s) back to root:root (book 3.1)" % n)
if mode_fixed:
print("reset %s to root:root 1777 (book 3.1)" % srcdir)
# the build's own progress lives outside the tree as well
for f in ("crosschain",):
d = os.path.join(store_dir(), f)
if os.path.isdir(d):
for sub in os.listdir(d):
prog = os.path.join(d, sub, "progress.json")
if os.path.isfile(prog):
os.remove(prog)
print("cleared the recorded build progress")
print()
next_step("%slfs build-system run" % sudo_prefix(), label="start again with:")
def _tree_state(lfs):
"""Everything worth knowing about a tree, as (label, value) pairs.
Judged from the tree itself, from the host, without entering the chroot.
This exists because a snapshot labelled "chapters 5-6: 22/22" said nothing
about chapter 8 -- so a tree that had already been damaged by a chapter-8
build looked pristine, and restoring it restored the damage.
"""
import glob as _glob
out = []
prog = _load_crosschain_progress(lfs) or {}
done = [d for d in prog.get("done", [])]
out.append(("chapters 5-6",
"%d of %d steps" % (len(done), len(CROSSCHAIN_STEPS))))
if prog.get("lfs_tgt"):
out.append(("built for", prog["lfs_tgt"]))
chroot_done = _chroot_progress(lfs)
order = _chroot_steporder(lfs)
if chroot_done or order:
out.append(("chapters 7-9",
"%d of %d steps" % (len(chroot_done), len(order) or 98)))
if chroot_done:
out.append(("last built", chroot_done[-1]))
out.append(("toolchain", _toolchain_state(lfs)))
# package users are the clearest sign chapter 8 actually ran
try:
n = len([d for d in os.listdir(os.path.join(lfs, "usr", "src"))
if os.path.isdir(os.path.join(lfs, "usr", "src", d))])
out.append(("package users", "%d" % n))
except OSError:
pass
native = _glob.glob(os.path.join(lfs, "usr", "lib", "gcc", "*-pc-linux-gnu"))
if native and not _build_is_finished(lfs):
# Only worth saying mid-build. On a finished tree the native gcc IS
# the answer -- printing it as a WARNING there says a completed system
# is wrong.
out.append(("WARNING",
"a native gcc is present (%s) -- chapter 8 has run"
% os.path.basename(native[0])))
return out
def cmd_bs_verify(args):
"""What state is this tree in?
Answers the question a snapshot label cannot: how far did the build get,
for which target, and is the toolchain usable."""
lfs = require_mounted_lfs()
print("State of %s\n" % lfs)
for label, value in _tree_state(lfs):
marker = " ! " if label == "WARNING" else " "
if label == "WARNING":
warn("%s%s" % (marker, value))
else:
print(" %-16s %s" % (label + ":", value))
print()
st = dict(_tree_state(lfs)).get("toolchain", "")
if st.startswith("BROKEN") or "unfinished" in st:
warn("This tree cannot build chapter 8 as it stands.")
print(" A snapshot taken now would capture that -- check with:")
print(" %slfs snapshot list" % sudo_prefix())
else:
ok_ = " The toolchain looks usable."
print(ok_)
def cmd_bs_set_book(args):
"""Choose the book the SYSTEM is built from (config key build_book).
Separate from `lfs set-default`, which chooses the book packages are
installed from on a running system. Having one command for both made it
impossible to tell which you were setting."""
ver = args.version
if not os.path.isfile(book_path(ver)):
known = []
try:
known = list(remote_versions())
except Exception:
known = []
if known and ver not in known:
sys.stderr.write("no such book: %s\n see them all: lfs books\n"
% ver)
sys.exit(1)
sys.stderr.write("note: %s isn't downloaded yet -- fetching it now\n"
% ver)
try:
_fetch_version(ver, set_default=False)
except Exception as e:
sys.stderr.write("fetch failed: %s\n" % e)
sys.exit(1)
cfg = load_config()
cfg["build_book"] = ver
save_config(cfg)
print("the system will be built from book %s" % ver)
if cfg.get("default") and cfg["default"] != ver:
print(" (packages are installed from %s -- lfs set-default changes that)"
% cfg["default"])
def cmd_bs_sync_tools(args):
"""Copy this machine's package-user environment into the tree.
Run after `layout` and again whenever anything here changes: the tools are
updated in place, so a chroot built earlier keeps running an old copy until
this is re-run. Copies:
* /etc/pkgusr/skel-package -- the package-user home skeleton
* /etc/pkgusr/skel-u_xdg -- the shared-user XDG profile (host's
verbatim, else the placeholder copy
beside the tools; never clobbered)
* /etc/pkgusr/{bash_profile,bashrc,build}
* the bash completion (lfs, with blfs/packagemanager/lfs-helper links)
* the hint's helper scripts (add_package_user and friends)
* lfs-helper itself
* a collector-group export, if one is configured
Nothing here is destructive: it overwrites the tooling, never the build."""
lfs = require_mounted_lfs()
print("Sync the package-user environment into %s\n" % lfs)
if not args.run:
_sync_report("package-user skeleton",
["/etc/pkgusr/skel-package",
"/etc/pkgusr/{bash_profile,bashrc,build}"])
_sync_report("shared-user XDG profile",
["/etc/pkgusr/skel-u_xdg/.bash_profile"])
_sync_report("bash completion",
["lfs-completion.bash -> completions/{lfs,blfs,...}"])
_sync_report("helper scripts",
["add_package_user, install_package, list_package, ...",
"the chown/chgrp/chmod/mkdir/install wrappers"])
_sync_report("the build driver", ["lfs-helper -> /usr/bin/lfs-helper"])
if load_config().get("collector_import_file"):
_sync_report("collector groups", ["the configured export"])
dry_run_note("copy them")
return
# the target directory must exist even when this machine has nothing to
# copy -- lfs-helper writes the default environment into it on first use,
# and the checklist uses its presence as "this step is done"
os.makedirs(os.path.join(lfs, "etc", "pkgusr"), exist_ok=True)
skel_ok = _install_pkgusr_skel(lfs, verbose=False)
_sync_report("package-user skeleton",
["/etc/pkgusr/skel-package" if skel_ok
else "none on this machine -- a default will be written"])
_sync_report("shared-user XDG profile",
[_install_xdg_skel(lfs, verbose=False)])
_sync_report("bash completion",
[_install_completion(lfs, verbose=False)])
copied = _install_pkgusr_helpers(lfs, verbose=False)
if copied:
_sync_report("helper scripts", copied)
else:
_sync_report("helper scripts",
["none on this machine"],
note="the tools use Shadow's useradd/groupadd instead")
# ONE SYNC PATH. `run` and `chroot enter` refresh the chroot through
# _refresh_chroot_tools; sync-tools installed the helper and the skels and
# left the other three tools and the stack files to whatever was there --
# so which command you used decided what got updated. It calls the same
# function now, and every entry point keeps the chroot current.
_refresh_chroot_tools(lfs, quiet=True)
install_helper(lfs, quiet=True)
_sync_report("the build driver",
["lfs-helper -> %s/usr/bin/lfs-helper" % lfs],
note="available as 'lfs-helper' inside the chroot")
_install_collector_groups(lfs)
# Everything above was copied in as root. Before the chroot exists the
# tree belongs to the lfs user, so hand these over too -- otherwise the
# next step refuses with "these build dirs are not owned by the 'lfs'
# user: /mnt/lfs/usr/lib/pkgusr, /mnt/lfs/etc/pkgusr, ...".
# (Book 7.2 gives the whole tree back to root when the chroot is prepared.)
_chown_tree_to_lfs(lfs, quiet=True)
print()
next_step("%slfs build-system next" % sudo_prefix())
def _fstab_script(cfg):
"""Write /etc/fstab from the session settings (book 10.2).
Without this the boot scripts cannot remount root read-write, so the very
first thing that tries to write -- /run/bootlog -- fails with
Read-only file system
and everything after it fails too. The device and filesystem were already
given to `lfs build-system session`, so there is no reason to make anyone
write this file by hand."""
dev = cfg.get("lfs_device") or ""
fstype = cfg.get("lfs_fstype") or "ext4"
swap = cfg.get("lfs_swap") or ""
home = cfg.get("lfs_home_device") or ""
out = ["#!/bin/bash", "set -e", "### book : LFS 12.4", "",
"# 10.2 -- /etc/fstab, from the settings given to "
"`lfs build-system session`"]
if not dev:
out += [
'echo "!! no root device configured -- cannot write /etc/fstab." >&2',
'echo " Set it, then regenerate:" >&2',
'echo " lfs config lfs_device /dev/sdXn" >&2',
'echo " lfs build-system gen-chroot-scripts --run --overwrite" >&2',
"exit 3",
]
return "\n".join(out) + "\n"
# btrfs/xfs do their own checking and must not be fsck'd at boot
fsck = "0 0" if fstype in ("btrfs", "xfs", "f2fs") else "1 1"
opts = "defaults,compress=zstd" if fstype == "btrfs" else "defaults"
rows = ["%-16s %-15s %-9s %-20s %s" % (dev, "/", fstype, opts, fsck)]
if swap:
rows.append("%-16s %-15s %-9s %-20s %s" % (swap, "swap", "swap", "pri=1", "0 0"))
if home:
rows.append("%-16s %-15s %-9s %-20s %s" % (home, "/home", fstype, "defaults", "0 2"))
for a, b, c, d, e in (
("proc", "/proc", "proc", "nosuid,noexec,nodev", "0 0"),
("sysfs", "/sys", "sysfs", "nosuid,noexec,nodev", "0 0"),
("devpts", "/dev/pts", "devpts", "gid=5,mode=620", "0 0"),
("tmpfs", "/run", "tmpfs", "defaults", "0 0"),
("devtmpfs", "/dev", "devtmpfs", "mode=0755,nosuid", "0 0"),
("tmpfs", "/dev/shm", "tmpfs", "nosuid,nodev", "0 0"),
("cgroup2", "/sys/fs/cgroup", "cgroup2", "nosuid,noexec,nodev", "0 0")):
rows.append("%-16s %-15s %-9s %-20s %s" % (a, b, c, d, e))
out += [
'# Never clobber an fstab someone else wrote.',
'if [ -s /etc/fstab ] && ! grep -q "generated by lfs" /etc/fstab; then',
' echo "# /etc/fstab exists and was not written by us -- left alone."',
' exit 4',
'fi',
"",
"cat > /etc/fstab <<'FSTAB_EOF'",
"# Begin /etc/fstab -- generated by lfs; edit freely",
"#",
"# file system mount-point type options dump fsck",
"# order",
"",
] + rows + [
"",
"# End /etc/fstab",
"FSTAB_EOF",
"",
'echo "# wrote /etc/fstab"',
"sed 's/^/ /' /etc/fstab",
"",
'# A device name can change between boots; a UUID cannot.',
'if command -v blkid >/dev/null 2>&1; then',
' _uuid="$(blkid -s UUID -o value ' + dev + ' 2>/dev/null || true)"',
' if [ -n "$_uuid" ]; then',
' echo ""',
' echo "# tip: to survive a device rename, replace ' + dev + '"',
' echo "# with UUID=$_uuid"',
' fi',
'fi',
]
return "\n".join(out) + "\n"
def cmd_bs_gen_chroot_scripts(args):
require_complete_config()
"""Write phased install scripts for the in-chroot packages (chapter 7's
remainder and chapter 8) into $LFS/usr/src/lfs-pkgusr/scripts, where lfs-helper
runs them. Same phased format as the packagemanager scripts, so
packagemanager can reuse them to update these packages later."""
ver, path = resolve_book(args)
lfs = require_mounted_lfs()
soup = load_soup(path)
# The parser harvests command blocks only. A few sections ship a FILE
# instead (9.6.8 prints rc.site as <pre class="auto">), so keep the raw
# text for _auto_file_block.
book_raw = str(soup)
secs = {sid: (title, sec) for sid, title, sec in iter_sections(soup)}
steps = list(CHROOT_INIT_STEPS) + list(CHROOT_STEPS_7)
if not args.chapter7_only:
for sid, (title, sec) in secs.items():
nv = package_name_version(sid, title)
if nv and sec["commands"]:
steps.append((nv[0], sid))
bl = bootloader()
if bl != "grub":
# GRUB's book instructions run `grub-install /dev/sda`, which writes a
# disk's boot sector. If it isn't the chosen bootloader, don't generate
# it at all -- there should be no script lying around that could do that
# to an existing boot partition by accident.
steps = [(n, sid) for n, sid in steps
if "grub" not in n.lower() and "grub" not in sid.lower()]
outdir = pkgusr_state(lfs, "scripts")
print(f"{len(steps)} script(s) -> {outdir}")
if not args.run:
for name, sid in steps[:10]:
print(f" {name}")
if len(steps) > 10:
print(f" ... and {len(steps) - 10} more")
dry_run_note("write them")
return
os.makedirs(outdir, exist_ok=True)
written, skipped = 0, 0
order = []
unresolved = []
for name, sid in steps:
entry = secs.get(sid)
if not entry:
continue
title, sec = entry
if not sec["commands"]:
continue
order.append(name)
dest = os.path.join(outdir, f"{name}.sh")
if os.path.isfile(dest) and not args.overwrite:
skipped += 1
continue
if name in [n for n, _s in CHROOT_INIT_STEPS]:
text = _plain_root_script(name, sid, title, sec["commands"])
else:
pkgver = _title_pkgver(title) or name
glob = _pkg_glob_for(pkgver, name)
text = _crosschain_script_body(name, sid, pkgver, glob,
sec["commands"])
text = _chrootify(text)
text, left = _fill_placeholders(text, name)
if left:
unresolved.append((name, left))
with open(dest, "w") as f:
f.write(text)
os.chmod(dest, 0o755)
written += 1
# chapter 9 configuration, after all the packages
pkg_like_cfg = []
cfg_steps_9 = list(CHROOT_CONFIG_STEPS_8) + list(CHROOT_CONFIG_STEPS_9)
if network_mode() == "dhcp":
# The book's network section writes a static interface config and a
# hand-made resolv.conf. With DHCP both are wrong -- the client
# supplies them -- so don't generate scripts that would overwrite what
# dhcpcd sets up.
cfg_steps_9 = [(n, sid) for n, sid in cfg_steps_9
if n not in ("cfg_network", "cfg_resolv")]
if not args.chapter7_only:
secs_all = secs
for cname, csid in cfg_steps_9:
entry = _resolve_section(secs_all, csid)
# A named step that produces nothing looked exactly like one that
# worked. /etc/shells was missing from finished systems for
# exactly this reason: cfg_shells resolved to zero blocks and was
# skipped without a word. Say so -- some of these are genuinely
# prose (cfg_rcsite), but silence cannot tell the two apart.
if not entry:
warn(f" {cname}: section '{csid}' not found in this book "
f"-- nothing generated")
continue
ctitle, csec = entry
ccmds = csec["commands"]
if not ccmds and cname in CONFIG_FILE_SECTIONS:
# no commands, but the section prints a file -- write that
_fsid, _fpath = CONFIG_FILE_SECTIONS[cname]
_blk = _auto_file_block(book_raw, _fsid, _fpath)
if _blk:
ccmds = [_blk]
if not ccmds:
warn(f" {cname}: '{csid}' has no command blocks in this book "
f"-- nothing generated")
continue
dest = os.path.join(outdir, f"{cname}.sh")
if os.path.isfile(dest) and not args.overwrite:
skipped += 1
order.append(cname)
continue
# Some chapter-9 "configuration" sections are actually PACKAGES:
# 9.2 LFS-Bootscripts-20250827 is a tarball to unpack and install,
# not a config file to write. Its title carries a version, so
# generate a normal phased build for it -- treating it as a plain
# script runs `make install` with nothing unpacked:
# make: *** No rule to make target 'install'. Stop.
cpkgver = _title_pkgver(ctitle)
if cpkgver and any(ch.isdigit() for ch in cpkgver):
cglob = _pkg_glob_for(cpkgver, cname)
ctext = _crosschain_script_body(cname, csid, cpkgver, cglob,
csec["commands"])
ctext = _chrootify(ctext)
pkg_like_cfg.append(cname)
else:
ctext = _plain_root_script(cname, csid, ctitle, csec["commands"])
ctext, cleft = _fill_placeholders(ctext, cname)
if cleft:
unresolved.append((cname, cleft))
with open(dest, "w") as f:
f.write(ctext)
os.chmod(dest, 0o755)
order.append(cname)
written += 1
# /etc/fstab, before the bootloader -- the system cannot boot read-write
# without it, and the bootloader step is where booting starts to matter
if not args.chapter7_only:
dest = os.path.join(outdir, "cfg_fstab.sh")
if not (os.path.isfile(dest) and not args.overwrite):
with open(dest, "w") as f:
f.write(_fstab_script(load_config()))
os.chmod(dest, 0o755)
written += 1
order.append("cfg_fstab")
if bl == "refind":
dest = os.path.join(outdir, "refind.sh")
with open(dest, "w") as f:
f.write(_refind_script())
os.chmod(dest, 0o755)
order.append("refind")
written += 1
elif bl == "none":
# Say that it was skipped, and how to change your mind. Silence here
# would be its own trap: someone who DOES need a bootloader would reach
# a finished, unbootable system without ever having been offered one.
hint("# no bootloader step: your machine already boots, so this build")
hint("# does not touch that. To add rEFInd to your existing ESP:")
hint("# lfs config bootloader refind")
hint("# lfs config esp /boot/efi")
hint("# lfs build-system gen-chroot-scripts --run --overwrite")
# /tools -- the cross-toolchain that built chapter 8. Book 7.13 removes it
# right after chapter 7; this runs it at the END instead, so the fallback
# survives the part of the build most likely to need redoing. Off unless
# asked for: deleting a working toolchain should never be a default.
if not args.chapter7_only and remove_tools():
entry = _resolve_section(secs, "ch-tools-cleanup")
if entry:
ctitle, csec = entry
cmds = [c for c in csec["commands"] if "/tools" in c]
if cmds:
dest = os.path.join(outdir, "cfg_rm-tools.sh")
if not (os.path.isfile(dest) and not args.overwrite):
text = _plain_root_script("cfg_rm-tools", "ch-tools-cleanup",
ctitle, cmds)
with open(dest, "w") as f:
f.write(text)
os.chmod(dest, 0o755)
written += 1
order.append("cfg_rm-tools")
else:
warn(" cfg_rm-tools: 7.13 has no /tools command in this book")
# The ownership epoch, as a STEP you can see.
#
# It ran as an invisible checkpoint inside build-all -- correct, but it
# never appeared in `lfs-helper list`, so the one place the whole scheme
# turns on was the one place you could not point at. It goes straight after
# init-files, which is book 7.6: the step that creates /etc/passwd, and
# therefore the first moment ownership means anything.
if "init-files" in order and "init-ownership" not in order:
order.insert(order.index("init-files") + 1, "init-ownership")
# Accounts you can log in with -- LAST, after everything else.
#
# A BUILT-IN step: lfs-helper performs it, there is no script. It sets
# the root password, so everything that could still fail should have
# failed by now -- and it is the last thing standing between a built
# system and a reboot you cannot log in after. Appended here rather
# than beside the bootloader, because cfg_rm-tools comes after that.
#
# This replaces last_build_step.sh entirely. wget and the Python modules
# move to `packagemanager setup`, which is where bootstrapping the tooling
# belongs; the root password stays here because setup runs on the BOOTED
# system and you need to log in to run it.
#
# It is a built-in rather than a user-editable script for the reason the
# old one kept failing: a file created once on the host never receives a
# fix, and `chown: invalid user: 'wget:wget'` survived three releases that
# way. Book 8.5 says to run `passwd root`; that is not a matter of taste.
if "init-accounts" in order:
order.remove("init-accounts")
order.append("init-accounts")
with open(pkgusr_state(lfs, "progress", "steporder"), "w") as f:
f.write("\n".join(order) + "\n")
# Stamp who generated this, and delete scripts for steps that are no
# longer in the order.
#
# The step order and the scripts live in the tree; the tools live outside
# it. Nothing connected the two, so a tree generated by an older version
# kept running steps that version had -- `last-step` was removed from the
# tools in 1.10.0 and went on failing in an existing tree, under a
# lfs-helper that no longer knew what it was. The chroot tools already
# carry a build id for exactly this reason; the generated scripts did not.
import time as _t
with open(pkgusr_state(lfs, "progress", "generated-by"), "w") as f:
f.write("version=%s\nbuild=%s\ndate=%s\n"
% (LFS_VERSION, _build_id(),
_t.strftime("%Y-%m-%d %H:%M:%S")))
stale = [f for f in sorted(os.listdir(outdir))
if f.endswith(".sh") and f[:-3] not in order]
for f in stale:
try:
os.remove(os.path.join(outdir, f))
except OSError:
continue
if stale:
note("# removed %d script(s) for steps no longer in the order:"
% len(stale))
for f in stale:
note("# %s" % f[:-3])
hint("# clear them from the progress file too, if they were built:")
for f in stale:
hint("# lfs-helper undone %s" % f[:-3])
# chapter 7's temporary tools are built as root, like the book does; the
# package-user system takes over for the real chapter 8 packages.
root_steps = [n for n, _s in CHROOT_INIT_STEPS + CHROOT_STEPS_7
if n in order]
root_steps += [n for n, _s in cfg_steps_9
if n in order and n not in pkg_like_cfg]
# init-accounts is performed by lfs-helper itself, not by a script, so it
# does not go in root_steps -- there is nothing to run as root.
if "cfg_fstab" in order:
root_steps.append("cfg_fstab") # writes /etc/fstab
if "cfg_rm-tools" in order:
root_steps.append("cfg_rm-tools") # deletes /tools, owned by root
if "refind" in order:
root_steps.append("refind") # writes to the ESP, so root
with open(pkgusr_state(lfs, "progress", "rootsteps"), "w") as f:
f.write("\n".join(root_steps) + "\n")
_run_bash(f'chmod -R a+rX "{os.path.join(lfs, PKGUSR_DIR)}" 2>/dev/null; true')
print(f"# wrote {written}, kept {skipped} existing "
f"(use --overwrite to replace)")
if unresolved:
fail('\n! these scripts still contain a placeholder the book expects you to fill in;\n edit the script before building them:')
for nm, left in unresolved:
sys.stderr.write(" %-18s %s\n"
% (nm, ", ".join("<%s>" % x for x in left)))
# SINGLE QUOTES INSIDE THE BRACES. Reusing the f-string's own quote
# character is legal only from Python 3.12, and `lfs` is the one tool that
# runs on the HOST -- where Python is whatever the distribution ships
# (Debian 12: 3.11). There it did not fail at this line, it failed to
# start at all, with a SyntaxError pointing at a debug print.
print(f"# step order -> {pkgusr_state(lfs, 'progress', 'steporder')}")
write_chroot_env(lfs)
install_helper(lfs)
_install_collector_groups(lfs)
live = _chroot_mounts(lfs)
print()
if len(live) < len(_KERNFS):
print("Next: prepare the chroot, then enter it:")
print(f" {sudo_prefix()}lfs build-system chroot prepare --run")
print(f" {sudo_prefix()}lfs build-system chroot enter")
else:
print("Next: enter the chroot:")
print(f" {sudo_prefix()}lfs build-system chroot enter")
def _copy_resolv_conf(lfs):
"""Give the chroot working DNS by copying the host's resolver config.
Routing works inside the chroot (a bare IP pings fine), but with no
/etc/resolv.conf nothing resolves, and every download fails with the
thoroughly unhelpful
Temporary failure in name resolution
This is a BUILD-TIME convenience only: the finished system gets its own
resolv.conf from dhcpcd (or from the static config, if you chose that)."""
dst = os.path.join(lfs, "etc", "resolv.conf")
src = "/etc/resolv.conf"
if not os.path.isfile(src):
return
def usable(path):
"""A resolv.conf is only usable if a nameserver line holds a real IP.
The book's template ships placeholders:
nameserver <IP address of your primary nameserver>
Checking merely that a line starts with "nameserver" accepts that and
leaves DNS just as broken as before."""
try:
with open(path) as f:
for line in f:
parts = line.split()
if len(parts) >= 2 and parts[0] == "nameserver":
try:
import ipaddress
ipaddress.ip_address(parts[1])
return True
except ValueError:
continue
except OSError:
return False
return False
try:
if os.path.isfile(dst):
if usable(dst):
return # already usable
if os.path.getsize(dst) > 0:
bak = dst + ".bak"
import shutil as _sh
_sh.copyfile(dst, bak)
print("# the chroot's /etc/resolv.conf has no real nameserver "
"(the book's placeholders);")
print("# replacing it with the host's -- old one kept at "
"%s" % bak)
os.makedirs(os.path.dirname(dst), exist_ok=True)
import shutil
# follow a symlink (a systemd-resolved stub, say) and copy the CONTENT,
# since the target does not exist inside the chroot
shutil.copyfile(os.path.realpath(src), dst)
print("# copied the host's /etc/resolv.conf into the chroot (for DNS "
"during the build)")
except OSError as e:
fail("! could not copy /etc/resolv.conf into the chroot: "
"%s" % e)
def cmd_bs_chroot(args):
require_complete_config()
"""7.2-7.4 -- hand the tree to root, mount the virtual kernel filesystems,
and enter (or leave) the chroot."""
lfs = require_mounted_lfs()
action = args.action or "status"
if action == "status":
live = _chroot_mounts(lfs)
print(f"LFS : {lfs}")
print(f"owned by root : {'yes' if _owner_is_root(lfs) else 'no (still lfs -- run: chroot prepare)'}")
print(f"virtual fs : {', '.join(live) if live else '(none mounted)'}")
rc_path = os.path.join(lfs, "etc", "resolv.conf")
has_dns = False
if os.path.isfile(rc_path):
import ipaddress
for _l in open(rc_path):
_p = _l.split()
if len(_p) >= 2 and _p[0] == "nameserver":
try:
ipaddress.ip_address(_p[1])
has_dns = True
break
except ValueError:
pass
print(f"DNS in chroot : {'yes' if has_dns else 'no (fix: lfs build-system chroot resolv)'}")
if len(live) < len(_KERNFS):
missing = [m for m in _KERNFS if m not in live]
print(f" not mounted : {', '.join(missing)}")
print(f"\n prepare: {sudo_prefix()}lfs build-system chroot prepare --run"
f"\n enter : {sudo_prefix()}lfs build-system chroot enter"
f"\n leave : exit, then {sudo_prefix()}lfs build-system chroot unmount --run")
return
if os.geteuid() != 0:
sys.stderr.write(f"! chroot {action} must be run as root.\n")
sys.exit(2)
if action == "prepare":
# Handing the tree to root is a one-way door: after it the lfs user can
# no longer build, and chapters 5-6 cannot be resumed. Refuse to do
# that on top of a half-built toolchain -- an interrupted chain used to
# sail straight through here into a chroot that could build nothing.
# A changed target triplet invalidates the whole toolchain: the
# headers were installed under the old one and the compiler will look
# under the new one. Catch it here, before the chroot, rather than
# forty packages later inside gcc.
mism = _crosschain_tgt_mismatch(lfs)
if mism and not getattr(args, "force", False):
was, now = mism
fail(
"\n! this tree was built for a different target:"
" built with : %s\n"
" configured : %s\n"
" The C++ headers are installed under the first name and the\n"
" compiler will look under the second, so chapter 8's gcc\n"
" will fail with 'bits/c++config.h: No such file or"
" directory'.\n\n"
" Put it back:\n %slfs config lfs_tgt %s\n"
" or rebuild chapters 5-6 for the new target:\n"
" %slfs build-system crosschain --run --restart\n"
% (was, now, sudo_prefix(), was, sudo_prefix()))
sys.exit(2)
cc_ok, cc_n = _crosschain_looks_done(lfs)
if not cc_ok and not getattr(args, "force", False):
fail(
"\n! chapters 5-6 are not finished: %d of %d steps built."
" Preparing the chroot gives the tree to root, after which\n"
" they cannot be resumed. Finish them first:\n"
" %slfs build-system crosschain --run\n"
" (it continues from where it stopped)\n"
" To prepare anyway: --force\n"
% (cc_n, len(CROSSCHAIN_STEPS), sudo_prefix()))
why = _crosschain_missing_output(lfs)
if why:
sys.stderr.write("\n %s\n" % why)
sys.exit(2)
steps = []
_need = _handover_needed(lfs)
if _need and not _handover_is_safe(lfs):
# A package user owns files here, so this is a chapter-8 tree and
# the RECURSIVE chown would strip every one of them. Refusing and
# stopping there left the tree with no way forward: 4.3 had taken
# /usr, /etc, /var and the root symlinks, 7.2 would not give them
# back, and `chroot enter` refused because /usr was not root's.
#
# Reclaim precisely what the build user holds instead. Selecting on
# its uid leaves every package user's files untouched, so this is
# safe here in a way `chown -R root:root` is not.
print("# 7.2: package users own files here, so this is a chapter-8")
print("# tree and chown -R root:root would strip all of them.")
print("# Taking back only what the build user holds:")
_n = _handover_build_user_only(lfs, run=True)
if _n:
print("# gave %d path(s) back to root (group left alone)" % _n)
else:
print("# nothing of the build user's left in the tree")
_need = False
if _need:
# Book 7.2 hands over the directories it created in 4.2, but the
# tree ROOT and the mount points are not in that list -- they were
# created before the handover and never came back. The result is
# a system where /, /dev, /proc, /sys, /run, /media, /opt and
# /tools belong to a user that does not exist in the chroot:
# drwxr-xr-x 1 lfs lfs 178 /
# Nothing owned by the build user should survive into the finished
# system. NOT recursive.
#
# /sources is skipped HERE, not exempted: a blanket `chown -h
# root:root` would take the directory but not the files inside it,
# and would say nothing about the 1777 mode. _normalize_sources
# below owns that rule in one place and applies all of it.
steps.append(("give the top level to root",
f'for d in "{lfs}" "{lfs}"/*; do '
f'case "$d" in *"/sources") continue ;; esac; '
f'[ -e "$d" ] || continue; '
f'chown -h root:root "$d" 2>/dev/null; done; true'))
steps.append(("give the tree to root",
f'for d in {" ".join(LFS_HANDOVER_DIRS)}; do '
f'[ -e "{lfs}/$d" ] && '
f'chown -h -R root:root "{lfs}/$d"; done; true'))
else:
print("# 7.2 already done (tree owned by root)")
steps.append(("create the mount points",
f'mkdir -pv "{lfs}"/{{dev,proc,sys,run}}'))
live = _chroot_mounts(lfs)
mounts = [
("dev", f'mount -v --bind /dev "{lfs}/dev"'),
("dev/pts", f'mkdir -pv "{lfs}/dev/pts"; '
f'mount -vt devpts devpts -o gid=5,mode=0620 "{lfs}/dev/pts"'),
("proc", f'mount -vt proc proc "{lfs}/proc"'),
("sys", f'mount -vt sysfs sysfs "{lfs}/sys"'),
("run", f'mount -vt tmpfs tmpfs "{lfs}/run"'),
("dev/shm", f'if [ -h "{lfs}/dev/shm" ]; then '
f'install -v -d -m 1777 "{lfs}$(realpath /dev/shm)"; '
f'else mkdir -pv "{lfs}/dev/shm"; '
f'mount -vt tmpfs -o nosuid,nodev tmpfs "{lfs}/dev/shm"; fi'),
]
for name, cmd in mounts:
if name in live:
print(f"# {name} already mounted -- skipping")
continue
steps.append((f"mount {name}", cmd))
for label, cmd in steps:
# On a real run the label is enough: the commands are the book's,
# they are about to run, and printing every one of them buries the
# mount output that actually matters. A dry run shows them, since
# seeing exactly what WOULD run is the whole point.
if args.run:
print(f"# {label}")
else:
print(f"# {label}")
print(f" {cmd}")
if not args.run:
dry_run_note("apply")
return
for label, cmd in steps:
rc = _run_bash(cmd)
if rc != 0:
sys.stderr.write(f"\n! failed: {label} (exit {rc})\n")
sys.exit(rc)
_normalize_sources(lfs, run=True)
write_chroot_env(lfs)
install_helper(lfs)
_copy_resolv_conf(lfs)
have_scripts = os.path.isdir(pkgusr_state(lfs, "scripts"))
print("\n# chroot environment ready.")
if not have_scripts:
next_step(f"{sudo_prefix()}lfs build-system gen-chroot-scripts --run",
f"{sudo_prefix()}lfs build-system chroot enter",
label="next run:")
else:
next_step(f"{sudo_prefix()}lfs build-system chroot enter")
return
if action == "resolv":
_copy_resolv_conf(lfs)
print("# DNS inside the chroot should work now:")
print("# ping -c1 archlinux.org")
return
if action == "unmount":
live = _chroot_mounts(lfs)
if not live:
print("# nothing of ours is mounted.")
return
# unmount deepest-first so nested mounts (dev/pts inside dev) come off
order = sorted(live, key=lambda p: p.count("/"), reverse=True)
for sub in order:
print(f"# umount {sub}")
if args.run:
_run_bash(f'umount -v "{lfs}/{sub}" || umount -lv "{lfs}/{sub}"')
if not args.run:
dry_run_note("unmount")
else:
left = _chroot_mounts(lfs)
print("# unmounted." if not left
else f"# still mounted: {', '.join(left)}")
return
if action == "enter":
# same reason as in `run`: entering with a stale lfs-helper means
# debugging a version that no longer exists out here
_refresh_chroot_tools(lfs)
# and the collector-group export, so editing it on the host reaches the
# next build without regenerating the chroot scripts. It only speaks
# up when the contents changed.
_install_collector_groups(lfs)
# Book 3.1 on every entry, not only on get-sources and restart.
#
# The host `lfs` account comes from book 4.3's own useradd, which pins
# no uid, so it lands wherever the host had a gap -- 10753 on a Debian
# host. Package users start at 10000, so that uid does not merely read
# as a number inside the chroot, it COLLIDES with a real package user
# and every tarball reports as owned by something unrelated.
#
# A tree prepared before _normalize_sources existed keeps that uid
# forever unless something applies the rule again. This is that
# something: entry is the one path every build goes through, whatever
# it did before.
_normalize_sources(lfs, run=True)
live = _chroot_mounts(lfs)
missing = [m for m in _KERNFS if m not in live]
if missing:
sys.stderr.write(
f"! the virtual kernel filesystems aren't mounted "
f"({', '.join(missing)}).\n"
f" Run first: {sudo_prefix()}lfs build-system chroot prepare --run\n")
sys.exit(2)
if not _owner_is_root(lfs):
# Repair it rather than sending you back to a command that just
# declined to do it. This guard refused while `chroot prepare`
# also refused -- correctly, since the recursive chown would have
# stripped every package user -- so the tree had no way forward and
# the message pointed at the command that had already said no.
#
# Only the build user's own paths are touched, so this is safe on a
# chapter-8 tree. If something else owns $LFS/usr, that is not
# ours to reassign and the guard still stops.
_n = _handover_build_user_only(lfs, run=True)
if _n:
print("# 7.2: took %d path(s) back from the build user "
"(group left alone)" % _n)
if not _owner_is_root(lfs):
sys.stderr.write(
"! %s/usr is owned by '%s', not root.\n"
% (lfs, _owner_name_of(os.path.join(lfs, "usr")))
+ " Book 7.2 hands the tree to root before the chroot.\n"
+ f" Run first: {sudo_prefix()}lfs build-system chroot prepare --run\n")
sys.exit(2)
cfg = load_config()
mkf = cfg.get("makeflags") or "-j$(nproc)"
install_helper(lfs, quiet=True)
write_chroot_env(lfs, quiet=True)
scripts_dir = pkgusr_state(lfs, "scripts")
print(f"# entering chroot at {lfs} (type 'exit' to leave)")
print("#")
print("# There's no Python in here yet -- use 'lfs-helper' (bash) to")
print("# drive the build. Start with:")
if not os.path.isdir(scripts_dir):
print("# (no build scripts yet -- leave the chroot and run:")
print(f"# {sudo_prefix()}lfs build-system gen-chroot-scripts --run)")
else:
print("# lfs-helper next # what to do, step by step")
print("# lfs-helper status # where the session stands")
print("# lfs-helper list # all steps + progress")
print("#")
# SAY that this is the chroot, rather than guessing later.
#
# inside_chroot() used to answer "does /usr/src/lfs-pkgusr exist" --
# which is true on ANY system these tools built, so a finished host
# was told it was inside the chroot and refused every build command.
# The shell that IS the chroot knows it; it says so.
inner = (f'chroot "{lfs}" /usr/bin/env -i '
f'HOME=/root TERM="$TERM" PS1="(lfs chroot) \\u:\\w\\$ " '
f'PATH=/usr/bin:/usr/sbin LFS_IN_CHROOT=1 '
f'MAKEFLAGS="{mkf}" TESTSUITEFLAGS="{mkf}" '
f'/bin/bash --login')
import subprocess
rc = subprocess.run(["bash", "-c", inner], cwd="/").returncode
print(f"# left the chroot (exit {rc}).")
print(f"# when you're done for now: {sudo_prefix()}lfs build-system chroot unmount --run")
return
def cmd_bs_crosschain(args):
"""Build the chapter-5 cross toolchain + chapter-6 temporary tools, in order,
AS the lfs user. Each
step is a real, phased script on disk (unpack/build/install/configure/test,
same shape as the BLFS install scripts) you can read and edit before
running, and re-run a single phase (e.g. just 'install') without rebuilding.
If a multi-step run is interrupted, the next run offers to continue where it
left off."""
ver, _bookpath = resolve_book(args)
if args.list:
# fast path: read the version straight out of already-saved scripts
# (a plain file read) instead of re-parsing the whole book per step --
# the book is only touched (once, for ALL missing steps together) if
# something hasn't been generated yet.
need_book = any(not os.path.isfile(crosschain_script_path(ver, n))
for n, _s, _t in CROSSCHAIN_STEPS)
book_info = _crosschain_all_pkgvers(args) if need_book else {}
outdated = []
for name, sid, tarpfx in CROSSCHAIN_STEPS:
p = crosschain_script_path(ver, name)
if os.path.isfile(p):
head = open(p, encoding="utf-8", errors="replace").read(2000)
m = re.search(r'^name_version="([^"]+)"', head, re.M)
pkgver = m.group(1) if m else "?"
vm = re.search(rf'{re.escape(CROSSCHAIN_MARK)} v(\d+):', head)
fv = int(vm.group(1)) if vm else 0
if fv < CROSSCHAIN_SCRIPT_VERSION:
saved = f"saved script -- OUTDATED FORMAT v{fv}, current is v{CROSSCHAIN_SCRIPT_VERSION}"
outdated.append(name)
else:
saved = "saved script"
else:
pkgver = book_info.get(name, (None, None))[0]
saved = "not generated yet"
print(f"{name:<16} {pkgver or '?':<20} [{saved}] {p}")
if outdated:
print(f"\n{len(outdated)} script(s) predate a bug fix -- regenerate "
f"them (this discards any edits; use --edit to preserve "
f"them manually first):")
print(" lfs build-system crosschain " + " ".join(outdated)
+ " --regenerate")
return
chosen = CROSSCHAIN_STEPS if args.step in (None, "all") else \
[(n, s, p) for n, s, p in CROSSCHAIN_STEPS if n == args.step]
if not chosen:
sys.stderr.write(f"unknown step: {args.step} (see: crosschain --list)\n")
sys.exit(2)
if args.path:
for name, sid, tarpfx in chosen:
print(crosschain_script_path(ver, name))
return
need_book = args.regenerate or any(
not os.path.isfile(crosschain_script_path(ver, name))
for name, _s, _t in chosen)
book_info = _crosschain_all_pkgvers(args) if need_book else {}
if args.edit:
editor = os.environ.get("EDITOR", "vim")
for name, sid, tarpfx in chosen:
path = ensure_crosschain_script(args, ver, name, sid, tarpfx,
book_info=book_info)
import subprocess
subprocess.run([editor, path], cwd="/")
return
if args.run and os.geteuid() != 0:
sys.stderr.write(
"! crosschain --run must be run as ROOT.\n"
" It builds each step AS the lfs user (via su - lfs) -- so run it "
"from a root shell, not as lfs:\n"
f" {sudo_prefix()}lfs build-system crosschain --run\n")
sys.exit(2)
lfs = require_mounted_lfs() if args.run else None
phase = args.phase or "all"
# ---- resume: skip steps already recorded as built. The record is a global
# set of finished steps for this book, so it survives building an individual
# package in between; only a multi-step run consults it.
chosen_names = [n for n, _s, _t in chosen]
skip = set()
if args.run and phase == "all" and not args.restart and len(chosen) > 1:
prog = _load_crosschain_progress(lfs) if lfs else None
done_all = [d for d in (prog or {}).get("done", [])
if (prog or {}).get("book") == ver and d in chosen_names]
if done_all and len(done_all) < len(chosen_names):
todo = [n for n in chosen_names if n not in done_all]
# Asked the same way round as lfs-helper's version of this
# question, and defaulting the same way: the safe answer is to
# KEEP work that is already done, so "rebuild everything" has to
# be chosen deliberately. The two prompts used to be inverted
# relative to each other -- same decision, opposite default.
print(f"\n{len(done_all)} of the {len(chosen_names)} step(s) in "
f"range are already built (last: {done_all[-1]}).")
print(f" y = rebuild all {len(chosen_names)} step(s) from the "
f"start of the range")
if todo:
print(f" N = keep them, build only the {len(todo)} remaining "
f"step(s), starting with '{todo[0]}'")
else:
print(" N = keep them; nothing would be built "
"(everything is done)")
rebuild = False
if sys.stdin.isatty() and not args.yes:
rebuild = input("Rebuild those too? [y/N]: ").strip().lower() \
in ("y", "yes")
if rebuild:
print("# rebuilding everything in range")
_clear_crosschain_progress(lfs)
else:
skip = set(done_all)
print(f"# keeping them; building the {len(todo)} remaining "
f"step(s)")
if args.run:
srcdir = os.path.join(lfs, "sources")
if not os.path.isdir(srcdir):
sys.stderr.write(
f"\n! {srcdir} doesn't exist -- packages haven't been "
f"downloaded yet.\n Run: lfs build-system get-sources --run\n")
sys.exit(2)
_require_lfs_writable_by_lfs(lfs)
total = len(chosen)
try:
for i, (name, sid, tarpfx) in enumerate(chosen, 1):
if name in skip:
continue
path = ensure_crosschain_script(args, ver, name, sid, tarpfx,
force_regen=args.regenerate,
book_info=book_info)
script = open(path).read()
print(f"\n# ===== STEP {i}/{total}: {name} ({sid}, phase: {phase}) =====")
print(f"# script: {path}")
if not args.run:
print(script)
continue
m = re.search(r'^pkg_glob="([^"]+)"', script, re.M)
glob = m.group(1) if m else f"{tarpfx}-*.tar.*"
# as ROOT: clear any leftover extracted dir (could be root-owned from
# a previous failed run, which lfs couldn't delete) and fix perms.
if phase in ("all", "unpack"):
print(f"# [root] clearing any old {tarpfx}-* source dir + "
f"fixing perms ...")
preclean_rc = _run_bash(
f'cd "$LFS/sources" && '
f'(for d in {tarpfx}-*/ ; do [ -d "$d" ] && rm -rf "$d"; done); '
f'chmod a+r {glob} 2>/dev/null; '
f'chmod a+wt "$LFS/sources" 2>/dev/null; true')
if preclean_rc != 0:
sys.stderr.write(
f"\n! STEP {i}/{total} ({name}) FAILED before the build "
f"even started -- couldn't prepare $LFS/sources.\n"
f" script: {path}\n"
f" This usually means the session isn't active. Check:\n"
f" lfs build-system session\n")
sys.exit(preclean_rc)
print(f"# [lfs] running {path} (phase: {phase}) ...")
# By PATH, not on stdin: the runner re-executes the script once
# per phase, so it has to be a file the lfs user can read.
full = f'exec bash "{path}" "{phase}"\n'
rc = _run_tracked(name, full, as_lfs=True, lfs=lfs)
if rc != 0:
bar = "!" * 58
sys.stderr.write(
f"\n{bar}\n"
f"! STEP {i}/{total} FAILED: {name} (phase '{phase}', "
f"exit {rc})\n"
f"! script: {path}\n"
f"! Open/edit that file to fix it, then retry just this "
f"phase:\n"
f"! lfs build-system crosschain {name} --edit\n"
f"! {sudo_prefix()}lfs build-system crosschain {name} --phase "
f"{phase} --run\n"
f"{bar}\n")
sys.exit(rc)
print(f"# STEP {i}/{total} ({name}): done")
if phase == "all": # a fully-built step -- remember it globally
_mark_crosschain_done(lfs, ver, name)
except KeyboardInterrupt:
sys.stderr.write(
"\n\n! interrupted -- progress has been saved. Re-run the same "
"command to continue from the next step:\n"
f" {sudo_prefix()}lfs build-system crosschain --run\n")
sys.exit(130)
if args.run:
# Do NOT clear the progress file here. It is the only exact record of
# which of the 22 steps ran, and clearing it made "finished" and
# "never started" indistinguishable -- which is why an interrupted
# chain could be mistaken for a complete one.
print("\ncrosschain: all requested steps done.")
# ---- package-user file tracking ------------------------------------------- #
# We record which files each package installs (snapshot diff of $LFS before/after
# each install) so ownership can later be handed to that package's user. The
# data lives under $LFS/var/lib/pkgusr so it persists into the final system.
# Build-tool metadata (manifests, snapshots, resume progress). This deliberately
# does NOT live under $LFS/var/lib: creating it as root would make $LFS/var/lib
# root-owned, and then packages building as the lfs user cannot create their own
# subdirs there (glibc's var/lib/nss_db fails with 'Permission denied'). It's
# our bookkeeping, not part of the LFS system being built, so it goes in its own
# top-level dir that we own outright.
# Where this toolchain keeps its working notes about the tree, RELATIVE to
# $LFS. Must match lfs-helper's $STATE exactly: both tools read and write the
# same manifests, the same step order and the same progress.
#
# Under /usr/src, beside the accounts it describes -- not in a hidden directory
# at the root of the filesystem, where it sat next to /boot and /etc as if it
# were part of the system being built. It is not; it is what the build knows
# about that system.
#
# It also means the tree scans exclude ONE directory instead of two, because
# everything under /usr/src already is.
PKGUSR_DIR = "usr/src/lfs-pkgusr"
# The sorted layout, so a name is written once and both tools agree.
PKGUSR_SUBDIRS = {
"scripts": "scripts", # one shell script per build step
"manifests": "manifests", # what each package installed
"logs": "logs", # one log per step, plus verify.log
"progress": "progress", # what has been built, and how far
"groups": "groups", # collector groups and their grants
"config": "config", # settings you may edit
}
def pkgusr_state(lfs, *parts):
"""A path inside the state directory: pkgusr_state(lfs, "progress",
"steporder")."""
return os.path.join(lfs, PKGUSR_DIR, *parts)
def _pkgusr_dir(lfs):
d = pkgusr_state(lfs, "manifests")
existed = os.path.isdir(d)
os.makedirs(d, exist_ok=True)
if not existed:
# we may be running as root; make sure the lfs user can write here too
_run_bash(f'chown -R lfs "{os.path.join(lfs, PKGUSR_DIR)}" 2>/dev/null; true')
return os.path.join(lfs, PKGUSR_DIR)
def _snapshot_cmd(lfs):
# /usr/src wholesale, not just the state directory inside it: the package
# users' homes and their build trees are under there too, and lfs-helper's
# own scans already exclude it. Two tools, one rule.
x = "usr/src"
return (f'find "{lfs}" -xdev \\( -type f -o -type l \\) '
f'-not -path "{lfs}/sources/*" -not -path "{lfs}/{x}/*" '
f'-not -path "{lfs}/dev/*" -not -path "{lfs}/proc/*" '
f'-not -path "{lfs}/sys/*" -not -path "{lfs}/run/*" '
f'-not -path "{lfs}/tmp/*" 2>/dev/null | sort')
def _snapshot_dirs_cmd(lfs):
x = "usr/src"
return (f'find "{lfs}" -xdev -type d '
f'-not -path "{lfs}/sources/*" -not -path "{lfs}/{x}/*" '
f'-not -path "{lfs}/dev/*" -not -path "{lfs}/proc/*" '
f'-not -path "{lfs}/sys/*" -not -path "{lfs}/run/*" '
f'-not -path "{lfs}/tmp/*" 2>/dev/null | sort')
def _take_snapshot(lfs, dirs=False):
import subprocess
cmd = _snapshot_dirs_cmd(lfs) if dirs else _snapshot_cmd(lfs)
r = subprocess.run(["bash", "-c", cmd], cwd="/",
capture_output=True, text=True)
return set(r.stdout.splitlines())
def _run_tracked(name, script, as_lfs, lfs):
"""Run an install (as lfs or root), then diff the $LFS snapshot to record the
files this package installed into $LFS/var/lib/pkgusr/manifests/<name>.files.
The manifest ACCUMULATES: a re-run of an already-installed step naturally
diffs to "0 new" (the files already exist), so we union with whatever was
already recorded instead of overwriting -- otherwise a harmless re-run would
silently erase a correct manifest from an earlier successful run.
Returns the install's exit code."""
import subprocess
# progress/tree-snapshot -- the SAME file lfs-helper reads.
#
# These two tools hand the tree over to each other at the end of chapter 6:
# `lfs` takes the last snapshot outside the chroot, `lfs-helper` diffs
# against it for the first package inside. When the state directory was
# sorted in 1.9.0 the reader moved to progress/tree-snapshot and this
# writer kept using .snapshot at the top level, so the handover snapshot
# was silently lost and the first chroot package diffed against nothing.
d = _pkgusr_dir(lfs) # state root; manifests hang off it
_prog = pkgusr_state(lfs, "progress")
os.makedirs(_prog, exist_ok=True)
snap = os.path.join(_prog, "tree-snapshot")
before = (set(open(snap).read().splitlines())
if os.path.isfile(snap) else _take_snapshot(lfs))
# The directory snapshot must be taken HERE, before the build runs -- the
# old code took it afterwards and then discarded the diff whenever there
# was no saved snapshot, so on a first run every package recorded zero
# directories. Its files got the package user while the directories
# holding them stayed root-owned.
snapd = os.path.join(_prog, "tree-snapshot-dirs")
before_d = (set(open(snapd).read().splitlines())
if os.path.isfile(snapd) else _take_snapshot(lfs, dirs=True))
if as_lfs:
rc = run_as_lfs(script, lfs=lfs)
else:
rc = _run_bash(script)
after = _take_snapshot(lfs) # capture even on failure
new = after - before
# directories too: a package owns the dirs it creates for itself, and
# without this every package looks like it created none
after_d = _take_snapshot(lfs, dirs=True)
new_d = after_d - before_d
mand = os.path.join(d, "manifests", f"{name}.dirs")
existing_d = (set(l for l in open(mand).read().splitlines() if l)
if os.path.isfile(mand) else set())
with open(mand, "w") as f:
tot = sorted(existing_d | new_d)
f.write(("\n".join(tot) + "\n") if tot else "")
with open(snapd, "w") as f:
f.write("\n".join(sorted(after_d)) + "\n")
man = os.path.join(d, "manifests", f"{name}.files")
existing = set(l for l in open(man).read().splitlines() if l) \
if os.path.isfile(man) else set()
total = sorted(existing | new)
with open(man, "w") as f:
f.write(("\n".join(total) + "\n") if total else "")
with open(snap, "w") as f:
f.write("\n".join(sorted(after)) + "\n")
print(f"# tracked {len(new)} new file(s) and {len(new_d)} new dir(s) this "
f"run ({len(total)} file(s) total known) for '{name}'")
return rc
def cmd_bs_manifests(args):
"""List tracked package manifests (or one package's file list)."""
lfs = _lfs_dir()
if not lfs:
sys.stderr.write("no LFS mount set (lfs build-system session)\n")
sys.exit(2)
mdir = pkgusr_state(lfs, "manifests")
if not os.path.isdir(mdir):
print("no manifests yet.")
return
if args.package:
shown = False
for suffix, label in ((".files", "files"), (".dirs", "directories")):
p = os.path.join(mdir, f"{args.package}{suffix}")
if os.path.isfile(p):
body = open(p).read()
if body.strip():
print(f"# {label}")
sys.stdout.write(body)
shown = True
if not shown:
sys.stderr.write(f"no manifest for {args.package}\n")
sys.exit(1)
return
def _count(path):
try:
return sum(1 for ln in open(path) if ln.strip())
except OSError:
return 0
# Report directories alongside files. A package owns the directories it
# creates as well as the files in them, so a package showing files but no
# directories is a bug worth seeing -- that is exactly how the C++ header
# tree ended up root-owned while the headers inside it belonged to gcc.
total_f = total_d = 0
rows = []
for f in sorted(os.listdir(mdir)):
if not f.endswith(".files"):
continue
name = f[:-6]
nf = _count(os.path.join(mdir, f))
nd = _count(os.path.join(mdir, name + ".dirs"))
total_f += nf
total_d += nd
rows.append((name, nf, nd))
for name, nf, nd in rows:
flag = " <- no directories recorded" if (nf and not nd) else ""
print(f" {name:<20} {nf:>6} files {nd:>5} dirs{flag}")
print(f"\n {total_f} file(s) and {total_d} directory(ies) tracked "
f"across {len(rows)} package(s) [{lfs}]")
# ---- installing the real toolchain into the chroot ------------------------ #
# The Python tooling needs two modules that base LFS does not ship:
# requests (blfs fetches the BLFS book)
# beautifulsoup4 (lfs and blfs parse the books)
# Without them the tools import-fail the moment you run them in the chroot.
TOOL_PY_DEPS = [("bs4", "beautifulsoup4"), ("requests", "requests")]
# The command wrappers, at the ONE location lfs-helper writes them to and
# /etc/pkgusr/bash_profile points at. Named here so this tool does not become a
# fourth place that spells the path out: the wrapper directory has already been
# two different directories once.
WRAPPERS = os.environ.get("LFS_WRAPPERS", "/usr/lib/pkgusr")
# The package-users hint ships helper scripts and command wrappers. A system
# built with this tooling should have the same ones, so `add_package_user`,
# `install_package` and friends work in the chroot exactly as they do outside.
PKGUSR_HELPERS = [
# (name, where it belongs)
("add_package_user", "usr/sbin"),
("install_package", "usr/sbin"),
("useradd", WRAPPERS.lstrip("/")),
("groupadd", WRAPPERS.lstrip("/")),
("list_package", "usr/bin"),
("uninstall_package", "usr/bin"),
("list_suspicious_files", "usr/bin"),
("list_suspicious_files_from", "usr/bin"),
("grep_all_regular_files_for", "usr/bin"),
("forall_direntries_from", "usr/bin"),
# the five command wrappers, which live first in a package user's PATH
("chgrp", WRAPPERS.lstrip("/")),
("chown", WRAPPERS.lstrip("/")),
("chmod", WRAPPERS.lstrip("/")),
("mkdir", WRAPPERS.lstrip("/")),
("install", WRAPPERS.lstrip("/")),
]
_HELPER_SEARCH = ["/usr/lib/pkgusr", "/usr/sbin", "/usr/bin", "/sbin", "/bin",
"/usr/local/sbin", "/usr/local/bin",
"/tools/more_control_helpers/sbin",
"/tools/more_control_helpers/bin",
"/tools/more_control_helpers/lib"]
def _find_host_helper(name, subdir):
"""Find a hint helper on THIS machine.
The wrappers share names with the real commands (chown, install, ...), so
only look for those under a pkgusr directory -- copying /usr/bin/chown into
the chroot as a "wrapper" would be a fine way to break the system."""
if subdir.endswith("pkgusr"):
cands = ["/usr/lib/pkgusr/" + name,
"/tools/more_control_helpers/lib/" + name]
else:
cands = [os.path.join(d, name) for d in _HELPER_SEARCH
if "pkgusr" not in d or name not in
("chown", "chgrp", "chmod", "mkdir", "install")]
for c in cands:
if os.path.isfile(c) and os.access(c, os.X_OK):
return c
return None
# The package-user skeleton. If this machine has one (the hint's
# /etc/pkgusr/skel-package), copy it verbatim -- matching the host exactly
# matters more than any default we could invent, and a mismatch shows up much
# later as a package user whose environment differs from every other one.
PKGUSR_SKEL_SRC = "/etc/pkgusr/skel-package"
def _install_pkgusr_skel(lfs, verbose=True):
if not os.path.isdir(PKGUSR_SKEL_SRC):
if verbose:
print("# no %s on this machine -- the tools will write a default "
"package-user environment instead" % PKGUSR_SKEL_SRC)
return False
import shutil
dst = os.path.join(lfs, PKGUSR_SKEL_SRC.lstrip("/"))
try:
if os.path.isdir(dst):
shutil.rmtree(dst)
shutil.copytree(PKGUSR_SKEL_SRC, dst, symlinks=True)
except OSError as e:
fail("! could not copy %s: %s" % (PKGUSR_SKEL_SRC, e))
return False
# the shared files the skeleton's symlinks point at
for name in ("bash_profile", "bashrc", "build"):
src = os.path.join("/etc/pkgusr", name)
if os.path.isfile(src):
d = os.path.join(lfs, "etc", "pkgusr", name)
os.makedirs(os.path.dirname(d), exist_ok=True)
shutil.copy(src, d)
os.chmod(d, 0o755 if name == "build" else 0o644)
if verbose:
print("# package-user skeleton copied from %s" % PKGUSR_SKEL_SRC)
return True
# The shared-user XDG profile. skel-package above covers PACKAGE users;
# shared application users get their session environment from this ONE file,
# which `packagemanager user create --shared` links from
# /etc/pkgusr/skel-u_xdg and fills in. It used to arrive only via `make
# install`, so a system that got its tools through this sync never had it and
# every --shared creation warned about the missing file.
#
# The host's copy is preferred VERBATIM, like skel-package: a host that has
# one has usually filled the runtime directory in already, and that answer is
# right for the machine being synced to. The placeholder copy beside the
# tools is the fallback; packagemanager only substitutes where the
# placeholder is still present, so a filled-in file passes through untouched.
# An existing copy in the tree is never clobbered -- same as the Makefile.
XDG_SKEL_REL = "etc/pkgusr/skel-u_xdg/.bash_profile"
def _install_xdg_skel(lfs, verbose=True):
"""Place the shared-user XDG profile in the tree. Returns a short
report string for the sync summary."""
import shutil
dst = os.path.join(lfs, XDG_SKEL_REL)
if os.path.isfile(dst):
return "kept the tree's existing copy"
src = "/" + XDG_SKEL_REL
if not os.path.isfile(src):
src = os.path.join(os.path.dirname(os.path.abspath(__file__)),
"skel-u_xdg", ".bash_profile")
if not os.path.isfile(src):
if verbose:
print("# no skel-u_xdg/.bash_profile on this machine or beside "
"the tools -- shared users will warn until one exists")
return "none found -- shared users will warn"
try:
os.makedirs(os.path.dirname(dst), exist_ok=True)
shutil.copy(src, dst)
os.chmod(dst, 0o644)
except OSError as e:
fail("! could not copy %s: %s" % (src, e))
return "copy failed: %s" % e
if verbose:
print("# shared-user XDG profile copied from %s" % src)
return "copied from %s" % src
# The bash completion. Same forgotten-file story as the XDG profile above:
# the Makefile installed it, but the chroot gets its tools from THIS sync,
# so a synced system never had tab completion. Tooling, so it is
# overwritten, not kept -- stale completion is worse than none.
COMPLETION_REL = "usr/share/bash-completion/completions"
def _install_completion(lfs, verbose=True):
"""Place the bash completion + its tool symlinks in the tree. Returns a
short report string for the sync summary."""
import shutil
src = os.path.join(os.path.dirname(os.path.abspath(__file__)),
"lfs-completion.bash")
if not os.path.isfile(src):
src = "/" + COMPLETION_REL + "/lfs"
if not os.path.isfile(src):
if verbose:
print("# no lfs-completion.bash beside the tools or installed "
"here -- skipping completion")
return "none found -- skipped"
dst_dir = os.path.join(lfs, COMPLETION_REL)
try:
os.makedirs(dst_dir, exist_ok=True)
shutil.copy(src, os.path.join(dst_dir, "lfs"))
os.chmod(os.path.join(dst_dir, "lfs"), 0o644)
for t in ("blfs", "packagemanager", "lfs-helper"):
link = os.path.join(dst_dir, t)
try:
if os.path.islink(link) or os.path.isfile(link):
os.remove(link)
os.symlink("lfs", link)
except OSError:
pass
except OSError as e:
fail("! could not install completion: %s" % e)
return "copy failed: %s" % e
if verbose:
print("# bash completion installed (lfs, blfs, packagemanager, "
"lfs-helper)")
return "lfs + 3 symlinks"
def _install_pkgusr_helpers(lfs, verbose=True):
"""Copy the hint's helper scripts into the chroot, where present."""
import shutil
copied, missing, claimed = [], [], []
for name, subdir in PKGUSR_HELPERS:
src = _find_host_helper(name, subdir)
if not src:
missing.append(name)
continue
dst_dir = os.path.join(lfs, subdir)
os.makedirs(dst_dir, exist_ok=True)
dst = os.path.join(dst_dir, name)
try:
shutil.copy(src, dst)
os.chmod(dst, 0o755)
copied.append("%s -> /%s" % (name, subdir))
claimed.append((name, subdir))
except OSError as e:
fail("! %s: %s" % (name, e))
# Give them an owner, the same as install_helper does for lfs/lfs-helper/
# packagemanager/blfs.
#
# This path copied and chmod'd but never chowned, so the hint's helpers came
# out root:root while the four tools beside them in /usr/bin belonged to
# p_pkgusr. Two copy routines, one claiming ownership and one not -- the
# same species as the two wrapper directories and the two chgrp rules.
#
# Everything in the WRAPPER directory is excluded and stays root:root. A
# package user that could rewrite chown, chgrp or install could rewrite the
# rule that constrains it, and useradd/groupadd live there too.
wrapper_sub = WRAPPERS.lstrip("/")
by_dir = {}
for name, subdir in claimed:
if subdir.rstrip("/") == wrapper_sub:
continue
by_dir.setdefault(subdir, []).append(name)
for subdir, names in sorted(by_dir.items()):
_chown_tools(lfs, os.path.join(lfs, subdir), names, quiet=not verbose)
if verbose:
if copied:
print("# package-user helpers copied from this machine:")
for c in copied:
print("# %s" % c)
if missing:
print("# not on this machine (the tools fall back to Shadow's own"
" useradd/groupadd): %s" % ", ".join(missing))
return copied
def _chroot_missing_tool_deps(lfs):
import glob as _glob
sitedirs = _glob.glob(os.path.join(lfs, "usr", "lib", "python3*",
"site-packages"))
missing = []
for mod, pkg in TOOL_PY_DEPS:
found = any(os.path.exists(os.path.join(d, mod))
or _glob.glob(os.path.join(d, mod + "-*"))
or _glob.glob(os.path.join(d, mod + ".py"))
for d in sitedirs)
if not found:
missing.append((mod, pkg))
return missing
def _chroot_has_python(lfs):
"""Is there a usable Python inside the tree yet?"""
for p in ("usr/bin/python3", "usr/bin/python"):
if os.path.exists(os.path.join(lfs, p)):
return True
return False
def cmd_bs_install_tools(args):
"""Put the Python tooling (lfs, packagemanager, blfs) inside the chroot.
lfs-helper exists as a bash stand-in for exactly one reason: there is no
Python in the chroot until chapter 8 builds it. Once it's there the real
tools can run inside, and packagemanager takes over package management --
including for BLFS packages later on."""
lfs = require_mounted_lfs()
if not _chroot_has_python(lfs):
fail('! there is no Python inside the chroot yet, so the Python tooling\n cannot run there. Build it first (inside the chroot):\n lfs-helper build Python')
sys.exit(2)
missing = _chroot_missing_tool_deps(lfs)
if missing and not args.force:
fail('! the Python tooling needs modules that base LFS does not ship:')
for mod, pkg in missing:
sys.stderr.write(" %-16s (pip install %s)\n" % (mod, pkg))
sys.stderr.write(
"\n Install them inside the chroot -- this needs network access:\n"
" pip3 install %s\n"
"\n No network in the chroot? Download the wheels on this machine\n"
" into %s/sources and install from there:\n"
" pip3 install --no-index --find-links /sources %s\n"
"\n packagemanager itself does not need them; only 'lfs' and 'blfs'\n"
" do (they parse the books). To install the tools anyway:\n"
" %slfs build-system install-tools --run --force\n"
% (" ".join(p for _m, p in missing), lfs,
" ".join(p for _m, p in missing), sudo_prefix()))
sys.exit(2)
here = os.path.dirname(os.path.abspath(__file__))
tools = []
for name in ("lfs", "packagemanager", "blfs", "lfs-helper"):
src = os.path.join(here, name)
if os.path.isfile(src):
tools.append((name, src))
if not tools:
fail("! no tools found next to %s" % here)
sys.exit(2)
dest_dir = os.path.join(lfs, "usr", "bin")
cfg_dir = os.path.join(lfs, "etc", "pkgusr")
store = os.path.join(lfs, "usr", "share", "lfs")
print("Install the Python tooling into %s:" % lfs)
for name, _src in tools:
print(" %-22s -> /usr/bin/%s" % (name, name))
print(" %-22s -> /usr/share/lfs" % "book + scripts")
print(" %-22s -> /etc/pkgusr/packagemanager.conf" % "packagemanager conf")
if not args.run:
dry_run_note("install")
return
import shutil
os.makedirs(dest_dir, exist_ok=True)
os.makedirs(cfg_dir, exist_ok=True)
os.makedirs(store, exist_ok=True)
for name, src in tools:
dst = os.path.join(dest_dir, name)
shutil.copy(src, dst)
os.chmod(dst, 0o755)
# The BLFS book lives in its OWN store (/usr/share/blfs), so copying only
# the LFS store left the chroot with no BLFS book at all -- and every
# `packagemanager install <pkg>` failed with "not found in book" when the
# real problem was that there was no book to look in.
_blfs_src = None
for cand in (os.environ.get("BLFS_STORE"), "/usr/share/blfs",
os.path.expanduser("~/.cache/blfs")):
if cand and os.path.isdir(os.path.join(cand, "books")):
_blfs_src = os.path.join(cand, "books")
break
if _blfs_src:
_blfs_dst = os.path.join(lfs, "usr", "share", "blfs", "books")
os.makedirs(_blfs_dst, exist_ok=True)
_n = 0
for f in os.listdir(_blfs_src):
if not f.endswith((".html", ".html.gz")):
continue
d = os.path.join(_blfs_dst, f)
if not os.path.isfile(d):
shutil.copy(os.path.join(_blfs_src, f), d)
_make_world_readable(d)
_n += 1
if _n:
print(" %-22s -> /usr/share/blfs/books" % ("BLFS book" if _n == 1
else "BLFS books"))
else:
# NOT an aside. This is the whole of `packagemanager bootstrap`
# failing later, announced early and quietly enough to scroll past:
# with no book here, `blfs sources wget` gave nothing, so get-sources
# staged no wget tarball either, and the tree ends up unable to fetch
# the book because it has no download tool and unable to install the
# download tool because it has no book.
warn("! BLFS book: none cached on this host, so none was copied in.")
warn(" The tree will have no book AND no wget tarball, and")
warn(" `packagemanager bootstrap` inside cannot break that circle.")
warn(" Fix it here, while there is still a network:")
warn(" blfs fetch # or: blfs import <book.html>")
warn(" lfs build-system get-sources --run")
warn(" lfs build-system install-tools --run")
for sub in ("books", "crosschain"):
s_dir = os.path.join(store_dir(), sub)
if os.path.isdir(s_dir):
d_dir = os.path.join(store, sub)
if os.path.isdir(d_dir):
shutil.rmtree(d_dir)
shutil.copytree(s_dir, d_dir)
cfgsrc = config_path()
if os.path.isfile(cfgsrc):
# The tree's OWN choices win. A system inside the chroot has its own
# `lfs set-default` / `blfs set-default`, and copying the host's
# config over it silently moved a built system back to the host's
# book -- the setting the person changed last, undone by the next
# `chroot enter`. So: merge, host values only where the tree has
# nothing to say.
cfgdst = os.path.join(store, "config.json")
merged = {}
try:
with open(cfgsrc) as f:
merged.update(json.load(f))
except Exception:
pass
# THE BUILD SYSTEM'S OWN CONFIG, WRITTEN BEFORE IT EXISTS.
#
# The user asked for this directly: a file on the host that IS the new
# system's config, so it can be configured before there is a tree to
# configure -- and the run copies THAT in, not the host's own
# settings. Otherwise the only way to give the new system a different
# prefix or layout was to wait for the tree, then edit it there.
#
# Precedence: the host's session config is the floor, the target
# config overrides it, and a config the TREE already has wins over
# both (it is the system's own choice, made after this file was
# written).
_target = os.path.join(store_dir(), "target-config.json")
if os.path.isfile(_target):
try:
with open(_target) as f:
_tgt = json.load(f)
merged.update(_tgt)
print(" %-22s from %s" % ("target config", _target))
for _k in sorted(_tgt):
print(" %-20s = %r" % (_k, _tgt[_k]))
except (OSError, ValueError) as e:
print(" could not read %s: %s" % (_target, e))
try:
with open(cfgdst) as f:
tree_cfg = json.load(f)
merged.update(tree_cfg) # the tree overrides both
except Exception:
tree_cfg = {}
try:
with open(cfgdst, "w") as f:
json.dump(merged, f, indent=2)
if tree_cfg.get("default") and \
tree_cfg.get("default") != load_config().get("default"):
print(" %-22s kept: %s (the tree's own choice)"
% ("book default", tree_cfg["default"]))
except OSError:
shutil.copy(cfgsrc, cfgdst)
# packagemanager's own config, matching the settings used out here so the
# collector groups it creates line up with the ones already on the system
pmconf = os.path.join(cfg_dir, "packagemanager.conf")
if not os.path.isfile(pmconf):
# key=value, NOT json: packagemanager parses this file line by line.
# Written as json it parsed to nothing, so the prefix never arrived and
# packagemanager fell back to its default -- every package installed
# afterwards joined `sysgroup_*` groups while the built system used
# whatever prefix was configured out here.
with open(pmconf, "w") as f:
f.write("# packagemanager configuration\n")
f.write("# written by lfs build-system install-tools\n")
f.write("collector_prefix=%s\n" % collector_prefix())
f.write("user_prefix=u\n")
# The rest of what the interview already asked for. Without these
# packagemanager's first run opened a wizard asking for the
# package-user prefix and the login account -- on a system built
# from the very answers it was asking for.
# WITHOUT the trailing underscore. lfs's accessors return `p_`
# ready to concatenate; packagemanager stores the bare word and
# adds the underscore itself. Writing `p_` here survives only
# because it rstrips on read -- an accident, not an agreement.
f.write("pkgusr_prefix=%s\n" % pkgusr_prefix().rstrip("_"))
f.write("cfguser_prefix=%s\n" % cfguser_prefix().rstrip("_"))
_mu = load_config().get("main_user")
if _mu:
f.write("main_user=%s\n" % _mu)
# the layout of the system being BUILT, not this machine's
f.write("pkgusr_home=%s\n" % layout("pkgusr_home", target=True))
f.write("appuser_home=%s\n" % layout("appuser_home", target=True))
_install_pkgusr_skel(lfs)
_install_xdg_skel(lfs)
_install_completion(lfs)
_install_pkgusr_helpers(lfs)
_write_tool_stamps(lfs, tools)
_run_bash('chmod -R a+rX "%s" "%s" 2>/dev/null; true' % (store, cfg_dir))
print("\n# installed version %s (build %s). Inside the chroot the real "
"tools are now available:" % (LFS_VERSION, _build_id()))
print(" lfs next # what to do next")
print(" packagemanager --help # package management from here on")
print(" blfs --help # BLFS install scripts")
# ---- installing and updating LFS packages --------------------------------- #
def _find_blfs():
here = os.path.dirname(os.path.abspath(__file__))
local = os.path.join(here, "blfs")
if os.path.isfile(local):
# Check the mode bits, not os.access(X_OK): for root that returns True
# for a file with no execute bit at all, and the exec then fails with a
# confusing "Permission denied" on a path that plainly exists.
executable = bool(os.stat(local).st_mode & 0o111)
return [local] if executable else [sys.executable or "python3", local]
import shutil as _sh
return ["blfs"] if _sh.which("blfs") else None
def _blfs_knows(name):
"""Does the BLFS book have this package?
BLFS is the better source when both books carry a package: its scripts
carry the dependency information and the configure options that matter for
a package used as a library, where LFS only builds what the base system
needs.
Asks `blfs versions` -- a fact, listing every anchor the book has. It
used to ask `blfs debug <name>`, which is a DIAGNOSTIC: it prints
"resolve: NOT FOUND" and exits 0, so this returned True for every string
ever passed to it. `lfs update` then told the user to run
`blfs update gcc-pass1 cfg_fstab init-dirs ...` -- build steps of the LFS
book, offered as BLFS packages, because the check could not say no."""
blfs = _find_blfs()
if not blfs:
return False
return name.lower() in _blfs_anchor_set()
_BLFS_ANCHORS = None
def _blfs_anchor_set():
"""Every anchor in the BLFS book, once per process."""
global _BLFS_ANCHORS
if _BLFS_ANCHORS is not None:
return _BLFS_ANCHORS
_BLFS_ANCHORS = set()
blfs = _find_blfs()
if blfs:
import subprocess
try:
r = subprocess.run(blfs + ["versions"], capture_output=True,
text=True, timeout=120, cwd="/")
if r.returncode == 0:
for line in r.stdout.splitlines():
if "\t" in line:
_BLFS_ANCHORS.add(line.split("\t", 1)[0].strip().lower())
except (OSError, subprocess.SubprocessError):
pass
return _BLFS_ANCHORS
def _lfs_script_path(lfs, name):
return pkgusr_state(lfs, "scripts", "%s.sh" % name)
def _find_lfs_package(args, name):
"""Locate a package in the LFS book by the name used for its script."""
ver, path = resolve_book(args)
soup = load_soup(path)
want = name.lower()
for sid, title, sec in iter_sections(soup):
nv = package_name_version(sid, title)
if nv and nv[0].lower() == want:
return sid, title, sec
# fall back to the chapter 5-7 step names
for n, sid in (list(CHROOT_STEPS_7) + [(a, b) for a, b in CHROOT_CONFIG_STEPS_9]):
if n.lower() == want:
for s2, t2, sec2 in iter_sections(soup):
if s2 == sid:
return sid, t2, sec2
return None, None, None
def _have_source_for(lfs, pkgver):
"""Is a tarball for this package version already in the tree's /sources?
Deliberately loose, and deliberately the same shape the generated script
globs for: the book calls it 'M4-1.4.21', the tarball is 'm4-1.4.21.tar.xz'.
A stricter test here than in the script would fetch a file the build then
refuses to see."""
srcdir = os.path.join(lfs, "sources")
if not os.path.isdir(srcdir):
return False
want = pkgver.lower().replace("_", "-")
try:
for f in os.listdir(srcdir):
fl = f.lower().replace("_", "-")
if fl.startswith(want + ".") or fl.startswith(want + "-") \
or fl.startswith(want):
if re.search(r"\.(tar\.[a-z0-9]+|tgz|tbz2|zip)$", fl):
return True
except OSError:
return False
return False
def _wget_list_url_for(ver, pkgver):
"""The wget-list URL whose filename matches this package version.
The book ships a wget-list with every source URL in it, and the tools
already cache it per book -- but only `get-sources` ever read it, in bulk.
So moving to a newer book left `lfs install --reinstall m4` failing with
"no M4-1.4.21.tar.* in /sources", when the URL for exactly that tarball
was sitting in a file on disk. Matched on the version token, so
'M4-1.4.21' finds 'm4-1.4.21.tar.xz' whatever the site calls its
directories."""
urls = _wget_list_urls(ver) or []
if not pkgver:
return None
want = pkgver.lower()
# 'M4-1.4.21' -> 'm4' + '1.4.21'; a tarball is <name>-<version>.<ext>
m = re.match(r"^(.*?)-([0-9][0-9a-z.+_-]*)$", want)
name, version = (m.group(1), m.group(2)) if m else (want, "")
name = name.replace("_", "-")
for u in urls:
base = os.path.basename(u).lower()
if version and version not in base:
continue
stem = re.split(r"-[0-9]", base, maxsplit=1)[0].replace("_", "-")
if stem == name or base.startswith(name + "-") or base.startswith(name):
return u
return None
def _fetch_one_source(args, lfs, pkgver, quiet=False):
"""Download one package's tarball into the tree's /sources. True if it is
there afterwards (including when it already was).
`lfs` is passed in rather than guessed: inside the chroot the tree is "/"
and /sources is /sources, outside it is $LFS/sources, and a fetch that
landed in the wrong one would look like it worked and change nothing."""
ver, _path = resolve_book(args)
srcdir = os.path.join(lfs, "sources")
url = _wget_list_url_for(ver, pkgver)
if not url:
return False
dest = os.path.join(srcdir, os.path.basename(url))
if os.path.isfile(dest):
return True
if not quiet:
print(" fetching %s" % os.path.basename(url))
print(" from %s" % url)
try:
os.makedirs(srcdir, exist_ok=True)
_download_to(url, dest)
except Exception as e:
fail(" ! download failed: %s" % e)
return False
return os.path.isfile(dest)
def _ensure_lfs_script(args, lfs, name, sid, title, sec, regenerate):
"""Generate (or keep) the LFS-book install script for one package.
Returns the script path, or None when the book has nothing to write.
The ONE generator: cmd_install builds through it and `lfs script fetch`
stops right after it, so the two can never drift apart."""
script = _lfs_script_path(lfs, name)
print("%s (%s)" % (name, title.strip()))
print(" script: %s" % script)
if regenerate or not os.path.isfile(script):
pkgver = _title_pkgver(title) or name
glob = _pkg_glob_for(pkgver, name)
text = _chrootify(_crosschain_script_body(name, sid, pkgver, glob,
sec["commands"]))
text, left = _fill_placeholders(text, name)
os.makedirs(os.path.dirname(script), exist_ok=True)
with open(script, "w") as f:
f.write(text)
os.chmod(script, 0o755)
print(" regenerated from the book")
if left:
fail(" ! still contains: %s -- edit the script "
"before building"
% ", ".join("<%s>" % x for x in left))
else:
# The existing script is KEPT -- it is yours, and regenerating would
# discard your edits. But the line above prints the BOOK's title, so
# a script written for an older book sat under a heading naming the
# new version, with nothing to say they differ. Say it.
book_ver = _title_pkgver(title)
have = None
try:
m = re.search(r'^name_version="([^"]+)"',
open(script, errors="replace").read(), re.M)
have = m.group(1) if m else None
except OSError:
pass
if have and book_ver and have != book_ver:
print(" kept your existing script, which builds %s" % have)
print(" (the book now has %s -- rebuild the script from it "
"with --regenerate," % book_ver)
print(" which discards any edits you made)")
else:
print(" kept your existing script")
return script
def cmd_install(args):
"""Install or reinstall an LFS package, as its package user.
The build itself is lfs-helper's job -- it owns the package users and the
file tracking -- so this generates a current script and hands over, rather
than growing a second, subtly different build path."""
# Are we running INSIDE the chroot? The state directory sits at the root of
# the built system, so it only exists at /usr/src/lfs-pkgusr when we are in there.
inside = os.path.isdir(os.path.join("/", PKGUSR_DIR))
lfs = "/" if inside else require_mounted_lfs()
reinstall = args.reinstall
for name in args.packages:
# BLFS first: if both books have the package, BLFS' script is the more
# complete one (dependencies, and the options a library build needs).
if not args.lfs_book and _blfs_knows(name):
print("%s is in the BLFS book -- using that (more complete than "
"the LFS one)." % name)
print(" blfs install %s" % name)
print(" (force the LFS book's version with: lfs install "
"--lfs-book %s)" % name)
blfs = _find_blfs()
if inside and blfs:
rc = _run_bash(" ".join(shlex.quote(c)
for c in blfs + ["install", name]))
if rc != 0:
sys.exit(rc)
continue
sid, title, sec = _find_lfs_package(args, name)
if not sid:
fail("! '%s' is not an LFS package."
" List them with: lfs packages\n"
" BLFS packages are blfs' job: blfs script %s\n"
% (name, name))
continue
if not sec["commands"]:
fail("! %s has no build commands in the book." % name)
continue
script = _ensure_lfs_script(args, lfs, name, sid, title, sec,
args.regenerate)
# The tarball has to be there before the build starts. Moving to a
# newer book means the new tarballs are NOT in /sources, and the build
# failed with "no M4-1.4.21.tar.* in /sources -- run: get-sources"
# while the URL for exactly that file sat in the book's cached
# wget-list. Fetch the one that is needed, and say so.
_pv = _title_pkgver(title)
if _pv and not _have_source_for(lfs, _pv):
if not _fetch_one_source(args, lfs, _pv):
fail(" ! no tarball for %s in %s/sources, and none found in"
" the book's wget-list." % (_pv, lfs))
fail(" Fetch everything the book needs with:")
fail(" %slfs build-system get-sources --run"
% sudo_prefix())
sys.exit(1)
# Reinstalling re-runs the CONFIGURE phase too, and that is not always
# harmless: those are the post-install steps that move files about,
# rewrite config and recreate symlinks. Ask rather than assume.
phases = "all"
if reinstall:
has_cfg = False
if os.path.isfile(script):
# a phase is a function (format v18+); older scripts carried a marker
has_cfg = bool(re.search(r"^configure_pkg\(\)|^#### CONFIGURE ####",
open(script).read(), re.M))
if has_cfg:
print()
print(" %s has a configure phase -- the post-install steps" % name)
print(" (moving files, rewriting config, recreating symlinks).")
print(" Re-running it is usually right, but it can overwrite")
print(" changes you made by hand since the first install.")
ans = ""
if args.yes:
ans = "y"
elif sys.stdin.isatty():
try:
ans = input(" Run the configure phase too? [y/N]: ").strip()
except EOFError:
ans = ""
if ans[:1].lower() != "y":
phases = "install"
print(" -> unpack/build/install only, leaving configure alone")
cmd = ["lfs-helper", "build", name, "--force"]
if phases != "all":
cmd += ["--phase", phases]
print()
if inside:
print(" running: %s" % " ".join(cmd))
rc = _run_bash(" ".join(shlex.quote(c) for c in cmd))
if rc != 0:
sys.exit(rc)
else:
print(" Run this inside the chroot:")
print(" %slfs build-system chroot enter" % sudo_prefix())
print(" %s" % " ".join(cmd))
def cmd_search(args):
"""Find a package or section in the LFS book by name fragment."""
label, path = resolve_book(args)
soup = load_soup(path)
needle = args.name.lower().replace(" ", "-")
hits = []
for sid, title, sec in iter_sections(soup):
hay = (sid + " " + title).lower().replace(" ", "-")
if needle not in hay:
continue
nv = package_name_version(sid, title)
kind = "package" if (nv and sec["commands"]) else "section"
hits.append((kind, sid, title.strip()))
if not hits:
print(f"No match for '{args.name}' in the {label} book.")
print(f" BLFS is searched separately: blfs search {args.name}")
sys.exit(1)
for kind, sid, title in hits:
print(f" [{kind:>7}] {sid:<34} {title}")
print(f"\n{len(hits)} result(s) [{label}].")
def cmd_script_fetch(args):
"""Get a package's install script from the book -- run nothing.
Stops exactly where cmd_install would start building: the script lands at
the same path, generated by the same code, so "fetch, edit, install" edits
the file the install will run."""
inside = os.path.isdir(os.path.join("/", PKGUSR_DIR))
lfs = "/" if inside else require_mounted_lfs()
name = args.package
if not args.lfs_book and _blfs_knows(name):
print("%s is in the BLFS book -- fetch that script (more complete):"
% name)
print(" packagemanager script fetch %s" % name)
print(" (force the LFS book's with: lfs script fetch --lfs-book %s)"
% name)
return
sid, title, sec = _find_lfs_package(args, name)
if not sid:
fail("! '%s' is not an LFS package. List them with: lfs packages"
% name)
sys.exit(1)
if not sec["commands"]:
fail("! %s has no build commands in the book." % name)
sys.exit(1)
script = _ensure_lfs_script(args, lfs, name, sid, title, sec,
args.regenerate)
print(" edit it, then: lfs install --reinstall %s" % name)
def cmd_update(args):
"""Rebuild packages whose installed version differs from the book's."""
inside = os.path.isdir(os.path.join("/", PKGUSR_DIR))
lfs = "/" if inside else require_mounted_lfs()
ver, path = resolve_book(args)
soup = load_soup(path)
manifests = pkgusr_state(lfs, "manifests")
installed = set()
if os.path.isdir(manifests):
installed = {f[:-6] for f in os.listdir(manifests) if f.endswith(".files")}
names = args.packages or sorted(installed)
# BOOK ORDER, not alphabetical. The book builds glibc before Python for
# a reason -- each package is compiled against the ones before it -- and
# an alphabetical list puts Python, binutils and coreutils ahead of
# glibc. Rebuilt in that order, the earlier ones link against the old
# glibc and are then stale again the moment it is replaced. Chapter 8's
# own sequence is the safe one, so the report is sorted by where each
# package appears in the book, and anything the book does not name goes
# last rather than jumping the queue.
_order = {}
for _i, (_sid, _title, _sec) in enumerate(iter_sections(soup)):
_nv = package_name_version(_sid, _title)
if _nv:
_order.setdefault(_nv[0].lower(), _i)
names = sorted(names, key=lambda n: (_order.get(n.lower(), 10 ** 6),
n.lower()))
if not names:
print("nothing is installed yet -- build the system first:")
print(" %slfs build-system next" % sudo_prefix())
return
stale, current, unknown = [], [], []
# The chapter 5-7 STAGE VARIANTS are not upgradable packages.
# gettext-tmp, python-tmp and util-linux-tmp are the TEMPORARY builds that
# existed to get chapter 8 built; binutils-pass1 and gcc-pass2 are cross
# stages. Comparing them with the current book reported "util-linux-tmp
# 2.41.1 -> 2.41.3" and put them in the reinstall line, which would
# rebuild the temporary toolchain onto a finished system.
#
# Only the VARIANT names are excluded, never the plain one. Chapter 6
# builds glibc and coreutils under their own names, so taking every name
# in the step tables would have silently hidden two of the packages most
# worth knowing about -- the first draft of this did exactly that.
_stage_steps = {n.lower() for n, _sid in CHROOT_STEPS_7
if n.lower().endswith("-tmp")}
_stage_steps |= {n.lower() for n, _sid, _pkg in CROSSCHAIN_STEPS
if re.search(r"-(pass\d+|tmp)$", n.lower())}
for name in names:
if name.lower() in _stage_steps:
continue
sid, title, sec = _find_lfs_package(args, name)
if not sid:
unknown.append(name)
continue
book_ver = _title_pkgver(title) or "?"
script = _lfs_script_path(lfs, name)
# What is INSTALLED comes from the package user's VERSION file. The
# script is regenerated whenever the book moves on, so its name_version
# says what the book has -- using it here would compare the book with
# itself and never report anything as out of date.
have, from_book = None, None
vpath = os.path.join(lfs, "usr", "src", name, "VERSION")
if os.path.isfile(vpath):
for line in open(vpath):
line = line.strip()
if line.startswith("# from:"):
from_book = line.split(":", 1)[1].strip()
elif line and not line.startswith("#") and have is None:
have = line
if have is None and os.path.isfile(script):
m = re.search(r'^name_version="([^"]+)"', open(script).read(), re.M)
if m:
have = m.group(1)
if have and have != book_ver:
stale.append((name, have, book_ver))
else:
current.append((name, have or book_ver))
if getattr(args, "porcelain", False):
# One fact per line, tab-separated, and NOTHING else -- no headings,
# no advice, no colour. packagemanager builds its plan from this;
# parsing the human report would break the first time a word changed.
#
# --include-current adds the packages whose version did NOT move.
# After glibc changes, "up to date" and "built against the current
# system" stop being the same statement: the version is right and the
# binary is still the old one. Rebuilding those, in book order, is
# the only way to know the whole system was built against the new
# toolchain. Emitted in the same shape, with installed == book.
rows = list(stale)
if getattr(args, "include_current", False):
_stale_names = {n for n, _h, _b in stale}
for name, v in current:
if name not in _stale_names:
rows.append((name, v or "?", v or "?"))
rows.sort(key=lambda r: (_order.get(r[0].lower(), 10 ** 6),
r[0].lower()))
for name, have, book_ver in rows:
print("%s\t%s\t%s" % (name, have, book_ver))
return
if stale:
print("Out of date -- comparing what is installed with the %s book:\n"
% ver)
_c = _C_OK if _tty_out() else ""
_o = _C_OFF if _tty_out() else ""
for name, have, book_ver in stale:
# the version you would move TO, in colour -- the same emphasis
# packagemanager's plan gives it, so the two reports read alike.
# Guarded by _tty_out() like every other colour here: escape codes
# in a piped report are noise in a file.
print(" %-20s %s -> %s%s%s"
% (name, have, _c, book_ver, _o))
print()
print("Rebuild them with:")
print(" lfs install --reinstall %s"
% " ".join(n for n, _h, _b in stale))
else:
print("Everything checked is current with the %s book." % ver)
if unknown:
in_blfs = [n for n in unknown if _blfs_knows(n)]
other = [n for n in unknown if n not in in_blfs]
if in_blfs:
print("\nIn the BLFS book -- update those with blfs:")
print(" blfs update %s" % " ".join(in_blfs))
if other:
print("\nIn neither book: %s" % ", ".join(other))
if args.verbose and current:
print("\nCurrent:")
for name, v in current:
print(" %-20s %s" % (name, v))
# ---- what to do next ------------------------------------------------------ #
def _chroot_progress(lfs):
"""Steps already built INSIDE the chroot (written by lfs-helper)."""
p = pkgusr_state(lfs, "progress", "steps-built")
if os.path.isfile(p):
return [l.strip() for l in open(p) if l.strip()]
return []
def _chroot_steporder(lfs):
p = pkgusr_state(lfs, "progress", "steporder")
if os.path.isfile(p):
return [l.strip() for l in open(p) if l.strip()]
return []
def _crosschain_looks_done(lfs):
"""Chapters 5-6 finished?
Returns (done, how_many_of_the_22).
This must be EXACT. A cross-gcc lands in $LFS/tools/bin after step 2 of
22, so treating its presence as "chapters 5-6 are done" declared a
toolchain finished when two thirds of it was missing -- and the run then
handed the tree to root and entered the chroot on top of it. Half a
toolchain is not a system you can build in.
The progress file is the only real record, so it is no longer cleared on
completion (an empty file used to mean both "not started" and "all done").
"""
names = {n for n, _s, _t in CROSSCHAIN_STEPS}
prog = _load_crosschain_progress(lfs) or {}
done = set(prog.get("done", [])) & names
if done >= names:
return True, len(done)
# Past this stage entirely: chapter 7/8 work exists in the chroot, which
# can only happen after chapters 5-6 completed.
if _chroot_progress(lfs):
return True, len(names)
# No progress file at all, but a full temporary system is present: an
# older run finished before progress was recorded. Require evidence from
# the END of the chain (chapter 6's last steps), not the beginning.
if not prog:
import glob as _glob
# Evidence must come from the LAST step, gcc-pass2. gawk/sed/tar/xz
# are steps 13-20, so they are present at 20 of 22 as well -- a tree
# missing only gcc-pass2 looked complete, and gcc-pass2 is what
# installs the C++ headers, so chapter 8's gcc then died with
# fatal error: bits/c++config.h: No such file or directory
if _glob.glob(os.path.join(lfs, "usr", "include", "c++", "*",
"*", "bits", "c++config.h")):
return True, len(names)
return False, len(done)
def _crosschain_missing_output(lfs):
"""A short, checkable reason why chapters 5-6 look unfinished, or ''."""
import glob as _glob
if not _glob.glob(os.path.join(lfs, "usr", "include", "c++", "*",
"*", "bits", "c++config.h")):
return ("the C++ headers are missing (gcc-pass2, the last step of "
"chapter 6, did not finish) -- chapter 8's gcc cannot build "
"without them")
return ""
def _step(done, title, *cmds, note=None):
return {"done": done, "title": title, "cmds": list(cmds), "note": note}
def _next_steps():
"""Work out where the build stands, as an ordered list of steps."""
cfg = load_config()
ver = build_version() # the checklist builds a system
steps = []
steps.append(_step(bool(ver), "choose and fetch a book",
"lfs books", "lfs fetch <version>"))
missing = [k for k, _w, _h in REQUIRED_CONFIG if not cfg.get(k)]
steps.append(_step(not missing, "complete the configuration",
*["%s%s" % (sudo_prefix(), h)
for k, _w, h in REQUIRED_CONFIG if k in missing]
or ["lfs config"],
note="see everything with: lfs config"))
lfs = _lfs_dir()
mounted = bool(lfs) and (not cfg.get("lfs_device") or _is_mounted(lfs))
steps.append(_step(mounted, "mount the LFS partition",
"%slfs build-system session --run" % sudo_prefix()))
if not lfs:
return steps, None
steps.append(_step(os.path.isdir(os.path.join(lfs, "tools")),
"create the directory layout",
"%slfs build-system layout --run" % sudo_prefix()))
steps.append(_step(os.path.isdir(os.path.join(lfs, "etc", "pkgusr")),
"copy the package-user environment into the tree",
"%slfs build-system sync-tools --run" % sudo_prefix(),
note="skel-package, the helper scripts and lfs-helper; "
"re-run it whenever they change here"))
import pwd as _pwd
try:
_pwd.getpwnam("lfs")
have_lfs_user = True
except KeyError:
have_lfs_user = False
steps.append(_step(have_lfs_user, "create the lfs user",
"%slfs build-system add-user --run" % sudo_prefix()))
srcdir = os.path.join(lfs, "sources")
# "done" is EVERY basename of the current book's wget-list being present,
# not "some tarball exists". Leftovers from an older book made the step
# look finished, and the download was skipped for a book whose tarballs
# were not there. (from the SysV-book session)
src_missing = 0
if os.path.isdir(srcdir):
wl_urls = _wget_list_urls(build_version())
if wl_urls is not None:
have = set(os.listdir(srcdir))
src_missing = sum(1 for u in wl_urls
if u.split("/")[-1] not in have)
have_sources = src_missing == 0
else:
# no wget-list cached yet: the old heuristic is all there is
have_sources = any(
f.endswith((".tar.xz", ".tar.gz", ".tar.bz2", ".tgz"))
for f in os.listdir(srcdir))
else:
have_sources = False
# BEFORE the sources, deliberately. `get-sources` asks
# `packagemanager bootstrap --sources`, which asks the BLFS book where
# wget's tarball is -- with no book that URL is silently absent from the
# download list, and the gap only shows up on the built system.
steps.append(_step(_blfs_book_on_host() or cfg.get("blfs_book") == "skip",
"fetch the BLFS book (for the built system)",
"blfs fetch",
note="the built system cannot fetch it itself until it "
"has wget, and wget is looked up in this book"))
steps.append(_step(have_sources, "download the sources"
+ (f" ({src_missing} missing)" if src_missing else ""),
"%slfs build-system get-sources --run" % sudo_prefix()))
cc_done, cc_n = _crosschain_looks_done(lfs)
steps.append(_step(cc_done,
"build the cross toolchain and temporary tools "
"(chapters 5-6, %d/%d)" % (cc_n, len(CROSSCHAIN_STEPS)),
"%slfs build-system crosschain --run" % sudo_prefix()))
live = _chroot_mounts(lfs)
steps.append(_step(len(live) == len(_KERNFS) and _owner_is_root(lfs),
"prepare the chroot (chapter 7)",
"%slfs build-system chroot prepare --run" % sudo_prefix()))
steps.append(_step(os.path.isdir(pkgusr_state(lfs, "scripts")),
"generate the in-chroot build scripts",
"%slfs build-system gen-chroot-scripts --run" % sudo_prefix()))
# inside the chroot
order = _chroot_steporder(lfs)
built = set(_chroot_progress(lfs))
inside_done = bool(order) and set(order) <= built
steps.append(_step(inside_done,
"build the system inside the chroot (chapters 7-9%s)"
% (", %d/%d" % (len(built & set(order)), len(order))
if order else ""),
"%slfs build-system chroot enter" % sudo_prefix(),
"lfs-helper next # (inside the chroot)",
note="lfs-helper is the bash stand-in used until Python "
"exists in the chroot"))
have_py = _chroot_has_python(lfs)
dep_missing = _chroot_missing_tool_deps(lfs) if have_py else TOOL_PY_DEPS
steps.append(_step(have_py and not dep_missing,
"install the Python modules the tooling needs (%s)"
% ", ".join(p for _m, p in TOOL_PY_DEPS),
"pip3 install %s # inside the chroot"
% " ".join(p for _m, p in TOOL_PY_DEPS),
note="base LFS has no requests/beautifulsoup4; "
"'lfs' and 'blfs' parse the books with them"))
tools_in = os.path.exists(os.path.join(lfs, "usr", "bin", "packagemanager"))
steps.append(_step(tools_in and have_py,
"hand over from lfs-helper to the real tooling",
"%slfs build-system install-tools --run" % sudo_prefix(),
note="Python is in the chroot now, so packagemanager can "
"take over from lfs-helper"))
return steps, lfs
def cmd_next(args):
"""Show where the build stands and what to do next."""
steps, lfs = _next_steps()
pending = [s for s in steps if not s["done"]]
done_n = len(steps) - len(pending)
if args.all:
print("Build progress:\n")
for s in steps:
mark = "[x]" if s["done"] else "[ ]"
print(" %s %s" % (mark, s["title"]))
print()
if not pending:
print("All %d steps are done -- the system is built.\n" % len(steps))
print("\nFrom here package management is packagemanager's job:")
print(" packagemanager list")
print(" packagemanager install <package>")
print(" blfs script <package> # BLFS install scripts")
return
nxt = pending[0]
# Say plainly how much is done and how much is left. "Step 11 of 11" reads
# like "finished" when it actually means "one still to go".
print("%d of %d done -- %d still to do.\n"
% (done_n, len(steps), len(pending)))
print("NEXT: %s" % nxt["title"])
for c in nxt["cmds"]:
print(" %s" % c)
if nxt["note"]:
print("\n (%s)" % nxt["note"])
if len(pending) > 1:
print("\n after that: %s" % pending[1]["title"])
if len(pending) > 2:
print(" and %d more -- see: lfs build-system next --all"
% (len(pending) - 2))
elif not args.all:
print("\n full checklist: lfs build-system next --all")
LFS_HELP_EPILOG = """\
commands, by what you are doing:
build a system -- everything about building lives under build-system
build-system run do all of it, from wherever you are, then
enter the chroot -- the only one you need
build-system next what is done, what comes next
build-system list every step and its state
build-system snapshot save the build, or put it back
build-system session mount the partition, set $LFS, ask for settings
build-system set-book which book to BUILD from
build-system chroot prepare / enter / resolv / unmount
build-system install-tools hand over to packagemanager
(session/layout/sync-tools/add-user/
get-sources/crosschain/gen-chroot-scripts
are the individual steps `run` performs)
manage packages on a machine (BLFS ones hand over to blfs)
install [--reinstall] build a package as its package user
update what the book has a newer version of
packages / commands list packages; print a section's commands
set-default which book to INSTALL from
books
books / fetch / import list, download, or add a book
sources a package's download URLs
settings and repair
config [--long] every setting, grouped by what it affects
reset clear the settings and start fresh
Inside the chroot, lfs-helper drives the build; `lfs` will tell you so.
Dry run is the default everywhere; --run applies.
"""
# Inside the chroot, `lfs` is the wrong tool for most things.
#
# The build is driven from OUTSIDE: `lfs` mounts the partition, creates the
# layout, builds the cross toolchain and generates the in-chroot scripts. From
# inside, none of that applies -- $LFS does not even exist as a path -- so
# `lfs build-system next` reads the empty state and cheerfully reports "2 of 12
# done", sending you back to the beginning of a finished build.
#
# Inside, lfs-helper is the tool.
_INSIDE_ONLY_ELSEWHERE = {
"next", "session", "layout", "add-user", "get-sources", "crosschain",
"chroot", "gen-chroot-scripts", "install-tools", "fix-ownership",
"version-check", "show",
}
_INSIDE_EQUIVALENT = {
"next": "lfs-helper next",
"show": "lfs-helper list",
"fix-ownership": "lfs-helper fix-ownership --run",
"manifests": "lfs-helper manifests",
}
def _root_differs_from_pid1():
"""Is this process's / a different directory from PID 1's?
THE KERNEL KNOWS. That is what a chroot IS -- a root that is not the
real one -- and /proc/1/root resolves to the real one, so comparing the
two answers the question outright instead of inferring it from files
that happen to be there. Returns None when it cannot be asked: no
/proc mounted, or not privileged enough to read PID 1's root.
"""
if os.geteuid() != 0:
return None # /proc/1/root is root's to read
try:
a, b = os.stat("/"), os.stat("/proc/1/root")
except OSError:
return None # no /proc, or a kernel that will not say
return (a.st_dev, a.st_ino) != (b.st_dev, b.st_ino)
def inside_chroot():
"""Are we running inside the LFS chroot?
The chroot SAYS SO: `lfs build-system chroot enter` sets LFS_IN_CHROOT in
the environment it starts. Failing that, the kernel is asked
(_root_differs_from_pid1).
It used to be inferred from "/usr/src/lfs-pkgusr exists", and that is true
of any system these tools have built -- so a finished host running its own
lfs refused every command with "you are inside the LFS chroot". An
inference that is true of the destination cannot identify the journey.
The guard on that inference was "...and /mnt/lfs/usr/src does not exist",
which fails at the one moment it matters: a FRESH build, where the
partition is mounted and still empty, on a host these tools built. That
is the report this fix comes from. The old test survives only as a last
resort -- no LFS_IN_CHROOT, no readable /proc/1 -- and then against the
CONFIGURED mount point, not a hardcoded one.
"""
if os.environ.get("LFS_IN_CHROOT"):
return True
differs = _root_differs_from_pid1()
if differs is not None:
return differs
if os.environ.get("LFS"):
return False # there is a build target: we are outside
mount = load_config().get("lfs_mount") or "/mnt/lfs"
return (os.path.isdir(os.path.join("/", PKGUSR_DIR))
and not os.path.isdir(mount))
def _refuse_inside_chroot(cmd, sub=None):
name = sub or cmd
sys.stderr.write(
"You are inside the LFS chroot, where `lfs %s` does not apply.\n"
"\n"
" The build is driven from OUTSIDE the chroot: `lfs` mounts the\n"
" partition, builds the toolchain and writes the scripts. From in\n"
" here there is no $LFS to work on, so this command would report\n"
" nonsense about a build that is actually finished.\n"
"\n" % (("build-system " + sub) if sub else cmd))
eq = _INSIDE_EQUIVALENT.get(name)
if eq:
sys.stderr.write(" In here, use: %s\n" % eq)
else:
sys.stderr.write(" In here, lfs-helper drives the build:\n"
" lfs-helper next # what to do next\n"
" lfs-helper status # where things stand\n"
" lfs-helper list # every step\n")
sys.stderr.write("\n To run this, leave the chroot first (type 'exit').\n")
# WHICH lfs SAID SO, AND WHY.
#
# A host was refused by this message for days. 1.14.1 fixed the
# detection, but the binary being run was 1.11.26 -- `make install` had
# been going somewhere else, and nothing on screen said which copy was
# talking or what convinced it. A refusal has to identify itself, or a
# stale install looks like a bug in the current one.
how = "LFS_IN_CHROOT is set" if os.environ.get("LFS_IN_CHROOT") \
else ("this process's / is not PID 1's"
if _root_differs_from_pid1() else "no $LFS, and no mounted tree")
sys.stderr.write("\n (said by %s %s, build %s -- because %s)\n"
% (os.path.abspath(sys.argv[0]), LFS_VERSION,
_build_id(), how))
sys.exit(2)
def main():
ap = argparse.ArgumentParser(prog="lfs", description="LFS book tool "
"(cache + follow one book)")
ap.add_argument("--version", action="version", version=f"lfs {LFS_VERSION} (build {_build_id()})")
ap.add_argument("--book", help="use this cached version for this run "
"(overrides the default)")
ap.add_argument("--book-file", help="use a local nochunks HTML file directly")
ap.epilog = _colour_epilog(LFS_HELP_EPILOG)
ap.formatter_class = argparse.RawDescriptionHelpFormatter
sub = ap.add_subparsers(dest="cmd", required=True, metavar="<command>")
p = sub.add_parser("next", help="-> lfs build-system next")
q = sub.add_parser("list", help="-> lfs build-system list", description="every build step and its state "
"(same as 'lfs-helper list' inside the chroot)")
q.set_defaults(func=cmd_next, all=True)
q = sub.add_parser("run", help="-> lfs build-system run", description="alias for 'lfs build-system run': do "
"everything up to the chroot, then enter it")
q.set_defaults(func=cmd_bs_run)
p.add_argument("--all", action="store_true",
help="show the whole checklist, not just the next step")
p.set_defaults(func=cmd_next)
# kept at the top level as a shorthand, like `next` and `run`
p = sub.add_parser("snapshot", help="-> lfs build-system snapshot")
p.add_argument("action", nargs="?",
choices=["list", "save", "restore", "remove"],
help="default: list")
p.add_argument("name", nargs="?", help="snapshot name")
p.add_argument("--note", help="what this snapshot is (shown in the list)")
p.add_argument("--run", action="store_true", help="apply (default: dry run)")
p.add_argument("--yes", action="store_true", help="skip the confirmation")
p.add_argument("--force", action="store_true", help="replace an existing one")
p.set_defaults(func=cmd_snapshot)
p = sub.add_parser("reset", help="clear the tools' configuration and start "
"fresh (does NOT touch the built system)")
p.add_argument("--run", action="store_true", help="delete (default: dry run)")
p.add_argument("--yes", action="store_true", help="skip the confirmation")
p.set_defaults(func=cmd_reset)
p = sub.add_parser("config", help="show or change settings (paths, $LFS, "
"LFS_TGT, mirror, ...)")
# values can start with '-' (e.g. makeflags -j4), so don't let argparse
# try to parse them as flags
# --target: the config the BUILT system will get, editable here, before
# the tree exists. `lfs run` copies it in (see target-config.json).
p.add_argument("--target", action="store_true",
help="show or change the config the BUILT system will get "
"(not this host's)")
p.add_argument("--ask", action="store_true",
help="walk the settings one at a time (with --target)")
p.add_argument("key", nargs="?", help="setting to show, or set with a value")
p.add_argument("value", nargs="?", help="new value")
p.usage = ("lfs config [key] [value] | lfs config --edit | "
"lfs config --unset KEY [KEY ...]")
p.add_argument("--edit", action="store_true",
help="open the config file in $EDITOR")
p.add_argument("--unset", nargs="+", metavar="KEY",
help="remove setting(s), reverting to the default")
p.add_argument("--long", action="store_true",
help="explain what each setting does")
p.set_defaults(func=cmd_config)
p = sub.add_parser("books", help="list cached books + all versions on the site")
p.add_argument("--local", action="store_true",
help="only show cached books (skip the site listing)")
p.set_defaults(func=cmd_books)
p = sub.add_parser("set-default",
help="which book to INSTALL PACKAGES from "
"(the build book: lfs build-system set-book)")
p.add_argument("version", help="e.g. 12.4 or 13.0")
p.set_defaults(func=cmd_set_default)
p = sub.add_parser("fetch", help="download + cache a book and its wget-list")
p.add_argument("version", help="e.g. 12.4, 13.0, stable, development")
p.add_argument("--set-default", action="store_true",
help="also make it the default")
p.set_defaults(func=cmd_fetch)
p = sub.add_parser("import", help="cache a LOCAL book file (offline)")
p.add_argument("file", help="path to an LFS ...-NOCHUNKS.html")
p.add_argument("--version", required=True, help="label to store it under")
p.add_argument("--wget-list", help="also import a wget-list file")
p.add_argument("--set-default", action="store_true")
p.set_defaults(func=cmd_import)
p = sub.add_parser("sources", help="print package source URLs (from wget-list)")
p.set_defaults(func=cmd_sources)
p = sub.add_parser("sections", help="list sect ids + titles of the main book")
p.add_argument("book_pos", nargs="?", metavar="BOOK",
help="version or file (default: the main book)")
p.set_defaults(func=cmd_sections)
p = sub.add_parser("install", help="install or reinstall an LFS package "
"as its package user")
p.add_argument("packages", nargs="+")
p.add_argument("--reinstall", action="store_true",
help="rebuild a package that is already installed")
p.add_argument("--regenerate", action="store_true",
help="regenerate the install script from the book first")
p.add_argument("--yes", "-y", action="store_true",
help="answer yes to the configure-phase question")
p.add_argument("--lfs-book", action="store_true",
help="use the LFS book even when BLFS also has the package")
p.set_defaults(func=cmd_install, action="install")
p = sub.add_parser("update", help="show which installed packages the book "
"has a newer version of")
p.add_argument("packages", nargs="*",
help="packages to check (default: everything installed)")
p.add_argument("--verbose", "-v", action="store_true",
help="list up-to-date packages too")
p.add_argument("--porcelain", action="store_true",
help="machine-readable: name<TAB>installed<TAB>book, one "
"per line, nothing else (for packagemanager)")
p.add_argument("--include-current", action="store_true",
help="with --porcelain: list up-to-date packages too, so a "
"caller can rebuild the whole system in book order")
p.set_defaults(func=cmd_update)
p = sub.add_parser("packages", help="list chapter-8 build packages "
"(name<TAB>version<TAB>id) of the main book")
p.set_defaults(func=cmd_packages)
p = sub.add_parser("search", help="find a package or section in the "
"LFS book by name")
p.add_argument("name", help="name fragment to look for")
p.set_defaults(func=cmd_search)
p = sub.add_parser("script", help="fetch a package's install script from "
"the book for editing (runs nothing)")
p.add_argument("action", choices=["fetch"],
help="'fetch': generate/keep the script and print its path")
p.add_argument("package")
p.add_argument("--regenerate", action="store_true",
help="rebuild from the book even if the script exists")
p.add_argument("--lfs-book", action="store_true",
help="use the LFS book even when BLFS also has the package")
p.set_defaults(func=cmd_script_fetch)
p = sub.add_parser("commands", help="print a section's shell commands")
p.add_argument("section", help="section id or exact title")
p.add_argument("book_pos", nargs="?", metavar="BOOK",
help="version or file (default: the main book)")
p.set_defaults(func=cmd_commands)
# ---- build-system: create the LFS system, step by step ----
bs = sub.add_parser("build-system",
help="create the LFS system (chapters 2 & 4 for now)")
bsub = bs.add_subparsers(dest="step", required=True)
q = bsub.add_parser("version-check",
help="2.2 run version-check.sh on the host (read-only)")
q.add_argument("--show", action="store_true", help="print the script, don't run")
q.set_defaults(func=cmd_bs_version_check)
q = bsub.add_parser("layout", help="4.2 create the $LFS directory layout")
q.add_argument("--run", action="store_true", help="execute (default: dry run)")
q.set_defaults(func=cmd_bs_layout)
q = bsub.add_parser("add-user", help="4.3 add the 'lfs' build user + 4.4 "
"write its environment (as lfs)")
q.add_argument("--run", action="store_true", help="execute (default: dry run)")
q.set_defaults(func=cmd_bs_add_user)
q = bsub.add_parser("session", help="start/rejoin a build session: configure "
"+ mount the partition, set $LFS, chown to lfs")
q.add_argument("--run", action="store_true", help="actually mount/chown")
q.add_argument("--reconfigure", action="store_true",
help="re-ask every setting")
q.set_defaults(func=cmd_bs_session)
q = bsub.add_parser("get-sources", help="3. download all packages + patches "
"into $LFS/sources and verify md5sums")
q.add_argument("--run", action="store_true", help="download (default: dry run)")
q.add_argument("--force", action="store_true", help="re-download existing files")
q.set_defaults(func=cmd_bs_get_sources)
q = bsub.add_parser("crosschain", help="5-6. build the cross toolchain + temporary tools "
"(binutils/gcc, glibc, m4..xz, binutils/gcc pass2) -- "
"each step is a saved, editable script")
q.add_argument("step", nargs="?",
help="one step name, or 'all' (default); see --list")
q.add_argument("--list", action="store_true", help="list the steps + versions")
q.add_argument("--run", action="store_true", help="build (default: dry run)")
q.add_argument("--phase", choices=["all", "unpack", "build", "install",
"configure", "test"], help="run just this phase (default: all; "
"use 'install' to retry after a failure without rebuilding)")
q.add_argument("--edit", action="store_true",
help="open the step's script in $EDITOR (creates it from the "
"book first if it doesn't exist yet)")
q.add_argument("--regenerate", action="store_true",
help="overwrite the saved script with a fresh one from the "
"book (old one kept as .bak)")
q.add_argument("--path", action="store_true",
help="print the script path(s) and exit")
q.add_argument("--restart", action="store_true",
help="ignore any saved resume progress and start over")
q.add_argument("--yes", action="store_true",
help="auto-confirm resuming a previously interrupted run "
"(skip the prompt)")
q.set_defaults(func=cmd_bs_crosschain)
q = bsub.add_parser("gen-chroot-scripts",
help="7-8. write phased install scripts for the packages "
"built inside the chroot (used by lfs-helper)")
q.add_argument("--run", action="store_true", help="write them (default: dry run)")
q.add_argument("--overwrite", action="store_true",
help="replace scripts that already exist")
q.add_argument("--chapter7-only", action="store_true",
help="only chapter 7's packages, not all of chapter 8")
q.set_defaults(func=cmd_bs_gen_chroot_scripts)
q = bsub.add_parser("run", help="do everything up to the chroot, from "
"wherever the build stands, then enter it")
q.set_defaults(func=cmd_bs_run)
q = bsub.add_parser("next", help="where the build stands and what to do next")
q.add_argument("--all", action="store_true",
help="show the whole checklist, not just the next step")
q.set_defaults(func=cmd_next)
q = bsub.add_parser("snapshot", help="save the build state, or put it back "
"(so a bad step can be undone)")
q.add_argument("action", nargs="?",
choices=["list", "save", "restore", "remove"],
help="default: list")
q.add_argument("name", nargs="?", help="snapshot name")
q.add_argument("--note", help="what this snapshot is (shown in the list)")
q.add_argument("--run", action="store_true", help="apply (default: dry run)")
q.add_argument("--yes", action="store_true", help="skip the confirmation")
q.add_argument("--force", action="store_true", help="replace an existing one")
q.set_defaults(func=cmd_snapshot)
# kept as an alias: `set-default` reads naturally under build-system too,
# and someone will type it
q = bsub.add_parser("set-default", help="-> lfs build-system set-book")
q.add_argument("version")
q.set_defaults(func=cmd_bs_set_default)
q = bsub.add_parser("list", help="every build step and its state "
"(same as 'lfs-helper list' inside the chroot)")
q.set_defaults(func=cmd_next, all=True)
q = bsub.add_parser("restart", help="delete the built system and start the "
"whole build again (asks first)")
q.add_argument("--run", action="store_true", help="do it (default: dry run)")
q.add_argument("--yes", action="store_true", help="skip the confirmation")
q.add_argument("--wipe-sources", action="store_true",
help="delete the downloaded tarballs too (they re-download)")
q.set_defaults(func=cmd_bs_restart)
q = bsub.add_parser("verify", help="what state is this tree in? how far "
"did the build get, and is the toolchain usable")
q.set_defaults(func=cmd_bs_verify)
q = bsub.add_parser("set-book", help="which book to BUILD THE SYSTEM from")
q.add_argument("version")
q.set_defaults(func=cmd_bs_set_book)
q = bsub.add_parser("sync-tools",
help="3b. copy this machine's package-user environment "
"(skel-package, helper scripts, lfs-helper) into the tree")
q.add_argument("--run", action="store_true", help="copy (default: dry run)")
q.set_defaults(func=cmd_bs_sync_tools)
q = bsub.add_parser("install-tools",
help="8. install the Python tooling (lfs, packagemanager, "
"blfs) inside the chroot, once Python exists there")
q.add_argument("--run", action="store_true", help="install (default: dry run)")
q.add_argument("--force", action="store_true",
help="install even if the Python modules the tools need "
"(requests, beautifulsoup4) are missing")
q.set_defaults(func=cmd_bs_install_tools)
q = bsub.add_parser("chroot", help="7. prepare / enter / leave the chroot "
"(chown to root, mount virtual kernel filesystems)")
q.add_argument("action", nargs="?",
choices=["status", "prepare", "enter", "unmount", "resolv"],
help="default: status")
q.add_argument("--run", action="store_true",
help="apply (prepare/unmount default to a dry run)")
q.add_argument("--force", action="store_true",
help="prepare even if chapters 5-6 are unfinished")
q.set_defaults(func=cmd_bs_chroot)
q = bsub.add_parser("fix-ownership", help="give the lfs user ownership of "
"the whole build tree again (after something ran as root)")
q.add_argument("--run", action="store_true", help="apply (default: dry run)")
q.set_defaults(func=cmd_bs_fix_ownership)
q = bsub.add_parser("manifests", help="show which files each package "
"installed (for later package-user ownership)")
q.add_argument("package", nargs="?", help="one package's file list")
q.set_defaults(func=cmd_bs_manifests)
q = bsub.add_parser("show", help="print any section's commands by id")
q.add_argument("section", help="section id (e.g. ch-partitioning-mounting)")
q.set_defaults(func=cmd_bs_show)
# `lfs config makeflags -j4`: the value legitimately starts with '-', which
# argparse would otherwise reject as an unknown option. Insert the '--'
# separator for that one case so values pass through untouched.
argv = sys.argv[1:]
if len(argv) >= 3 and argv[0] == "config" and argv[2].startswith("-") \
and argv[2] != "--" and not argv[1].startswith("-"):
argv = argv[:2] + ["--"] + argv[2:]
args = ap.parse_args(argv)
if inside_chroot():
cmd = getattr(args, "cmd", None)
sub = getattr(args, "step", None)
if cmd == "build-system" and sub in _INSIDE_ONLY_ELSEWHERE:
_refuse_inside_chroot(cmd, sub)
if cmd in _INSIDE_ONLY_ELSEWHERE:
_refuse_inside_chroot(cmd)
# Build-system commands read the BUILD book; everything else reads the
# default one. Set it in one place so a new subcommand cannot forget.
if getattr(args, "cmd", None) == "build-system":
setattr(args, "_use_build_book", True)
args.func(args)
if __name__ == "__main__":
try:
main()
except BrokenPipeError:
sys.exit(0)
except KeyboardInterrupt:
sys.exit(130)