Packagemanager-LFS-PackageUser-System git · main
Package-user based package manager for LFS/BLFS
Shell 63.7% Markdown 35.8%git clone https://git.christianimmanuel.de/linux-from-scratch/Packagemanager-LFS-PackageUser-System.gitwget https://git.christianimmanuel.de/linux-from-scratch/Packagemanager-LFS-PackageUser-System/archive/Packagemanager-LFS-PackageUser-System.tar.gzpackagemanager raw
#!/usr/bin/python3
#
# packagemanager -- manage packages under the LFS/BLFS "pkgusr" model.
# Copyright (C) 2025 packagemanager contributors
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, see <https://www.gnu.org/licenses/>.
"""
packagemanager -- one tool for managing packages under the LFS/BLFS "pkgusr"
model, rewritten to sit on top of the new offline `blfs` tool.
The pkgusr model
----------------
Every package is installed by its own dedicated *package-user* whose home is
/usr/src/<name>. Because that user runs `make install`, every file the package
installs ends up owned by that user, so `list_package <name>` records the exact
manifest into /usr/src/<name>/pkg.lst. Collector groups (nimgnu_*) share
directories between package-users; u_* users run GUI apps for the human.
The commands
------------
`packagemanager --help` lists them, grouped by what you are doing, and that
list is generated beside the parsers themselves.
THERE WAS A SECOND LIST HERE. A hand-kept "Roadmap" marking check, search,
dependencies, install, remove and reload-pkg-list as [TODO] -- all six had
been implemented for releases, and none of the twenty commands added since
appeared at all. A list of commands maintained anywhere but next to the
commands goes stale silently, and this file is the place a reader starts.
Design decisions still open:
* should the CONFIGURE phase run as root? Every BLFS "Configuring" section
is written for root, and files it creates would be root-owned, which the
file tracking does not expect.
* the `.new` mechanism: three functions move stack files and it still
surprises people.
"""
import argparse
import hashlib
import glob
import json
import os
import re
import shlex
import shutil
import stat
import subprocess
import sys
import time
try:
import readline # enables arrow-key line editing in input() prompts
except ImportError:
readline = None
try: # used by `info` (and later commands) to look up users
import pwd
import grp
except ImportError: # non-unix: degrade gracefully
pwd = grp = None
# --------------------------------------------------------------------------- #
# configuration
# --------------------------------------------------------------------------- #
# /usr/src holds one directory (== home) per package-user. Overridable so the
# tool can be exercised against a fixture tree in testing.
# Where package users live. Config wins over the built-in default, so a
# system built with a different layout keeps it; the environment still wins
# over both, for tests and one-off runs.
def _base_dir_default():
"""The BASE the account roots hang under -- /usr/src, not /usr/src/pkgusr.
ONE KEY, TWO MEANINGS. `lfs` writes `pkgusr_home` as the FULL package-user
root (its LAYOUT_DEFAULTS say /usr/src/pkgusr); this read the same key as
the base and then appended PKGUSR_SUBDIR itself, so a tree configured by
lfs got both:
logs: /usr/src/pkgusr/pkgusr/p_wget/log
Taking the parent when the value already ends in the subdirectory makes the
two readings agree without either tool having to change what it writes --
and a base that does NOT end in it is still taken literally, which is what
a hand-written config means.
"""
# CONFIG_PATH is defined further down, so read the path the same way it
# does rather than depending on definition order.
path = os.environ.get("PKGUSR_CONFIG") or "/etc/pkgusr/packagemanager.conf"
sub = os.environ.get("PKGUSR_SUBDIR", "pkgusr")
try:
with open(path) as f:
for line in f:
k, _, v = line.partition("=")
if k.strip() == "pkgusr_home" and v.strip():
v = v.strip().rstrip("/")
if sub and os.path.basename(v) == sub:
return os.path.dirname(v) or "/"
return v
except OSError:
pass
return "/usr/src"
BASE_DIR = os.environ.get("PKGUSR_BASE") or _base_dir_default()
def appuser_home():
"""Where application users live -- separate from package users, because
they are accounts a person uses rather than build trees.
DEFAULT /home, NOT the package base. This returned BASE_DIR, so every
application user's home, shared directory and XDG profile link were
built under /usr/src -- `mkdir -p /usr/src/u_claude/Shared`, and a
~/Shared/Claude link pointing into the package tree. A build tree is
not a home. Configurable as before (`appuser_home` in the config), but
the default is now the one that is right.
"""
return load_config().get("appuser_home") or APPUSER_HOME_DEFAULT
# ONE SUBDIRECTORY FOR ALL OF THEM. The user's layout, recorded on the
# machine as appuser_home=/home/shared_user -- and ignored: the account was
# moved to a hard-coded "/home/<user>", so /home filled up with u_* beside
# the real logins. The default is that subdirectory; the config key still
# overrides it.
APPUSER_HOME_DEFAULT = "/home/shared_user"
# /home/Shared (1.14.315). 1.14.312 put a hub of LINKS in
# /home/shared_user/Shared and argued against /home: a directory there that
# is not a home looks like one. Once the hub holds the DATA, that argument
# turns round: the shared directories are the one thing in /home/shared_user
# that is not private to an account, and taking them out is what lets every
# account home be 0700.
SHARED_HUB_DEFAULT = "/home/Shared"
def shared_hub():
"""Where every shared directory lives (1.14.312; the data itself since
1.14.315 -- see SHARED_HUB_DEFAULT).
It was ~/Shared of the main user. Two things were wrong with that:
* ONE PERSON'S HOME IS NOT A SYSTEM PATH. A program of your own, a
script, a launcher, an app account -- anything that wants "the
shared dirs" had to know whose home to look in. A fixed path is the
same answer for every account.
* The SHARED_* variables were enumerated from $HOME/Shared by EVERY
login -- and in an app account $HOME/Shared is its own data
directory, so u_claude got SHARED_<FILENAME> for each file in it.
Default /home/Shared: the real directories, one per account
(/home/Shared/Claude, 2770 u_claude:u_claude). Each account's ~/Shared
and the main user's ~/Shared are links into it, so nothing you type
changes. `shared_hub=` in packagemanager.conf moves it.
"""
return (load_config().get("shared_hub") or "").rstrip("/") \
or SHARED_HUB_DEFAULT
def _app_home(u):
"""An application user's home: what passwd says, or where they go.
passwd FIRST, because an account whose home has already been moved
(usermod -d) must not have its shared directory rebuilt somewhere else
-- which is exactly what happened: the homes were moved to /home and the
next `cfg app-users` run recreated Shared under /usr/src.
"""
return _home_of(u) or os.path.join(appuser_home(), u)
INSTALL_SCRIPTS_DIR = os.environ.get("PKGUSR_INSTALL_SCRIPTS", "/etc/pkgusr/install_scripts")
TMP_DIR = "/tmp/packagemanager"
# Configurable group-name prefixes, stored in a small config file. The
# collector prefix names shared install-dir groups (was hard-coded "nimgnu");
# the user prefix names application-user accounts (was hard-coded "u").
CONFIG_PATH = os.environ.get("PKGUSR_CONFIG", "/etc/pkgusr/packagemanager.conf")
# Version stamps. Bump PLAN_VERSION when the cached dry-run plan structure
# changes (so stale plans are ignored); TOOL_VERSION is the program version.
TOOL_VERSION = "1.14.371"
def _build_id():
"""A short fingerprint of this file's own contents.
A hand-maintained version number goes stale the moment someone forgets to
bump it -- and the whole reason for printing a version is to answer "am I
running the code that was just fixed?". This cannot go stale: it changes
whenever the file does.
"""
import hashlib
try:
with open(os.path.abspath(__file__), "rb") as fh:
return hashlib.md5(fh.read()).hexdigest()[:7]
except OSError:
return "unknown"
PLAN_VERSION = 1
_DEFAULT_CONFIG = {"collector_prefix": "sysgroup", "user_prefix": "u",
"pkgusr_prefix": "p", "cfguser_prefix": "cfg",
"main_user": ""}
_config = None
def load_config():
global _config
if _config is not None:
return _config
_config = dict(_DEFAULT_CONFIG)
# WHICH KEYS CAME FROM THE FILE, as opposed to the defaults above. The
# conflict warning below compared the lfs config against whatever was in
# _config, and a default counted -- so a system with no packagemanager.conf
# at all was told its settings disagreed:
# note: collector_prefix is 'sysgroup' here but 'nimgnu' in the lfs
# config -- using 'nimgnu'
# Nothing said sysgroup. That is the built-in default being reported as an
# opinion someone held.
from_file = set()
try:
with open(CONFIG_PATH) as f:
for line in f:
line = line.strip()
if line and not line.startswith("#") and "=" in line:
k, v = line.split("=", 1)
_config[k.strip()] = v.strip()
from_file.add(k.strip())
except OSError:
pass
# Settings that MUST match across the tools come from one place.
#
# collector_prefix decides which groups shared directories carry. The
# system was built with the prefix in the lfs config; if packagemanager
# used its own default instead, every package installed afterwards would
# join `sysgroup_*` groups while the built system uses `nimgnu_*` -- two
# parallel sets of groups over the same directories, and no error to say
# so. The lfs config is authoritative for these.
shared = _shared_lfs_config()
for key in _SHARED_WITH_LFS:
val = shared.get(key)
if not val:
continue
if key in from_file and _config.get(key) and _config[key] != val:
printWarning(
f"note: {key} is '{_config[key]}' here but '{val}' in the lfs "
f"config -- using '{val}', which is what the system was built "
f"with.")
_config[key] = val
return _config
# Keys owned by the lfs config, not by packagemanager's own file.
# EVERY setting both tools have. `lfs build-system session` asks for these
# once, on the host, and the answers are what the system was actually built
# with -- so they are authoritative here, not a default to fall back to.
#
# This was `collector_prefix` alone. The other three were collected by the
# same interview, written into the same config.json, copied into the same
# chroot, and then asked for again by first_run_setup as though nothing knew
# them.
# The layout keys travel with the tree too, so a build system with a
# different directory layout keeps it without the environment having to be
# right on every invocation (1.13.69 did this for the prefixes).
_SHARED_WITH_LFS = ("collector_prefix", "pkgusr_prefix", "cfguser_prefix",
"main_user", "pkgusr_subdir", "cfguser_subdir")
def _shared_lfs_config():
"""The lfs tool's settings, if it has any on this system."""
for cand in (os.environ.get("LFS_STORE"), "/usr/share/lfs",
os.path.expanduser("~/.local/share/lfs")):
if not cand:
continue
p = os.path.join(cand, "config.json")
try:
with open(p) as f:
return json.load(f)
except (OSError, ValueError):
continue
return {}
def _save_shared_key(key, value):
"""Write a key where the READER will look for it.
`config --pkgusr-prefix ""` wrote packagemanager.conf and reported
success -- while the value is read from /usr/share/lfs/config.json, which
is authoritative for the shared keys. `--show` then still said "p_, the
lfs config", and nothing had changed.
A setting that is saved somewhere the tool does not read is not saved.
"""
for cand in (os.environ.get("LFS_STORE"), "/usr/share/lfs",
os.path.expanduser("~/.local/share/lfs")):
if not cand:
continue
path = os.path.join(cand, "config.json")
if not os.path.isfile(path):
continue
try:
with open(path) as f:
data = json.load(f)
data[key] = value
with open(path, "w") as f:
json.dump(data, f, indent=2, sort_keys=True)
f.write("\n")
printInfo(" wrote %s to %s (it wins over packagemanager.conf)"
% (key, path))
return True
except (OSError, ValueError) as e:
printWarning(" could not write %s: %s" % (path, e))
return False
def save_config(cfg):
try:
os.makedirs(os.path.dirname(CONFIG_PATH) or ".", exist_ok=True)
with open(CONFIG_PATH, "w") as f:
f.write("# packagemanager configuration\n")
for k, v in cfg.items():
f.write(f"{k}={v}\n")
return True
except OSError:
return False
def config_exists():
return os.path.isfile(CONFIG_PATH)
def collector_prefix():
return load_config()["collector_prefix"].rstrip("_") + "_"
def user_prefix():
return load_config()["user_prefix"].rstrip("_") + "_"
def pkgusr_prefix():
"""Prefix on PACKAGE users: `p_` gives p_gcc, p_zlib.
Three kinds of account share one passwd file and one namespace, and
without prefixes they are indistinguishable from each other and from the
system's own accounts -- a package called `man` or `news` collides with a
real one. `p_` for packages, `u_` for application users, `<collector>_`
for the shared-directory groups.
An explicitly EMPTY value means "no prefix", which is how a tree built
before prefixes keeps working; it is not the same as unset.
"""
v = load_config().get("pkgusr_prefix", _DEFAULT_CONFIG["pkgusr_prefix"])
v = (v or "").rstrip("_")
return (v + "_") if v else ""
def cfguser_prefix():
"""Prefix on CONFIG-STEP users: `cfg_` gives cfg_bootscripts.
A config step used to be `p_cfg_bootscripts` -- the package prefix stacked
on top of the step name's own `cfg_`. Two prefixes on one account means
neither one identifies it: `p_` said "package" while the account lived in
the config root. One account, one prefix.
"""
v = load_config().get("cfguser_prefix", _DEFAULT_CONFIG["cfguser_prefix"])
v = (v or "").rstrip("_")
return (v + "_") if v else ""
def pkgusr_kind(name):
"""What KIND of account is this -- 'pkg', 'cfg' or 'app'?
The one place that decides. pkgusr_name and pkgusr_root both answer from
it, so a name cannot be given one kind's prefix and placed in another
kind's root -- which is exactly what produced `p_cfg_bootscripts` under
/usr/src/cfg.
Takes a bare name (`gcc`, `cfg_bootscripts`), an account name (`p_gcc`),
or a legacy stacked name (`p_cfg_bootscripts`).
"""
n = name or ""
ppfx = pkgusr_prefix()
if ppfx and n.startswith(ppfx):
n = n[len(ppfx):]
upfx = user_prefix()
if upfx and n.startswith(upfx):
return "app"
cpfx = cfguser_prefix()
if cpfx and n.startswith(cpfx):
return "cfg"
return "pkg"
def pkgusr_name(name):
"""Package name -> account name, with the prefix ITS KIND carries.
IDEMPOTENT, and that is the whole point: this is called on names typed by
a person (`gcc`), on directory names read back from disk (`p_gcc`), and on
values that already went through it. A second pass must not give p_p_gcc.
Because it is idempotent, every caller can apply it without first knowing
which kind of name it is holding.
A legacy stacked name is repaired on the way through: the package prefix is
stripped before the kind's own prefix is applied, so `p_cfg_bootscripts`
comes back as `cfg_bootscripts`.
"""
if not name:
return name
# Lower-case FIRST, exactly like lfs-helper's pkg_owner_name: the book's
# anchors are not consistent (DocBook, Wget, Which), and p_DocBook next
# to the helper's p_docbook is two accounts for one package -- a failure
# box this session printed both names in one breath.
name = name.lower()
kind = pkgusr_kind(name)
# an application user or a collector group is not a package and is never
# renamed here
if kind == "app" or name.startswith(collector_prefix()):
return name
pfx = cfguser_prefix() if kind == "cfg" else pkgusr_prefix()
n = name
ppfx = pkgusr_prefix()
if ppfx and n.startswith(ppfx):
n = n[len(ppfx):]
if not pfx or n.startswith(pfx):
return n
return pfx + n
def unprefix_pkgusr(name):
"""Package-user name -> package name, for display."""
pfx = pkgusr_prefix()
if pfx and name.startswith(pfx):
return name[len(pfx):]
return name
# Accounts live in subdirectories of BASE_DIR by kind, so `ls /usr/src` says
# what a thing IS rather than listing a hundred packages and a few config
# steps as one flat set:
#
# /usr/src/pkgusr/p_gcc a package
# /usr/src/cfg/cfg_bootscripts a config step that installs files
# /usr/src/u_firefox an application user
def _early_conf(key, default):
"""A value from packagemanager.conf, read before load_config() exists.
THE DIRECTORIES HAD NO CONFIG KEYS. BASE_DIR has read `pkgusr_home` from
this file for a long time; the two subdirectories were ENVIRONMENT ONLY,
so a tree with a different layout depended on the environment being right
every time -- and could not be made tree-specific the way the prefixes
now are (1.13.69).
Precedence, as everywhere else: environment, then the file, then the
built-in default.
"""
env = {"pkgusr_subdir": "PKGUSR_SUBDIR",
"cfguser_subdir": "CFGUSR_SUBDIR"}.get(key)
if env and os.environ.get(env) is not None:
return os.environ[env]
# The LFS config is authoritative for the layout as it is for the
# prefixes -- that is what makes it travel with the tree rather than
# depending on the environment of whoever runs the command.
for cand in (os.environ.get("LFS_STORE"), "/usr/share/lfs",
os.path.expanduser("~/.local/share/lfs")):
if not cand:
continue
try:
import json as _json
with open(os.path.join(cand, "config.json")) as f:
v = _json.load(f).get(key)
if v is not None:
return str(v)
except (OSError, ValueError):
continue
path = os.environ.get("PKGUSR_CONFIG") or "/etc/pkgusr/packagemanager.conf"
try:
with open(path) as f:
for line in f:
k, _, v = line.partition("=")
if k.strip() == key:
return v.strip() # an empty value is an answer
except OSError:
pass
return default
PKGUSR_SUBDIR = _early_conf("pkgusr_subdir", "pkgusr")
CFGUSR_SUBDIR = _early_conf("cfguser_subdir", "cfg")
def pkgusr_root(name):
"""Which root an account belongs under. Takes any kind of name.
Answers from pkgusr_kind, the same function pkgusr_name uses, so the
prefix an account is given and the root it is placed in cannot disagree.
"""
kind = pkgusr_kind(name)
if kind == "app":
return BASE_DIR
sub = CFGUSR_SUBDIR if kind == "cfg" else PKGUSR_SUBDIR
return os.path.join(BASE_DIR, sub) if sub else BASE_DIR
def pkgusr_roots():
"""Every account root, for the passes that walk all of them."""
seen, out = set(), []
for sub in (PKGUSR_SUBDIR, CFGUSR_SUBDIR):
d = os.path.join(BASE_DIR, sub) if sub else BASE_DIR
if d not in seen:
seen.add(d)
out.append(d)
return out
def pkgusr_home(name):
"""Where a package user lives. Takes either kind of name."""
return os.path.join(pkgusr_root(name), pkgusr_name(name))
def main_user():
return load_config().get("main_user", "").strip()
def editor():
return (load_config().get("editor") or os.environ.get("EDITOR") or "vim")
def difftool():
return (load_config().get("difftool") or "vimdiff")
# --------------------------------------------------------------------------- #
# colored output
# --------------------------------------------------------------------------- #
_USE_COLOR = sys.stdout.isatty() and not os.environ.get("NO_COLOR")
class colors:
normal = "\033[0m" if _USE_COLOR else ""
red = "\033[31m" if _USE_COLOR else ""
green = "\033[32m" if _USE_COLOR else ""
yellow = "\033[33m" if _USE_COLOR else ""
blue = "\033[34m" if _USE_COLOR else ""
cyan = "\033[36m" if _USE_COLOR else ""
dim = "\033[2m" if _USE_COLOR else ""
HEADER_WIDTH = 60
def printHeader(msg):
inner = f"### {msg} "
tail = "#" * max(HEADER_WIDTH - len(inner), 3)
print(f"\n{colors.blue}{inner}{tail}{colors.normal}")
def printLine():
line = f"{colors.blue}{'-' * HEADER_WIDTH}{colors.normal}"
print(line)
return line
def printSuccess(msg): print(f"{colors.green}{msg}{colors.normal}")
def printInfo(msg): print(f"{colors.blue}{msg}{colors.normal}")
# Warnings go to STDERR, like lfs-helper's warn(). A note printed to stdout
# lands inside whatever a caller is capturing -- the config-conflict note was
# read back as part of `print(pm.BASE_DIR)` by anything scripting this module,
# and the value it polluted was correct.
def printWarning(msg): print(f"{colors.yellow}{msg}{colors.normal}", file=sys.stderr)
def printError(msg): print(f"{colors.red}{msg}{colors.normal}")
def ask_yes_no(question):
while True:
resp = input(f"{question} [y|n]: ").strip().lower()
if resp == "y":
return True
if resp == "n":
return False
print("Please enter 'y' for yes or 'n' for no.")
# --------------------------------------------------------------------------- #
# shared helpers
# --------------------------------------------------------------------------- #
# BASE_DIR is /usr/src, which holds more than package users -- lfs-pkgusr,
# a stray root/, whatever else lives there. Listing every directory reported
# them as packages ("root broken has a home dir but no user"). An account
# either exists in the user database or carries the skeleton every package
# user is created with.
_ACCOUNT_MARKERS = ("build.conf", ".project", "install_last", "pkg.lst",
".bash_profile")
def _looks_like_an_account(path, name):
# the prefix IS the evidence: p_gcc is a package user by construction
for pfx in (pkgusr_prefix(), user_prefix(), cfguser_prefix()):
if pfx and name.startswith(pfx):
return True
# NOT "a user of this name exists" -- /usr/src/root passed that test
# because root exists, and came out of verify as a broken package. A
# system user is not evidence that a DIRECTORY is a package home.
return any(os.path.lexists(os.path.join(path, m)) for m in _ACCOUNT_MARKERS)
def list_all_users():
"""Every package-user, i.e. every directory directly under BASE_DIR."""
# Directory names ARE user names, so they carry the prefix. Hand back
# package names: everything user-facing speaks in package names, and
# pkgusr_name() puts the prefix back wherever it is needed.
#
# A root that does not exist is normal -- a system with no config-step
# packages has no cfg/ directory. ALL of them missing is not, and must
# still be reported: swallowing it turns "nothing is installed here" into
# a silent empty list, which reads as "no packages".
out, found = [], False
for root in pkgusr_roots():
try:
names = os.listdir(root)
except OSError:
continue
found = True
# A COLLECTOR GROUP IS NOT A PACKAGE.
#
# `nimgnu_perl-modules`, `nimgnu_python-modules` and friends are
# GROUPS that own shared install directories. They were listed as
# package-users and reported "broken -- has a user but no home dir",
# so a verify of a healthy system ended with four scary lines about
# things that are working exactly as designed. A false alarm in a
# health check is worse than no check: it is what teaches people to
# skip the output.
# BOTH SPELLINGS. lfs-helper fell back to "nimgnu" while this tool
# and `lfs` fell back to "sysgroup" (fixed in 1.14.35), so a system
# whose config never set the key has groups under the OTHER name.
# Recognise them however they were created; the config still decides
# what NEW ones are called.
_cps = {collector_prefix(), "sysgroup_", "nimgnu_"}
# ONE UNDERSCORE. collector_prefix() already ends in "_", so this
# tested startswith("sysgroup__") and never matched anything -- the
# filter above has never worked, and every collector group was still
# listed as a "broken" package.
# ...and check the name BOTH ways. A directory may be `p_nimgnu_x`
# (the package prefix stacked on a group name) or plain `nimgnu_x`;
# the raw name misses the first, the unprefixed one misses nothing.
for d in names:
if not os.path.isdir(os.path.join(root, d)):
continue
bare = unprefix_pkgusr(d)
if any(c and (d.startswith(c) or bare.startswith(c)) for c in _cps):
continue
if not _looks_like_an_account(os.path.join(root, d), d):
continue
out.append(bare)
if not found:
printError(f"{BASE_DIR} does not exist (looked in "
f"{', '.join(pkgusr_roots())}).")
return []
return sorted(set(out))
# --------------------------------------------------------------------------- #
# bridge to the `blfs` tool (offline book parser / script generator)
#
# packagemanager no longer parses the book itself -- it shells out to `blfs`,
# which is fast (cached) and knows the real 13.0 structure. Location/book can
# be steered with env vars so the same code works in production and in tests:
# BLFS_BIN e.g. "blfs" or "python3 /path/blfs" (default: auto-detect)
# BLFS_BOOK_FILE passed through as `--book-file <path>` (offline/testing)
# BLFS_BOOK passed through as `--book <selector>`
# --------------------------------------------------------------------------- #
def find_blfs():
override = os.environ.get("BLFS_BIN")
if override:
return override.split()
here = os.path.dirname(os.path.abspath(__file__))
local = os.path.join(here, "blfs")
if os.path.isfile(local):
# os.access(X_OK) answers "yes" for root even without any execute bit,
# so check the mode bits themselves -- exactly the situation in a
# fresh copy of the tools, and the exec then dies with EACCES.
ex = bool(os.stat(local).st_mode & 0o111)
return [local] if ex else [sys.executable or "python3", local]
return ["blfs"] # rely on $PATH
# A PACKAGE MAY NEED A DIFFERENT BOOK, AND ONLY A WHOLE PAGE IS COHERENT.
#
# The systemd book's linux-pam page lists systemd as required; the 12.4 SysV
# book's linux-pam page has no Required section at all -- it wants only
# Shadow. Same package, two books, two dependency graphs, and this machine
# is SysV.
#
# NOT a merge. Taking the dependency list from one book and the commands
# from another produces a script neither book has ever tested: the versions
# differ (1.7.1 vs 1.7.2), the configure switches differ, the doc tarball
# differs. A whole page from the older book is something upstream shipped
# and someone ran. Half of each is not.
#
# machine.conf:
# [linux-pam]
# book = /usr/share/blfs/books/BLFS-BOOK-12_4-nochunks.html
#
# ...and everything that asks the book about THAT package -- versions,
# dependencies, the generated script -- asks the one named there.
_BOOK_FOR_PKG = {}
# A STACK CAN READ ITS PACKAGES FROM ANOTHER BOOK (1.14.321).
#
# media.stack: from-book svn
#
# ...and every book entry of THAT file (not of files it includes) is read
# from the development book: versions, dependencies, the generated script.
# It is a property of the package, not of one run -- `update`, `info` and
# `install --reinstall` ask the same book the stack built it from, or an
# update-all would "update" mpv back to the stable book's older page.
#
# Dependencies outside the stack stay on their own book: an installed one is
# left alone (the svn page naming a newer glib is not a reason to rebuild
# glib), a missing one is built from the default book, and only a page the
# default book does not have at all is read from the stack's book.
_STACK_BOOKS = None # anchor (lower) -> selector
_PLAN_BOOK = None # the stack book a running plan's target reads
_BOOK_PATHS = {} # selector -> local file ("" = could not resolve)
def _stack_book_map():
"""anchor -> selector, from every installed stack's `from-book`."""
global _STACK_BOOKS
if _STACK_BOOKS is not None:
return _STACK_BOOKS
_STACK_BOOKS = {}
# a stack run hands its own file's books to the installs it starts
# (it may be a checkout, not the installed copy)
for kv in (os.environ.get("PM_STACK_BOOKS") or "").split(","):
if "=" in kv:
a, sel = kv.split("=", 1)
_STACK_BOOKS[a.strip().lower()] = sel.strip()
d = STACK_DIR if "STACK_DIR" in globals() else "/etc/pkgusr/stacks"
for f in sorted(glob.glob(os.path.join(d, "*.stack"))):
try:
for r in _parse_stack_file(f):
sel = r["opts"].get("from-book")
if sel and r["kind"] == "book":
_STACK_BOOKS.setdefault(r["target"].lower(), sel)
except (ValueError, OSError):
continue
return _STACK_BOOKS
def _book_path_for(selector, refresh=False):
"""The local file for a book selector, asked of `blfs path` ONCE.
With refresh, `blfs fetch` first -- a stack run on the svn book wants
today's svn book; every other command reads the cached copy and never
touches the network."""
if not selector:
return None
if os.path.isfile(selector):
return selector
if refresh or selector not in _BOOK_PATHS:
if refresh:
set_status("refreshing the %s book ..." % selector)
try:
r = subprocess.run(find_blfs() + ["fetch", selector],
capture_output=True, text=True)
finally:
clear_status()
if r.returncode != 0:
printWarning(" could not refresh the %s book; using the "
"cached one" % selector)
r = subprocess.run(find_blfs() + ["path", selector],
capture_output=True, text=True)
out = (r.stdout or "").strip().splitlines()
_BOOK_PATHS[selector] = out[-1] if r.returncode == 0 and out \
and os.path.isfile(out[-1]) else ""
if not _BOOK_PATHS[selector]:
printWarning(" no %s book on disk (blfs fetch %s)"
% (selector, selector))
return _BOOK_PATHS[selector] or None
def _book_override(anchor):
"""The book this package should be read from, or None.
machine.conf `book =` first, then a stack's `from-book`, then -- inside
a plan for a stack-book package -- that book for a page the default
book does not have at all."""
if not anchor:
return None
if anchor not in _BOOK_FOR_PKG:
opts = _machine_opts_for(anchor)
sel = opts.get("book") or _stack_book_map().get(anchor.lower()) or ""
_BOOK_FOR_PKG[anchor] = (_book_path_for(sel) or sel) if sel else ""
if _BOOK_FOR_PKG[anchor]:
return _BOOK_FOR_PKG[anchor]
if _PLAN_BOOK and not _map_lookup(_versions_for(""), [anchor]) \
and _map_lookup(_versions_for(_PLAN_BOOK), [anchor]):
return _PLAN_BOOK
return None
def _blfs_book_flags(anchor=None):
over = _book_override(anchor) if anchor else None
if over:
if os.path.isfile(over):
return ["--book-file", over]
return ["--book", over]
if os.environ.get("BLFS_BOOK_FILE"):
return ["--book-file", os.environ["BLFS_BOOK_FILE"]]
if os.environ.get("BLFS_BOOK"):
return ["--book", os.environ["BLFS_BOOK"]]
return []
# what each blfs subcommand is actually doing, for the status line
_BLFS_ACTIVITY = {
"order": "working out the build order for %s",
"debug": "looking up %s in the book",
"deps": "reading %s's dependencies",
"script": "writing the install script for %s",
"versions": "reading versions from the book",
"search": "searching the book for %s",
}
# distinct blfs notes already shown this invocation (see run_blfs)
_FORWARDED_NOTES = set()
_SAID_BOOK = set()
def run_blfs(args, book=None):
"""Run blfs; return (returncode, stdout, stderr). rc 127 => blfs missing.
book= reads that book file for this call, whatever the package.
Each call is a subprocess, and the first one for a given book INDEXES it,
which takes many seconds. Announce what is happening: an install that sits
silent for half a minute before its first output looks frozen."""
# the package a call is ABOUT is its first non-flag argument, which is
# what decides whether an override applies
_anchor = next((a for a in args[1:] if not a.startswith("-")), None) \
if len(args) > 1 else None
if book:
cmd = find_blfs() + ["--book-file", book] + list(args)
else:
cmd = find_blfs() + _blfs_book_flags(_anchor) + list(args)
# ONCE per package: a plan asks blfs several things about each one
if not book and _anchor and _book_override(_anchor) \
and _anchor not in _SAID_BOOK:
_SAID_BOOK.add(_anchor)
printInfo(" %s: read from %s"
% (_anchor, os.path.basename(_book_override(_anchor))))
verb = args[0] if args else ""
if verb in _BLFS_ACTIVITY:
arg = args[1] if len(args) > 1 and not args[1].startswith("-") else ""
try:
set_status(_BLFS_ACTIVITY[verb] % arg if "%s" in _BLFS_ACTIVITY[verb]
else _BLFS_ACTIVITY[verb])
except Exception:
pass
try:
r = subprocess.run(cmd, capture_output=True, text=True)
# blfs says important things on stderr -- "note: no book matched
# '13.0-systemd'; using the cached ..." is a decision being
# overridden, and capturing it silently is how a wrong book got used
# without a word. Forward each DISTINCT note once: a plan touching
# ninety packages makes ninety blfs calls, and the same note ninety
# times is noise wearing a warning's colour.
_notes = [ln for ln in (r.stderr or "").splitlines()
if ln.startswith("note:") and ln not in _FORWARDED_NOTES]
if _notes:
try:
clear_status()
except Exception:
pass
for ln in _notes:
_FORWARDED_NOTES.add(ln)
printWarning(ln)
return r.returncode, r.stdout, r.stderr
except FileNotFoundError:
return 127, "", "blfs executable not found"
finally:
# the status line this call set is ITS to clear: left standing, the
# caller's next print lands on the same line
# ("reading versions from the bookChecking 97 package(s)...")
if verb in _BLFS_ACTIVITY:
try:
clear_status()
except Exception:
pass
_VERSION_MAPS = {} # book file ("" = default) -> {anchor: nv}
_BOOK_MISSES = set() # names no book had: ask the slow way once
def _versions_for(book):
"""The whole version map of one book, loaded once per process."""
if book not in _VERSION_MAPS:
try:
_VERSION_MAPS[book] = book_versions_all(book or None) or {}
except Exception:
_VERSION_MAPS[book] = {}
return _VERSION_MAPS[book]
def _map_lookup(vmap, names):
for a in names:
if a in vmap:
return vmap[a]
for a in names:
low = a.lower()
for k, nv in vmap.items():
if k.lower() == low:
return nv
return None
def book_version(anchor):
"""Current name-version of a package in ITS book, or None.
One version map per book, loaded once (see book_versions_all); a name
no map has falls back to `blfs debug` once, and a miss is remembered --
`install imv` asked three slow questions to learn the same "no" three
times. The package's book is machine.conf's, else its stack's
from-book, else the default.
THE PREFIX IS NOT PART OF THE BOOK'S NAME: `info webkitgtk` asks with
the account name p_webkitgtk, which no anchor matches.
"""
_tries = [anchor]
for _pre in (pkgusr_prefix(), user_prefix(), collector_prefix()):
if _pre and anchor.startswith(_pre):
_tries.append(anchor[len(_pre):])
own = _book_override(_tries[-1])
books = ([own] if own and os.path.isfile(own) else []) + [""]
if _PLAN_BOOK and _PLAN_BOOK not in books:
books.append(_PLAN_BOOK)
for b in books:
hit = _map_lookup(_versions_for(b), _tries)
if hit:
return hit
if _tries[-1] in _BOOK_MISSES:
return None
rc, out, _ = run_blfs(["debug", _tries[-1]])
m = re.search(r"^title\s*:\s*(.+)$", out, re.M) if rc == 0 else None
nv = m.group(1).strip() if m else None
if nv:
_versions_for(books[0])[anchor] = nv # ask once, even for an alias
else:
_BOOK_MISSES.add(_tries[-1])
return nv
def default_book_version(anchor):
"""The version the DEFAULT book has, whatever book the package reads."""
return _map_lookup(_versions_for(""), [anchor])
def book_rdeps(target, no_recommended=False, optional=False):
"""anchors of packages that recursively depend on `target`, via blfs rdeps."""
argv = ["rdeps", target, "--anchors"]
if no_recommended:
argv.append("--no-recommended")
if optional:
argv.append("--optional")
rc, out, _ = run_blfs(argv)
result = {}
if rc == 0:
for line in out.splitlines():
if "\t" in line:
a, nv = line.split("\t", 1)
if a.strip():
result[a.strip()] = nv.strip()
return result
def _versions_cache_path(book=""):
"""Where the anchor -> version map is kept between runs: one per book."""
d = os.path.join("/var", "cache", "pkgusr")
try:
os.makedirs(d, exist_ok=True)
except OSError:
d = "/tmp"
tag = hashlib.md5(book.encode()).hexdigest()[:8] if book else "default"
return os.path.join(d, "book-versions-%s.json" % tag)
def _default_book_file():
"""The file the default book is: BLFS_BOOK_FILE, BLFS_BOOK, `blfs path`.
NOT the newest file on disk: once a stack fetched the svn book it was
the newest, and every version question on the machine would have been
answered from svn."""
_bf = os.environ.get("BLFS_BOOK_FILE") or ""
if _bf and os.path.isfile(_bf):
return _bf
if not hasattr(_default_book_file, "_p"):
_p = ""
try:
r = subprocess.run(find_blfs() + (
["--book", os.environ["BLFS_BOOK"]]
if os.environ.get("BLFS_BOOK") else []) + ["path", "--offline"],
capture_output=True, text=True)
out = (r.stdout or "").strip().splitlines()
if r.returncode == 0 and out and os.path.isfile(out[-1]):
_p = out[-1]
except OSError:
pass
if not _p: # an older blfs without `path`
_books = sorted((f for f in glob.glob("/usr/share/blfs/books/*.html")
if "svn" not in os.path.basename(f)),
key=lambda f: os.stat(f).st_mtime, reverse=True)
_p = _books[0] if _books else ""
_default_book_file._p = _p
return _default_book_file._p
def book_versions_all(book=None):
"""Map anchor -> name_version for the whole book (default, or `book`).
ONE CALL WAS NOT ENOUGH: IT IS ONE CALL PER PROCESS. `blfs versions`
re-reads the whole book to build this map, which is eighteen seconds on
a 12 MB book -- and `info` pays it to answer one question. The page
index is cached; this map was not, so every command that wanted a
version rebuilt it from scratch.
Cached on disk against the book's own size and mtime: the moment the
book file changes the cache is ignored, so a stale map cannot outlive
the book it came from.
"""
# WHICH book: the one asked for, else the default book's file.
_bf = book if book and os.path.isfile(book) else _default_book_file()
_key = None
if _bf:
try:
_st = os.stat(_bf)
_key = "%s:%d:%d" % (_bf, _st.st_size, int(_st.st_mtime))
except OSError:
_key = None
_cp = _versions_cache_path(_bf)
if _key:
try:
with open(_cp, encoding="utf-8") as f:
_c = json.load(f)
if _c.get("key") == _key and isinstance(_c.get("versions"), dict):
return _c["versions"]
except (OSError, ValueError):
pass
rc, out, _ = run_blfs(["versions"], book=_bf or None)
versions = {}
if rc == 0:
for line in out.splitlines():
if "\t" in line:
a, nv = line.split("\t", 1)
a, nv = a.strip(), nv.strip()
if a and nv and " " not in a:
versions[a] = nv
if versions and _key:
# CONFIRM IT CAN BE READ BACK, not just written: a cache that fails
# to load silently turns into the 18 seconds all over again, on
# every command, invisibly.
try:
with open(_cp, "w", encoding="utf-8") as f:
json.dump({"key": _key, "versions": versions}, f)
with open(_cp, encoding="utf-8") as f:
if json.load(f).get("key") != _key:
printWarning(" the version cache did not read back; "
"every command will rebuild it")
except (OSError, ValueError) as e:
printInfo(" (not caching the book version map: %s)" % e)
return versions
def cmd_blfs(args):
"""Pass through to the blfs book tool: books, set-default, search, etc.
e.g. packagemanager blfs books --discover / packagemanager blfs set-default svn"""
passthrough = args.args
if passthrough and passthrough[0] == "--":
passthrough = passthrough[1:]
cmd = find_blfs() + _blfs_book_flags() + passthrough
try:
sys.exit(subprocess.run(cmd).returncode)
except FileNotFoundError:
printError("blfs not found (set BLFS_BIN or put blfs on PATH).")
sys.exit(127)
def find_pm_install():
"""The install front door. packagemanager_install is retired (item 6,
step 7); the front-matter and the loop both live in lfs-helper now, behind
`lfs-helper pm-install <account> <script>`. Overridable with
PM_INSTALL_BIN so tests can substitute a stub."""
override = os.environ.get("PM_INSTALL_BIN")
if override:
return override.split()
here = os.path.dirname(os.path.abspath(__file__))
local = os.path.join(here, "lfs-helper")
if os.path.isfile(local):
base = [local] if os.access(local, os.X_OK) else ["bash", local]
return base + ["pm-install"]
return ["lfs-helper", "pm-install"]
# =========================================================================== #
# command: info [IMPLEMENTED]
# =========================================================================== #
#
# Mirrors the original `info`: a table of package-users with the columns
# name / uid / gid / home / pkg_list. A value is shown in red when it is
# missing, and uid/gid are shown in yellow when they differ. A user is
# "broken" when any column is missing (e.g. a /usr/src dir with no matching
# system user, or a user that has never been installed so has no pkg.lst);
# broken users are repeated in their own section.
#
# Column selection: -name -uid -gid -home -pkg-list pick specific columns;
# with none given, all columns are shown. `info all` == every package-user.
# =========================================================================== #
# command: info [IMPLEMENTED]
# =========================================================================== #
#
# A table of package-users. Beyond the system-user facts (name/uid/gid) we also
# read /usr/src/<name>/install_last -- the install script copied into the user's
# home at install time -- to show the installed version, and (with -l/--long)
# the link, md5, info text, and required/recommended/optional deps + groups.
#
# State (refined):
# installed user + home dir + non-empty pkg.lst
# not installed user + home dir, but no pkg.lst yet
# broken an inconsistency: a /usr/src dir with no matching system
# user, or a named user with no /usr/src home
# uid != gid is shown yellow as a warning but is not by itself "broken".
#
# Column selection: -name -uid -gid -home -pkg-list -version -state pick an
# explicit set; with none given the default overview is
# name/uid/gid/version/state. `info all` == every package-user.
# fields we read out of an install_last script
_LAST_SCALARS = ("name", "name_version", "url", "link", "pkg", "md5_sum", "info",
"installed_program", "installed_directory", "validate_cmd")
_LAST_ARRAYS = ("required", "recommended", "optional",
"install_groups", "additional_links",
"installed_programs", "installed_libraries",
"installed_directories", "installed_content")
def _parse_bash_vars(text, scalars=(), arrays=()):
"""Pull simple key="value" and single-line key=(a b c) assignments out
of a generated install script. Good enough for our own install_last files
(all header vars are single lines at the top)."""
# GREEDY, AND THE LINE HAS A COMMENT ON IT.
#
# The stack's generated scripts document their own metadata:
# installed_programs=() # e.g. ('gst-launch-1.0' 'gst-inspect-1.0')
# installed_program="sway" # e.g. "ffmpeg" (checked on PATH)
# `\((.*)\)` ran from the first "(" to the LAST ")" -- into the comment --
# and `"(.*)"` likewise from the first quote to the last. Every git and
# tar package in `verify` therefore reported
# missing: program ), program #, program e.g., program ('gst-launch-1.0
# for twenty packages, in every run, all session. Non-greedy, and a
# trailing comment is allowed and ignored.
out = {}
for key in scalars:
# SINGLE QUOTES TOO, AND THE BOOK USES THEM.
#
# This read only key="value". The book-generated scripts write
# name_version='Rustc-1.93.1'
# so name_version was never parsed for any package built from the
# book -- and that is where the installed version comes from.
# `info webkitgtk` printed uid, home, 1727 files and every
# dependency, and not WHICH webkitgtk; `update` filed it under
# "cannot judge"; late-deps reported 28 packages with no recorded
# version. One quote character, three symptoms.
m = re.search(rf'^{key}=(?:"(?P<d>.*?)"|\x27(?P<s>.*?)\x27)[ \\t]*(#.*)?$', text, re.M)
if m:
out[key] = m.group("d") if m.group("d") is not None \
else m.group("s")
for key in arrays:
m = re.search(rf'^{key}=\((.*?)\)[ \t]*(#.*)?$', text, re.M)
if m:
items = re.findall(r"'[^']*'|\"[^\"]*\"|\S+", m.group(1))
out[key] = [it.strip("'\"") for it in items]
return out
def _same_version(a, b):
"""Do these two name_version strings describe the same release?
The books capitalize differently -- BLFS anchors are lowercase
('which-2.23'), the LFS book titles its packages ('Which-2.23') -- and
the VERSION file records the LFS spelling. Comparing exactly would
invent an update for every package both books carry, which is precisely
the risk of reading VERSION at all."""
if not a or not b:
return False
return a.strip().lower() == b.strip().lower()
def read_install_last(home):
"""Parsed vars from /usr/src/<name>/install_last (or install_<name>).
Falls back to the VERSION file when there is no install script record.
Both build paths write down what they installed -- packagemanager into
install_last, lfs-helper into VERSION -- and reading only the first meant
every package the LFS build produced came back with no version at all:
"97 checked, 0 judged" on a system where `lfs update` compares all 97
happily. Same fact, two files; this reads whichever exists.
(The real fix is one build loop -- open item 6 -- but a tool must not be
blind to the other loop's records in the meantime.)"""
if not home:
return {}
# install_last first: it is the record of the script AS IT LAST RAN, the
# only file that states what is actually installed.
path = os.path.join(home, "install_last")
if os.path.isfile(path):
try:
text = open(path, encoding="utf-8", errors="replace").read()
except OSError:
return {}
return _parse_bash_vars(text, _LAST_SCALARS, _LAST_ARRAYS)
# Then lfs-helper's record. It must come BEFORE the install script: that
# script is regenerated whenever the book moves on, so its name_version is
# what the BOOK has, and reading it would compare the book with itself and
# call everything current forever. (`lfs update` avoids the same trap.)
vpath = os.path.join(home, "VERSION")
if os.path.isfile(vpath):
try:
for line in open(vpath, encoding="utf-8", errors="replace"):
line = line.strip()
if line and not line.startswith("#"):
return {"name_version": line, "_source": "VERSION"}
except OSError:
pass
# Last resort, and only for a home with no record of its own.
path = os.path.join(home, "install_" + os.path.basename(home))
if os.path.isfile(path):
try:
text = open(path, encoding="utf-8", errors="replace").read()
except OSError:
return {}
return _parse_bash_vars(text, _LAST_SCALARS, _LAST_ARRAYS)
return {}
def _is_account_furniture(path):
"""Paths that exist because the ACCOUNT exists, not because anything was
installed: useradd's mail spool, and the home directory itself."""
return path.startswith("/var/mail/") or path.startswith("/var/spool/mail/")
class User:
__slots__ = ("search", "name", "uid", "gid", "home", "pkg_list", "meta")
def __init__(self, search):
self.search = search
self.name = self.uid = self.gid = None
self.home = self.pkg_list = None
self.meta = {}
@property
def has_user(self):
return self.uid is not None
@property
def state(self):
"""How we decide a package is INSTALLED (same rule everywhere):
there must be a package-user AND its /usr/src/<name> home, AND evidence
that an install actually completed -- either an 'install_last' recording
a version, or a 'pkg.lst' manifest listing at least one path that still
exists on disk. User+home but no such evidence => 'not installed' (the
user was created but nothing is installed yet). Only one of user/home
=> 'broken'."""
if not self.home and not self.has_user:
return "not installed"
if self.home and self.has_user:
# install_last says a version was installed -- but if the manifest
# beside it is mostly gone, the files are not there and the claim
# is stale. The manifest wins when there is one.
if self.pkg_list:
if self._manifest_has_real_paths():
return "installed"
return "not installed"
if self._declares_content_that_is_missing():
# the script says where it installs, and it is not there
return "not installed"
if self.meta.get("name_version"):
return "installed"
return "not installed"
return "broken"
def _declares_content_that_is_missing(self):
"""The book says what this package installs -- is any of it there?
A SECOND OPINION, for when there is no manifest to consult. Every
generated script carries the page's own Installed-programs /
-libraries / -directories, e.g. for Pathspec
installed_directories='/usr/lib/python3.14/site-packages/pathspec ...'
so "installed" can be checked against the system instead of inferred
from the existence of an install_last record. Only decides NOT
installed when the script names paths and none of them exist; a
package whose page names nothing is left to the older rule.
"""
paths = []
for key in ("installed_directories", "installed_directory",
"installed_libraries", "installed_library"):
for p in (self.meta.get(key) or "").split():
if p.startswith("/"):
paths.append(p)
if not paths:
return False
return not any(os.path.exists(p) for p in paths)
def _manifest_has_real_paths(self): # noqa: D401 (see _is_account_furniture)
"""Does this manifest still DESCRIBE the system?
One surviving path used to be enough, and that is how a record kept
lying for three sessions: binutils claimed 330 paths with 190 gone,
140 survivors said "installed", the planner skipped it, and packages
that needed it failed somewhere else entirely (xmllint, glib,
-lfreetype). `lfs-helper verify` had the right rule all along -- a
record with most of its files missing is not describing an installed
package -- but only the manual sweep applied it. The planner uses the
same rule now, so an ordinary `packagemanager install` / `stack --run`
rebuilds such a package instead of skipping it.
Counted over the WHOLE manifest (capped at 5000 paths, which no
package reaches by much). Sampling the first N is wrong: pkg.lst is
path-sorted, so a package whose /usr/bin survived and whose
/usr/share died would look healthy from the top."""
if not self.pkg_list:
return False
seen = alive = 0
try:
with open(self.pkg_list, encoding="utf-8", errors="replace") as f:
for line in f:
p = line.strip()
if not p.startswith("/"):
continue
# NOT AN INSTALLED FILE, WHOEVER WROTE THE RECORD.
# useradd's mail spool is owned by the package user, so
# older manifests contain it -- and for a package that
# installed nothing it is the ONLY line, so the record
# counted 1 of 1 alive and said "installed". Excluding it
# from the scan (1.14.16) only fixed records written after;
# every pkg.lst already on disk still claimed it, which is
# why pathspec was skipped again on the next run.
if _is_account_furniture(p):
continue
seen += 1
if os.path.lexists(p):
alive += 1
if seen >= 5000:
break
except OSError:
return False
if not seen:
return False
return alive * 2 > seen # more than half still there
@property
def broken(self):
return self.state == "broken"
@property
def uid_gid_mismatch(self):
return self.has_user and self.uid != self.gid
@property
def version(self):
# THE VERSION IS WHAT FINISHED INSTALLING.
#
# install_last is staged into the home BEFORE the build, so reading
# the version out of it meant a FAILED build advanced it: webkitgtk
# 2.52.5 died at configure and `update` then said "already up to
# date (2.52.5)" with 2.50.5's libraries still on disk.
#
# .pkgusr-installed is written by lfs-helper only after every phase
# succeeded. Where it is absent -- every package built before this
# release -- fall back to install_last, which is the old behaviour
# and no worse than it was.
if self.home:
try:
with open(os.path.join(self.home, ".pkgusr-installed"),
encoding="utf-8", errors="replace") as f:
done = f.read().strip()
if done:
return done
except OSError:
pass
return self.meta.get("name_version")
def _resolve_need(name):
"""A needs= name -> the book's anchor for it (1.14.317).
The book files a package under an ANCHOR that is not always its name:
libjpeg-turbo-3.2.0 is `libjpeg`. needs=libjpeg-turbo therefore found
no page, the stack tried to INSTALL it, and stopped on
ERROR: 'libjpeg-turbo' not found
with the package installed all along -- the "did you mean" under it even
named the right anchor. So: a name that is not an anchor, and matches
exactly ONE page's own package name (its version string without the
version), is that page. More than one, or none, and the name is used as
given, which fails the way it always did.
"""
try:
rc, out, _e = run_blfs(["search", name])
except Exception:
return name
anchors, by_pkg = set(), []
for ln in (out or "").splitlines():
m = re.match(r"\s*\[package\]\s+(\S+)\s+(\S+)", ln)
if not m:
continue
anchor, ver = m.group(1), m.group(2)
anchors.add(anchor)
base = re.sub(r"-\d[\w.+~-]*$", "", ver)
if base.lower() == name.lower():
by_pkg.append(anchor)
if name in anchors or len(set(by_pkg)) != 1:
return name
printInfo(" (%s is the book's `%s`)" % (name, by_pkg[0]))
return by_pkg[0]
def gather_user(name):
"""Populate a User from the system user db, the on-disk package dir, and
its install_last script.
The lookup goes through pkgusr_name(): callers speak in PACKAGE names
('glibc') and list_all_users hands them back unprefixed, while the account
is 'p_glibc'. Looking the raw name up got a KeyError, so has_user stayed
false, so state came out 'broken' -- and every package the LFS build made
landed in the "cannot judge" bucket of `packagemanager update` with its
home, VERSION and manifest sitting right there. pkgusr_name is
idempotent, so this is safe for a name that already carries the prefix."""
u = User(name)
if pwd is not None:
try:
pw = pwd.getpwnam(pkgusr_name(name))
u.name, u.uid, u.gid = pw.pw_name, str(pw.pw_uid), str(pw.pw_gid)
except KeyError:
pass
home = pkgusr_home(name)
if os.path.isdir(home):
u.home = home
pkg_lst = os.path.join(home, "pkg.lst")
if os.path.isfile(pkg_lst) and os.path.getsize(pkg_lst) > 0:
u.pkg_list = pkg_lst
u.meta = read_install_last(u.home)
return u
def count_pkg_files(pkg_list):
if not pkg_list:
return None
try:
with open(pkg_list, encoding="utf-8", errors="replace") as f:
return sum(1 for _ in f)
except OSError:
return None
_STATE_COLOR = {"installed": colors.green,
"unvalidated": colors.cyan,
"not installed": colors.yellow,
"broken": colors.red}
def print_user_details(u):
m = u.meta
expected = book_version(u.name or u.search)
state, why = assess(u, expected)
head = f"{u.name or u.search}"
if u.version:
head += f" ({u.version})"
print(f"\n{colors.cyan}{head}{colors.normal} "
f"[{_STATE_COLOR.get(state, '')}{state}{colors.normal}]")
# `why` EXISTS TO EXPLAIN A VERDICT THAT NEEDS EXPLAINING.
#
# For an installed package it was printing the routine sentence every
# time -- "recorded files present (the page names no targets to verify
# against)" above the very fields it is talking about -- so the one line
# that ever matters was indistinguishable from noise. Routine reasons
# are dropped; anything unusual is kept and labelled `note`, because for
# an installed package it is a note, not a justification.
_routine = ("all installed_* targets present",
"recorded files present",
"its script names no verifiable target")
_dull = any(r in (why or "") for r in _routine)
if why and not (state == "installed" and _dull):
_label = "note" if state == "installed" else "why"
print(f" {colors.dim}{_label:<12}{colors.normal}: {why}")
# THE VERSION IS THE FIRST THING ANYONE WANTS, and it was only in the
# header -- shown when `u.version` happened to be set, and silent
# otherwise. webkitgtk printed uid, home, 1727 files and every
# dependency, and not which webkitgtk.
#
# `name_version` is in the install script itself (WebKitGTK-2.50.2), so
# when the recorded version is missing there is still an answer; it just
# has to say which answer it is. And the BOOK's version belongs beside
# it: "is this current" is the question behind the question.
# JUST THE NUMBER. The header already says WebKitGTK-2.50.5, so
# repeating the name in the field made the two lines look like two
# different facts.
def _ver_only(nv):
m2 = re.search(r"-([0-9][0-9A-Za-z.\-_+]*)$", nv or "")
return m2.group(1) if m2 else (nv or "")
_nv = m.get("name_version") or ""
_inst = _ver_only(u.version or "")
if not _inst and _nv:
_inst = _ver_only(_nv)
if _inst:
_inst += " (from the install script, not recorded)"
_book = _ver_only(expected or "")
rows = [
("version", _inst or "not recorded, and the script does not name one"),
("book", _book if _book and _book not in _inst else None),
("uid/gid", f"{u.uid}/{u.gid}" if u.has_user else None),
("home", u.home),
("link", m.get("link")),
("md5", m.get("md5_sum")),
("info", m.get("info")),
("required", ", ".join(m.get("required", [])) or None),
("recommended", ", ".join(m.get("recommended", [])) or None),
("optional", ", ".join(m.get("optional", [])) or None),
("groups", ", ".join(m.get("install_groups", [])) or None),
("files", count_pkg_files(u.pkg_list)),
]
for label, value in rows:
if value is None or value == "":
continue
if label == "info":
lines = [ln.strip() for ln in str(value).replace("\\n", "\n").split("\n")
if ln.strip()]
# THE BOOK'S BLURB REPEATS THE DEPENDENCY LISTS, which are three
# rows further down: webkitgtk printed 18 required packages
# twice, in different spellings, and the description they were
# attached to scrolled away. Keep the prose, drop the lines that
# are just a dependency list.
_deps = set()
for _k in ("required", "recommended", "optional"):
for _d in m.get(_k, []):
_deps.add(_d.split("-")[0].lower())
def _is_dep_list(ln):
parts = [x.strip() for x in ln.replace(" and ", ", ").split(",")
if x.strip()]
if len(parts) < 3:
return False
hit = sum(1 for x in parts
if x.split("-")[0].lower().replace(" ", "-") in _deps
or x.split("-")[0].lower() in _deps)
return hit >= max(2, len(parts) // 2)
lines = [ln for ln in lines if not _is_dep_list(ln)]
if not lines:
continue
print(f" {label:<12}: {lines[0]}")
for extra in lines[1:]:
print(f" {'':<12} {extra}")
else:
print(f" {label:<12}: {value}")
def print_user_short(users, versions=None):
versions = versions or {}
width = max((len(u.name or u.search) for u in users), default=4)
for u in users:
name = (u.name or u.search)
anchor = u.meta.get("name") or name
st, _ = assess(u, versions.get(anchor), run_cmd=False)
color = _STATE_COLOR.get(st, "")
print(f"{name.ljust(width)} {color}{st:<12}{colors.normal} "
f"{u.version or ''}".rstrip())
def book_info(anchor):
"""Parse `blfs debug <anchor>` into a dict (works for any book package,
installed or not)."""
rc, out, _ = run_blfs(["debug", anchor])
if rc != 0 or "resolve" in out and "not an installable" in out:
# still return whatever fields parsed (title may be absent)
pass
info = {}
for line in (out or "").splitlines():
m = re.match(r"^([\w /]+?)\s*:\s*(.*)$", line)
if m:
info[m.group(1).strip()] = m.group(2).strip()
return info or None
def print_book_info(anchor):
"""Show the book's view of a package (version, link, deps, description) --
useful when the package isn't installed yet."""
bi = book_info(anchor)
if not bi or not bi.get("title"):
return
print(f" {colors.dim}(from the BLFS book){colors.normal}")
for label, key in (("book version", "title"), ("link", "link"),
("md5", "md5"), ("required", "required"),
("recommended", "recommended"), ("optional", "optional")):
val = bi.get(key)
if val and val not in ("[]", ""):
print(f" {label:<12}: {val}")
def _script_name_version(path):
try:
with open(path) as f:
for line in f:
# EITHER QUOTE. Since 1.12.18 the generator emits book strings
# SINGLE-quoted (a dollar sign in prose is not a variable), and this
# matched only double quotes -- so _script_name_version returned None,
# the local script never matched, and every edit a user made in
# install_<name>-<version> was quietly ignored for the generated one.
m = re.match(r"""\s*name_version=["']([^"']+)["']""", line)
if m:
return m.group(1)
except OSError:
pass
return None
def _pending_edit_conflict(home, iv, bv):
"""Would updating iv -> bv silently orphan local edits?
`install_last` is the read-only record of the script AS IT LAST RAN;
the canonical `install_<iv>` beside it is where edits live. If they
differ, the user has changes that never ran -- and the update will
generate a fresh script for `bv` that carries none of them. The make-ca
systemctl edit was exactly this, and --regenerate discarding it is what
the handoff warned about.
Self-clearing: once a local `install_<bv>` exists in the home (the user
merged their edits into the new version's script -- resolve_script will
prefer it), the conflict is resolved and this answers False. Missing
files answer False too: with no record there is nothing to judge."""
if not iv or not bv or iv == bv or iv == "(reinstall)":
return False
if os.path.isfile(os.path.join(home, f"install_{bv}")):
return False
canon = os.path.join(home, f"install_{iv}")
last = os.path.join(home, "install_last")
if not (os.path.isfile(canon) and os.path.isfile(last)):
return False
try:
return open(canon, errors="replace").read() \
!= open(last, errors="replace").read()
except OSError:
return False
def _warn_if_wrong_file_edited(user, chosen):
"""Point out an edit made to the wrong file.
A package user's home holds two files with nearly identical names:
install_<name>-<version> the source -- edit THIS one
install_<name> the engine's runtime copy, overwritten each run
Editing the second is an easy mistake, and the symptom is silent: the
install runs the unedited script and fails in exactly the same way as
before, so it looks as though the edit was thrown away."""
runtime = os.path.join(pkgusr_home(user), f"install_{user}")
if not os.path.isfile(runtime):
return
try:
if os.path.samefile(runtime, chosen):
return
if os.path.getmtime(runtime) <= os.path.getmtime(chosen):
return # not edited more recently: nothing odd
except OSError:
return
printWarning(f" note: {runtime} was changed more recently than the script "
f"in use.")
printWarning(f" That file is a runtime copy and is overwritten every "
f"run -- edits to it are lost.")
printWarning(f" Edit this instead: {chosen}")
_GENVER_RE = re.compile(r"^### generated by blfs (\S+)", re.M)
def _blfs_gen_version():
"""The generator's own version string, asked once."""
if not hasattr(_blfs_gen_version, "_v"):
rc, out, _ = run_blfs(["--version"])
m = re.search(r"blfs (\S+)", out or "")
_blfs_gen_version._v = m.group(1) if (rc == 0 and m) else None
return _blfs_gen_version._v
def _script_is_user_edited(path):
"""Did somebody change this script after it was generated?
The generator stamps every script and never writes it again, so an mtime
newer than the stamp line's own file creation is the only signal there
is. We use a marker instead: a script the user has edited gets a
".edited" sibling the first time it is regenerated, and we treat the
presence of a `# EDITED` line, or of that sibling, as "hands off".
"""
if os.path.exists(path + ".edited"):
return True
try:
with open(path) as f:
head = f.read(4000)
except OSError:
return False
return "# EDITED" in head or "# do not regenerate" in head.lower()
def _script_gen_version(path):
try:
head = open(path).read(400)
except OSError:
return None
m = _GENVER_RE.search(head)
return m.group(1) if m else None
def _prewarm_scripts(plan, script_dir, force_regen=False):
"""Generate every script the plan needs, in ONE blfs call.
resolve_script generates per package, and a plan of 151 packages therefore
ran 151 subprocesses that each loaded and indexed the whole book. With a
warm cache that is a fifth of a second each; on a fresh system, where the
first call INDEXES the book, it is the difference between a wait and a
hang -- and it looked like a hang, because the only output was one
"-- regenerating" line per package with minutes of silence between them.
Nothing else changes: this writes the same files into the same directory,
and resolve_script then finds them cached. A local script in a package
user's home still wins; those are not touched here.
"""
wanted = []
for anchor, nv, _reason in plan:
local = os.path.join(pkgusr_home(pkgusr_name(anchor)), f"install_{nv}")
if os.path.isfile(local) and not force_regen:
continue # the user's own copy wins
cached = os.path.join(script_dir, f"install_{nv}")
if not force_regen and os.path.isfile(cached) \
and _script_name_version(cached) == nv:
_cv, _gv = _script_gen_version(cached), _blfs_gen_version()
if not (_cv and _gv and _cv != _gv):
continue # current, keep it
wanted.append(anchor)
if len(wanted) < 2:
return # nothing to gain
printInfo(" generating %d install script(s) from the book "
"(one pass) ..." % len(wanted))
os.makedirs(script_dir, exist_ok=True)
rc, out, err = run_blfs(["script"] + wanted + ["-o", script_dir])
made = len(re.findall(r"^Created ", out, re.M))
if rc != 0 and made == 0:
# not fatal: resolve_script will generate them one at a time, as before
printWarning(" could not generate them in one pass (%s); "
"falling back to one at a time"
% (err or out).strip().splitlines()[-1:] or "no output")
return
printInfo(" %d written" % made)
def _report_stale_requires(names, _verbose=False):
"""For a single-package update: which of its requirements are older here.
The prereq check catches a missing TOOL. It says nothing about a
dependency that is present but older than the version the book's page
was written against -- and that is the real question behind "can I
update webkitgtk without updating glib2":
webkitgtk 2.52.5 requires GLib-2.88.3; this system has GLib-2.86.4.
That may build perfectly. It may also fail at configure, or worse,
build and misbehave. Nobody can tell from here, so this reports the
numbers and leaves the decision alone -- refusing would break the one
workflow it exists to serve, which is updating one package at a time.
"""
for name in names[:4]:
u = gather_user(name)
if not u.meta:
continue
rows = []
for kind in ("required", "recommended"):
for dep in u.meta.get(kind, []):
a = _resolve_dep(dep, _pkg_dirs()) if "_resolve_dep" in globals() \
else None
want = book_version(a or dep.rsplit("-", 1)[0]) or dep
have = None
if a:
have = gather_user(a).version or \
(gather_user(a).meta or {}).get("name_version")
if have and want and not _same_version(have, want):
_row = (kind, a or dep, have, want)
if _row not in rows: # two deps can resolve alike
rows.append(_row)
if not rows:
continue
print()
printWarning(" %s's requirements that differ from the book here:"
% name)
_show = rows if _verbose else rows[:12]
for kind, a, have, want in _show:
printWarning(" %-11s %-22s installed %-24s book %s"
% (kind, a, have, want))
if len(rows) > len(_show):
# A TRUNCATED LIST MUST SAY HOW TO SEE THE REST. "... and 5
# more" with no way to ask is a dead end.
printInfo(" ... and %d more -- all of them: packagemanager "
"update %s -v" % (len(rows) - len(_show), name))
printInfo(" This is not a refusal: one of them may be exactly what")
printInfo(" you do not want to touch. It is what the page was")
printInfo(" written against, so if the build fails at configure,")
printInfo(" start with these.")
def _stop_if_prereqs_missing(args):
"""A plan that cannot finish should not start.
Warning and building anyway is how a missing cmake got found twenty-two
packages deep, after the warning had already scrolled off. The check is
exact -- these are the scripts' own guards and their own command words --
so stopping is safe, and --ignore-prereqs is there for the case where it
is wrong.
"""
if not (_MISSING_TOOLS or _MISSING_ENV):
return
if not getattr(args, "run", False):
return # a dry run is allowed to show the plan
if getattr(args, "ignore_prereqs", False):
printWarning(" --ignore-prereqs: starting anyway; the packages above"
" will fail where they need what is missing.")
return
printError("Not starting: this plan is missing what %d of its packages"
" need." % len({p for v in list(_MISSING_TOOLS.values())
+ list(_MISSING_ENV.values()) for p in v}))
# SAY WHEN THE PACKAGE IS THERE AND ONLY THE TOOL IS NOT FOUND.
for _t in sorted(_MISSING_TOOLS):
_prov = _provider_of(_t)
if _prov and gather_user(_prov).state == "installed":
printInfo(" NOTE: %s is installed, so `%s` exists but is not on"
" PATH here." % (_prov, _t))
printInfo(" BLFS puts it outside /usr (e.g. /opt/%s/bin)"
" and adds it in" % _prov)
printInfo(" /etc/profile.d -- which a login shell reads and"
" this one did not.")
printInfo(" Check: bash -lc 'command -v %s'" % _t)
printInfo(" Fix the lines above, then run the same command again --"
" finished packages are skipped.")
printInfo(" To start regardless: --ignore-prereqs")
sys.exit(1)
def _report_ignored_required(steps):
"""A package in the plan whose REQUIRED dependency is being ignored.
webkitgtk requires Ruby; the stack's avoid list contains ruby; so the plan
left out a hard requirement of a package the user asked for, and cmake
said "Ruby 2.5 or higher is required" after an hour of building. Skipping
an avoided branch is right (1.14.24) -- doing it without a word, when the
thing skipped is REQUIRED by something being built, is not.
"""
if not _IGNORED_REQUIRED:
return
_in = {a.lower() for a, _n, _r, _p in steps}
pairs = [(w, d) for w, d in _IGNORED_REQUIRED if w.lower() in _in]
if not pairs:
return
print()
printWarning("%d package(s) in this plan REQUIRE something the ignore list"
" excludes:" % len(pairs))
for w, d in pairs:
printWarning(" %-20s requires %s, which is ignored" % (w, d))
printWarning(" They will fail where they need it. Either drop it from the")
printWarning(" ignore list (in the stack file, or --ignore), or install it:")
printWarning(" packagemanager install %s --run --recursive"
% " ".join(sorted({d for _w, d in pairs})))
print()
def _report_cycles(steps):
"""Name the packages a dependency cycle forces out of order.
vala recommends graphviz; graphviz -> gegl -> babl -> librsvg -> vala
closes the loop, so one recommended edge has to give and vala is built
first -- without graphviz, and (for vala) with configure stopping on
"Package 'libgvc' not found". There is no order that satisfies a cycle;
what there is, is a rebuild afterwards, and the person can only ask for it
if they are told.
"""
if not _CYCLE_PAIRS:
return
# Only the direction that ACTUALLY happened. The breaker drops both
# edges of a two-package cycle, so freetype2/harfbuzz was reported in both
# directions and one of the two lines was always false. The final order
# decides which one is true.
_pos = {a.lower(): i for i, (a, _n, _r, _p) in enumerate(steps)}
_pairs = [(a, d) for a, d in _CYCLE_PAIRS
if a.lower() in _pos and d.lower() in _pos
and _pos[a.lower()] < _pos[d.lower()]]
if not _pairs:
return
print()
printWarning("%d package(s) are built before something they recommend"
" (a dependency cycle):" % len(_pairs))
for a, d in _pairs:
printWarning(" %-16s is built before %s" % (a, d))
printWarning(" That is unavoidable -- the recommendation runs both ways."
" They build")
printWarning(" without the optional feature; rebuild afterwards if you"
" want it:")
printWarning(" packagemanager install %s --run --reinstall"
% " ".join(sorted({a for a, _d in _pairs})))
print()
def _report_missing_env():
"""One block naming every variable the plan's scripts will refuse without.
Reported per package this was one to three lines each: in a 151-package
plan the same XORG_PREFIX notice appeared dozens of times, scrolled past,
and the run stopped on it seventeen packages in.
"""
if not _MISSING_ENV:
return
print()
printWarning("This plan cannot finish: %d build variable(s) are not set."
% len(_MISSING_ENV))
for name in sorted(_MISSING_ENV):
users = _MISSING_ENV[name]
printWarning(" %-14s needed by %d package(s), first: %s"
% (name, len(users), users[0]))
printWarning(" Set them once -- they apply to every build:")
for name in sorted(_MISSING_ENV):
printWarning(" packagemanager env --set %s=%s"
% (name, _ENV_SUGGESTIONS.get(name, "...")))
printWarning(" (exporting them in your shell does not reach a package"
" user; %s does)" % BUILD_ENV_FILE)
print()
def _report_missing_tools():
"""One block, under the plan, naming every build-system driver the plan's
own commands call and this system does not have."""
_report_missing_env()
if not _MISSING_TOOLS:
return
print()
printWarning("This plan cannot finish: %d build tool(s) are missing."
% len(_MISSING_TOOLS))
for tool in sorted(_MISSING_TOOLS):
users = _MISSING_TOOLS[tool]
printWarning(" %-8s needed by %d package(s), first: %s"
% (tool, len(users), users[0]))
_provs = sorted({p for p in (_provider_of(t) for t in _MISSING_TOOLS) if p})
if _provs:
printWarning(" Install them first, with their own dependencies:")
printWarning(" packagemanager install %s --run --recursive"
% " ".join(_provs))
_noprov = sorted(t for t in _MISSING_TOOLS if not _provider_of(t))
if _noprov:
printWarning(" No BLFS page provides %s -- these come from LFS;"
% ", ".join(_noprov))
printWarning(" check they were built and are on PATH.")
print()
def resolve_script(user, name_version, script_dir, assume_yes=False,
force_regen=False):
"""Path to the install script to use for `user`.
By default, if the package user's home has the canonical, edited script
'install_<name_version>', that LOCAL one is used (so your edits are kept) and
we say so. 'install_last' (a read-only record) and 'install_<user>' (the
engine's runtime copy) are never treated as sources. --regenerate forces a
fresh build from the book."""
local = os.path.join(pkgusr_home(user), f"install_{name_version}")
# A STALE SCRIPT IS REFRESHED, NOT JUST REPORTED.
#
# Telling the user to pass --regenerate has cost several rounds each on
# brotli, rust, xcb-utilities and ffmpeg: the fix existed, the warning
# printed, and the build still ran the old script -- because spotting one
# warning in a 200-package plan and acting on it per package is not a
# reasonable thing to ask. A script older than the generator and NOT
# edited by the user is simply regenerated below.
_stale_unedited = False
if os.path.isfile(local):
_lgv, _cgv = _script_gen_version(local), _blfs_gen_version()
_stale_unedited = bool(_lgv and _cgv and _lgv != _cgv
and not _script_is_user_edited(local))
if _stale_unedited:
printWarning(f" {os.path.basename(local)}: generated by blfs "
f"{_lgv}, this is {_cgv} -- regenerating")
if (not force_regen and not _stale_unedited and os.path.isfile(local)
and _script_name_version(local) == name_version):
printInfo(f" using LOCAL script (your edits, not the book): {local}")
_warn_if_wrong_file_edited(user, local)
try:
body = open(local).read()
except OSError:
body = ""
# a real build script has the phased functions or actual build commands;
# warn if it looks like a stub so a "reinstall that does nothing" is obvious
if ("unpack_pkg" not in body and "build_pkg" not in body
and not re.search(r"\b(make|ninja|meson|cmake|configure)\b", body)):
printWarning(f" ^ this script has no build commands -- it may be a "
f"stub. If a reinstall does nothing, rebuild it from the "
f"book with --regenerate.")
else:
printInfo(f" -- pass --regenerate to rebuild {name_version} from the book")
# A SCRIPT FROM AN OLDER GENERATOR CARRIES THE OLD BUGS.
#
# brotli failed twice on the same `pip3 install --find-links dist`
# after the generator had been fixed: the local script was written by
# the previous blfs, resolve_script prefers it, and the only notice
# was a one-line hint that is easy to miss in a 200-package run.
# Say it loudly, and offer the command that fixes it.
# A STALE SCRIPT IS REFRESHED, NOT JUST REPORTED.
#
# Telling the user to pass --regenerate has now cost several rounds
# each on brotli, rust, xcb-utilities and ffmpeg: the fix existed,
# the message was printed, and the build still ran the old script --
# because noticing one warning in a 200-package plan and acting on it
# per package is not a reasonable thing to ask.
#
# A script older than the generator is regenerated automatically.
# Nothing is lost: if the file differs from what the generator
# produces, the old one is kept as .edited (1.13.4) and named here.
_lg, _cg = _script_gen_version(local), _blfs_gen_version()
if _lg and _cg and _lg != _cg:
printWarning(f" ^ generated by blfs {_lg}; the generator is now "
f"{_cg}, and this file has YOUR EDITS -- keeping it")
printWarning(f" to take the newer generator instead (your copy "
f"is saved as install_{name_version}.edited):")
printWarning(f" packagemanager install {user} --run --regenerate")
# THE OPTIONS APPLY TO A LOCAL SCRIPT TOO.
#
# They were merged only when a script was GENERATED, so a package
# whose script already sat in its home -- every package that has
# failed once -- was reused untouched, and libassuan kept building
# its TeX manual with the fix sitting in package-fixes.conf. The
# merge is idempotent, so applying it on every resolve is safe.
apply_machine_opts(user, local)
# THE CHECKS BELONG TO EVERY PATH OUT OF HERE.
#
# They were on the generate path (1.14.9 added the cache path), and a
# script already sitting in the package user's home returned before
# either -- which is EVERY package that has been attempted once. So a
# plan whose libaom needed a missing cmake said nothing, built
# twenty-two other packages, and then stopped exactly where the check
# would have pointed.
_check_script_prereqs(user, local)
return local
# reuse a script a PRIOR DRY-RUN already generated into script_dir, so a
# follow-up --run doesn't regenerate all of them (version-keyed = safe).
cached = os.path.join(script_dir, f"install_{name_version}")
if (not force_regen and os.path.isfile(cached)
and _script_name_version(cached) == name_version):
# "version-keyed = safe" was only half true: it caught a moved BOOK,
# not a moved GENERATOR. A pre-fix cached docbook-xsl script ran the
# optional doc block a newer blfs comments out. A cache from an
# older generator is regenerated.
_cv, _gv = _script_gen_version(cached), _blfs_gen_version()
if _cv and _gv and _cv != _gv:
printInfo(f" cached script is from blfs {_cv}, this is {_gv} "
f"-- regenerating")
else:
_check_script_prereqs(user, cached)
return cached
rc, out, err = run_blfs(["script", user, "-o", script_dir])
m = re.search(r"^Created (.+)$", out, re.M)
if rc != 0 or not m:
printError(f"{name_version}: blfs script failed -- {(err or out).strip()}")
return None
path = m.group(1).strip()
# the machine's options, merged the moment the script is born
apply_machine_opts(user, path)
_check_script_prereqs(user, path)
return path
# --------------------------------------------------------------------------- #
# machine option overrides
# --------------------------------------------------------------------------- #
# The BOOK decides how a package is built; the MACHINE decides its options.
# Mesa's gallium drivers belong to the GPU in the case, not to the stack file
# or the book -- so /etc/pkgusr/stacks/machine.conf carries them:
#
# [mesa]
# gallium-drivers = radeonsi
# vulkan-drivers = amd
#
# Every key of an [anchor] section is merged into the generated script's
# `meson setup` line as -Dkey=value -- an existing -Dkey is replaced, a new
# one is appended right after `meson setup`. Only meson for now (cmake
# pending); only freshly generated scripts are touched, and the change makes
# the script a LOCAL edit, which resolve_script then deliberately keeps.
MACHINE_CONF = "/etc/pkgusr/stacks/machine.conf"
# Two files, and the difference matters.
#
# package-fixes.conf -- workarounds the TOOLS ship (cairo's ctime_r,
# libassuan's TeX manual). Refreshed with the other
# stack files, so a fix reaches an existing system.
# machine.conf -- the MACHINE'S own choices (mesa's drivers). Never
# overwritten, and it wins where both mention a key.
#
# They were one file, which meant a workaround shipped in machine.conf could
# never arrive anywhere: the sync refuses to overwrite it, correctly, because
# it is the user's. libassuan kept failing on a fix that was in the release
# and not on the system.
PACKAGE_FIXES = "/etc/pkgusr/stacks/package-fixes.conf"
def _read_conf_section(path, anchor):
if not os.path.isfile(path):
return {}
import configparser
# strict=False: A REPEATED SECTION IS NOT A BROKEN FILE.
#
# Appending the same block twice -- exactly what `cat >> machine.conf`
# does when a recipe is followed twice -- made configparser raise, and
# this discarded the ENTIRE file. So [webrtc-audio-processing] did
# nothing, and so did [mesa]: the driver selection quietly reverted to
# "build everything" because of a duplicate heading further down. One
# typo disabling every option in the file is not a proportionate answer.
# Later keys win, which is what someone appending a correction expects.
cp = configparser.ConfigParser(strict=False)
# CMAKE VARIABLES ARE CASE-SENSITIVE.
#
# configparser lower-cases keys by default, so
# [webkitgtk]
# ENABLE_JOURNALD_LOG = OFF
# reached cmake as -Denable_journald_log=OFF -- a different variable,
# which cmake accepted and ignored, and webkitgtk went on building
# against systemd/sd-journal.h on a system that has no systemd. meson's
# options are lower-case anyway, which is why this went unnoticed for as
# long as the file only held [mesa].
cp.optionxform = str
try:
cp.read(path)
except configparser.Error as e:
printWarning(f"{os.path.basename(path)} is broken ({e})")
printWarning(" NO options from that file were applied -- not for this"
" package and not for any other.")
printWarning(f" Fix it and re-run: {path}")
return {}
for sect in cp.sections():
if sect.lower() == anchor.lower():
return dict(cp[sect])
return {}
def _opt_sections(anchor, keys, first_only=False):
"""Which sections actually define these keys -- [all], the package, or both.
Both messages below used to name the package section whatever the truth
was: `machine.conf [rust]: x.py --jobs 13` for a file with no [rust]
section, and `[somepkg] and [all] set weird_option` where only one of
them did. A message that names a thing must be checked against the
thing, and here the thing is two files and two sections.
"""
keys = set(keys)
out = []
for sec in (anchor, "all"):
for f in (os.environ.get("PM_MACHINE_CONF", MACHINE_CONF),
os.environ.get("PM_PACKAGE_FIXES", PACKAGE_FIXES)):
if keys & set(_read_conf_section(f, sec)):
out.append(sec)
break
# the package section overrides [all], so for a value that was USED
# only the winner is worth naming: "[webkitgtk] and [all]: ninja -j8"
# credited a section whose 13 had just been overridden by 8.
if out and first_only:
break
return out
def _machine_opts_for(anchor):
# [all] FIRST, THEN THE PACKAGE.
#
# `jobs` had to be written per package, because ninja ignores MAKEFLAGS
# and there was no way to say "never more than 13 on this machine" once.
# An [all] section in either file applies to every package, and a named
# section overrides it -- so [webkitgtk] jobs = 8 still wins over
# [all] jobs = 13.
opts = _read_conf_section(
os.environ.get("PM_PACKAGE_FIXES", PACKAGE_FIXES), "all")
opts.update(_read_conf_section(
os.environ.get("PM_MACHINE_CONF", MACHINE_CONF), "all"))
opts.update(_read_conf_section(
os.environ.get("PM_PACKAGE_FIXES", PACKAGE_FIXES), anchor))
opts.update(_read_conf_section(
os.environ.get("PM_MACHINE_CONF", MACHINE_CONF), anchor))
return opts
_ENV_GUARD_RE = re.compile(r'needs ([A-Z][A-Z0-9_]{2,}) to be set')
def _warn_unset_env(anchor, script_path):
"""Say it BEFORE the build, not in the log afterwards.
The generated script refuses if a variable the book expects is unset --
but it refuses after unpacking, patching and configuring, with the
failure buried in a log. The script is generated here, so the question
can be asked here: which variables does it need, and are they set?
Applies to every package, not just TeX Live: whatever the book expects
the user to export, this finds it the moment the script exists."""
try:
body = open(script_path, encoding="utf-8", errors="replace").read()
except OSError:
return
# SET WHERE THE BUILD LOOKS, NOT WHERE WE HAPPEN TO BE.
#
# This asked os.environ only, so after `packagemanager env --set
# XORG_CONFIG=...` -- which writes /etc/pkgusr/build.env and IS what the
# build reads -- every Xorg package in the plan was still announced as
# missing it. And the advice it gave, `export`, is the one thing that
# does NOT work: builds run as the package user through `su -`, which
# drops the caller's environment. A warning that is wrong and then
# recommends the wrong cure teaches people to ignore warnings.
_build_env = _read_build_env()
missing = [n for n in dict.fromkeys(_ENV_GUARD_RE.findall(body))
if not (_build_env.get(n) or os.environ.get(n))]
if not missing:
return
# once per VARIABLE, like the build tools -- the rest goes in the summary
fresh = [n for n in missing if n not in _MISSING_ENV]
for n in missing:
_MISSING_ENV.setdefault(n, []).append(anchor)
if not fresh:
return
printWarning("%s needs %s set before it can build:"
% (anchor, ", ".join(fresh)))
for n in fresh:
_sugg = _ENV_SUGGESTIONS.get(n)
printWarning(" packagemanager env --set %s=%s" % (n, _sugg or "...")
+ ("" if _sugg else " (the book's page for %s says what)"
% anchor))
printWarning(" that writes %s, which every build reads;" % BUILD_ENV_FILE)
printWarning(" exporting it in your shell does not reach a package user.")
printWarning(" the build will stop at that check rather than configure with"
" an empty value")
# A build-system driver the script CALLS but the page never lists. libheif's
# BLFS page has no Required section at all: CMake reaches it only through
# x265, so on a system where x265 was already installed nothing pulled cmake
# into the plan, and the build died 33 lines in with
# install_libheif: line 33: cmake: command not found
# after three other packages had been built. The dependency list is the
# book's opinion; the COMMANDS are the fact, and they are right here.
_MISSING_TOOLS = {} # tool -> [packages in this plan that call it]
_MISSING_ENV = {} # variable -> [packages in this plan that need it]
_CYCLE_PAIRS = [] # (dependant, dependency) the cycle forced out of order
_IGNORED_REQUIRED = [] # (wanted package, its REQUIRED dep that is ignored)
# tool -> the BLFS ANCHOR that provides it. These are checked against the
# book at use time (_provider_in_book), because a suggestion naming a package
# the book does not have is the 1.14.10 mistake again: `cargo` was reported
# with "packagemanager install rustc --run --recursive", and the book's anchor
# is `rust`, so the command would have answered "not in the book".
# Meson, Ninja, gperf and libtool are LFS packages, not BLFS ones -- they have
# no anchor here, and are reported without a command rather than a wrong one.
_BUILD_TOOLS = {
"cmake": "cmake", "scons": "scons", "qmake6": "qt6",
"cargo": "rust", "rustc": "rust",
"ant": "apache-ant", "b2": "boost", "bjam": "boost",
"itstool": "itstool", "xsltproc": "libxslt", "doxygen": "doxygen",
"valac": "vala", "node": "nodejs", "npm": "nodejs", "rake": "ruby",
"yasm": "yasm", "nasm": "nasm",
# a stack entry of kind=git clones with it; nothing in the book's command
# text mentions it, so only the tool map can catch it before the build
"git": "git",
# no BLFS anchor: LFS builds these, or the book has no page
"meson": "", "ninja": "", "gperf": "", "libtoolize": "",
"autoreconf": "", "go": "", "mvn": "", "gradle": "", "qmake": "",
}
_TOOL_SEEN = {}
def _tool_available(tool):
"""Is the tool there FOR THE BUILD -- not just for us?
`shutil.which` reads OUR PATH. A build runs through `su -`, a LOGIN
shell, which reads /etc/profile and /etc/profile.d -- and that is exactly
where BLFS puts the packages it installs outside /usr:
/opt/rustc/bin (rustc.sh) -> cargo, rustc
/opt/qt6/bin (qt6.sh) -> qmake6
So Rust was installed, cargo was on the build's PATH, and this reported
it missing and refused to start. Worse, the plan could not repair it:
rust WAS installed, so adding it changed nothing and the same stop came
back on the second pass -- the loop the user saw.
Asked the way the build will ask.
"""
if tool in _TOOL_SEEN:
return _TOOL_SEEN[tool]
found = bool(shutil.which(tool))
if not found:
try:
found = subprocess.run(["bash", "-lc", "command -v " + shlex.quote(tool)],
capture_output=True, timeout=20).returncode == 0
except (OSError, subprocess.SubprocessError):
found = False
if not found:
found = any(os.access(os.path.join(d, tool), os.X_OK)
for d in glob.glob("/opt/*/bin"))
_TOOL_SEEN[tool] = found
return found
def _build_tools_used(script_path):
"""The build-system drivers a script's own commands call."""
try:
body = open(script_path, encoding="utf-8", errors="replace").read()
except OSError:
return []
used = []
for line in body.splitlines():
line = line.strip()
if not line or line.startswith("#"):
continue
word = line.split()[0]
while word in ("env", "sudo", "nice", "time") and len(line.split()) > 1:
line = line.split(None, 1)[1]
word = line.split()[0]
if "/" in word or word not in _BUILD_TOOLS or word in used:
continue
used.append(word)
return used
def _order_by_build_tools(steps):
"""Build a package AFTER the tool its commands call.
The dependency graph is the book's, and the book does not say that libaom
builds with cmake -- libaom's page lists no Required at all. So a plan
could contain BOTH, correctly, and still schedule libaom at position 89
and CMake at 151, and stop at 89 with `cmake: command not found`. The
resolver was right; it was ordering on the wrong facts.
The CONSUMER moves down to just after its provider, never the provider up:
moving a package later keeps every dependency it already has in front of
it, while moving one earlier could jump it over its own.
"""
if not steps:
return steps, []
moved = []
for _round in range(len(steps)): # bounded: each pass fixes >= 1
index = {}
for i, (anchor, nv, _r, _p) in enumerate(steps):
index[anchor.lower()] = i
index[nv.lower()] = i
index[nv.rsplit("-", 1)[0].lower()] = i
for i, (anchor, nv, reason, path) in enumerate(steps):
for tool in _build_tools_used(path):
if _tool_available(tool):
continue # already on the system
j = index.get((_BUILD_TOOLS.get(tool) or "").lower())
if j is None or j <= i:
continue # absent, or already earlier
steps.insert(j + 1, steps.pop(i))
moved.append((nv, steps[j][1], tool))
break
else:
continue
break
else:
break
return steps, moved
def _provider_of(tool):
"""The BLFS package that provides a tool, if the book actually has one."""
name = _BUILD_TOOLS.get(tool) or ""
if not name:
return ""
try:
return name if book_version(name) else ""
except Exception:
return ""
def cmd_provider(args):
"""Which BLFS package provides a program -- for scripts and for lfs-helper.
The knowledge already existed here (_BUILD_TOOLS + the book); nothing
outside could ask. lfs-helper needs it the moment a configure stops on
`itstool not found`: the tool's name is in the log, the package that
provides it is in this map, and one lookup turns a dead stack into a
build that carries on."""
rc = 1
for tool in args.tools:
prov = _provider_of(tool)
if prov:
print(prov)
rc = 0
return rc
def _install_cmd_for(tool):
prov = _provider_of(tool)
if prov:
return "packagemanager install %s --run --recursive" % prov
return ""
def _warn_missing_build_tools(anchor, script_path):
"""Name the tool the build will die on, before the build starts."""
try:
body = open(script_path, encoding="utf-8", errors="replace").read()
except OSError:
return []
missing = []
# A CLONE IS NOT A COMMAND IN THE SCRIPT.
#
# `is_git="1"` tells the runner to clone $link, so `git` never appears in
# the command text and the scan could not see it. elogind's entry died on
# `git: command not found` after the account had been created and the
# phase started.
if re.search(r'^is_git=[\'"]?1', body, re.M) and not _tool_available("git"):
missing.append("git")
for line in body.splitlines():
line = line.strip()
if not line or line.startswith("#"):
continue
# the command word: skip env prefixes and anything path-qualified
# (./configure is the package's own, not something to install)
word = line.split()[0]
while word in ("env", "sudo", "nice", "time") and len(line.split()) > 1:
line = line.split(None, 1)[1]
word = line.split()[0]
if "/" in word or word not in _BUILD_TOOLS:
continue
if _tool_available(word) or word in missing:
continue
missing.append(word)
if not missing:
return []
# ONCE PER TOOL, not once per package: a 151-package plan would otherwise
# print the same three lines about cmake thirty-seven times, and a warning
# repeated thirty-seven times is scrollback, not a warning. The rest are
# collected for the summary below the plan.
for w in missing:
first = w not in _MISSING_TOOLS
_MISSING_TOOLS.setdefault(w, []).append(anchor)
if not first:
continue
printWarning("%s builds with `%s`, which is not installed:" % (anchor, w))
# --recursive, ALWAYS. A tool from the book has its own tree, and
# `packagemanager install cmake --run` builds cmake alone, which then
# fails at "CMAKE_USE_SYSTEM_CURL is ON but a curl is not found" --
# cURL is one of its four recommended deps. A suggested command that
# does not work is worse than no suggestion, which is also why a tool
# with no BLFS page gets no command at all.
_cmd = _install_cmd_for(w)
printWarning(" " + (_cmd if _cmd else
"(no BLFS page provides `%s` -- it comes from LFS; "
"check it is built and on PATH)" % w))
printWarning(" (the book's page does not list it as a dependency, so"
" the plan cannot know -- its own commands do)")
return missing
def _check_script_prereqs(anchor, script_path):
"""Everything knowable from a script before it runs.
Called for a script whether it was just generated or came from the cache:
the checks belong to USING a script, not to writing one, and pre-generating
a whole plan in one pass (1.14.8) would otherwise have skipped them.
"""
_warn_unset_env(anchor, script_path)
_warn_missing_build_tools(anchor, script_path)
def _has_command(body, cmd):
r"""Is `cmd` really invoked here -- not mentioned in a comment?
THE BOOK PUTS IT AFTER `&&`. webkitgtk's page writes
mkdir -vp build && cd build && cmake -D CMAKE_BUILD_TYPE=Release \\
and the detector used `^\s*cmake`, so it found no cmake line and
reported "machine.conf [webkitgtk] set USE_WOFF2 ... but the script has
no `meson setup` or `cmake` line -- NOT applied". Seven options,
silently dropped, on the one package where they matter most.
`_insert_on_command` had already learnt about `&&`; only the test that
decides whether to call it had not.
Comment lines do not count: the same page opens with a NOTE about "the
cmake command", which is prose.
"""
for line in body.split("\n"):
if line.lstrip().startswith("#"):
continue
if re.search(r"(?:^|\s|&&|;)%s\b" % re.escape(cmd), line):
return True
return False
def _insert_on_command(body, cmd, arg):
"""Add an argument to the first REAL invocation of `cmd`, skipping
comments -- a book NOTE that mentions the command by name is prose."""
out, done = [], False
for line in body.split("\n"):
if (not done and not line.lstrip().startswith("#")
and re.search(r"(?:^|\s|&&|;)%s\b" % re.escape(cmd), line)):
line = re.sub(r"(%s)\b" % re.escape(cmd), r"\1 " + arg, line, count=1)
done = True
out.append(line)
return "\n".join(out)
def _meson_option_names(anchor):
"""The options this package's own source declares, if it is unpacked.
meson fails the whole build on an unknown option, and a machine.conf
section is written by hand: `[gst10-plugins-bad] alsa = enabled` looks
perfectly reasonable and is wrong, because `alsa` is a
gst-plugins-BASE option. The answer is in the source tree, which is on
disk by the time anything is built.
"""
home = pkgusr_home(anchor)
src = os.path.join(home, "src")
if not os.path.isdir(src):
return None
names = set()
found = False
for d in sorted(os.listdir(src)):
for f in ("meson.options", "meson_options.txt"):
path = os.path.join(src, d, f)
if not os.path.isfile(path):
continue
found = True
try:
body = open(path, encoding="utf-8", errors="replace").read()
except OSError:
continue
names |= set(re.findall(r"""^\s*option\(\s*['"]([^'"]+)['"]""",
body, re.M))
return names if found else None
def _warn_unknown_meson_opts(anchor, opts):
"""Drop options the package does not have, and say which they are."""
known = _meson_option_names(anchor)
if not known:
return opts # not unpacked yet, or not a meson package
# meson's own built-ins are not in meson.options and are always valid
builtin = {"wrap_mode", "buildtype", "prefix", "libdir", "default_library",
"c_args", "cpp_args", "c_link_args", "cpp_link_args",
"b_lto", "b_ndebug", "werror", "optimization", "strip",
"auto_features", "backend", "jobs", "configure_args"}
bad = {k: v for k, v in opts.items() if k not in known and k not in builtin}
if not bad:
return opts
for k in sorted(bad):
printWarning(" machine.conf [%s]: '%s' is not an option of this "
"package -- NOT passed" % (anchor, k))
# the near misses are what makes this useful: `alsa` under
# [gst10-plugins-bad] belongs under [gst10-plugins-base]
near = sorted(n for n in known if k in n or n in k)[:4]
if near:
printInfo(" it does have: %s" % ", ".join(near))
printInfo(" meson would have failed the whole build on this"
" (\"Unknown option\").")
printInfo(" the options it does have: grep -h \"^option(\" %s"
% os.path.join(pkgusr_home(anchor), "src", "*",
"meson.options"))
return {k: v for k, v in opts.items() if k not in bad}
def _restamp_md5(path):
"""A fresh pm_md5 for a script, WITHOUT touching its format stamp."""
try:
text = open(path, encoding="utf-8", errors="replace").read()
except OSError:
return False
if not re.search(r'### generated by blfs[^\n]*script format v\d+', text):
return False
text = re.sub(r'\n### pm_md5=[0-9a-f]+', '', text)
digest = hashlib.md5(text.encode()).hexdigest()
text = re.sub(r'(### generated by blfs[^\n]*script format v\d+)',
lambda m: m.group(1) + f'\n### pm_md5={digest}', text, count=1)
try:
with open(path, "w") as f:
f.write(text)
return True
except OSError:
return False
def apply_machine_opts(anchor, script_path):
"""Merge machine.conf's options into the script, and keep it PRISTINE.
THE TOOL'S OWN EDIT IS NOT YOURS. A generated script carries a checksum
of what the book produced; machine.conf's options are merged in AFTER
that, so every package with a machine.conf section read as "edited" in
`script-status` -- gstreamer10, the three plugin sets with switches, and
webkitgtk -- and the person was told to hand-merge edits they never
made. A script that was pristine before the merge is re-stamped after
it. One the person really edited keeps its old checksum, so a later
migration still stops and asks."""
_before = None
try:
_before = open(script_path, encoding="utf-8", errors="replace").read()
except (OSError, TypeError):
pass
_was_pristine = _before is not None and script_edited(_before) is False
_apply_machine_opts_inner(anchor, script_path)
if _was_pristine:
try:
_after = open(script_path, encoding="utf-8", errors="replace").read()
except OSError:
return
if _after != _before:
_restamp_md5(script_path)
def _apply_machine_opts_inner(anchor, script_path):
"""Merge machine.conf's [anchor] options into the script's meson line."""
opts = _warn_unknown_meson_opts(anchor, _machine_opts_for(anchor))
if not opts or not script_path or not os.path.isfile(script_path):
return
try:
body = open(script_path).read()
except OSError:
return
# AUTOTOOLS TOO. This only knew `meson setup`, so a package built with
# ./configure had no way to take a machine option -- and libassuan needs
# one: its `make` builds a PDF manual that wants TeX, which this system
# deliberately does not have. configure_args is appended to the
# ./configure line; every other key stays a meson -Dkey=value.
# `needs` is for the DEPENDENCY GRAPH, not the build command. blfs reads
# it from the same file (1.13.69) to add an edge the book omits; this
# function turns every remaining key into -Dkey=value, so it passed
# `-Dneeds=glslang` to meson and mesa died with
# ERROR: Unknown option: "needs".
# One file, two readers: each must ignore what belongs to the other.
opts.pop("needs", None)
# Same hazard, same answer: expect= is a post-install feature probe read
# by the stack runner. It belongs in a stack line, but this file is
# where a key that is not a build option does damage, so drop it here too.
opts.pop("expect", None)
# ...and REMOVE it if an earlier version already wrote it into the script.
# 1.13.69 stopped adding -Dneeds=, but every script generated while the
# bug existed still carries it, and merging is idempotent so nothing took
# it out again. A key that is not a build option has no business on a
# meson line, whoever put it there.
if "-Dneeds=" in body:
body = re.sub(r"\s*-Dneeds=\S+", "", body)
with open(script_path, "w") as f:
f.write(body)
printInfo(" removed a stale -Dneeds= from %s (it is a dependency "
"key, not a build option)" % os.path.basename(script_path))
# WRAP MODE, PER PACKAGE.
#
# blfs adds --wrap-mode=nofallback to every `meson setup` so a package
# cannot silently build and install a bundled copy of a dependency (cairo
# once installed its own freetype that way, and the planner then thought
# freetype2 was done). For a package whose subproject is NOT a BLFS
# package at all, that flag is fatal instead of protective:
# gegl: ERROR: Dependency 'libnsgif' is required but not found.
# Use of fallback dependencies is disabled.
# libnsgif and poly2tri-c have no book page; the fallback is the only way
# to get them. So the flag stays the default and becomes overridable:
# [gegl]
# wrap_mode = default
# NINJA DOES NOT READ MAKEFLAGS.
#
# WebKitGTK's page: "some source files require more than 4 GiB of RAM to
# be built ... you should pass -j<N> to ninja (N = RAM / 4 GiB) to avoid
# the job being killed by the kernel OOM killer". MAKEFLAGS cannot do
# that -- ninja ignores it and defaults to nproc+2 -- so there was no way
# to follow the book's advice short of editing the script by hand, and
# the failure it prevents arrives hours in, as a killed compiler.
_jobs = opts.pop("jobs", None)
if _jobs:
_n = str(_jobs).lstrip("-j")
# NINJA IS NOT THE ONLY ONE THAT IGNORES THE ENVIRONMENT.
#
# rust's own page says it: the build "can be mostly limited to a
# specified number of processors by a COMBINATION of adding the
# switch --jobs <N> on each invocation of ./x.py and using the
# CARGO_BUILD_JOBS environment variable". With only the variable
# set, `./x.py build` used every core.
#
# Each pattern refuses to touch a line that already names -j or
# --jobs, so an explicit choice in a script or a stack entry wins.
_subs = (
# ninja: no environment variable at all
# ...and not only at the start of a line: `cmake .. && ninja` is
# how half the pages are written, and that ninja was untouched
(r"(?m)(^|&&[ \t]*|;[ \t]*)(ninja\b(?![^\n]*(\s-j|\s--jobs)))",
r"\1\2 -j" + _n, "ninja -j%s"),
# ./x.py build|install|test|dist
(r"(?m)^(\s*\.?/?x\.py\s+\w+(?![^\n]*(\s-j|\s--jobs)))",
r"\1 --jobs " + _n, "x.py --jobs %s"),
# cargo build / cbuild / install / cinstall
(r"(?m)^(\s*cargo\s+c?(?:build|install)\b(?![^\n]*(\s-j|\s--jobs)))",
r"\1 -j " + _n, "cargo -j %s"),
)
_changed = []
for _pat, _rep, _what in _subs:
_nb = re.sub(_pat, _rep, body)
if _nb != body:
body = _nb
_changed.append(_what % _n)
if _changed:
with open(script_path, "w") as f:
f.write(body)
printInfo(" machine.conf %s: %s"
% (" and ".join("[%s]" % w
for w in _opt_sections(anchor, ["jobs"],
first_only=True))
or "[%s]" % anchor,
", ".join(_changed)))
_wrap = opts.pop("wrap_mode", None)
if _wrap:
if "meson setup" not in body:
printWarning(" machine.conf has [%s] wrap_mode but the script has"
" no `meson setup` line -- not applied" % anchor)
else:
new_body = re.sub(r"--wrap-mode=\S+", "--wrap-mode=" + _wrap, body)
if new_body == body: # none there yet: add one
new_body = re.sub(r"(\bmeson\s+setup\b)",
r"\1 --wrap-mode=" + _wrap, body, count=1)
if new_body != body:
body = new_body
with open(script_path, "w") as f:
f.write(body)
printInfo(" machine.conf [%s]: wrap-mode %s" % (anchor, _wrap))
# KEYS THAT ADDRESS THE TOOL, NOT THE BUILD.
#
# `[linux-pam] book = .../BLFS-BOOK-12_4-nochunks.html` tells the tool
# WHICH BOOK to read that package from -- and it went straight onto the
# build line:
# meson setup -Dbook=/usr/share/blfs/books/... ..
# ERROR: Unknown option: "book".
# jobs and configure_args were already popped out one at a time; a
# reserved SET means the next such key cannot leak the same way, and
# `_warn_unknown_meson_opts` cannot save us here because at script
# generation time the source is not unpacked yet and it has no
# meson.options to check against.
for _reserved in ("book",):
opts.pop(_reserved, None)
_conf_args = opts.pop("configure_args", None)
if _conf_args:
if "./configure" not in body:
printWarning(" machine.conf has [%s] configure_args but the script"
" has no ./configure line -- not applied" % anchor)
else:
if _conf_args in body:
pass # already applied; merging is idempotent
else:
# AN --enable-X MUST BEAT THE BOOK'S --disable-X.
#
# configure_args was PREPENDED, and autoconf takes the last
# occurrence: `[iptables] configure_args = --enable-nftables`
# produced
# ./configure --enable-nftables --prefix=/usr \
# --disable-nftables ...
# so the book's flag won and the option silently did nothing.
# Drop the opposite form of each flag we are adding first.
# ...line by line, so a continuation that held nothing but
# the removed flag goes with it. Deleting just the token left
# ./configure --enable-nftables --prefix=/usr \ \
# -- a backslash-escaped space, which is an argument.
for _flag in _conf_args.split():
_m = re.match(r"^--(enable|disable|with|without)-(.+)$", _flag)
if not _m:
continue
_opp = {"enable": "disable", "disable": "enable",
"with": "without", "without": "with"}[_m.group(1)]
# BY NAME, NOT NAME=VALUE, AND THE SAME FLAG TOO.
# `--with-systemdsystemunitdir=no` looked for its
# opposite as `--without-systemdsystemunitdir=no`, which
# no page writes; and a book line carrying
# `--with-systemdsystemunitdir=/usr/lib/systemd/system`
# kept it -- after ours, so configure took the book's.
# Both forms, any value, go before ours is put in.
_name = _m.group(2).split("=", 1)[0]
_pat = re.compile(r"--(?:%s|%s)-%s(?:=[^\s\\]*)?(?=[\s\\]|$)"
% (_m.group(1), _opp, re.escape(_name)))
_out = []
for _line in body.split("\n"):
if not _pat.search(_line):
_out.append(_line)
continue
_stripped = _pat.sub("", _line)
if _stripped.strip() in ("", "\\"):
continue # the line was only that flag
_out.append(_stripped)
body = "\n".join(_out)
body = re.sub(r"(\./configure\b)", r"\1 " + _conf_args, body, count=1)
if _conf_args not in open(script_path).read():
with open(script_path, "w") as f:
f.write(body)
printInfo(" machine.conf [%s]: configure %s" % (anchor, _conf_args))
if not opts:
return
# CMAKE TAKES -D OPTIONS TOO.
#
# This only knew `meson setup`, so for a cmake package it printed
# machine.conf has [webkitgtk] but the script has no `meson setup`
# line -- options NOT applied
# and the person had nowhere to put -D ENABLE_SPELLCHECK=OFF. Both build
# systems take -Dkey=value; only the command word differs. (autotools is
# already handled above, by configure_args.)
# NOTHING LEFT TO APPLY IS NOT A PROBLEM.
#
# `[all] jobs = 13` was consumed above (x.py --jobs 13, printed and
# correct) and then this warned that rust's options had NOT been
# applied -- with nothing left in opts to apply.
if not opts:
return
_anchor_cmd = ("meson setup" if _has_command(body, "meson setup")
else ("cmake" if _has_command(body, "cmake") else ""))
if not _anchor_cmd:
# ...AND NAME THE SECTION THAT ACTUALLY EXISTS. It said
# machine.conf has [rust] but ...
# about a file with no [rust] section at all: the keys had come from
# [all]. A message that names a thing must be checked against the
# thing.
_where = _opt_sections(anchor, opts)
printWarning(" machine.conf %s set %s for %s, but the script has no "
"`meson setup` or `cmake` line -- NOT applied"
% (" and ".join("[%s]" % w for w in _where) or "(unknown)",
", ".join(sorted(opts)), anchor))
printWarning(" (an autotools package takes configure_args instead)")
return
new = body
applied = []
for k, v in opts.items():
# `-D key=value` WITH A SPACE is how the book writes them.
#
# This matched only "-Dkey=", so the book's own `-D gallium-drivers=auto`
# was left in place and OUR value was merely prepended -- and meson
# takes the LAST occurrence. mesa built for every driver instead of
# radeonsi and then failed on a dependency it should never have wanted:
# ERROR: Dependency "libdrm_intel" not found
# A VALUE WITH A SPACE IS ONE VALUE.
#
# The fix for the commonest GCC-15 breakage is
# [webrtc-audio-processing]
# cpp_args = -include cstdint
# and this pasted it in unquoted, so the shell split it and meson saw
# `-Dcpp_args=-include` plus a stray `cstdint`. Every option here that
# takes a flag LIST -- c_args, cpp_args, c_link_args -- has this shape;
# the single-word examples in the help text hid it.
_v = v if not re.search(r"\s", v) else "'%s'" % v.replace("'", "'\\''")
pat = re.compile(r"-D\s*%s\s*=(?:'[^']*'|\S+)" % re.escape(k))
if pat.search(new):
new = pat.sub("-D%s=%s" % (k, _v), new)
else:
# ...and put it on the COMMAND, not in the first line that happens
# to contain the word. WebKitGTK's page opens with a NOTE
# mentioning "the cmake command", so every option landed inside
# that comment and none of them reached the build.
new = _insert_on_command(new, _anchor_cmd, "-D%s=%s" % (k, _v))
applied.append("%s=%s" % (k, v))
if new != body:
with open(script_path, "w") as f:
f.write(new)
printInfo(" machine.conf [%s]: %s" % (anchor, ", ".join(applied)))
def _which_program(prog):
"""Does an executable `prog` exist (PATH or the usual bindirs)?"""
if not prog:
return None
if "/" in prog:
return prog if os.access(prog, os.X_OK) else None
for d in os.environ.get("PATH", "").split(":") + [
"/usr/bin", "/usr/sbin", "/bin", "/sbin", "/usr/local/bin"]:
p = os.path.join(d, prog)
if os.path.isfile(p) and os.access(p, os.X_OK):
return p
return None
def _find_library(lib):
"""Does a shared library `lib` exist in the usual lib dirs?"""
if not lib:
return None
if "/" in lib:
return lib if os.path.exists(lib) else None
for d in ("/usr/lib", "/usr/lib64", "/lib", "/lib64", "/usr/local/lib",
"/usr/libexec"):
p = os.path.join(d, lib)
if os.path.exists(p):
return p
return None
_STACK_TARGETS = None # name -> (prog, have) from every stack file, once
_STACK_TARGET_SOURCE = {} # name -> the stack file the declaration came from
def _stack_declared_targets():
"""What every git/tar entry in every stack file says proves it installed.
A stack entry's `prog=` and `have=` are its own statement of what to
check. Until 1.14.87 they were not carried into the generated script, and
install_last -- the record of the script AS IT RAN -- keeps whatever it
was given at the time. Reading the stack files directly means verify
checks the right thing for an entry built under any version, without
rewriting a record. Read once per run; a dozen small files.
"""
global _STACK_TARGETS
if _STACK_TARGETS is not None:
return _STACK_TARGETS
out = {}
dirs = [os.environ.get("PM_STACK_DIR") or "/etc/pkgusr/stacks",
os.path.join(os.path.dirname(os.path.abspath(__file__)), "stacks")]
for d in dirs:
try:
names = sorted(os.listdir(d))
except OSError:
continue
for f in names:
if not f.endswith(".stack"):
continue
try:
rows = _parse_stack_file(os.path.join(d, f))
except (OSError, ValueError):
continue
for r in rows:
if r.get("kind") not in ("git", "tar"):
continue
o = r["opts"]
if not (o.get("prog") or o.get("have")):
continue
# ALL declarations, not the first: the user's stack in
# /etc/pkgusr/stacks is never overwritten (1.14.58), so a
# corrected prog=/have= shipped in a newer stack sits in a
# .new file beside it -- and the first-found rule meant the
# stale one always won, and the user was asked to hand-merge
# a metadata fix. Read every copy; prefer the one whose
# proof is actually on disk.
out.setdefault(r["target"], []).append(
(o.get("prog", ""), o.get("have", ""), os.path.join(d, f)))
# the .new files a protected install leaves beside the user's copies
# carry the shipped fixes; they count as declarations too
for f in names:
if not f.endswith(".stack.new"):
continue
try:
rows = _parse_stack_file(os.path.join(d, f))
except (OSError, ValueError):
continue
for r in rows:
if r.get("kind") not in ("git", "tar"):
continue
o = r["opts"]
if o.get("prog") or o.get("have"):
out.setdefault(r["target"], []).append(
(o.get("prog", ""), o.get("have", ""), os.path.join(d, f)))
chosen = {}
for name, alts in out.items():
pick = None
for prog, have, src in alts:
ok = True
if prog and not shutil.which(prog):
ok = False
if have and not os.path.exists(have):
ok = False
if ok:
pick = (prog, have, src)
break
if pick is None:
pick = alts[0] # nothing satisfied: report the first
chosen[name] = (pick[0], pick[1])
_STACK_TARGET_SOURCE[name] = pick[2]
_STACK_TARGETS = chosen
return chosen
def install_targets(meta):
"""Collect concrete things the install script says it installs:
programs, libraries, directories (from installed_program(s)/-libraries/
-director(y|ies))."""
# ONE SCALAR, TWO PATHS. The book writes "Installed Directories:
# /usr/include/opus and /usr/share/doc/opus-1.6.1", and both ended up in
# a single `installed_directory=`. The check then looked for a
# directory literally called
# "/usr/include/opus /usr/share/doc/opus-1.6.1"
# which of course does not exist -- so opus read "unvalidated: missing:
# dir /usr/include/opus ..." with /usr/include/opus sitting right there.
# Same for a program or library field with two names in it.
def _paths(v):
return [x for x in str(v or "").split() if x]
progs, libs, dirs = [], [], []
progs += _paths(meta.get("installed_program"))
for _p2 in meta.get("installed_programs", []):
progs += _paths(_p2)
for _l in meta.get("installed_libraries", []):
libs += _paths(_l)
dirs += _paths(meta.get("installed_directory"))
for _d in meta.get("installed_directories", []):
dirs += _paths(_d)
# A STACK ENTRY'S OWN DECLARATION WINS OVER A STALE RECORD.
#
# If the stack says what proves this package installed, use that: it is
# current, it is the entry's own word, and it is what a rebuilt script
# would carry anyway. A self-named program left in an old record
# (installed_program="libsrtp") is dropped in favour of it.
name = (meta.get("name") or "").strip()
decl = _stack_declared_targets().get(name) if name else None
if decl:
prog, have = decl
progs = [p for p in progs if p != name] + ([prog] if prog else [])
if have and have not in dirs:
dirs.append(have)
return ([p for p in progs if p], [l for l in libs if l],
[d for d in dirs if d])
def run_validate_cmd(user, cmd):
"""Run a package's validate_cmd AS the user; return its stdout (version), or
None on failure."""
if not cmd:
return None
run = ["su", "-", user, "-c", cmd] if _can_su(user) else ["bash", "-c", cmd]
try:
r = subprocess.run(run, capture_output=True, text=True, timeout=30)
except (OSError, subprocess.SubprocessError):
return None
return (r.stdout or r.stderr).strip() if r.returncode == 0 else None
def _version_number(name_version):
m = re.search(r"[0-9][0-9.]*[0-9]|[0-9]", name_version or "")
return m.group(0) if m else None
def assess(u, expected=None, run_cmd=True):
"""Layered install assessment. Returns (status, reason):
installed -- evidence + validated (files exist / version confirmed)
unvalidated -- evidence it's installed, but we could NOT confirm it
not installed / broken
`expected` is the book name_version, used for version checks. With
run_cmd=False the (possibly slow) validate command is skipped -- used for
bulk/short views."""
if not u.home and not u.has_user:
return "not installed", "no package-user and no home"
if not (u.home and u.has_user):
# NAME THE DIRECTORY. "root broken has a home dir but no user" sent
# someone looking for /usr/src/root, which is not where it was; the
# path is right here and is the whole of what they need to act.
return "broken", (("no account for the directory %s" % u.home)
if u.home else "has a user but no home dir")
m = u.meta
if not (m.get("name_version") or u._manifest_has_real_paths()):
return "not installed", "user + home exist, but nothing installed yet"
# 1) strongest signal: a validate command that prints the installed version.
# A per-package 'validate' file in the home persists across versions and
# wins over any validate_cmd inside the (regenerated) install script.
vfile = os.path.join(u.home or pkgusr_home(u.name or u.search),
"validate")
vcmd = None
if os.path.isfile(vfile):
vcmd = f"bash {shlex.quote(vfile)}"
elif m.get("validate_cmd"):
vcmd = m["validate_cmd"]
if vcmd and run_cmd:
out = run_validate_cmd(u.name or u.search, vcmd)
if out is None:
return "unvalidated", "validate command failed to run"
want = _version_number(expected or m.get("name_version"))
if want and want in out:
return "installed", f"validate -> {out.splitlines()[0][:60]} (matches {want})"
return "unvalidated", (f"validate -> {out.splitlines()[0][:60]} "
f"(expected {want})")
# 2) check the concrete targets the script says it installs
progs, libs, dirs = install_targets(m)
missing = []
for p in progs:
if not _which_program(p):
missing.append(f"program {p}")
for l in libs:
if not _find_library(l):
missing.append(f"lib {l}")
dir_version_ok = None
for d in dirs:
# exists, not isdir: a stack entry's `have=` is whatever proves the
# package landed -- often a .pc file or a header, not a directory --
# and it is recorded here (1.14.87). A path that is there is proof
# either way; a book's installed_directories are directories anyway.
if not os.path.exists(d):
missing.append(f"dir {d}")
elif expected: # versioned dir check
want = _version_number(expected)
if want and re.search(r"\d", os.path.basename(d)):
dir_version_ok = (want in d)
if progs or libs or dirs:
# A TARGET THAT WAS NEVER A REAL TARGET.
#
# Until 1.14.87 a stack entry's script got installed_program=<its own
# name>, so libsrtp, tllist and dejavu-fonts were checked for binaries
# that do not exist. Their install_last files still say that -- it is
# the record of the script AS IT RAN, and rewriting it would be a lie
# about history -- so verify would report "missing: program libsrtp"
# for a package that is perfectly installed. When the ONLY thing
# missing is a program named exactly like the package, and the
# manifest's files are on disk, the script named no real check.
_pkgname = (m.get("name") or "").strip()
if (missing and len(missing) == 1 and _pkgname
and missing[0] == f"program {_pkgname}"):
return "unvalidated", ("its script names no verifiable target"
" (rebuilt scripts declare one)")
# A DOC DIRECTORY NOBODY WAS GOING TO BUILD IS NOT A FAULT.
#
# /usr/share/doc/opus-1.6.1 exists only if Doxygen and texlive are
# installed, and this system deliberately has neither -- the
# generator comments those steps out. So the page names a directory
# that was never going to appear, and opus read "unvalidated"
# because of it while every header and library was in place.
# the path COMPONENT, not a /usr prefix: a doc directory is a doc
# directory under any prefix, and anchoring on /usr/share/doc made
# this untestable anywhere else
_docs = [x for x in missing
if x.startswith("dir ")
and ("/share/doc/" in x or "/share/gtk-doc" in x
or "/share/info" in x or "/share/man" in x)]
if _docs and len(_docs) == len(missing):
return "installed", ("docs not built (no doxygen/texlive here): %s"
% ", ".join(d[4:] for d in _docs[:2]))
if _docs:
missing = [x for x in missing if x not in _docs]
if missing:
# SAY WHERE THE EXPECTATION CAME FROM. "missing: dir
# /usr/include/wlr" sent the user to look for a directory; the
# useful fact was that /etc/pkgusr/stacks/sway.stack still said
# have=/usr/include/wlr while sway.stack.new beside it -- the
# shipped fix -- said the .pc file. The user's copy is never
# overwritten (1.14.58), so the place to fix it must be named.
_src = _STACK_TARGET_SOURCE.get(_pkgname)
_where = (" (declared by %s)" % _src) if _src else ""
return "unvalidated", "missing: " + ", ".join(missing[:4]) + (
" ..." if len(missing) > 4 else "") + _where
if dir_version_ok is False:
return "unvalidated", "installed dir version differs from the book"
return "installed", "all installed_* targets present" + (
" (dir version matches)" if dir_version_ok else "")
# 3) NOTHING TO CHECK IS NOT THE SAME AS A FAILED CHECK.
#
# Most pages name no installed_programs/libraries/directories, so this
# answered "unvalidated" for 226 of 242 packages on a healthy system --
# and "unvalidated" reads like a problem. A health check whose normal
# output is two hundred warnings is one nobody reads, and the twelve lines
# that DID matter were buried in it.
#
# The manifest has already been checked at this point: its paths exist
# (that is what got us here). So the honest answer is installed, with the
# limit stated -- and anything genuinely wrong still comes out above as
# "not installed", "broken" or a missing target.
if m.get("name_version"):
return "installed", (f"{m['name_version']}, recorded files present "
f"(the page names no targets to verify against)")
return "installed", ("recorded files present "
"(the page names no targets to verify against)")
def _install_evidence(u, expected=None):
return assess(u, expected)[1]
def cmd_verify(args):
"""Re-check install state for all (or named) package-users, showing WHY each
is/ isn't considered installed. --fix regenerates pkg.lst for users that own
files but lack a manifest (which can flip them to 'installed')."""
names = args.packages if args.packages else list_all_users()
# edit a package's persistent validate command in $EDITOR
if getattr(args, "set_validate_command", False):
if not args.packages:
printError("verify --set-validate-command needs a package name.")
return
for name in args.packages:
home = pkgusr_home(name)
if not os.path.isdir(home):
printError(f"{name}: no /usr/src/{name} -- add-user it first.")
continue
vfile = os.path.join(home, "validate")
if not os.path.isfile(vfile):
with open(vfile, "w") as f:
f.write(
"#!/bin/bash\n"
f"# Print the INSTALLED version of {name}.\n"
"# packagemanager marks the package 'installed' if this\n"
"# output contains the book's version number.\n"
"# Example:\n"
"# ffmpeg -version | grep -oP '(?<=version ).*(?= Copyright)'\n\n")
os.chmod(vfile, 0o755)
if pwd is not None:
try:
pw = pwd.getpwnam(name)
os.chown(vfile, pw.pw_uid, pw.pw_gid)
except KeyError:
pass
printInfo(f"opening {vfile} in {editor()} ...")
subprocess.run([editor(), vfile])
out = run_validate_cmd(name, f"bash {shlex.quote(vfile)}")
printInfo(f" test run -> {out if out else '(no output / failed)'}")
return
versions = book_versions_all()
printHeader("Verify install state")
print()
print(f" {colors.dim}installed = evidence + validated (installed_* files "
f"exist / validate_cmd matches);\n"
f" unvalidated = looks installed but couldn't confirm; "
f"not installed = nothing built{colors.normal}\n")
counts, _states = {}, {}
for name in sorted(set(names)):
u = gather_user(name)
anchor = u.meta.get("name") or name
st, why = assess(u, versions.get(anchor))
counts[st] = counts.get(st, 0) + 1
_states[name] = st
color = _STATE_COLOR.get(st, "")
print(f" {name:<24} {color}{st:<12}{colors.normal} {why}")
# A PIP MODULE INSTALLED BEFORE 1.14.37 HAS NO RECORD AT ALL.
#
# `packagemanager pip` wrote neither pkg.lst nor install_last, so the
# bootstrap set reads "nothing installed yet" while being in daily use.
# If the module IMPORTS, the account is real and only its paperwork is
# missing -- say so, and offer the one command that fixes it.
if st == "not installed" and u.has_user and u.home \
and not u.meta.get("name_version") and _python_module_here(name):
st, why = "unvalidated", (
"installed as a Python module, but with no record -- run:"
" packagemanager pip record %s" % name)
counts["not installed"] -= 1
counts[st] = counts.get(st, 0) + 1
_states[name] = st
color = _STATE_COLOR.get(st, "")
print(f" {name:<24} {color}{st:<12}{colors.normal} {why}")
continue
if args.fix and st == "not installed" and u.has_user and u.home \
and not u.meta.get("name_version"):
printInfo(f" fixing: regenerating pkg.lst for {name}...")
write_pkg_list(name, background=False)
print("\n " + " ".join(f"{k}: {v}" for k, v in sorted(counts.items())))
#
# `unvalidated` is the normal state for most of a system -- it means only
# that the page named no file to check, not that anything is wrong -- and
# on a real tree it is 226 of 242 lines. The dozen that need action
# scroll past inside it. Repeat them at the end, where the eye lands,
# with the command that fixes them.
# ONE PASS, NOT TWO. This re-assessed every package to find the ones to
# repeat -- a second validate command per package on a 242-package tree.
# The states were already computed in the loop above.
_act = [n for n, st in _states.items() if st in ("not installed", "broken")]
if _act:
print()
printWarning("%d package(s) need attention (everything else is fine "
"-- 'unvalidated' just means" % len(_act))
printWarning("the book named no file to check):")
for n in _act:
u = gather_user(n)
st, why = assess(u, None, run_cmd=False)
printWarning(" %-24s %-14s %s" % (n, st, why))
# ONLY WHAT THIS COMMAND CAN ACTUALLY BUILD.
#
# The list was pasted straight into an install command that could not
# work: collector groups and a stray `root` are not packages at all,
# and pkgusr / linux-headers are LFS steps with no BLFS page. The one
# bad name then killed the whole run (fixed in 1.14.36), but even
# without that, offering a command that cannot succeed is worse than
# offering none.
_vers = book_versions_all() or {}
_book = [n for n in _act if _vers.get(n) or _vers.get(n.lower())]
_other = [n for n in _act if n not in _book]
if _book:
printInfo(" Rebuild the BLFS ones:")
printInfo(" packagemanager install %s --run --recursive"
% " ".join(_book))
if _other:
printInfo(" Not BLFS packages, and not rebuilt by that command:")
printInfo(" " + ", ".join(_other))
printInfo(" LFS steps (linux-headers, pkgusr, ...) belong to the"
" chroot build: lfs-helper build <name> --force")
# NOT `packagemanager remove root`. For a directory with no
# account the fix is to delete the DIRECTORY, and suggesting a
# remove command against a name like `root` is an invitation to
# an accident.
printInfo(" Anything else is a directory or account with no"
" package behind it -- the line")
printInfo(" above names it; a leftover directory can simply be"
" removed.")
else:
printSuccess(" nothing needs attention.")
def cmd_info(args):
names = list_all_users() if args.packages == ["all"] else args.packages
if not names:
printError("No package-users to show. Try: packagemanager info all")
return
users = [gather_user(n) for n in names]
# short for `info all` (or -s); detailed for named packages (or -l)
if args.long:
short = False
elif args.short:
short = True
else:
short = args.packages == ["all"]
printHeader("Package-user info")
print()
versions = book_versions_all()
if short:
print_user_short(users, versions)
else:
for u in users:
print_user_details(u)
st, _ = assess(u, versions.get(u.meta.get("name") or u.name or u.search),
run_cmd=False)
if st in ("not installed", "unvalidated") and not args.no_book:
print_book_info(u.name or u.search)
# THREE BUGS IN TWO LINES, and `info` crashed after printing everything
# it was asked for:
# NameError: name '_states' is not defined
# `_states` was never initialised here (it belongs to verify's loop,
# where this was copied from), `name` does not exist in this scope --
# `anchor` does -- and the line was there twice. The dict is worth
# having: it is what names the packages that need attention.
counts, _states = {}, {}
for u in users:
anchor = u.meta.get("name") or u.name or u.search
st, _ = assess(u, versions.get(anchor), run_cmd=False)
counts[st] = counts.get(st, 0) + 1
_states[anchor] = st
print(f"\nTotal: {len(users)} " +
" ".join(f"{k}: {v}" for k, v in sorted(counts.items())))
# ...and say which ones, where the eye lands, rather than leaving the
# reader to match a count against the list above.
_act = [n for n, st in _states.items()
if st in ("not installed", "broken")]
if _act:
print()
printWarning("%d of these need attention: %s"
% (len(_act), ", ".join(sorted(_act))))
printInfo(" what and why: packagemanager verify %s" % " ".join(sorted(_act)))
# =========================================================================== #
# command: nimgnu (collector groups) [IMPLEMENTED]
# =========================================================================== #
#
# nimgnu_* groups let one package install into another package's directories:
# a shared dir is owned by group nimgnu_<x> and made group-writable, and every
# package-user that needs to write there is added to nimgnu_<x>. The prefix
# "nimgnu_" is fixed; the suffix is free-form. Mirrors the engine:
# create : add_package_user <g> <g> 10000 20000 <g> 10000 20000
# add : usermod -a -G <g> <pkg>
# remove : gpasswd -d <pkg> <g>
# Collector groups live at 90000+, by convention and by the sanity check.
COLLECTOR_GID_MIN = 90000
COLLECTOR_GID_MAX = 99000
# Application users (u_*) get their OWN range too. They were created with
# the package-user range, so u_claude took 10349 and the next package user
# started after it -- an application account in the middle of the package
# accounts, uid and private gid alike.
APP_UID_MIN = 30000
APP_UID_MAX = 39999
def nimgnu_name(name):
"""The collector group for a package or directory name.
ONE ACCOUNT, ONE PREFIX -- and a group named after a package must not carry
the package prefix as well. Given the owner `p_openssl` this returned
`nimgnu_p_openssl`, which the prompt did not even offer (it showed
`nimgnu_openssl`) and which the sanity report flags as outside every
convention. Strip the owner's prefix before adding this one.
"""
cp = collector_prefix()
if name.startswith(cp) or name.startswith(user_prefix()) or name == "audio":
return name
for pfx in (pkgusr_prefix(), cfguser_prefix()):
if pfx and name.startswith(pfx):
name = name[len(pfx):]
break
return cp + name
def _next_collector_gid():
"""The first free gid in the collector range.
`groupadd <name>` with no -g takes the next free system gid, which put a
collector group at 10093 -- in the middle of the package-user accounts,
where nothing expects it:
!! group nimgnu_p_openssl (gid 10093) is outside every convention
"""
used = set()
if grp is not None:
try:
used = {g.gr_gid for g in grp.getgrall()}
except Exception:
used = set()
for gid in range(COLLECTOR_GID_MIN, COLLECTOR_GID_MAX):
if gid not in used:
return gid
return None
def _create_collector_group(group):
"""Create a collector group IN ITS RANGE. True if it exists afterwards."""
if _group_exists(group):
return True
gid = _next_collector_gid()
cmd = [_cmd_path("groupadd")]
if gid is not None:
cmd += ["-g", str(gid)]
cmd.append(group)
return _run(cmd) or _group_exists(group)
def find_add_package_user():
return os.environ.get("ADD_PACKAGE_USER_BIN", "add_package_user").split()
# useradd/groupadd live in /usr/sbin, which is often absent from a non-login
# PATH -- look there explicitly rather than concluding they do not exist.
_SBIN_PATH = os.environ.get("PATH", "") + ":/usr/sbin:/sbin:/usr/local/sbin"
def _have_cmd(name):
return shutil.which(name, path=_SBIN_PATH) is not None
def _cmd_path(name):
return shutil.which(name, path=_SBIN_PATH) or name
PKGUSR_ETC = os.environ.get("PKGUSR_ETC", "/etc/pkgusr")
# The command wrappers, at the ONE location lfs-helper writes them to.
#
# This used to be spelled out inside the profile heredoc below while lfs-helper
# wrote its wrappers somewhere else entirely, so a package user's PATH led with
# a directory that did not exist. Same name, same variable, both tools.
WRAPPERS = os.environ.get("LFS_WRAPPERS", "/usr/lib/pkgusr")
# The shared environment every package user gets, as the package-users hint
# describes it: .bash_profile and .bashrc are SYMLINKS into a central directory
# so all package users share one environment, and `build` is the helper each
# package's build.conf is fed to. Creating a user without these leaves an
# account that cannot build anything.
_PKGUSR_BASH_PROFILE = """\
# /etc/pkgusr/bash_profile -- shared by every package user.
# Symlinked as ~/.bash_profile. Keep the environment identical for all of
# them, so a build behaves the same whichever package user runs it.
umask 022
export LC_ALL=POSIX
# /usr/local/bin comes FIRST after the wrappers: locally installed
# overrides belong ahead of /usr/bin, and a temporary wrapper there
# (the no-verify wget, say) is useless if nothing ever finds it.
PATH=@WRAPPERS@:/usr/local/bin:/usr/local/sbin:/usr/bin:/bin:/usr/sbin:/sbin
export PATH
# a short prompt that makes it obvious which package user you are
export PS1='\\u:\\w\\$ '
# where this package's sources live
export PKG_HOME="$HOME"
cd "$HOME" 2>/dev/null || true
[ -f "$HOME/build.conf" ] && . "$HOME/build.conf"
"""
_PKGUSR_BASHRC = """\
# /etc/pkgusr/bashrc -- shared by every package user.
# Symlinked as ~/.bashrc. Package users are not used for logging in, so there
# is no reason for login and non-login shells to differ: just source the
# profile, and `su pkg` and `su - pkg` then behave identically.
[ -f "$HOME/.bash_profile" ] && . "$HOME/.bash_profile"
"""
_PKGUSR_BUILD = """\
#!/bin/bash
# /etc/pkgusr/build -- shared by every package user, symlinked as ~/build.
#
# Reads build.conf from the package user's home and runs the usual
# unpack/configure/make/install sequence with it. It is deliberately small:
# anything unusual belongs in the package's own install script.
set -e
conf="${1:-$HOME/build.conf}"
[ -f "$conf" ] || { echo "no build.conf at $conf" >&2; exit 1; }
. "$conf"
: "${SRC_DIR:=$HOME}"
: "${CONFIGURE_OPTS:=--prefix=/usr}"
: "${MAKE_OPTS:=}"
cd "$SRC_DIR"
[ -x ./configure ] && ./configure $CONFIGURE_OPTS
make $MAKE_OPTS
make install
"""
_PKGUSR_BUILD_CONF = """\
# build.conf for package user '%(name)s'
#
# Read by ~/build (a symlink to /etc/pkgusr/build). Override anything you need
# and leave the rest alone.
# where the unpacked source is
#SRC_DIR="$HOME/%(name)s-1.0"
CONFIGURE_OPTS="--prefix=/usr"
MAKE_OPTS=""
"""
_PKGUSR_PROJECT = """\
%(name)s
Package user for '%(name)s'. Everything this package installed is owned by
this account; `packagemanager info %(name)s` lists it.
"""
def ensure_pkgusr_etc(verbose=True):
"""Make sure the shared package-user environment exists in /etc/pkgusr.
Copied from the host when this runs there; written from scratch otherwise
(a fresh LFS system has no /etc/pkgusr at all)."""
made = []
try:
os.makedirs(PKGUSR_ETC, exist_ok=True)
except OSError as e:
printError(f"cannot create {PKGUSR_ETC}: {e}")
return False
for fname, content, mode in (
# @WRAPPERS@ substituted here, not written into the template:
# the template is a plain string so $HOME and \\u reach the file
# untouched, and the wrapper path is the one value that is ours.
("bash_profile", _PKGUSR_BASH_PROFILE.replace("@WRAPPERS@", WRAPPERS), 0o644),
("bashrc", _PKGUSR_BASHRC, 0o644),
("build", _PKGUSR_BUILD, 0o755)):
path = os.path.join(PKGUSR_ETC, fname)
if os.path.exists(path):
continue
with open(path, "w") as f:
f.write(content)
os.chmod(path, mode)
made.append(fname)
if made and verbose:
printInfo(f"created the shared package-user environment in "
f"{PKGUSR_ETC}: {', '.join(made)}")
return True
def init_package_user_home(name, home=None, verbose=True):
name = pkgusr_name(name) # the home belongs to the ACCOUNT, not the package
"""Give a package user the environment the scheme expects.
Without this the account exists but has no .bash_profile, no .bashrc and no
build helper -- `su - <pkg>` lands in a bare shell with the wrong umask and
no PATH to the wrapper commands."""
home = home or pkgusr_home(name)
if not ensure_pkgusr_etc(verbose=verbose):
return False
try:
os.makedirs(home, exist_ok=True)
except OSError as e:
printError(f"cannot create {home}: {e}")
return False
# symlinks to the shared files, so changing one changes them all
for link, target in ((".bash_profile", "bash_profile"),
(".bashrc", "bashrc"),
("build", "build")):
dst = os.path.join(home, link)
src = os.path.join(PKGUSR_ETC, target)
try:
if os.path.islink(dst) or os.path.exists(dst):
if os.path.islink(dst) and os.readlink(dst) == src:
continue
os.remove(dst)
os.symlink(src, dst)
except OSError as e:
printWarning(f" {dst}: {e}")
# per-package files
for fname, content in (("build.conf", _PKGUSR_BUILD_CONF % {"name": name}),
(".project", _PKGUSR_PROJECT % {"name": name})):
path = os.path.join(home, fname)
if not os.path.exists(path):
with open(path, "w") as f:
f.write(content)
_run(["chown", "-R", "-h", f"{name}:{name}", home])
_run([_cmd_path("chmod"), "0750", home]) if _have_cmd("chmod") else None
if verbose:
printInfo(f" initialised {home} "
f"(.bash_profile, .bashrc, build -> {PKGUSR_ETC}; build.conf)")
return True
def create_package_user(name, home=None):
"""Create a package user, with or without the hint's add_package_user.
add_package_user comes from the package-users hint's helper tarball, which
a fresh LFS system does not have -- and without a fallback nothing can be
installed as a package user at all. Shadow's own tools do the same job:
one user, a private group of the same name, plus the supplementary
'install' group."""
# Callers hand us a PACKAGE name ("gcc"); the account is p_gcc. Normalise
# once here rather than at each call site -- pkgusr_name is idempotent, so
# a caller that already prefixed is unaffected.
name = pkgusr_name(name)
if _user_exists(name):
return True
home = home or pkgusr_home(name)
if _have_cmd(find_add_package_user()[0]):
cmd = find_add_package_user() + [name, name, "10000", "20000",
name, "10000", "20000"]
printInfo(f"creating package-user {name}")
if _run(cmd):
# THE RECORDED HOME MUST BE THE REAL ONE.
#
# add_package_user takes no home argument, so the hint's script
# uses its own default -- /usr/src/<name>. init_package_user_home
# then created and chowned the home this tool computes,
# /usr/src/pkgusr/p_name, and the two never met. A real run said
# both in the same report:
# $ chown -R -h p_urllib3:p_urllib3 /usr/src/pkgusr/p_urllib3
# home directory created: /usr/src/p_urllib3
# An account whose passwd entry points somewhere its files are not
# lands `su - p_urllib3` in a directory that does not exist.
#
# And repointing is only HALF the repair. usermod -d moves the
# entry, not the directory -- so every bootstrap left a husk
# beside the roots (/usr/src/p_wget, /usr/src/p_requests, ...)
# holding the skeleton add_package_user had copied in. lfs-helper
# fix-home is the one implementation of the whole repair: repoint,
# merge the stray into the real home, delete nothing that differs.
if shutil.which("lfs-helper"):
_run(["lfs-helper", "fix-home", name, "--run"])
else:
_relocate_stray_home(name, home)
_fix_recorded_home(name, home)
init_package_user_home(name, home)
return True
printWarning("add_package_user failed -- falling back to useradd")
if not _have_cmd("useradd"):
printError("neither add_package_user nor useradd is available; "
"cannot create package users on this system")
return False
if not _group_exists("install"):
printInfo("creating the 'install' group")
_run([_cmd_path("groupadd"), "-g", "9999", "install"])
printInfo(f"creating package-user {name} (useradd)")
# -U makes the private group alongside the user so uid and gid match;
# -K keeps package users in their own id range
ok = _run([_cmd_path("useradd"), "-c", f"package {name}", "-d", home, "-U",
"-G", "install", "-s", "/bin/bash",
"-K", "UID_MIN=10000", "-K", "GID_MIN=10000", name])
if ok:
init_package_user_home(name, home)
return ok
def _relocate_stray_home(name, home):
"""Move the directory add_package_user just created onto the real home.
Fallback only -- lfs-helper fix-home is the full repair (it can merge).
This handles the clean case: the wrong home exists, the right one does
not yet, so the whole directory simply moves. Anything messier is left
for fix-home and reported by verify."""
try:
recorded = pwd.getpwnam(name).pw_dir
except (KeyError, TypeError):
return
if os.path.normpath(recorded) == os.path.normpath(home):
return
if os.path.isdir(recorded) and not os.path.exists(home):
try:
os.makedirs(os.path.dirname(home), exist_ok=True)
shutil.move(recorded, home)
printInfo(f" {name}: moved {recorded} -> {home}")
except OSError as e:
printWarning(f" {name}: could not move {recorded} -> {home}: {e}")
elif os.path.isdir(recorded):
printWarning(f" {name}: both {recorded} and {home} exist -- "
f"merge with: lfs-helper fix-home {name} --run")
def _fix_recorded_home(name, home):
"""Point the account's passwd entry at the home this tool actually uses."""
try:
recorded = pwd.getpwnam(name).pw_dir
except (KeyError, TypeError):
return
if os.path.normpath(recorded) == os.path.normpath(home):
return
if not _have_cmd("usermod"):
printWarning(f" {name}: home is recorded as {recorded}, but its files "
f"are in {home} -- and usermod is not available to fix it")
return
if _run([_cmd_path("usermod"), "-d", home, name]):
printInfo(f" {name}: home corrected to {home} (was {recorded})")
else:
printWarning(f" {name}: could not correct the recorded home "
f"({recorded} -> {home})")
def _group_exists(name):
try:
grp.getgrnam(name)
return True
except (KeyError, TypeError):
return False
def _user_exists(name):
"""Does this account exist? Accepts an account name or a package name --
the raw name is tried first, so a real account ('root', 'lfs') is never
reinterpreted as a package."""
for n in (name, pkgusr_name(name)):
try:
pwd.getpwnam(n)
return True
except (KeyError, TypeError):
continue
return False
def _in_group(user, group):
try:
return user in grp.getgrnam(group).gr_mem
except (KeyError, TypeError):
return False
def cmd_nimgnu_list(args):
if grp is None:
printError("no group database available.")
return
rows = [(g.gr_name, g.gr_gid, sorted(g.gr_mem)) for g in grp.getgrall()
if g.gr_name.startswith(collector_prefix())]
if args.filter:
needle = args.filter.lower()
rows = [r for r in rows if needle in r[0].lower()]
rows.sort()
printHeader("collector groups")
print()
for name, gid, members in rows:
manifest = os.path.join(pkgusr_home(name), "pkg.lst")
entries = pkglist_entries(manifest)
owns = f" owns {len(entries)} dirs/files" if os.path.isfile(manifest) else ""
print(f" {name:<26} gid {gid:<8} members: "
f"{', '.join(members) if members else '-'}{owns}")
if args.long:
if members:
print(f" members : {', '.join(members)}")
if entries:
print(f" manifest: {manifest}")
for e in entries[:40]:
print(f" {e}")
if len(entries) > 40:
print(f" ... (+{len(entries) - 40} more)")
else:
print(" (no manifest yet -- assign a dir with "
"'add-dir-to-sysgroup', or run 'reload-pkg-list "
f"{name}')")
print(f"\n{len(rows)} group(s).")
def cmd_nimgnu_create(args):
g = nimgnu_name(args.name)
if _group_exists(g):
printInfo(f"{g}: already exists.")
return
printHeader(f"Create collector {g}")
print()
# A GROUP, IN THE COLLECTOR RANGE -- no account. This ran
# the package-user helper with the package range, which made a login account
# named after the group (home /usr/src/pkgusr/p_nimgnu_python-modules)
# and a gid in the middle of the package users (10096). Every other
# collector group comes from _create_collector_group at 90000+.
if _create_collector_group(g):
printSuccess(f"{g} created (gid {grp.getgrnam(g).gr_gid if grp else '?'}).")
printInfo(f" assign directories to it with: "
f"packagemanager add-dir-to-sysgroup {args.name} <dir>")
else:
printError(f"could not create {g}")
return 1
def cmd_nimgnu_add(args):
g = nimgnu_name(args.group)
if not _group_exists(g):
printError(f"{g}: no such collector group "
f"(create it first: packagemanager sysgroup create {args.group}).")
return
for pkg in args.users:
if not _user_exists(pkg):
printError(f"{pkg}: no such package-user.")
continue
if _in_group(pkg, g):
printInfo(f"{pkg} is already a member of {g}.")
continue
if _run(["usermod", "-a", "-G", g, pkg]):
printSuccess(f"Added {pkg} to {g} -- {pkg} may now install into "
f"directories owned by group {g}.")
def cmd_nimgnu_remove(args):
g = nimgnu_name(args.group)
for pkg in args.users:
if not _in_group(pkg, g):
printInfo(f"{pkg} is not a member of {g}.")
continue
if _run(["gpasswd", "-d", pkg, g]):
printSuccess(f"Removed {pkg} from {g}.")
def cmd_nimgnu_delete(args):
g = nimgnu_name(args.name)
if not g.startswith(collector_prefix()):
printError(f"refusing to delete '{g}': not a collector (sysgroup) group.")
return
has_group, has_user = _group_exists(g), _user_exists(g)
if not has_group and not has_user:
printInfo(f"{g}: does not exist.")
return
members = sorted(grp.getgrnam(g).gr_mem) if has_group else []
printHeader(f"Delete collector {g}")
print()
if has_group:
print(f" group : {g} (gid {grp.getgrnam(g).gr_gid})")
if has_user:
print(f" user : {g} (uid {pwd.getpwnam(g).pw_uid})")
if members:
printWarning(f" {len(members)} member(s) still in the group: "
f"{', '.join(members)}")
print(" (they will simply lose this group; their own files are untouched)")
if not args.run:
printInfo("\nDry run -- re-run with --run to delete.")
return
if not args.yes and not ask_yes_no(f"\nReally delete collector group {g}?"):
printWarning("Aborted.")
return
# give every node owned by this group back to its file-owner's own group,
# so nothing is left with a dangling gid after the group is deleted.
printInfo(f"Reassigning nodes owned by group {g} to their owners' groups "
f"(runs 'find /', may take a while)...")
forall = " ".join(shlex.quote(x) for x in find_forall())
reassign = (f"{forall} {shlex.quote(g)} -exec bash -c "
f"'for f in \"$@\"; do chgrp \"$(stat -c %U \"$f\")\" \"$f\" "
f"2>/dev/null; done' _ {{}} +")
subprocess.run(["sh", "-c", reassign], stderr=subprocess.DEVNULL)
if has_user:
_run(["userdel", g])
if _group_exists(g):
_run(["groupdel", g])
printSuccess(f"{g} deleted.")
# =========================================================================== #
# command: make-group-dir / fix-group-dir (collector & install dirs)
# =========================================================================== #
#
# Two flavours of shared directory (per the package-user hint):
# install dir : chgrp install <dir> ; chmod g+w,o+t <dir> (STICKY)
# group-writable so package-users can add files, sticky so they
# can't remove each other's. Files stay user:user (install is
# a supplementary group), so recursion only regroups DIRS.
# nimgnu dir : chgrp nimgnu_<x> <dir> ; chmod g+rwx <dir> (group rwx, no setgid)
# new files/dirs inherit group nimgnu_<x> (owner = creator), so
# members of nimgnu_<x> can install here. Recursion regroups
# files too (they belong to the collector group).
#
# Each nimgnu group also gets a manifest at /usr/src/nimgnu_<x>/pkg.lst listing
# everything connected to the group (forall_direntries_from by group), refreshed
# whenever we (re)assign a directory to it.
def _dir_group_name(path):
try:
return grp.getgrgid(os.stat(path).st_gid).gr_name
except (KeyError, OSError, TypeError):
return None
def _perm_kind(group):
if group == "install":
return "g+w,o-t", "group-writable" # no sticky (per your call)
if group.startswith(collector_prefix()):
return "g+rwx,o-t", "group-rwx" # group rwx, NO setgid
return "g+w", "group-writable"
def _assign_group_dir(path, group, recursive):
"""chgrp + canonical mode for one dir (optionally its tree)."""
mode, _ = _perm_kind(group)
is_nimgnu = group.startswith(collector_prefix())
if recursive:
if is_nimgnu:
_run(["chgrp", "-R", group, path]) # files + dirs
else:
_run(["find", path, "-type", "d", "-exec", "chgrp", group, "{}", "+"])
_run(["find", path, "-type", "d", "-exec", "chmod", mode, "{}", "+"])
else:
_run(["chgrp", group, path])
_run(["chmod", mode, path])
def _resolve_group_arg(value):
"""'install' stays install; anything else becomes a nimgnu_ group."""
if value == "install":
return "install"
return nimgnu_name(value)
def write_group_manifest(group, background=True):
"""Write /usr/src/<group>/pkg.lst = everything owned by that group, via
forall_direntries_from (run as root so the whole tree is scanned)."""
home = pkgusr_home(group)
if not os.path.isdir(home):
return False
forall = " ".join(shlex.quote(x) for x in find_forall())
tmp = shlex.quote(os.path.join(home, "pkg.lst.new"))
dst = shlex.quote(os.path.join(home, "pkg.lst"))
inner = f"{forall} {shlex.quote(group)} > {tmp} 2>/dev/null && mv {tmp} {dst}"
cmd = ["sh", "-c", inner]
if background:
subprocess.Popen(cmd, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL, start_new_session=True)
return True
return subprocess.run(cmd).returncode == 0
# Directories that are shared by nature, matched as patterns.
#
# Kept in step with lfs-helper's list: many packages drop a single file into
# /usr/share/zsh/site-functions or /usr/share/applications, so treating those
# as belonging to whichever package created them first makes every later
# package fail there. /usr/libexec is shared; /usr/libexec/p11-kit is not.
SHARED_DIR_PATTERNS = [
"/usr/share/man/man[1-9]", "/usr/share/man/*/man[1-9]", "/usr/share/man/*",
"/usr/share/locale/*/LC_MESSAGES", "/usr/share/locale/*",
"/usr/share/info", "/usr/share/doc",
"/usr/lib/pkgconfig", "/usr/share/pkgconfig",
"/usr/share/bash-completion/completions",
"/usr/share/zsh/site-functions", "/usr/share/zsh/vendor-completions",
"/usr/share/fish/completions", "/usr/share/fish/vendor_completions.d",
"/usr/share/applications", "/usr/share/mime/packages",
"/usr/share/dbus-1/services", "/usr/share/dbus-1/system-services",
"/usr/share/metainfo", "/usr/share/appdata",
"/usr/share/icons/hicolor/*", "/usr/share/pixmaps",
"/usr/share/glib-2.0/schemas", "/usr/share/vala/vapi",
"/usr/share/gir-1.0", "/usr/lib/girepository-1.0",
"/usr/share/aclocal", "/usr/share/gtk-doc/html",
"/usr/lib/systemd/system", "/usr/lib/udev/rules.d",
"/etc/xdg/autostart", "/usr/libexec",
# /etc/profile.d is shared by nature: rust, bash-completion, gettext and
# the book's own config pages each drop ONE file in it. Without this the
# first package to create the directory owns it -- p_libnice here -- and
# every later package has to be granted it one at a time.
"/etc/profile.d",
"/usr/share/terminfo/*",
]
def is_shared_dir(path):
import fnmatch
p = path.rstrip("/")
return any(fnmatch.fnmatch(p, pat) for pat in SHARED_DIR_PATTERNS)
# One cheap pre-filter, then the precise patterns.
#
# These run over a whole build log, which for something like p11-kit is tens of
# megabytes. Running five regexes across all of it -- once per retry round --
# burned ~90 seconds of CPU and made the machine crawl. So: only read the tail
# (errors are at the end), skip any line without the words we care about, and
# anchor the patterns per line so there is nothing to backtrack over.
# p11-kit reports errno 13 as a NUMBER -- "Unknown error 13" -- so a filter
# looking for the word "denied" skips the line entirely:
# p11-kit: couldn't create file: /etc/ssl/certs/X.pem: Unknown error 13
# 13 is EACCES. make-ca then exits 0 having written the other stores, so
# nothing else flags it either.
_PERM_HINT = ("Permission denied", "Operation not permitted", "PermissionError",
"Unknown error 13", "Errno 13")
# THE QUOTES ARE NOT ALWAYS APOSTROPHES.
#
# coreutils quotes paths with the locale's quoting style, which under a UTF-8
# locale is U+2018/U+2019 -- so the real failure
#
# install: cannot change permissions of '/usr/sbin': Operation not permitted
#
# matched nothing, no directory was extracted, and the auto-grant that exists
# precisely for this case never fired. packagemanager printed the manual
# command instead, as though it had no opinion.
_Q = r"['\u2018\u2019\"\u201c\u201d`]"
_PERM_PATTERNS = [
re.compile(r"PermissionError[^'\u2018]*" + _Q + r"([^'\u2018\u2019\"]+)" + _Q),
re.compile(r"(?:Permission denied|Operation not permitted)[:,]?\s*" + _Q
+ r"([^'\u2018\u2019\"]+)" + _Q),
# `change permissions` was missing, which is the one install(1) uses when
# the directory already exists and only its mode cannot be set -- the
# commonest way a package meets a directory another package owns.
re.compile(r"cannot (?:create|remove|touch|open|change permissions of)"
+ r"[^'\u2018]*" + _Q + r"([^'\u2018\u2019\"]+)" + _Q),
re.compile(r"^\s*(/[^\s:]+):\s*Permission denied"),
# p11-kit, and anything else that prints "<verb>: /path: <errno text>"
re.compile(r"couldn't (?:create|write|open)[^/]*(/[^\s:]+)"),
re.compile(r"(/[^\s:]+):\s*(?:Unknown error 13|Errno 13)"),
]
_LOG_TAIL_BYTES = 512 * 1024 # the error is never at the start
def _paths_from_permission_errors(log_path, max_paths=20):
if not log_path or not os.path.isfile(log_path):
return []
try:
size = os.path.getsize(log_path)
with open(log_path, errors="replace") as f:
if size > _LOG_TAIL_BYTES:
f.seek(size - _LOG_TAIL_BYTES)
f.readline() # drop the partial first line
lines = f.readlines()
except OSError:
return []
seen, out = set(), []
for line in lines:
# the pre-filter: a plain substring test skips ~all of a build log
if not any(h in line for h in _PERM_HINT):
continue
for pat in _PERM_PATTERNS:
m = pat.search(line)
if m:
path = m.group(1)
if path.startswith("/") and path not in seen:
seen.add(path)
out.append(path)
break
if len(out) >= max_paths:
break
return out
def _dir_of(path):
return path if os.path.isdir(path) else os.path.dirname(path)
def _user_can_write(user, path):
try:
r = subprocess.run(["su", "-s", "/bin/bash", user, "-c",
"test -w %s" % shlex.quote(path)],
capture_output=True)
return r.returncode == 0
except OSError:
return False
def auto_grant_from_log(user, log_path, verbose=True):
"""Give `user` the access a failed install needed, and say what changed.
This is what lfs-helper does during the base build, and there is no reason
for packagemanager to behave differently afterwards: a package that has to
write into a directory another package owns joins that directory's
collector group.
Returns the list of directories granted (empty if there was nothing to do).
"""
granted = []
dirs = []
for p in _paths_from_permission_errors(log_path):
d = _dir_of(p)
if d and os.path.isdir(d) and d not in dirs:
dirs.append(d)
if not dirs:
return granted
for d in dirs:
if _user_can_write(user, d):
continue # already fine; something else failed
cur = None
try:
cur = grp.getgrgid(os.stat(d).st_gid).gr_name
except (KeyError, OSError):
pass
# ALREADY IN THE GROUP: granting again cannot be the answer.
#
# /usr/sbin is root:install and the user is in install, so it could
# create files there -- the failure was `install -vdm755` trying to
# CHMOD a directory root owns, which is the wrappers' job, not the
# group's. Granting a group the user already has, twice, looks like
# progress and is not.
try:
if cur and cur in user_supplementary_groups(user):
printWarning(f" {user} is already in group '{cur}' on {d} --")
printWarning( " this is not a group problem. A book command like")
printWarning( " install -vdm755 " + d)
printWarning( " chmods a directory root owns, which no package user")
printWarning( " may do; the package-user wrappers make it succeed and")
printWarning( " change nothing. Check them with:")
printWarning( " lfs-helper make-wrappers --run")
continue
except Exception:
pass
if is_shared_dir(d) and cur != "install":
# shared by nature: it belongs to the install group, and every
# package user is already in that -- no collector group needed
if verbose:
printInfo(f" {d} is a shared directory -> group install")
_assign_group_dir(d, "install", recursive=False)
granted.append((d, "install"))
continue
if cur == "install" or (cur and cur.startswith(collector_prefix())):
# The directory is ALREADY shared -- the package just isn't in the
# group yet. Adding it is the whole fix; don't touch the directory.
group = cur
if verbose:
printInfo(f" {d} is already group {group} -- adding '{user}' to it")
else:
owner = "root"
try:
owner = pwd.getpwuid(os.stat(d).st_uid).pw_name
except (KeyError, OSError):
pass
group = nimgnu_name(owner if owner != "root" else os.path.basename(d))
if not _group_exists(group):
if verbose:
printInfo(f" creating collector group {group}")
if not _create_collector_group(group):
continue
if owner not in ("root", "") and _user_exists(owner):
_run([_cmd_path("usermod"), "-a", "-G", group, owner])
_assign_group_dir(d, group, recursive=False)
if verbose:
printInfo(f" {d} -> group {group} (owner {owner})")
if not _run([_cmd_path("usermod"), "-a", "-G", group, user]):
continue
granted.append((d, group))
if granted and verbose:
printWarning("\nCollector groups changed so '%s' can install:" % user)
for d, g in granted:
print(f" {d:<44} -> {g}")
print(" These are permanent; review with: "
f"getent group | grep '^{collector_prefix()}'")
return granted
def _latest_log(name):
"""The log the most recent install run wrote."""
d = _pkg_log_dir(name)
try:
logs = [os.path.join(d, f) for f in os.listdir(d) if f.endswith(".log")]
except OSError:
return None
return max(logs, key=os.path.getmtime) if logs else None
def _run_pm_install_once(name, path, env):
"""Run the install engine exactly once, with no recovery.
THE ENGINE IS HANDED AN ACCOUNT NAME, NOT A PACKAGE NAME.
Argument 1 is used by packagemanager_install for exactly one thing: the
user to create, own the files and run the build as. This passed the bare
package name, so the engine did
id wget || addUser wget
and created an account literally called `wget`, living in /usr/src/wget --
no prefix, no subdirectory, colliding with any real account of that name.
A fresh tree proved it: `id wget` said no such user until the install ran.
Half this codebase asks for the name -- pkgusr_name, owner-name,
pkgusr-home -- and this one call assumed it. Same species as
`chown: invalid user: 'wget:wget'` and `usermod: user 'requests' does not
exist`. pkgusr_name is idempotent, so a caller that already resolved is
unaffected.
"""
# Staging the downloaded tarball used to happen here, and could not: for a
# new package the home does not exist yet, because the engine creates it.
# It lives in packagemanager_install now, after the account is made.
cmd = find_pm_install() + [pkgusr_name(name), path]
printInfo(" $ " + " ".join(shlex.quote(c) for c in cmd))
try:
return subprocess.run(cmd, env=env)
except FileNotFoundError:
printError(f" {cmd[0]}: command not found")
return subprocess.CompletedProcess(cmd, 127)
def run_pm_install(name, path, env, auto_fix=True, max_rounds=4):
"""Run the install engine, granting access and retrying on a permission
failure -- the same recovery lfs-helper does during the base build.
An install reveals only the FIRST directory it cannot write, so one grant
is rarely enough: a package that drops files into several shared places
fails again on the next one. Keep going while each round actually grants
something, bounded so a real build error cannot loop.
Note the two functions: the retry wrapper must call the RAW runner, never
itself. Calling itself is an infinite recursion that ends in
RecursionError: maximum recursion depth exceeded
before the install is even attempted."""
result = _run_pm_install_once(name, path, env)
rounds = 0
while result.returncode != 0 and auto_fix and rounds < max_rounds:
log = _latest_log(name)
# THE ACCOUNT, NOT THE PACKAGE. This passed `make-ca`, so the grant
# it worked out was correct and then applied to a user that does not
# exist:
# usermod: user 'make-ca' does not exist
# Third time this species has surfaced in this file.
granted = auto_grant_from_log(pkgusr_name(name), log)
if not granted:
break # nothing to grant: a real build error
rounds += 1
printInfo(f"\nretrying {name} (round {rounds}) ...")
result = _run_pm_install_once(name, path, env)
if result.returncode != 0 and rounds >= max_rounds:
printWarning(f"gave up after {max_rounds} permission grants -- "
f"something else is wrong")
if result.returncode == 0:
warn_if_nothing_installed(name)
return result
def warn_if_nothing_installed(name):
"""A package that installs nothing has usually failed quietly.
pip is the worst offender -- it can report success while writing nowhere
useful -- but any build whose install step is a no-op leaves a package user
owning nothing, and the problem only surfaces later when something tries to
use it."""
pkglist = os.path.join(pkgusr_home(name), "pkg.lst")
n = 0
if os.path.isfile(pkglist):
try:
with open(pkglist) as f:
n = sum(1 for line in f if line.strip())
except OSError:
return
else:
return # manifest not written yet: nothing to judge
if n == 0:
printWarning(
f"\n!! {name} reported success but owns no files.\n"
f" That normally means the install silently did nothing.\n"
f" Check before trusting it: packagemanager info {name}\n")
def cmd_make_group_dir(args):
group = _resolve_group_arg(args.group)
if group != "install" and not _group_exists(group):
printError(f"{group}: no such group "
f"(create it: packagemanager sysgroup create {args.group}).")
return
_, kind = _perm_kind(group)
printHeader(f"Make {'install' if group == 'install' else group} dir "
f"({kind})")
print()
for d in args.dirs:
if not os.path.isdir(d):
printError(f"{d}: not a directory.")
continue
_assign_group_dir(d, group, args.recursive)
printSuccess(f"{d} -> group {group}, {kind}"
+ (" (recursive)" if args.recursive else ""))
if group.startswith(collector_prefix()):
printInfo(f"Refreshing {group} manifest in the background...")
write_group_manifest(group, background=True)
def cmd_fix_group_dir(args):
forced = _resolve_group_arg(args.group) if args.group else None
if forced and forced != "install" and not _group_exists(forced):
printError(f"{forced}: no such group.")
return
touched_nimgnu = set()
for d in args.dirs:
if not os.path.isdir(d):
printError(f"{d}: not a directory.")
continue
targets = [d]
if args.recursive:
for root, dirs, _files in os.walk(d):
targets.extend(os.path.join(root, sub) for sub in dirs)
fixed = 0
for path in targets:
group = forced or _dir_group_name(path)
if not group:
continue
if group != "install" and not group.startswith(collector_prefix()) \
and not forced:
continue # leave unrelated dirs alone
mode, _ = _perm_kind(group)
if forced:
_run(["chgrp", group, path])
_run(["chmod", mode, path])
if group.startswith(collector_prefix()):
touched_nimgnu.add(group)
fixed += 1
printSuccess(f"{d}: re-applied permissions to {fixed} dir(s)"
+ (" (recursive)" if args.recursive else ""))
for group in touched_nimgnu:
write_group_manifest(group, background=True)
def _todo(name):
def run(args):
printWarning(f"'{name}' is not implemented yet -- coming next.")
sys.exit(2)
return run
# =========================================================================== #
# command: check [IMPLEMENTED]
# =========================================================================== #
#
# Report the install state of one or more packages and, when possible, compare
# the installed version (from install_last) against the current book version
# (via `blfs debug`). The package's book anchor is taken from install_last's
# recorded name when present, else the package-user name.
#
# Result per package (also reflected in the exit code):
# installed (up to date) user+home+pkg.lst, installed == book
# update: X -> Y installed, but book has a different version
# installed installed, book version unknown/unchecked
# not installed user+home exist, no pkg.lst yet
# broken user/dir inconsistency
# Exit code: 0 if every requested package is installed & up to date, else 1.
def check_one(name, use_book=True):
"""Return (User, book_ver, status) for one package."""
u = gather_user(name)
anchor = u.meta.get("name") or name
book_ver = book_version(anchor) if use_book else None
if u.state == "broken":
status = "broken"
elif u.state == "not installed":
status = "not installed"
elif book_ver and u.version and book_ver != u.version:
status = f"update: {u.version} -> {book_ver}"
elif book_ver and u.version and book_ver == u.version:
status = "installed (up to date)"
else:
status = "installed"
return u, book_ver, status
def _status_color(status):
if status.startswith("update:"):
return colors.cyan
if status.startswith("installed"):
return colors.green
if status == "not installed":
return colors.yellow
return colors.red # broken
def cmd_check(args):
all_ok = True
for name in args.packages:
u, book_ver, status = check_one(name, use_book=not args.no_book)
if not status.startswith("installed"): # update:/not installed/broken
all_ok = False
color = _status_color(status)
tail = ""
if u.version and status in ("installed", "broken", "not installed"):
tail = f" [{u.version}]"
print(f"{name:<20} {color}{status}{colors.normal}{tail}")
sys.exit(0 if all_ok else 1)
# =========================================================================== #
# command: search [IMPLEMENTED]
# =========================================================================== #
#
# `blfs search` hits, each annotated with the book version (the title), the
# local install state, and the installed version when present.
_SEARCH_RE = re.compile(r"^\s*\[(package|module)\s*\]\s+(\S+)\s+(.+?)\s*$")
def install_tag(anchor, book_version_str):
"""Colored 'installed/update/not installed/broken' tag for an anchor whose
current book version is book_version_str."""
u = gather_user(anchor)
if u.state == "broken":
return colors.red + "broken" + colors.normal
if u.state != "installed":
return colors.yellow + "not installed" + colors.normal
if u.version and book_version_str and u.version != book_version_str:
# "the book has moved on", not "this will be updated": nothing in a
# stack run changes an installed package.
return (colors.cyan + f"installed {u.version} (book is newer)"
+ colors.normal)
return colors.green + f"installed {u.version or ''}".rstrip() + colors.normal
def cmd_search(args):
rc, out, err = run_blfs(["search"] + args.packages)
if rc == 127:
printError("blfs not found (set BLFS_BIN or put blfs on PATH).")
sys.exit(1)
hits = 0
for line in out.splitlines():
m = _SEARCH_RE.match(line)
if not m:
continue # blfs summary / no-match
hits += 1
kind, anchor, book_ver = m.group(1), m.group(2), m.group(3)
print(f" [{kind:<7}] {anchor:<26} {book_ver:<24} "
f"{install_tag(anchor, book_ver)}")
if hits == 0:
msg = (out or err).strip()
print(msg if msg else
f"No package matching {', '.join(args.packages)!r}.")
# =========================================================================== #
# command: dependencies [IMPLEMENTED]
# =========================================================================== #
#
# Recursive dependency order, each line annotated with local install state and,
# for dependency packages, HOW it was pulled in: the kind (required/recommended/
# optional) and the parent package that required it. --tree shows blfs's tree
# verbatim; --no-recommended / --optional pass through.
def blfs_order_anchors(name, no_recommended=False, optional=False, ignore=None):
"""Return (order, missing): order is [(anchor, name_version, kind, parent)],
missing is [(name, kind, parent)], via blfs order --anchors.
`ignore` is passed to the WALK, not applied to the result: a package that
is not built should not drag its own dependencies in behind it.
"""
argv = ["order", name, "--anchors"]
if no_recommended:
argv.append("--no-recommended")
if optional:
argv.append("--optional")
if ignore:
argv += ["--ignore", ",".join(sorted(ignore))]
rc, out, err = run_blfs(argv)
if rc == 127:
printError("blfs not found (set BLFS_BIN or put blfs on PATH).")
sys.exit(1)
if rc != 0:
# ONE BAD NAME MUST NOT KILL THE WHOLE COMMAND.
#
# `blfs order` exits 1 for a name the book does not have, and this
# turned that into sys.exit(1) for the entire run. So
# packagemanager install <12 names> --run --recursive
# died on the first name that is not a BLFS package -- and since the
# exit came before any plan was printed, the person saw the book note
# and nothing else. Report it and hand back an empty order; the
# caller records it in `not_found` and carries on with the rest.
printError((err or out).strip() or f"blfs order {name} failed")
return [], []
order, missing = [], []
for line in out.splitlines():
if "\t" not in line: # skip any stray output
continue
cols = line.split("\t")
cols += [""] * (4 - len(cols))
left, nv, kind, parent = cols[:4]
left = left.strip()
if left == "#ignored-required":
_IGNORED_REQUIRED.append((nv.strip(), kind.strip()))
elif left == "#cycle":
# a dependant the walker had to build BEFORE something it
# recommends, because the cycle admits no order satisfying both
_CYCLE_PAIRS.append((nv.strip(), kind.strip()))
elif left == "#missing":
missing.append((nv.strip(), kind, parent))
elif left and " " not in left and nv.strip():
order.append((left, nv.strip(), kind, parent))
return order, missing
def cmd_dependencies(args):
name = args.packages[0]
if args.tree:
argv = ["deps", name, "--tree"]
if args.no_recommended:
argv.append("--no-recommended")
if args.optional:
argv.append("--optional")
rc, out, err = run_blfs(argv)
print(out if rc == 0 else (err or out).strip())
return
order, missing = blfs_order_anchors(name, args.no_recommended, args.optional)
# anchor -> name_version, to show the parent by its versioned name
nv_of = {a: nv for a, nv, _, _ in order}
printHeader(f"Install order for {name} ({len(order)} packages)")
print()
need = 0
for anchor, nv, kind, parent in order:
u = gather_user(anchor)
if u.state == "installed" and (not u.version or u.version == nv):
state = colors.green + "installed" + colors.normal
elif u.state == "installed":
state = colors.cyan + f"update ({u.version})" + colors.normal
need += 1
else:
state = colors.yellow + "needs install" + colors.normal
need += 1
origin = ""
if kind and parent:
origin = f" ({kind} by {nv_of.get(parent, parent)})"
print(f" {nv:<30} {state}{origin}")
if missing:
print()
printWarning("Out-of-book / missing (handle manually):")
for m, kind, parent in sorted(set(missing)):
extra = f" ({kind} by {nv_of.get(parent, parent)})" if kind and parent else ""
print(f" {m}{extra}")
print(f"\n{len(order)} in order, {need} to build"
+ (f", {len(set(m for m, _, _ in missing))} missing" if missing else ""))
# =========================================================================== #
# command: install [IMPLEMENTED]
# =========================================================================== #
#
# Resolve the recursive install order (via blfs), work out which packages
# actually need building (state != installed, or --reinstall), generate an
# install script for each with `blfs script`, and either print the plan
# (default, safe) or run lfs-helper pm-install for each in order (--run).
#
# -f/--force just the named package(s), skip dependency resolution
# -r/--reinstall rebuild even packages that are already up to date
# -e/--ignore-recommended required deps only
# --optional also pull in optional deps
# -i/--ignore a,b,c skip these anchors
# --run actually execute (default is a dry run / plan)
# --yes don't ask for confirmation before --run
# -o/--outdir DIR where to write the generated scripts
def _blfs_book_available():
"""Is there a BLFS book on disk to look packages up in?
Asking `blfs books` is no good: it also lists every version available on
the site, so the output is never empty and the answer was always yes."""
if os.environ.get("BLFS_BOOK_FILE"):
return os.path.isfile(os.environ["BLFS_BOOK_FILE"])
for cand in (os.environ.get("BLFS_STORE"), "/usr/share/blfs",
os.path.expanduser("~/.cache/blfs")):
if not cand:
continue
d = os.path.join(cand, "books")
try:
if any(f.endswith((".html", ".html.gz")) for f in os.listdir(d)):
return True
except OSError:
continue
return False
def _download_tool_available():
return bool(shutil.which("wget") or shutil.which("curl"))
def _explain_no_blfs_book(target):
"""No book, and say so in terms of what can actually be done from here.
"run `blfs fetch`" is sound advice on a system that can reach the network.
On a fresh chroot it is the circle itself: fetching needs the download
tool, the download tool is looked up in the book, and the book is what is
missing. Telling someone to close a circle by going round it once more is
worse than saying nothing.
The host resolves wget's URL FROM THE BOOK too (`blfs sources wget`, via
lfs's bootstrap_sources), so a host with no cached book staged no wget
tarball either -- both halves are missing together, and both are fixed in
the same trip. Say which of them is actually absent rather than making
the person find out one at a time.
"""
srcdir = setup_sources_dir()
have_tool = _download_tool_available()
try:
have_tarball = any(f.startswith("wget-") and ".tar" in f
for f in os.listdir(srcdir))
except OSError:
have_tarball = False
printError(f"{target}: there is no BLFS book on this system to look "
f"it up in.")
printInfo("")
if have_tool:
printInfo(" Fetch one:")
printInfo(" blfs fetch")
printInfo(" or import a copy:")
printInfo(" blfs import /path/to/BLFS-BOOK-...-nochunks.html")
printInfo("")
return
printInfo(" There is no download tool on this system either, so")
printInfo(" `blfs fetch` cannot work from in here -- that circle is what")
printInfo(" bootstrap exists to break, and it is broken from the HOST.")
printInfo("")
printInfo(" The host stages both halves into the tree's own sources")
printInfo(f" directory, which it sees as $LFS{srcdir}:")
printInfo("")
printInfo(" # on the host, which still has a network")
printInfo(" blfs fetch # or: blfs import <book.html>")
if not have_tarball:
printInfo(" blfs sources wget # the URL to download")
printInfo(f" wget -P $LFS{srcdir} <that URL>")
printInfo(f" cp <book.html> $LFS{srcdir}/")
printInfo("")
printInfo(" # back in here")
printInfo(f" blfs import {srcdir}/<book.html>")
printInfo(" packagemanager bootstrap --run")
printInfo("")
printInfo(" Already on this system: "
+ ("the wget tarball is in %s" % srcdir if have_tarball
else "no wget tarball in %s either" % srcdir))
printInfo(" (`lfs build-system install-tools` copies a book the host has")
printInfo(" cached; it said so at the time if there was none.)")
printInfo("")
def _explain_not_in_book(target):
"""Say why a package could not be found, and what to do about it.
"not found in book" is unhelpful when the real problem is that there IS no
book -- a fresh chroot has the LFS book but not the BLFS one, and every
lookup fails the same way."""
if not _blfs_book_available():
_explain_no_blfs_book(target)
return
printError(f"{target}: not in the BLFS book.")
rc, out, _e = run_blfs(["search", target])
hits = [ln for ln in out.splitlines() if ln.strip().startswith("[")][:5]
if hits:
printInfo(" Did you mean:")
for h in hits:
printInfo(" " + h.strip())
else:
printInfo(f" Nothing similar found either. Try: blfs search {target}")
def _install_finished(u):
"""Positive evidence that an install FINISHED -- not that one started.
THIS IS WHY THE STACK NEEDED HAND-HOLDING. `install_last` is copied
into the package home BEFORE the build runs (lfs-helper: "RECORD WHAT
IS INSTALLED BEFORE OVERWRITING THE RECORD"), so a build that dies at
configure leaves one behind. For a package whose page declares no
programs or libraries to check -- cairomm is one -- `state` then has
nothing else to go on and answers "installed".
A manifest with real paths, or the marker the runner writes after a
SUCCESSFUL install, is evidence. A script that was copied in before
the build is not.
"""
if getattr(u, "pkg_list", None):
return True
home = getattr(u, "home", "") or ""
return bool(home) and os.path.isfile(os.path.join(home, ".pkgusr-installed"))
def build_install_plan(args):
"""Return (plan, missing, why). plan = [(anchor, name_version, reason)] in
install order (deps first); reason in {new, update, reinstall}. why maps
anchor -> (kind, parent) so we can show 'required by <parent>'."""
ignore = {x for x in (args.ignore or "").split(",") if x}
# THE AVOID LIST IS A PROPERTY OF THE MACHINE, NOT OF A STACK RUN.
#
# `install linux-pam --run --recursive` planned **Systemd-261.2** on a
# SysV system. avoid.stack says `avoid systemd` and has since 1.14.224 --
# but only `stack` ever read it, so a bare install walked the book's
# graph with no exclusions at all and offered to install the one thing
# this machine must never have.
#
# A package named ON THE COMMAND LINE is still built: asking for it
# explicitly is an instruction, and refusing that would be second-
# guessing. It is dependencies that are filtered.
_named = {a for a in (args.packages or [])}
_avoided = _machine_avoids() - _named
if _avoided:
ignore |= _avoided
# SAY IT, because an avoided REQUIRED dependency changes the build.
# linux-pam's page (in the systemd book) lists systemd as required;
# skipping it is the right call on a SysV machine and the reader
# still needs to know it happened.
printInfo(" %d package(s) this machine never builds are excluded "
"from the plan" % len(_avoided))
printInfo(" (avoid.stack; a package you name yourself is still "
"built). If the plan")
printInfo(" then looks short, one of them was a dependency: "
"packagemanager deps <pkg>")
printInfo(" the list: packagemanager stack avoid --avoided")
plan, seen, missing_all, why = [], set(), [], {}
# WHAT IS ALREADY DONE, counted as we go.
#
# A re-run recomputes the whole tree and prints only what is LEFT, so a
# plan that has shrunk 181 -> 151 -> 135 across three runs reads as three
# identical fresh starts. The progress was real and invisible.
build_install_plan.already = set()
build_install_plan.behind = [] # installed deps the book has moved past
build_install_plan.unproven = [] # "installed", with nothing to show for it
_not_found = []
# Reading the book can take many seconds the first time (it is indexed
# once per version). Without a word on screen that looks like a hang, so
# say what is happening as it happens.
set_status("reading the book ...")
def consider(anchor, nv, kind="", parent=""):
if anchor not in why:
why[anchor] = (kind, parent)
if anchor in seen or anchor in ignore:
return
seen.add(anchor)
# A DEPENDENCY OUTSIDE A STACK BOOK READS ITS OWN BOOK. The order
# came from the svn book, so it carries svn's versions for
# everything; a dependency the stack does not own is judged -- and
# built, if missing -- against the book it always reads.
if _PLAN_BOOK and _book_override(anchor) != _PLAN_BOOK:
nv = default_book_version(anchor) or nv
u = gather_user(anchor)
_done = u.state == "installed" and _install_finished(u)
if u.state == "installed" and not _done:
# recorded, but nothing says the install ever finished
build_install_plan.unproven.append(anchor)
up_to_date = (_done and (not u.version or u.version == nv))
if up_to_date:
build_install_plan.already.add(anchor)
# AN INSTALLED DEPENDENCY IS NOT A THING TO UPGRADE.
#
# `install gparted --recursive` planned 53 packages: gparted, its
# real missing dependencies -- and UPDATES of bubblewrap, alsa-lib,
# gtk3, gtk4 and thirty others that were installed and working,
# purely because the book had moved on. A DEPENDENCY that is
# installed is left alone; a package NAMED on the command line still
# goes through; --update-deps restores the old behaviour.
elif (_done and anchor not in _named
and not getattr(args, "update_deps", False)):
build_install_plan.already.add(anchor)
build_install_plan.behind.append((anchor, u.version or "?", nv))
return
if args.reinstall:
plan.append((anchor, nv, "reinstall" if up_to_date else
("update" if u.state == "installed" else "new")))
elif not up_to_date:
plan.append((anchor, nv,
"update" if u.state == "installed" else "new"))
global _PLAN_BOOK
for target in args.packages:
set_status(f"working out what {target} needs ...")
_PLAN_BOOK = None
_tb = _book_override(target)
_PLAN_BOOK = _tb if _tb and os.path.isfile(_tb) else None
if not getattr(args, "recursive", False):
# DEFAULT: just this package, no dependency walk
nv = book_version(target)
if nv is None:
clear_status()
_explain_not_in_book(target)
_not_found.append(target)
continue
consider(target, nv, "requested", "")
else:
# THE PRUNE BELONGS TO THE WALK, AND NOW IT IS THERE (1.14.24).
#
# 1.12.82 pruned HERE, on the finished list, and was reverted for
# a good reason: `blfs order` reports ONE parent per package, so
# graphite2 -- recorded under an ignored parent but genuinely
# needed by harfbuzz -- was dropped, and the build failed with
# Dependency 'graphite2' is required but not found
# The note then read "pruning needs every edge, which this output
# does not carry". Right, so it moved to where the edges are:
# walk_deps does not descend into an ignored package, and anything
# another package needs is still reached through that package.
#
# Until then --ignore dropped only the named package and kept
# everything it alone wanted: harfbuzz with libreoffice ignored
# planned 230 packages, of which PostgreSQL, unixODBC, CLucene,
# Redland, OpenJDK, apache-ant and Cups were nobody else's. It
# comes to 46 now, with icu, libxml2, fontconfig, curl and
# graphite2 all still present. sway.stack's 80-name avoid list
# was the workaround.
order, missing = blfs_order_anchors(
target, args.ignore_recommended, args.optional, ignore=ignore)
# A NAME THE BOOK DOES NOT HAVE, IN THE RECURSIVE PATH.
#
# The non-recursive branch explains and records it; this one got
# an empty order back and moved on WITHOUT A WORD. So
# packagemanager install <12 names> --run --recursive
# printed the book note and nothing else, and the person could not
# tell whether it had worked, refused, or crashed. Silence is the
# one answer a command must never give.
if not order:
clear_status()
_explain_not_in_book(target)
_not_found.append(target)
continue
missing_all.extend(m for m, _, _ in missing)
for anchor, nv, kind, parent in order:
consider(anchor, nv,
"requested" if not parent else (kind or "required"),
parent or target)
if getattr(args, "dependents", False): # also rebuild reverse deps
for danchor, dnv in book_rdeps(
target, args.ignore_recommended, args.optional).items():
consider(danchor, dnv, "dependent-of", target)
return plan, missing_all, why, _not_found
def _select_plan_rows(rows, render):
"""Interactive subset picker. Everything is selected by default; type a
selection to narrow it. Syntax:
(empty) / all keep everything
none keep nothing
1,3,5-8 keep only these
all,!4,!7 everything except 4 and 7
!4,!7 (no positives) => everything except 4 and 7
"""
print("\nPlan -- all selected by default. Enter to keep all, or edit the set")
print("(e.g. '1,3,5-8' | 'all,!4,!7' | '!4,!7' | 'none'):")
for i, row in enumerate(rows, 1):
print(f" [{i:>3}] {render(row)}")
def expand(tok):
out = set()
tok = tok.strip().lstrip("!")
if "-" in tok:
try:
lo, hi = tok.split("-", 1)
out.update(range(int(lo), int(hi) + 1))
except ValueError:
pass
elif tok.isdigit():
out.add(int(tok))
return out
try:
ans = input("select> ").strip().lower()
except EOFError:
ans = ""
if ans in ("", "all", "a"):
return rows
if ans in ("none", "n", "q"):
return []
parts = [p for p in re.split(r"[,\s]+", ans) if p]
positives = [p for p in parts if not p.startswith("!")]
negatives = [p for p in parts if p.startswith("!")]
numeric_pos = [p for p in positives if p not in ("all", "a")]
# start from "all" if no numeric positives were given (or 'all' was typed)
if not numeric_pos or "all" in positives or "a" in positives:
keep = set(range(1, len(rows) + 1))
else:
keep = set()
for p in numeric_pos:
keep |= expand(p)
for n in negatives:
keep -= expand(n)
return [r for i, r in enumerate(rows, 1) if i in keep]
def find_local_script(name):
"""Look for a local install script for `name` (e.g. one made with
`template`) in INSTALL_SCRIPTS_DIR or the current directory."""
for d in (INSTALL_SCRIPTS_DIR, "."):
try:
for fn in sorted(os.listdir(d)):
if fn == f"install_{name}" or fn.startswith(f"install_{name}-"):
return os.path.abspath(os.path.join(d, fn))
except OSError:
continue
return None
def install_local(name, path, run, yes, verb="install", test=False, clean=False):
"""Build/install a single package from a local install script (custom or
git-based). Used by both install --local and update --local."""
if not path or not os.path.isfile(path):
printError(f"{name}: no local install script found "
f"(make one with: packagemanager template {name}).")
sys.exit(1)
printHeader(f"Local {verb}: {name}")
print(f"\n script : {path}")
if not run:
printInfo("\nDry run -- to proceed, re-run with --run, or execute:\n"
f" lfs-helper pm-install {name} {path}")
return
_acct = pkgusr_name(name)
if not _user_exists(_acct) and (yes or ask_yes_no(
f"\nPackage-user {_acct} doesn't exist -- create it first?")):
# THE SAME DOOR AS EVERY OTHER NEW ACCOUNT. This called
# add_package_user directly, skipping the home repair and set-up
# create_package_user does -- so pm-install found the hint's
# /usr/src/p_gitsync and opened with "misplaced home" warnings about
# an account one second old.
create_package_user(_acct)
if not yes and not ask_yes_no(f"Build and {verb} {name} now?"):
printWarning("Aborted.")
return
printHeader(f"{verb.capitalize()}ing {name} (local)")
env = {**os.environ, "PM_YES": "1" if yes else "0",
"PM_MODE": "update" if verb == "update" else "all"}
if test:
env["PM_TEST"] = env["PKGUSR_TESTS"] = "1"
old_paths = _pkglist_paths(name) if clean else None
t0 = time.time()
if clean:
env = {**env, "PM_NO_REFRESH": "1"}
result = run_pm_install(name, path, env)
if result.returncode != 0:
printError(f"{verb} failed for {name} (exit {result.returncode}).")
printInfo(f" logs: {_pkg_log_dir(name)}")
sys.exit(result.returncode)
if clean:
_clean_after_install(name, old_paths, t0)
printSuccess(f"Done -- {name} {verb}ed. logs: {_pkg_log_dir(name)}")
def _stack_git_entry(name, with_file=False):
"""The git/tar entry for `name` in an installed stack, or None.
with_file: (entry, stack file) instead."""
d = STACK_DIR if "STACK_DIR" in globals() else "/etc/pkgusr/stacks"
for f in sorted(glob.glob(os.path.join(d, "*.stack"))):
try:
for r in _parse_stack_file(f):
if r["kind"] in ("git", "tar") and r["target"] == name:
return (r, f) if with_file else r
except (ValueError, OSError):
continue
return (None, None) if with_file else None
# --------------------------------------------------------------------------- #
# update a stack TARBALL package (yt-dlp) [IMPLEMENTED]
# --------------------------------------------------------------------------- #
#
# `update --run yt-dlp` said "looking up yt-dlp in the book", then "1 not in
# the BLFS book at all" -- and did nothing. yt-dlp is a `tar` entry of
# media.stack, a pinned release tarball, and update knew two kinds of
# package: book pages and git clones. A tarball has a version in its URL;
# for a GitHub release URL the project's own "latest release" says what the
# newest one is. The pin moves in the stack file too, after the build
# succeeds -- otherwise a replay of the stack would build the old one.
_GH_RELEASE_RE = re.compile(
r"^https://github\.com/([^/]+)/([^/]+)/releases/download/([^/]+)/")
def _github_latest_tag(owner, repo):
"""The tag GitHub's /releases/latest points at (pre-releases excluded),
read from the redirect without following it. '' when unknown."""
import urllib.request, urllib.error
class _NoRedirect(urllib.request.HTTPRedirectHandler):
def redirect_request(self, *a, **k):
return None
url = "https://github.com/%s/%s/releases/latest" % (owner, repo)
try:
urllib.request.build_opener(_NoRedirect).open(
urllib.request.Request(url, method="HEAD",
headers={"User-Agent": "packagemanager"}),
timeout=30)
except urllib.error.HTTPError as e:
loc = e.headers.get("Location", "") if e.code in (301, 302, 303, 307) else ""
m = re.search(r"/releases/tag/([^/?#]+)", loc)
if m:
return urllib.request.unquote(m.group(1))
except (OSError, ValueError):
pass
# no redirect answer (offline, rate limit): the newest-looking tag
tags = [r.rsplit("/", 1)[-1] for _s, r in
_ls_remote("https://github.com/%s/%s" % (owner, repo), "tags")]
tags = [t for t in tags if not re.search(r"(rc|alpha|beta|nightly|dev)", t, re.I)]
return sorted(tags, key=_vsort_key)[-1] if tags else ""
def _update_tar_packages(args):
"""update for stack `tar` entries. Returns the names it handled."""
done = []
for name in list(args.packages or []):
entry, sfile = _stack_git_entry(name, with_file=True)
if not entry or entry["kind"] != "tar":
continue
done.append(name)
url = entry["opts"].get("url", "")
m = _GH_RELEASE_RE.match(url)
if not m:
printInfo("%s: a tarball pinned to %s" % (name, url or "(no url)"))
printInfo(" to move it: edit the version in %s, then" % sfile)
printInfo(" packagemanager stack %s --only %s --run"
% (os.path.basename(sfile)[:-6], name))
continue
owner, repo, tag = m.groups()
latest = _github_latest_tag(owner, repo)
u = gather_user(name)
inst = u.version or "-"
if not latest:
printWarning("%s: could not ask GitHub for the latest release of "
"%s/%s" % (name, owner, repo))
continue
if latest == tag:
printSuccess("%s: %s is the latest release (installed: %s)"
% (name, tag, inst))
continue
printInfo("%s: pinned to %s, latest release is %s (installed: %s)"
% (name, tag, latest, inst))
new = {"kind": "tar", "target": name,
"opts": {k: (v.replace(tag, latest) if isinstance(v, str) else v)
for k, v in entry["opts"].items()}}
if not args.run:
printInfo(" would build %s from %s" % (latest, new["opts"]["url"]))
printInfo(" and move the pin in %s -- add --run" % sfile)
continue
script, _ = _stack_git_script(new, os.path.join(TMP_DIR, "stack_files"))
apply_machine_opts(name, script)
install_local(name, script, True, args.yes, "update",
test=getattr(args, "test", False),
clean=getattr(args, "clean", False))
# still here: install_local exits on failure. Move the pin.
try:
t = open(sfile, encoding="utf-8").read()
lines = t.split("\n")
out, inside = [], False
for ln in lines:
if re.match(r"^\s*tar\s+%s(\s|$)" % re.escape(name), ln):
inside = True
if inside:
ln = ln.replace(tag, latest)
if not ln.rstrip().endswith("\\"):
inside = False
out.append(ln)
if out != lines:
shutil.copy2(sfile, sfile + ".bak")
open(sfile, "w", encoding="utf-8").write("\n".join(out))
printInfo(" %s: %s pinned to %s now (old file: %s.bak)"
% (os.path.basename(sfile), name, latest,
os.path.basename(sfile)))
except OSError as e:
printWarning(" could not move the pin in %s (%s) -- a stack replay "
"would build %s" % (sfile, e, tag))
return done
def _install_git_packages(args):
"""Build the git packages among args.packages from their own scripts.
NO BOOK. `install --reinstall imv` asked the book about imv three
times (an index lookup, `blfs debug`, `blfs search`), took a minute to
say "not in the BLFS book", and built nothing -- imv is a git entry of
apps.stack, and its script is in its home. A git package is recognised
from that script (or from a stack's git/tar entry, the first time),
before anything asks the book. --clean and --test go through as they
do for `update`. Returns the names it handled."""
done = []
for name in list(args.packages or []):
script = _canonical_script_path(name)
if not script:
p = os.path.join(pkgusr_home(name), "install_%s" % name)
script = p if os.path.isfile(p) else None
is_git = bool(script) and _is_git_package(name)
entry = None if is_git else _stack_git_entry(name)
if not is_git and not entry:
continue
done.append(name)
if entry and not script:
script, _ = _stack_git_script(
entry, os.path.join(TMP_DIR, "stack_files"))
u = gather_user(name)
if u.state == "installed" and not args.reinstall:
printSuccess("%s: installed (a %s package). Rebuild it: "
"--reinstall [--clean]; new commits: update %s"
% (name, entry["kind"] if entry else "git", name))
continue
printInfo("%s: %s package -- building from %s (no book lookup)"
% (name, entry["kind"] if entry else "git", script))
if script != (_canonical_script_path(name) or "") and entry:
apply_machine_opts(name, script)
install_local(name, script, args.run, args.yes, "install",
test=getattr(args, "test", False),
clean=getattr(args, "clean", False))
return done
def cmd_install(args, _replanned=False):
# ---- git packages: their own script, never the book ----
if not args.local and args.packages:
_git_done = _install_git_packages(args)
if _git_done:
args.packages = [p for p in args.packages if p not in _git_done]
if not args.packages:
return
# ---- local package: install a custom script directly, no BLFS ----
if args.local or (len(args.packages) == 1
and book_version(args.packages[0]) is None
and find_local_script(args.packages[0])):
name = args.packages[0]
path = os.path.abspath(args.local) if args.local else find_local_script(name)
install_local(name, path, args.run, args.yes, "install",
test=getattr(args, "test", False),
clean=getattr(args, "clean", False))
return
plan, missing, why, not_found = build_install_plan(args)
if missing:
printWarning("Out-of-book / missing (install these manually): "
+ ", ".join(sorted(set(missing))))
if not plan:
if not_found:
# NOT success: the package could not be looked up at all. Saying
# "everything is installed and up to date" here told the user the
# opposite of the truth.
printError("Could not install: " + ", ".join(not_found))
sys.exit(1)
printSuccess("Nothing to do -- everything is installed and up to date.")
return
def why_str(anchor):
kp = why.get(anchor)
if not kp:
return ""
kind, parent = kp
if kind == "requested":
return "requested"
if kind == "dependent-of":
return f"depends on {parent}"
return f"{kind} by {parent}"
_unproven = getattr(build_install_plan, "unproven", []) or []
if _unproven:
printWarning(" %d package(s) are recorded as installed but have no"
" finished-install record:" % len(_unproven))
printWarning(" " + ", ".join(sorted(set(_unproven))[:8])
+ (" ..." if len(set(_unproven)) > 8 else ""))
printInfo(" Their install_last was written before a build that did"
" not finish, so they")
printInfo(" are in this plan rather than skipped -- which is what"
" lets the stack run")
printInfo(" through without naming them by hand.")
_behind = getattr(build_install_plan, "behind", []) or []
if _behind:
printInfo(" %d installed dependency(ies) are older than the book and"
" are NOT touched:" % len(_behind))
printInfo(" " + ", ".join("%s (%s -> %s)" % (a, iv, bv)
for a, iv, bv in _behind[:6])
+ (" ..." if len(_behind) > 6 else ""))
printInfo(" that is deliberate: one program is not a reason to"
" rebuild what works.")
printInfo(" move them forward on purpose: packagemanager update"
" <name> --run")
printInfo(" or build this plan against them: add --update-deps")
if getattr(args, "select", False):
plan = _select_plan_rows(
plan, lambda r: f"{r[1]:<30} {r[2]:<10} {why_str(r[0])}")
if not plan:
printWarning("Nothing selected.")
return
script_dir = args.outdir or os.path.join(TMP_DIR, "install_files")
os.makedirs(script_dir, exist_ok=True)
# A ONE-PACKAGE PLAN WITH MISSING DEPENDENCIES IS A TRAP.
#
# `packagemanager install ffmpeg --run` (no --recursive) plans exactly
# one package and says "1 to build" -- then the build dies on
# ERROR: libass >= 0.11.0 not found using pkg-config
# because libass was never installed and was never going to be. The
# plan knows; it just did not say.
if not getattr(args, "recursive", False) and len(plan) == 1:
# A SWALLOWED EXCEPTION HERE IS SILENCE WHERE THE WARNING SHOULD BE.
#
# `except (SystemExit, Exception): _missing = []` made a failed
# dependency lookup indistinguishable from "no missing
# dependencies", and the trap this whole block exists to catch went
# uncaught -- numpy planned alone and died on
# BackendUnavailable: Cannot import 'mesonpy'
# with meson_python (a book page, and numpy's Required) never built.
_missing, _why_not = [], ""
try:
# the SAME ignore set the plan was built with (build_install_plan
# has its own local; this referenced a name that does not exist
# here, so every non-recursive install printed "NameError: name
# 'ignore' is not defined" instead of its missing dependencies)
_ignore = {x for x in (getattr(args, "ignore", "") or "").split(",") if x}
_deps, _ = blfs_order_anchors(plan[0][0], ignore=_ignore)
_missing = [nv for a, nv, _k, parent in _deps
if parent and gather_user(a).state != "installed"]
except SystemExit as e:
_why_not = "the book lookup exited (%s)" % e
except Exception as e:
_why_not = "%s: %s" % (type(e).__name__, e)
if _why_not:
printWarning(" could not work out what %s depends on -- %s"
% (plan[0][1], _why_not))
printWarning(" so this plan builds ONLY %s, and a missing"
" dependency will not be" % plan[0][1])
printWarning(" reported until the build fails on it. Safer:"
" --recursive")
if _missing:
printWarning(" %d dependenc%s of %s %s NOT installed:"
% (len(_missing), "y" if len(_missing) == 1 else "ies",
plan[0][1], "is" if len(_missing) == 1 else "are"))
printWarning(" " + ", ".join(_missing[:8])
+ (" ..." if len(_missing) > 8 else ""))
printWarning(" this plan builds ONLY %s and will fail on them."
% plan[0][1])
printWarning(" to build them too: packagemanager install %s "
"--run --recursive" % plan[0][0])
_done = len(getattr(build_install_plan, "already", ()))
printHeader(f"Install plan ({len(plan)} to build)")
if _done:
printSuccess(" %d of the %d packages in this tree are already"
" installed -- they are skipped."
% (_done, _done + len(plan)))
print()
_prewarm_scripts(plan, script_dir,
force_regen=getattr(args, "regenerate", False))
steps = [] # (anchor, nv, reason, path)
for anchor, nv, reason in plan:
path = resolve_script(anchor, nv, script_dir,
assume_yes=args.yes,
force_regen=getattr(args, "regenerate", False))
if not path:
sys.exit(1)
steps.append((anchor, nv, reason, path))
# A TOOL THE PLAN DOES NOT HAVE AT ALL: add it and plan again.
#
# 1.14.12 orders a package after the tool it calls when both are in the
# plan. When the provider is absent entirely the run just stopped:
# cargo needed by 3 package(s), first: rust-bindgen
# and rust-bindgen's page does not list Rust, so no amount of reordering
# would have helped. The provider is a book package like any other, so
# add it to what was asked for and rebuild the plan -- once, so a tool
# whose own provider is missing cannot loop.
_add = sorted({_provider_of(t) for t in _MISSING_TOOLS if _provider_of(t)}
- {a.lower() for a, _n, _r, _p in steps})
# A provider that is ALREADY INSTALLED cannot be added -- the plan would
# come back identical and stop again, which is what happened with rust:
# installed in /opt/rustc, invisible to our PATH, so "add it" produced no
# change and the same refusal appeared twice.
_add = [a for a in _add if gather_user(a).state != "installed"]
if _add and getattr(args, "recursive", False) and not _replanned:
printInfo(" adding to the plan: %s -- %s call %s, which the book's"
" pages do not list"
% (", ".join(_add),
"their commands" if len(_add) > 1 else "its commands",
", ".join("`%s`" % t for t in sorted(_MISSING_TOOLS))))
args.packages = list(args.packages) + _add
_MISSING_TOOLS.clear()
_MISSING_ENV.clear()
return cmd_install(args, _replanned=True)
# ORDER BEFORE PRINTING. The list is a promise about what happens next,
# so it has to be the order that will actually run -- printing as each
# script resolved and reordering afterwards showed libaom above CMake and
# then built it below.
steps, _moved = _order_by_build_tools(steps)
_in_plan = {a.lower() for a, _n, _r, _p in steps}
for anchor, nv, reason, _path in steps:
color = {"new": colors.yellow, "update": colors.cyan,
"reinstall": colors.blue}.get(reason, "")
w = why_str(anchor)
print(f" {nv:<30} {color}{reason:<10}{colors.normal} "
f"{colors.dim}{w}{colors.normal}")
for nv, after, tool in _moved:
printInfo(" ordering: %s moved after %s (its commands call `%s`,"
" which the book's page does not list)" % (nv, after, tool))
# a tool the plan itself builds is no longer missing -- it is just later
for _t in list(_MISSING_TOOLS):
if (_BUILD_TOOLS.get(_t) or "").lower() in _in_plan:
_MISSING_TOOLS.pop(_t, None)
_report_missing_tools()
_report_ignored_required(steps)
_report_cycles(steps)
_stop_if_prereqs_missing(args)
if not args.run:
print()
printInfo("Dry run -- to install, re-run with --run, or execute:")
for anchor, nv, reason, path in steps:
print(f" lfs-helper pm-install {anchor} {path}")
return
print()
if not args.yes and not ask_yes_no(f"Build and install {len(steps)} "
f"package(s) now?"):
printWarning("Aborted.")
return
pm = find_pm_install()
env = {**os.environ, "PM_MODE": "all", "PM_YES": "1" if args.yes else "0"}
if getattr(args, "test", False):
env["PM_TEST"] = env["PKGUSR_TESTS"] = "1"
if getattr(args, "reinstall", False):
env["PM_REINSTALL"] = "1"
total = len(steps)
errors = 0
run_log = start_run_log("install", ",".join(args.packages))
for i, (anchor, nv, reason, path) in enumerate(steps, 1):
set_status(f"[{i}/{total}] installing {nv}"
+ (f" ({errors} error(s))" if errors else ""))
printHeader(f"({i}/{total}) Installing {nv} ({reason})")
cleaning = getattr(args, "clean", False)
old_paths = _pkglist_paths(anchor) if cleaning else None
t0 = time.time()
penv = {**env, "PM_NO_REFRESH": "1"} if cleaning else env
result = run_pm_install(anchor, path, penv)
add_run_entry(run_log, anchor, "install", result.returncode == 0,
result.returncode, _pkg_log_dir(anchor))
if result.returncode != 0:
errors += 1
clear_status()
printError(f"install failed for {nv} (exit {result.returncode}); "
f"stopping ({i - 1}/{total} done).")
printInfo(f" logs: {_pkg_log_dir(anchor)} "
f"(see all: packagemanager errors)")
printInfo(f" Then re-run ONLY the install step (no recompile):\n"
f" packagemanager script install {anchor}")
# THE FILE TO EDIT IS THE LAST THING SAID.
#
# It is what you do next, and it was buried three messages up. A
# path you have to scroll back for is a path you retype wrong.
#
# And it must be the copy in the package user's HOME, not the one
# under /tmp/packagemanager/install_files. That is where a script
# freshly generated from the book lands; the engine copies it into
# the home, and the next run regenerates the /tmp one -- so an edit
# there is silently discarded, which is the exact failure this
# message exists to prevent.
_canon = os.path.join(pkgusr_home(anchor),
os.path.basename(path))
_edit = _canon if os.path.isfile(_canon) else path
_runtime = os.path.join(pkgusr_home(anchor),
"install_" + pkgusr_name(anchor))
print()
printInfo(" To change what it does, edit:")
printError(f" {_edit}")
# Since 1.14.16 the canonical script is the one that runs, so
# this only fires for a leftover copy from an older version.
if os.path.isfile(_runtime) and _runtime != _edit:
printInfo(f" ({os.path.basename(_runtime)} is a leftover copy"
f" from an older version;\n"
f" it is no longer used and can be deleted)")
if _edit != path:
printInfo(f" (not {path} --\n"
f" that one is regenerated from the book each run)")
sys.exit(result.returncode)
if getattr(args, "clean", False):
_clean_after_install(anchor, old_paths, t0)
_maybe_remove_old_sources(anchor, nv, getattr(args, "keep_sources", False))
clear_status()
printSuccess(f"Done -- {total} package(s) installed.")
# THE PATTERN HAS TO BE A REAL PATH.
#
# This built the hint by hand -- os.path.join(BASE_DIR, ...) -- and got it
# wrong twice over: it printed "/usr/src/p_<pkg>/log", missing the pkgusr/
# component that every package home actually has. Someone following it
# found nothing, twice: the directory does not exist, and until 1.14.71
# the logs were not there either. _pkg_log_dir is the function that knows.
printInfo(f" results: packagemanager errors logs: "
f"{_pkg_log_dir('<name>')}")
# =========================================================================== #
# command: update [IMPLEMENTED]
# =========================================================================== #
#
# Update installed packages whose book version differs from what's installed.
# With no arguments it checks EVERY installed package-user; with package names it
# checks those packages plus their (recursive) dependencies. Required and
# recommended deps are followed by default (-e to skip recommended, --optional to
# add optional). Dry-run by default; --run rebuilds each outdated package via
# the engine.
_PROTECTED_PREFIXES = ("/etc/", "/var/", "/boot/", "/home/", "/root/", "/srv/",
"/opt/etc/")
def _pkglist_paths(user):
pl = os.path.join(pkgusr_home(user), "pkg.lst")
return {p for p in pkglist_entries(pl) if p.startswith("/")}
def _orphan_sweep(user, old_paths, since, remove=True):
"""Remove files that were in the package's OLD manifest but were NOT rewritten
by the new version (mtime older than the install start `since`). Only old-
manifest files are considered; config paths + the source tree are kept. Does
NOT regenerate the manifest (caller does, after any reapply). Returns the
number of files removed."""
if old_paths is None:
return 0
user = pkgusr_name(user) # the ACCOUNT -- see _clean_after_install
protected = _PROTECTED_PREFIXES + (BASE_DIR.rstrip("/") + "/",)
# NEVER ANOTHER PACKAGE'S FILE. An old manifest can list a path that
# has since moved to someone else: switching Mesa to glvnd hands
# libGL.so.1 to libglvnd, which installs it BEFORE the Mesa rebuild --
# so it is old, in Mesa's old list, and not rewritten by Mesa. mtime
# alone would delete the GL library. A package cleans only what it
# still owns.
try:
own_uid = pwd.getpwnam(user).pw_uid if pwd else None
except KeyError:
own_uid = None
victims = []
for p in sorted(old_paths):
if any(p == pre.rstrip("/") or p.startswith(pre) for pre in protected):
continue
if not os.path.lexists(p):
continue
try:
st = os.lstat(p)
if own_uid is not None and st.st_uid != own_uid:
continue
if st.st_mtime < since:
victims.append(p)
except OSError:
continue
# A LIBRARY SOMETHING IS STILL LINKED TO IS NOT AN ORPHAN.
#
# webkitgtk rebuilt from the GTK4 port to the GTK3 one leaves
# libwebkitgtk-6.0.so.4 in the old manifest, unrewritten and owned by this
# package -- every test above says "orphan", and deleting it breaks every
# program built against it, with no warning and no way back but a rebuild.
# So each library about to go is asked the only question that matters:
# does anything else still record its soname? Files of this package do
# not count -- they go with it.
kept = []
if victims:
libs = [p for p in victims if re.search(r"\.so(\.\d+)*$", p)
and os.path.isfile(p) and not os.path.islink(p)]
victim_set = set(victims)
for p in libs:
s = _soname_of(p)
if not s:
continue
# AN OLD FILE BEHIND A SONAME THE NEW VERSION SERVES IS NOT NEEDED.
# minibrowser links libjavascriptcoregtk-6.0.so.1; the upgrade
# pointed that name at .so.1.8.x, so .so.1.6.10 and .so.1.7.13 --
# both kept "because minibrowser links against it" -- were loaded
# by nothing. Only a soname that would VANISH with this file
# needs its users asked.
_cur = os.path.join(os.path.dirname(p), s)
_real = os.path.realpath(_cur) if os.path.lexists(_cur) else ""
if _real and os.path.isfile(_real) and _real != os.path.realpath(p) \
and _real not in victim_set and _cur not in victim_set:
continue
users = [(f, o) for f, o in _needed_by(s, skip_uid=own_uid)
if f not in victim_set]
if users:
pkgs = sorted({o for _f, o in users})
printWarning(" --clean: KEEPING %s -- %d file(s) in %s still "
"link against %s"
% (p, len(users), ", ".join(pkgs[:4]), s))
kept.append((p, s, pkgs))
if kept:
keep_paths = {p for p, _s, _pk in kept}
# the symlinks that point at a kept library stay with it, or the
# soname it is found by disappears and nothing loads anyway
for p in list(victims):
if os.path.islink(p):
t = os.path.realpath(p)
if t in keep_paths:
keep_paths.add(p)
victims = [p for p in victims if p not in keep_paths]
printInfo(" rebuild those packages, then clean again:")
for _p, _s, pkgs in kept:
printInfo(" packagemanager update %s --reinstall --run"
% " ".join(p[2:] if p.startswith("p_") else p
for p in pkgs[:4]))
break
printInfo(" or see the whole picture: packagemanager used-by %s"
% kept[0][1])
if not remove:
# the caller decides (_clean_after_install parks them first)
return victims
if victims:
printInfo(f" --clean: removing {len(victims)} old-version file(s) the "
f"upgrade no longer installs (configs / source kept)")
remove_paths_as_user(user, victims)
elif not kept:
printInfo(" --clean: no leftover files from the old version.")
return len(victims)
def _reapply_install(user):
"""Re-run ONLY the install phase (no recompile), as the user, to put back
the current-version files the mtime test mistook for old ones.
Returns True only when the phase ran and exited 0.
IT HAD NOT RUN SINCE 1.14.16 (fixed 1.14.311). It called
`bash ~/install_<account>` -- the runtime copy 1.14.16 removed everywhere
else -- with stdout and stderr thrown away and the exit status ignored.
So every --clean deleted the files the mtime test misjudged, the "restore"
failed on `No such file or directory` where nobody could see it, and the
run ended green. webkitgtk lost /usr/include/webkitgtk-6.0/jsc/*.h that
way: CMake's install skips a file whose content is unchanged ("--
Up-to-date:"), so the headers kept their old mtime and read as orphans.
Now: the same script and the same door as `packagemanager script
install` (lfs-helper build), its output kept in a log, and its result
returned so the caller can undo the clean when it fails.
"""
user = pkgusr_name(user)
home = pkgusr_home(user)
script = (_versioned_script_in(home)
or os.path.join(home, f"install_{user}"))
if not os.path.isfile(script):
printError(f" --clean: no install script in {home} -- cannot reapply")
return False
helper = shutil.which("lfs-helper")
if helper and _can_su(user):
cmd = [helper, "build", user, "--script", script,
"--phase", "install", "--force"]
else:
inner = f"cd ~ && bash {shlex.quote(script)} install"
cmd = (["su", "-", user, "-c", inner] if _can_su(user)
else ["sh", "-c", inner])
log = os.path.join(_plans_dir(),
re.sub(r"[^A-Za-z0-9_.-]+", "_", user) + ".clean-reapply.log")
printInfo(" --clean: reapplying the install step to restore current files...")
printInfo(" $ " + " ".join(cmd[1:] if cmd[0] == helper else cmd))
try:
with open(log, "w") as lf:
rc = subprocess.run(cmd, stdout=lf, stderr=subprocess.STDOUT).returncode
except OSError as e:
printError(f" --clean: could not run the install step ({e})")
return False
if rc != 0:
printError(f" --clean: the install step FAILED (exit {rc}) -- log: {log}")
try:
for line in open(log, errors="replace").read().splitlines()[-12:]:
printInfo(" | " + line)
except OSError:
pass
return False
return True
_PARK_SUFFIX = ".pkgusr-clean"
def _park_ledger(user):
return os.path.join(_plans_dir(),
re.sub(r"[^A-Za-z0-9_.-]+", "_", user) + ".parked")
def _unpark_all(user):
"""Put every parked file back, from the ledger. Returns how many."""
ledger = _park_ledger(user)
n = 0
try:
lines = open(ledger).read().splitlines()
except OSError:
return 0
for p in lines:
if not p:
continue
q = p + _PARK_SUFFIX
if not os.path.lexists(q):
continue
if os.path.lexists(p):
# the install put the current version there: that one stays
try:
os.unlink(q)
except OSError:
pass
continue
try:
os.rename(q, p)
n += 1
except OSError as e:
printError(f" could not put back {p} ({e}) -- it is at {q}")
try:
os.unlink(ledger)
except OSError:
pass
return n
# =========================================================================== #
# git packages: what they follow, and whether it moved [IMPLEMENTED]
# =========================================================================== #
#
# A book package has a version, and `update` compares it with the book's. A
# git package has a REF, and a ref answers a different question depending on
# what it is:
# a branch -- a moving target. "Is it behind?" is asked of the remote, and
# the answer changes without anyone doing anything.
# a tag -- a point in time. It is never behind; the question is whether
# a NEWER tag exists, and moving to it is a decision.
# a commit -- the same, without even a name to compare.
# Telling them apart is the whole of this.
_GIT_ENV = dict(os.environ, GIT_TERMINAL_PROMPT="0", GIT_ASKPASS="/bin/true",
GIT_SSH_COMMAND="ssh -oBatchMode=yes")
def _script_var(text, var):
"""A shell assignment out of an install script: link="..." / git_ref=..."""
m = re.search(r'^\s*%s\s*=\s*(?:"([^"]*)"|\'([^\']*)\'|([^\s#]*))'
% re.escape(var), text or "", re.M)
if not m:
return ""
return (m.group(1) or m.group(2) or m.group(3) or "").strip()
def _git_clone_dir(name, link=""):
"""Where this package's checkout is. lfs-phases builds in $HOME/src, so
that is where to look -- and the remote is what confirms it, because a
home can hold more than one tree."""
home = pkgusr_home(name)
for base in (os.path.join(home, "src"), home):
if not os.path.isdir(base):
continue
try:
entries = sorted(os.listdir(base))
except OSError:
continue
cands = [os.path.join(base, d) for d in entries
if os.path.isdir(os.path.join(base, d, ".git"))]
if link:
for c in cands:
r = subprocess.run(["git"] + _git_at(c) + ["remote", "get-url", "origin"],
capture_output=True, text=True, env=_GIT_ENV)
if r.returncode == 0 and r.stdout.strip().rstrip("/") == link.rstrip("/"):
return c
if len(cands) == 1:
return cands[0]
return ""
def _git_at(path):
"""git arguments that read `path` WITHOUT discovering it.
Root reading p_swas's clone with `git -C` is refused ("detected dubious
ownership"), and _git returned "" -- which is why `update swas` said
`main moved ? -> 0ec52012`. An explicit --git-dir is not discovery, so
no ownership check applies; nothing is configured to allow it."""
if os.path.isdir(os.path.join(path, ".git")):
return ["--git-dir=" + os.path.join(path, ".git"), "--work-tree=" + path]
return ["--git-dir=" + path]
def _git(clone, *args, timeout=60):
try:
r = subprocess.run(["git"] + _git_at(clone) + list(args), capture_output=True,
text=True, env=_GIT_ENV, timeout=timeout)
except (OSError, subprocess.SubprocessError):
return ""
return r.stdout.strip() if r.returncode == 0 else ""
def _ls_remote(link, pattern=None, timeout=90):
"""[(sha, refname)] from the remote, without cloning anything."""
cmd = ["git", "ls-remote"]
if link and link.startswith(("/", "file://")):
# A DEV SOURCE IS SOMEONE ELSE'S REPOSITORY. ls-remote on it is
# refused ("dubious ownership") and -c safe.directory does not reach
# the reading process. An explicit --git-dir is not "discovering"
# a repository, so root may read the refs directly.
p = link[len("file://"):] if link.startswith("file://") else link
gd = os.path.join(p, ".git") if os.path.isdir(os.path.join(p, ".git")) else p
want = ("refs/tags/*" if pattern == "tags" else pattern) or ""
try:
r = subprocess.run(["git", "--git-dir=" + gd, "for-each-ref",
"--format=%(objectname) %(refname)"],
capture_output=True, text=True, timeout=timeout)
except (OSError, subprocess.SubprocessError):
return []
import fnmatch as _fn
out = []
for ln in (r.stdout or "").splitlines():
sha, _, ref = ln.partition(" ")
if not want or _fn.fnmatch(ref, want):
out.append((sha, ref))
return out
if pattern == "tags":
cmd += ["--tags", "--refs"]
cmd.append(link)
if pattern and pattern != "tags":
cmd.append(pattern)
try:
r = subprocess.run(cmd, capture_output=True, text=True, env=_GIT_ENV,
timeout=timeout)
except (OSError, subprocess.SubprocessError):
return []
if r.returncode != 0:
return []
out = []
for ln in r.stdout.splitlines():
parts = ln.split(None, 1)
if len(parts) == 2:
out.append((parts[0].strip(), parts[1].strip()))
return out
def _vsort_key(tag):
"""Sort 1.9.0 before 1.10.0, which a plain string sort does not."""
return [int(x) if x.isdigit() else x
for x in re.split(r"(\d+)", tag.lstrip("vV"))]
def _git_pkg_state(name, remote=True):
"""Everything worth saying about one git package.
Returns a dict, or None when this is not a git package. `remote=False`
asks the disk only -- no network, for the bulk view."""
script = (_canonical_script_path(name)
or os.path.join(pkgusr_home(name), "install_%s" % name))
if not script or not os.path.isfile(script):
return None
try:
text = open(script, encoding="utf-8", errors="replace").read()
except OSError:
return None
if not (re.search(r'is_git="?1"?', text) or re.search(r"\bgit\s+clone\b", text)):
return None
link = _dev_source(name) or _script_var(text, "link")
ref = _script_var(text, "git_ref")
clone = _git_clone_dir(name, link)
st = {"name": name, "script": script, "link": link, "ref": ref,
"clone": clone, "kind": "?", "local": "", "remote": "",
"newer_tags": [], "behind": None, "note": ""}
if clone:
st["local"] = _git(clone, "rev-parse", "HEAD")
if not link:
st["note"] = "the script names no repository (link=)"
return st
if not remote:
return st
# WHAT KIND OF REF IS IT? The remote knows; guessing from the string
# ("master" is a branch, "2.18.0" is a tag) is how a tag named `main`
# or a branch named `1.x` goes wrong.
heads = {r.rsplit("/", 1)[-1]: s for s, r in _ls_remote(link, "refs/heads/*")}
tags = [r.rsplit("/", 1)[-1] for _s, r in _ls_remote(link, "tags")]
st["tags"] = tags
if ref and ref in heads:
st["kind"] = "branch"
st["remote"] = heads[ref]
elif ref and ref in tags:
st["kind"] = "tag"
newer = [t for t in tags if _vsort_key(t) > _vsort_key(ref)]
st["newer_tags"] = sorted(newer, key=_vsort_key)
elif not ref:
st["kind"] = "default branch"
for d in ("main", "master"):
if d in heads:
st["remote"] = heads[d]
st["ref"] = d
break
elif ref:
st["kind"] = "commit or missing ref"
st["note"] = "the remote has no branch or tag called %s" % ref
# how far behind, when the clone already has the objects
if st["kind"].endswith("branch") and st["local"] and st["remote"] \
and st["local"] != st["remote"] and clone:
n = _git(clone, "rev-list", "--count", "%s..%s" % (st["local"], st["remote"]))
st["behind"] = int(n) if n.isdigit() else None
return st
def _git_pkg_line(st):
n = st["name"]
ref = st["ref"] or "(default)"
if st["note"] and not st["remote"] and not st["newer_tags"]:
return " %-20s %-14s %s" % (n, ref, st["note"])
if st["kind"].endswith("branch"):
if not st["remote"]:
return " %-20s %-14s branch -- the remote could not be reached" % (n, ref)
if st["local"][:12] == st["remote"][:12]:
return " %-20s %-14s branch, up to date (%s)" % (n, ref, st["local"][:8])
behind = ("%d commit(s) behind" % st["behind"]) if st["behind"] \
else "moved"
return " %-20s %-14s branch, %s: %s -> %s" % (
n, ref, behind, (st["local"] or "?")[:8], st["remote"][:8])
if st["kind"] == "tag":
if not st["newer_tags"]:
return " %-20s %-14s tag, nothing newer released" % (n, ref)
nt = st["newer_tags"]
return " %-20s %-14s tag -- %d newer: %s" % (
n, ref, len(nt), ", ".join(nt[-4:]))
return " %-20s %-14s %s %s" % (n, ref, st["kind"], st["note"])
def cmd_git_status(args):
"""What every git package follows, and whether it has moved."""
names = args.packages or list_all_users()
states = []
for n in sorted(set(names)):
st = _git_pkg_state(n, remote=not getattr(args, "local_only", False))
if st:
states.append(st)
if not states:
printInfo("no git-based packages here"
if not args.packages else
"none of those are git-based packages")
return 0
printHeader("git packages")
print()
print(" %-20s %-14s %s" % ("package", "ref", "state"))
print(" " + "-" * 62)
moved, taggy = [], []
for st in states:
print(_git_pkg_line(st))
if st["kind"].endswith("branch") and st["remote"] \
and st["local"][:12] != st["remote"][:12]:
moved.append(st["name"])
if st["newer_tags"]:
taggy.append(st)
print()
if moved:
printInfo(" follow the branch (fetch, rebuild, install):")
printInfo(" packagemanager update %s --run" % " ".join(moved))
for st in taggy:
printInfo(" %s is pinned to %s; to move it:" % (st["name"], st["ref"]))
printInfo(" packagemanager git-ref %s %s"
% (st["name"], st["newer_tags"][-1]))
if not moved and not taggy:
printInfo(" nothing to do: every git package is where its ref says")
return 0
# =========================================================================== #
# command: dev -- a package you develop on this machine [IMPLEMENTED]
# =========================================================================== #
#
# swas, swbr and the browser are written HERE. Their git entries cloned the
# server, so a change had to be pushed before the system could have it, and
# the clone in p_swas's home is not a place to work: it is the package's.
#
# A dev source keeps both ownerships intact. The checkout is yours (edit,
# build and run it as yourself); the package user clones FROM it, and what
# it installs stays its own. One line per package in DEV_SOURCES:
# p_swas /home/nim/dev/SwAS
# lfs-phases reads it when it fetches, _git_pkg_state when `update` asks
# whether the package moved. What is installed is the COMMITTED state of the
# followed branch -- never the working tree.
DEV_SOURCES = "/etc/pkgusr/dev-sources.conf"
DEV_HUB = "/srv/git" # where `dev --hub` keeps the bare repositories
def _dev_entries():
"""{account: (source, checkout)}. A line is
p_swas /srv/git/SwAS.git <- /home/n76310/Git/SwAS
the part after `<-` is where you work (hub setups only)."""
out = {}
try:
lines = open(os.environ.get("PM_DEV_SOURCES", DEV_SOURCES),
encoding="utf-8").read().splitlines()
except OSError:
return out
for ln in lines:
ln = ln.split("#", 1)[0].strip()
parts = ln.split(None, 1)
if len(parts) == 2:
src, _, co = parts[1].partition("<-")
out[parts[0]] = (src.strip(), co.strip())
return out
def _dev_sources():
"""{account: source} -- what the package is built from."""
return {a: v[0] for a, v in _dev_entries().items()}
def _dev_source(name):
return _dev_sources().get(pkgusr_name(name), "")
def _write_dev_sources(ent):
"""ent: {account: (source, checkout)}"""
path = os.environ.get("PM_DEV_SOURCES", DEV_SOURCES)
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w", encoding="utf-8") as f:
f.write("# package account -> the repository it is built from"
" [<- where you work]\n"
"# managed by: packagemanager dev <name> --hub <checkout> --run\n")
for k in sorted(ent):
src, co = ent[k]
f.write("%-24s %s%s\n" % (k, src, (" <- " + co) if co else ""))
os.chmod(path, 0o644)
def _dev_trust(acct, path):
"""Name the checkout in the account's own git config, as git requires
for a repository someone else owns -- the .git path is what it checks."""
if not _can_su(acct):
return
for sd in (path, os.path.join(path, ".git")):
subprocess.run(["su", "-s", "/bin/sh", acct, "-c",
"git config --global --get-all safe.directory 2>/dev/null"
" | grep -qxF %s || git config --global --add safe.directory %s"
% (shlex.quote(sd), shlex.quote(sd))],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
def _dev_can_read(acct, path):
"""Can the package account read this repository? Asked as it."""
if not _can_su(acct):
return None
r = subprocess.run(["su", "-s", "/bin/sh", acct, "-c",
"git ls-remote %s HEAD >/dev/null 2>&1"
% shlex.quote(path)],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
return r.returncode == 0
def _dev_grant(acct, path):
"""Read access for the account: x on the parents, rX on the tree.
ACLs, so nothing becomes world-readable and your own group and modes
stay what they are. The default ACL covers files you create later."""
if not shutil.which("setfacl"):
printError(" setfacl is not here (the Acl package) -- grant it by hand:")
printInfo(" chmod o+x each parent directory, and o+rX on the tree")
return False
ok = True
d = os.path.dirname(path)
while d and d != "/":
try:
if not os.stat(d).st_mode & 0o001:
subprocess.run(["setfacl", "-m", "u:%s:x" % acct, d], check=True)
printInfo(" setfacl -m u:%s:x %s" % (acct, d))
except (OSError, subprocess.CalledProcessError) as e:
printWarning(" could not open %s for %s (%s)" % (d, acct, e))
ok = False
d = os.path.dirname(d)
for flags in (["-R", "-m"], ["-R", "-d", "-m"]):
try:
subprocess.run(["setfacl"] + flags + ["u:%s:rX" % acct, path], check=True)
except (OSError, subprocess.CalledProcessError) as e:
printWarning(" setfacl %s on %s failed (%s)" % (" ".join(flags), path, e))
ok = False
printInfo(" setfacl -R -m u:%s:rX %s (and the same as default ACL)" % (acct, path))
return ok
def _dev_path_arg(p):
"""A path root typed: ~/ means the desktop user's home, not /root."""
if p.startswith("~/") and os.geteuid() == 0 and main_user():
try:
p = os.path.join(pwd.getpwnam(main_user()).pw_dir, p[2:])
except KeyError:
pass
return os.path.realpath(os.path.expanduser(p))
def _is_repo(p):
return os.path.isdir(os.path.join(p, ".git")) or os.path.isfile(
os.path.join(p, "HEAD"))
def _dev_ungrant(acct, path):
"""Take back what _dev_grant gave for `path` (this account's entries only;
ACLs another dev account holds on a shared parent stay)."""
if not shutil.which("setfacl") or not os.path.isdir(path):
return
for flags in (["-R", "-x"], ["-R", "-d", "-x"]):
subprocess.run(["setfacl"] + flags + ["u:%s" % acct, path],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
d = os.path.dirname(path)
while d and d != "/":
subprocess.run(["setfacl", "-x", "u:%s" % acct, d],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
d = os.path.dirname(d)
printInfo(" removed %s's read access to %s (and its parents)" % (acct, path))
def _dev_status(name, acct, src, co):
"""The lines that answer "which one is newest?" for one package."""
st = _git_pkg_state(name, remote=False) or {}
ref = st.get("ref") or _git(src, "symbolic-ref", "--short", "HEAD") or "HEAD"
inst = (st.get("local") or "")[:7]
at_src = _git(src, "rev-parse", "--short", ref)
out = [" source %s (%s)" % (src, ref)]
if not inst:
out.append(" installed - (not built from it yet)")
else:
out.append(" installed %s" % inst)
if at_src and inst and not at_src.startswith(inst[:7]) \
and not inst.startswith(at_src):
out.append(" source at %s <- newer: packagemanager update %s --run"
% (at_src, name))
elif at_src:
out.append(" source at %s (= installed)" % at_src if inst else
" source at %s" % at_src)
work = co or src
if _is_repo(work) and os.path.isdir(os.path.join(work, ".git")):
br = _git(work, "symbolic-ref", "--short", "HEAD") or "detached"
head = _git(work, "rev-parse", "--short", "HEAD")
dirty = [l for l in _git(work, "status", "--porcelain",
"--untracked-files=no").splitlines() if l.strip()]
note = []
if co:
ahead = _git(work, "rev-list", "--count", "%s..%s" % (at_src or ref, br)) \
if at_src else ""
if ahead and ahead != "0":
note.append("%s commit(s) not pushed: git push local %s" % (ahead, br))
if dirty:
note.append("%d file(s) not committed" % len(dirty))
out.append(" checkout %s %s %s%s" % (work, br, head,
(" -- " + "; ".join(note)) if note else ""))
return out
def cmd_dev(args):
"""List, set or drop the repository a git package is built from."""
ent = _dev_entries()
if not args.name:
printHeader("dev sources (%s)" % os.environ.get("PM_DEV_SOURCES", DEV_SOURCES))
if not ent:
print(" none. set one up: packagemanager dev <package> --hub <checkout> --run")
return 0
for acct, (src, co) in sorted(ent.items()):
pre = pkgusr_prefix()
name = acct[len(pre):] if pre and acct.startswith(pre) else acct
here = _is_repo(src)
can = _dev_can_read(acct, src) if here else False
print()
print(" %s (%s)%s" % (name, acct, "" if can or can is None else
(" NO REPOSITORY AT %s" % src if not here
else " NOT READABLE by %s -- re-run the dev command" % acct)))
for ln in _dev_status(name, acct, src, co):
print(ln)
print()
return 0
acct = pkgusr_name(args.name)
old_src, old_co = ent.get(acct, ("", ""))
if args.off:
if acct not in ent:
printInfo("%s has no dev source" % acct)
return 0
if not args.run:
printInfo("would stop building %s from %s (back to its url=) -- add --run"
% (acct, old_src))
return 0
del ent[acct]
_write_dev_sources(ent)
printSuccess("%s builds from its url= again" % acct)
return 0
hub_co = getattr(args, "hub", None)
if hub_co:
# ---- a bare repository in DEV_HUB, fed by `git push local`
co = _dev_path_arg(hub_co)
if not os.path.isdir(os.path.join(co, ".git")):
printError("%s is not a git checkout" % co)
return 1
owner = pwd.getpwuid(os.stat(co).st_uid).pw_name
hubdir = os.environ.get("PM_DEV_HUB", DEV_HUB)
bare = os.path.join(hubdir, os.path.basename(co.rstrip("/")) + ".git")
if not args.run:
printInfo("would:")
if not os.path.isdir(hubdir):
printInfo(" create %s (owner %s, readable by everyone)" % (hubdir, owner))
printInfo(" %s %s from %s (as %s)"
% ("use" if os.path.isdir(bare) else "create", bare, co, owner))
printInfo(" add the remote `local` -> %s in %s" % (bare, co))
printInfo(" build %s from %s" % (acct, bare))
if old_src and old_src != bare and old_src.startswith("/home/"):
printInfo(" remove %s's ACL access to %s" % (acct, old_src))
printInfo(" dry run -- add --run")
return 0
if os.geteuid() != 0:
printError("needs root")
return 1
def as_owner(cmd):
return subprocess.run(["su", "-s", "/bin/sh", owner, "-c", cmd],
capture_output=True, text=True)
if not os.path.isdir(hubdir):
os.makedirs(hubdir, mode=0o755)
shutil.chown(hubdir, user=owner)
os.chmod(hubdir, 0o755)
printInfo(" created %s (owner %s)" % (hubdir, owner))
if not os.path.isdir(bare):
r = as_owner("git clone -q --bare %s %s" % (shlex.quote(co), shlex.quote(bare)))
if r.returncode != 0:
printError(" could not create %s: %s" % (bare, r.stderr.strip()))
return 1
printInfo(" created %s" % bare)
# readable by package users, whatever the owner's umask: existing
# files now, and everything later pushes write
as_owner("git --git-dir=%s config core.sharedRepository 0644 && chmod -R a+rX %s"
% (shlex.quote(bare), shlex.quote(bare)))
cur = as_owner("git -C %s remote get-url local" % shlex.quote(co))
if cur.returncode != 0:
as_owner("git -C %s remote add local %s" % (shlex.quote(co), shlex.quote(bare)))
printInfo(" remote `local` added to %s" % co)
elif cur.stdout.strip() != bare:
as_owner("git -C %s remote set-url local %s" % (shlex.quote(co), shlex.quote(bare)))
printInfo(" remote `local` now points at %s" % bare)
ent[acct] = (bare, co)
_write_dev_sources(ent)
_dev_trust(acct, bare)
if old_src and old_src != bare and old_src.startswith("/home/"):
_dev_ungrant(acct, old_src)
if _dev_can_read(acct, bare) is False:
printError(" %s cannot read %s -- check its permissions" % (acct, bare))
return 1
printSuccess("%s builds from %s" % (acct, bare))
print()
printInfo(" work in %s; to install:" % co)
printInfo(" git push local <branch> (as %s, in the checkout)" % owner)
printInfo(" packagemanager update %s --run" % args.name)
return 0
if not args.path:
if acct in ent:
for ln in _dev_status(args.name, acct, old_src, old_co):
print(ln)
else:
printInfo("%s: no dev source" % acct)
return 0
# ---- build straight from a checkout (ACL access into it)
path = _dev_path_arg(args.path)
if not _is_repo(path):
printError("%s is not a git repository" % path)
return 1
if _git_pkg_state(args.name, remote=False) is None:
printWarning("%s is not installed as a git package yet -- the dev source"
" applies once it is (its stack git entry)" % args.name)
if not args.run:
printInfo("would build %s from %s, and give %s read access to it"
% (acct, path, acct))
printInfo(" (a hub keeps your home closed: packagemanager dev %s --hub %s)"
% (args.name, args.path))
printInfo(" dry run -- add --run")
return 0
if os.geteuid() != 0:
printError("needs root (it writes %s and sets ACLs)" % DEV_SOURCES)
return 1
ent[acct] = (path, "")
_write_dev_sources(ent)
_dev_trust(acct, path)
printSuccess("%s now builds from %s" % (acct, path))
if _dev_can_read(acct, path) is False:
printInfo(" %s cannot read it yet -- granting read access:" % acct)
_dev_grant(acct, path)
if _dev_can_read(acct, path) is False:
printError(" %s still cannot read %s -- see the lines above" % (acct, path))
return 1
print()
printInfo(" Commit, then install what you committed:")
printInfo(" packagemanager update %s --run" % args.name)
printInfo(" (the working tree is never installed -- only the followed branch)")
return 0
# =========================================================================== #
# command: rename -- a package under another name [IMPLEMENTED]
# =========================================================================== #
#
# The browser's entry was `minibrowser`, kept that way only because
# p_minibrowser existed: the program is Browser, it builds browser-mini and
# browser-big, and the name was a leftover. Renaming by hand is eight
# places -- account, group, home, the script's name lines, the clone
# directory, the stack entry, the stack's done-record, the dev source -- and
# missing one leaves an orphan the next stack run rebuilds under the new
# name. Files keep their owner across the rename: ownership is by uid, and
# usermod keeps the uid.
def _rename_plan(old, new):
oa, na = pkgusr_name(old), pkgusr_name(new)
oh, nh = pkgusr_home(oa), os.path.join(os.path.dirname(pkgusr_home(oa)), na)
pre = pkgusr_prefix()
on = oa[len(pre):] if pre and oa.startswith(pre) else oa
nn = na[len(pre):] if pre and na.startswith(pre) else na
return oa, na, oh, nh, on, nn
def _rename_in_scripts(home, on, nn):
"""install_<on>* -> install_<nn>*, and the name lines inside them; the
clone src/<on>-* -> src/<nn>-*. Only these lines: a script's commands
may mention the old word for other reasons."""
done = []
for f in sorted(os.listdir(home)):
p = os.path.join(home, f)
if not (f.startswith("install_") and os.path.isfile(p)):
continue
try:
t = open(p, encoding="utf-8", errors="replace").read()
except OSError:
continue
t2 = re.sub(r"(?m)^((?:name|name_version|pkg)=['\"]?)%s(?=[-'\"\s]|$)"
% re.escape(on), r"\g<1>" + nn, t)
t2 = re.sub(r"(?m)^(### )%s(?=-|\s|$)" % re.escape(on), r"\g<1>" + nn, t2)
if t2 != t:
st = os.stat(p)
with open(p, "w", encoding="utf-8") as fh:
fh.write(t2)
os.chown(p, st.st_uid, st.st_gid)
nf = f.replace("install_" + on, "install_" + nn, 1) \
if f.startswith("install_" + on) else f
if nf != f:
os.rename(p, os.path.join(home, nf))
if t2 != t or nf != f:
done.append(nf)
src = os.path.join(home, "src")
if os.path.isdir(src):
for d in sorted(os.listdir(src)):
if d == on or d.startswith(on + "-"):
os.rename(os.path.join(src, d), os.path.join(src, nn + d[len(on):]))
done.append("src/" + nn + d[len(on):])
vf = os.path.join(home, "VERSION")
if os.path.isfile(vf):
t = open(vf, encoding="utf-8", errors="replace").read()
t2 = re.sub(r"\A%s(?=-)" % re.escape(on), nn, t)
if t2 != t:
st = os.stat(vf)
open(vf, "w", encoding="utf-8").write(t2)
os.chown(vf, st.st_uid, st.st_gid)
done.append("VERSION")
return done
def cmd_rename(args):
"""Rename a package: account, group, home, script, stack entry, records."""
oa, na, oh, nh, on, nn = _rename_plan(args.old, args.new)
try:
ouid = pwd.getpwnam(oa).pw_uid
except KeyError:
printError("no account %s" % oa)
return 1
try:
pwd.getpwnam(na)
printError("%s already exists" % na)
return 1
except KeyError:
pass
if os.path.exists(nh):
printError("%s already exists" % nh)
return 1
_sd = STACK_DIR if "STACK_DIR" in globals() else "/etc/pkgusr/stacks"
ent_re = re.compile(r"^(\s*(?:book|git|tar|check|cfg)\s+)%s(?=\s|$)" % re.escape(on), re.M)
stacks = [f for f in sorted(glob.glob(os.path.join(_sd, "*.stack")))
if ent_re.search(open(f, encoding="utf-8", errors="replace").read())]
printHeader("rename %s -> %s" % (on, nn))
print(" account %s -> %s (uid %d: every file keeps its owner)" % (oa, na, ouid))
print(" home %s -> %s" % (oh, nh))
print(" scripts name lines and file names in the home; the clone dir")
for f in stacks:
print(" stack %s: entry %s -> %s" % (f, on, nn))
if oa in _dev_entries():
print(" dev %s" % os.environ.get("PM_DEV_SOURCES", DEV_SOURCES))
if not args.run:
printInfo("\n dry run -- add --run")
return 0
if os.geteuid() != 0:
printError("needs root")
return 1
busy = []
for pid in os.listdir("/proc"):
if pid.isdigit():
try:
if os.stat("/proc/" + pid).st_uid == ouid:
busy.append(pid)
except OSError:
pass
if busy:
printError("%s has running processes (%s) -- a build? Not renaming."
% (oa, ", ".join(busy[:5])))
return 1
r = subprocess.run(["usermod", "-l", na, "-d", nh, "-m", oa],
capture_output=True, text=True)
if r.returncode != 0:
printError("usermod failed: %s" % r.stderr.strip())
return 1
printSuccess(" account and home renamed")
try:
import grp
grp.getgrnam(oa)
r = subprocess.run(["groupmod", "-n", na, oa], capture_output=True, text=True)
if r.returncode == 0:
printInfo(" group %s -> %s" % (oa, na))
else:
printWarning(" groupmod failed: %s" % r.stderr.strip())
except (KeyError, ImportError):
pass
for x in _rename_in_scripts(nh, on, nn):
printInfo(" %s" % x)
for f in stacks:
t = open(f, encoding="utf-8").read()
shutil.copy2(f, f + ".bak")
open(f, "w", encoding="utf-8").write(ent_re.sub(r"\g<1>" + nn, t))
printInfo(" %s: entry renamed (old file: %s.bak)" % (f, os.path.basename(f)))
for sf in glob.glob(os.path.join(os.environ.get("LFS_PKGUSR_DIR", "/usr/src/lfs-pkgusr"),
"progress", "stacks", "*.done")):
t = open(sf, encoding="utf-8").read()
t2 = re.sub(r"(?m)^(\w+ )%s$" % re.escape(on), r"\g<1>" + nn, t)
if t2 != t:
open(sf, "w", encoding="utf-8").write(t2)
printInfo(" %s: done-record renamed" % os.path.basename(sf))
ent = _dev_entries()
if oa in ent:
ent[na] = ent.pop(oa)
_write_dev_sources(ent)
_dev_trust(na, ent[na][0])
printInfo(" dev source moved to %s" % na)
print()
printSuccess("%s is now %s" % (on, nn))
printInfo(" check: packagemanager info %s packagemanager stack sway" % nn)
if on == "minibrowser":
printInfo(" your sway config may still say `exec minibrowser` -- the"
" program is browser-mini")
return 0
def cmd_git_ref(args):
"""Show what a git package could follow, or move it to another ref."""
name = args.package
st = _git_pkg_state(name)
if st is None:
printError("%s is not a git-based package" % name)
return 1
if not st["link"]:
printError("%s: its script names no repository" % name)
return 1
if not args.ref:
tags = sorted(st.get("tags") or [], key=_vsort_key)
heads = [r.rsplit("/", 1)[-1] for _s, r in _ls_remote(st["link"], "refs/heads/*")]
printHeader("%s -- %s" % (name, st["link"]))
print()
print(" now: %s (%s)" % (st["ref"] or "(default)", st["kind"]))
print()
print(" branches: " + (", ".join(heads[:8]) or "(none)"))
print(" tags, newest last:")
for t in tags[-12:]:
print(" %s%s" % (t, " <- current" if t == st["ref"] else ""))
if not tags:
print(" (none -- this repository is not tagged)")
print()
printInfo(" pick one: packagemanager git-ref %s <ref>" % name)
return 0
want = args.ref
known = set(st.get("tags") or []) | {
r.rsplit("/", 1)[-1] for _s, r in _ls_remote(st["link"], "refs/heads/*")}
if want not in known and not re.fullmatch(r"[0-9a-f]{7,40}", want):
printError("%s: the remote has no tag or branch called %s" % (name, want))
printInfo(" what it has: packagemanager git-ref %s" % name)
return 1
# THE SCRIPT IS ONE OF TWO PLACES THE REF LIVES. A stack entry writes it
# into the generated script, so editing only the script is undone by the
# next stack run -- say so, and name the file.
try:
text = open(st["script"], encoding="utf-8", errors="replace").read()
except OSError as e:
printError("%s: %s" % (st["script"], e))
return 1
new, n = re.subn(r'^(\s*git_ref\s*=\s*).*$', r'\1"%s"' % want, text,
count=1, flags=re.M)
if not n:
printError("%s has no git_ref= line to change" % st["script"])
return 1
if not getattr(args, "run", False):
printInfo(" would set git_ref=\"%s\" in %s (was \"%s\")"
% (want, st["script"], st["ref"]))
printInfo(" dry run -- add --run")
return 0
bak = st["script"] + ".bak-" + time.strftime("%Y%m%d-%H%M%S")
try:
shutil.copy2(st["script"], bak)
open(st["script"], "w").write(new)
except OSError as e:
printError("could not write %s: %s" % (st["script"], e))
return 1
printSuccess("%s: git_ref is now %s (was %s; backup %s)"
% (name, want, st["ref"] or "(default)", os.path.basename(bak)))
src = _STACK_TARGET_SOURCE.get(name)
if src:
printWarning(" %s also carries this ref: a stack run regenerates the"
" script and puts the old one back." % src)
printInfo(" change the entry there too: ref=%s" % want)
printInfo(" then build it: packagemanager update %s --run" % name)
return 0
# =========================================================================== #
# who still links against this? [IMPLEMENTED]
# =========================================================================== #
#
# A package-user system records who INSTALLED every file. It records nothing
# about who USES one -- and a library is used by things that were built
# against it long ago. webkitgtk rebuilt from the GTK4 port to the GTK3 one
# leaves libwebkitgtk-6.0.so.4 behind with nothing to rebuild it, and
# `--clean` is then one keystroke from deleting a library other programs are
# still linked to. ldd answers it for ONE file; this answers it for a
# library.
_ELF_SCAN_DIRS = ("/usr/bin", "/usr/sbin", "/usr/lib", "/usr/libexec",
"/bin", "/sbin", "/lib", "/opt")
_ELF_CONFIRM_MAX = 600 # readelf calls; a grep hit is cheap, this is not
def _soname_of(path):
"""The SONAME an ELF library announces -- libwebkitgtk-6.0.so.4, not the
file name libwebkitgtk-6.0.so.4.13.8. That is the name other binaries
record, so it is the name to search for."""
if not os.path.isfile(path):
return None
try:
r = subprocess.run(["readelf", "-d", path], capture_output=True,
text=True, timeout=30)
except (OSError, subprocess.SubprocessError):
r = None
if r and r.returncode == 0:
m = re.search(r"\(SONAME\)\s+Library soname: \[([^\]]+)\]", r.stdout)
if m:
return m.group(1)
b = os.path.basename(path)
m = re.match(r"^(lib.+?\.so(?:\.\d+)?)(?:\.|$)", b)
return m.group(1) if m else None
def _needed_by(soname, skip_uid=None, dirs=None):
"""Every ELF that records this soname in DT_NEEDED: [(path, owner)].
grep first -- a soname is a literal string in .dynstr, and grep over
/usr/lib is seconds -- then readelf only on the hits, because a string can
also appear in a script, a .la file or another library's rpath.
"""
# RESOLVED HERE, NOT IN THE SIGNATURE. A default argument is bound at
# import, so `dirs=_ELF_SCAN_DIRS` could never be overridden -- including
# by a test, which is how this function first appeared to find nothing.
live = [d for d in (dirs or _ELF_SCAN_DIRS) if os.path.isdir(d)]
if not live:
return []
try:
r = subprocess.run(["grep", "-rlZ", "--binary-files=binary", "-F",
"--", soname] + live,
capture_output=True, timeout=600)
except (OSError, subprocess.SubprocessError):
return []
_all = [c.decode("utf-8", "replace") for c in r.stdout.split(b"\0") if c]
cands = _all[:_ELF_CONFIRM_MAX]
if len(_all) > _ELF_CONFIRM_MAX:
# A TRUNCATED ANSWER MUST NOT READ AS A COMPLETE ONE.
printWarning(" %s: %d candidates, only the first %d were confirmed"
% (soname, len(_all), _ELF_CONFIRM_MAX))
out = []
for p in cands:
if os.path.islink(p) or not os.path.isfile(p):
continue
if os.path.basename(p).startswith(soname.split(".so")[0] + ".so"):
continue # the library itself, or its siblings
try:
st = os.lstat(p)
if skip_uid is not None and st.st_uid == skip_uid:
continue # its own package may rebuild freely
d = subprocess.run(["readelf", "-d", p], capture_output=True,
text=True, timeout=30)
except (OSError, subprocess.SubprocessError):
continue
if d.returncode != 0:
continue
if re.search(r"\(NEEDED\).*\[%s\]" % re.escape(soname), d.stdout):
owner = "?"
try:
owner = pwd.getpwuid(st.st_uid).pw_name if pwd else "?"
except KeyError:
owner = str(st.st_uid)
out.append((p, owner))
return out
def cmd_used_by(args):
"""What still links against this library, or this package's libraries."""
target = args.name
sonames = []
if re.search(r"\.so(\.|$)", target):
sonames = [target if os.sep not in target
else (_soname_of(target) or os.path.basename(target))]
else:
# a package: every shared library in its manifest
libs = [p for p in _pkglist_paths(target) or []
if re.search(r"/lib[^/]*\.so(\.\d+)*$", p)
and os.path.isfile(p) and not os.path.islink(p)]
for p in sorted(libs):
s = _soname_of(p)
if s and s not in sonames:
sonames.append(s)
if not sonames:
printInfo("%s installs no shared library (or has no manifest)" % target)
return 0
printHeader("What links against %s" % ", ".join(sonames))
print()
total = 0
for s in sonames:
users = _needed_by(s)
by_pkg = {}
for p, owner in users:
by_pkg.setdefault(owner, []).append(p)
if not users:
print(" %-34s nothing links against it" % s)
continue
total += len(users)
print(" %s -- %d file(s), %d package(s):" % (s, len(users), len(by_pkg)))
for owner in sorted(by_pkg):
ex = ", ".join(os.path.basename(x) for x in sorted(by_pkg[owner])[:3])
more = "" if len(by_pkg[owner]) <= 3 else " ..."
print(" %-22s %s%s" % (owner, ex, more))
print()
if total:
printInfo(" removing a library these were built against breaks them"
" until they are rebuilt:")
printInfo(" packagemanager update <package> --reinstall --run")
else:
printInfo(" nothing needs these; they are safe to remove")
return 0
def _clean_after_install(user, old_paths, since):
"""Full --clean sequence run AFTER a successful install: remove old-version
orphans, reapply install (to undo any over-removal), then regenerate the
manifest so it reflects the final state.
PARK, REINSTALL, THEN JUDGE (1.14.311). "Older than the install" is
not "no longer installed": CMake skips a file whose content has not
changed, meson and `install -p` keep timestamps, so a current file can
look old. This used to delete first and trust a reapply to repair the
damage -- a reapply that had not run since 1.14.16, so the damage stayed
(webkitgtk's jsc/*.h, and every build against webkitgtk-6.0 broke).
Now nothing is deleted on the mtime test's word alone. The candidates
are renamed aside (same directory, so it is one rename each and nothing
is copied), the install phase runs, and only then:
* a path the install put back IS current -- the parked copy goes;
* a path it did not put back is a real orphan -- that one goes;
* if the install step FAILS, every parked file is put back, and the
clean is reported as not done. Nothing is half-cleaned.
"""
# THE ACCOUNT, NOT THE BOOK NAME. `update` passed `webkitgtk`: su to it
# failed, so the reapply ran as root in /root ("no unpacked source in
# /root") and the clean was undone -- and getpwnam failed too, which
# turned off both "only files this package still owns" and "its own
# files do not count as users of its libraries".
user = pkgusr_name(user)
# an earlier clean interrupted between park and judge: finish undoing it
if os.path.exists(_park_ledger(user)):
n = _unpark_all(user)
printWarning(f" --clean: an interrupted clean had parked files -- "
f"{n} put back first")
victims = _orphan_sweep(user, old_paths, since, remove=False)
if not victims:
printInfo(" --clean: no leftover files from the old version.")
printInfo(" --clean: regenerating the file manifest...")
write_pkg_list(user, background=False)
return 0
files = [p for p in victims if os.path.islink(p) or not os.path.isdir(p)]
dirs = [p for p in victims if p not in set(files)]
parked = []
with open(_park_ledger(user), "w") as led:
for p in files:
try:
os.rename(p, p + _PARK_SUFFIX)
parked.append(p)
led.write(p + "\n")
led.flush()
except OSError as e:
printWarning(f" could not set {p} aside ({e}) -- left in place")
printInfo(f" --clean: {len(parked)} file(s) look older than this install "
f"-- set aside while the install step confirms which are gone")
if not _reapply_install(user):
n = _unpark_all(user)
printError(f" --clean: NOT DONE -- put {n} file(s) back as they were.")
printInfo(" The upgrade itself is installed; only the clean-up was undone.")
_pre = pkgusr_prefix()
printInfo(" Fix the install step, then: packagemanager update "
f"{user[len(_pre):] if _pre and user.startswith(_pre) else user}"
" --reinstall --clean --run")
write_pkg_list(user, background=False)
return 0
still = [p for p in parked if os.path.lexists(p)]
gone = [p for p in parked if not os.path.lexists(p)]
# both kinds of parked copy go: for `still` the install wrote the
# current one, for `gone` nothing replaced it -- that is the orphan
remove_paths_as_user(user, [p + _PARK_SUFFIX for p in parked] + dirs)
try:
os.unlink(_park_ledger(user))
except OSError:
pass
if still:
printInfo(f" --clean: {len(still)} of them are still installed by this "
f"version (unchanged, so the install left their old date) -- kept")
if gone:
printInfo(f" --clean: removed {len(gone)} file(s) the new version no "
f"longer installs")
for p in gone[:10]:
printInfo(" " + p)
if len(gone) > 10:
printInfo(" ... and %d more" % (len(gone) - 10))
else:
printInfo(" --clean: nothing was an orphan after all")
printInfo(" --clean: regenerating the file manifest...")
write_pkg_list(user, background=False)
return len(gone)
def _is_phased_script(text):
"""Does this install script hand its phases to the runner?
The runner is what fetches, unpacks, resumes in the build directory and
re-runs one phase -- so "phased" means "ends in pkgusr_run", not "defines
some particular function". Three spellings, because the header and the
library name are each written by a different generation of the tools.
"""
t = text or ""
return bool(re.search(r"\bpkgusr_run\b", t)
or re.search(r"PKGUSR_LIB|lfs-phases", t)
or re.search(r"^###\s*phases:", t, re.M))
def _is_git_package(name):
"""A package whose install script builds from a git repo (is_git=1, or it
clones/pulls) rather than a book tarball."""
p = (_canonical_script_path(name)
or os.path.join(pkgusr_home(name), f"install_{name}"))
if not p or not os.path.isfile(p):
return False
try:
t = open(p, encoding="utf-8", errors="replace").read(8000)
except OSError:
return False
return bool(re.search(r'is_git="?1"?', t)
or re.search(r'\bgit\s+(clone|pull|fetch)\b', t))
# A plan entry whose "script" is this is built by `lfs`, not by us: the LFS
# book owns it, and lfs already knows how to build one of its packages.
_LFS_STEP = ":lfs-book:"
_TOOLCHAIN_RISK = ("glibc", "binutils", "gcc", "libstdcpp", "ld", "linux-headers")
def _lfs_stale_list(include_current=False):
"""[(name, installed, book)] for packages the LFS book has newer.
With include_current, the packages whose version did NOT move are listed
too (installed == book). After glibc changes, "up to date" and "built
against the current system" stop being the same statement -- so a full
rebuild in book order is a thing a person may legitimately ask for.
Read from `lfs update --porcelain` -- one tab-separated fact per line.
Parsing the human report instead would break the first time a word in it
changed, and this tool has been bitten by exactly that before."""
lfs_bin = shutil.which("lfs") or ("/usr/bin/lfs" if os.access(
"/usr/bin/lfs", os.X_OK) else None)
if not lfs_bin:
return []
try:
_cmd = [lfs_bin, "update", "--porcelain"]
if include_current:
_cmd.append("--include-current")
r = subprocess.run(_cmd, capture_output=True, text=True, timeout=600)
except Exception:
return []
out = []
for ln in (r.stdout or "").splitlines():
parts = ln.split("\t")
if len(parts) == 3 and parts[0]:
out.append((parts[0], parts[1], parts[2]))
return out
def _lfs_update_report(names):
"""Ask `lfs update` about the packages the BLFS book cannot answer for,
and relay WHAT IT SAYS.
Returns (kind, payload):
("current", None) -- lfs said so, in those words
("lines", [...]) -- lfs's own report, verbatim
("failed", why) -- no answer; the caller must say so, not assume
The previous version returned [] both for "everything is current" and for
"the call produced nothing", and the caller printed "all current with the
LFS book" for both -- a clean bill of health manufactured out of silence,
on a system with 30 stale packages. Empty-as-pass, in the very code that
was written to stop packages disappearing quietly.
It is called with no package names: `lfs update` already covers every
installed LFS package and labels what belongs to the other book, so this
asks the same question the user would ask by hand, and shows the same
answer. Passing 77 names asked the same tool a longer question with more
ways to go wrong."""
lfs_bin = shutil.which("lfs") or ("/usr/bin/lfs" if os.access(
"/usr/bin/lfs", os.X_OK) else None)
if not lfs_bin:
return ("failed", "lfs is not installed here")
try:
r = subprocess.run([lfs_bin, "update"],
capture_output=True, text=True, timeout=600)
except Exception as e:
return ("failed", str(e))
out = (r.stdout or "").strip()
if not out:
tail = (r.stderr or "").strip().splitlines()
why = tail[-1] if tail else f"no output (exit {r.returncode})"
return ("failed", why)
if "Everything checked is current" in out and "Out of date" not in out:
return ("current", None)
return ("lines", [ln for ln in out.splitlines() if ln.strip()])
def _report_no_book(no_book, consult_lfs, no_record=(), unjudged=(),
uptodate_n=None, total=None, named=()):
"""Account for every package the BLFS plan could not judge, and let the
LFS book answer for them.
Listing them only when the plan was non-empty made the commonest state
(BLFS subset current, glibc stale) read as a clean bill of health -- and
two silent `continue`s meant the numbers never had to add up. They do
now: checked = planned + current + these."""
rest = sorted(set(no_book) | set(no_record) | set(unjudged))
if uptodate_n is not None and total is not None:
print()
printInfo(f"Of {total} checked: {uptodate_n} current with the BLFS "
f"book, {len(rest)} it cannot judge.")
if not rest:
return
if not (uptodate_n is not None and total is not None):
print()
if no_record:
printInfo(f" {len(no_record)} installed, but no recorded version "
f"here -- what the book carries is shown beside each:")
for _nr in sorted(no_record)[:12]:
printInfo(" " + _nr)
if len(no_record) > 12:
printInfo(" ... and %d more" % (len(no_record) - 12))
printInfo(" the installed version, if the script still names it:"
" packagemanager info <name>")
if unjudged:
printInfo(f" {len(unjudged)} with no completed-install record here "
f"(the LFS build tracks these in its own manifests):")
printInfo(" " + ", ".join(sorted(unjudged)[:12])
+ (" ..." if len(unjudged) > 12 else ""))
# SAY WHAT TO DO INSTEAD. Asking to update a package this system has
# never installed is a reasonable thing to try, and "cannot judge" is
# a true answer to a question the user did not mean to ask. If they
# NAMED the packages, tell them the command that does what they want.
_named = list(named or ())
if _named:
_want = [u for u in sorted(unjudged)
if any(u.lower() == n.lower() for n in _named)]
if _want:
printInfo(" These have no record here because they are NOT "
"INSTALLED. To build them:")
printInfo(" packagemanager install %s --run --recursive"
% " ".join(_want[:6]))
if no_book:
printInfo(f" {len(no_book)} not in the BLFS book at all.")
printInfo(" The LFS book answers for these:")
if not consult_lfs:
printInfo(" check them with: lfs update")
return
kind, payload = _lfs_update_report(rest)
if kind == "failed":
printWarning(f" could not ask lfs ({payload}) -- these are UNCHECKED.")
printInfo(" run it yourself: lfs update")
elif kind == "current":
printInfo(" all current with the LFS book.")
else:
for ln in payload:
print(" " + ln)
def _systemctl_users(steps):
"""Planned packages whose install script runs systemctl.
Generated scripts shim a missing systemctl into a visible skip (1.11.30),
so a systemd book builds on a SysV system -- but "skipped" means the
package's service was NOT enabled, and only the person can decide whether
that matters for this package. So: name them, once, rather than leaving
the notices to scroll past mid-build."""
if os.path.isdir("/usr/lib/systemd/system") or shutil.which("systemctl"):
return [] # this system HAS systemd: nothing to say
out = []
for _anchor, name, _iv, _bv, path in steps:
cands = []
if path and path != _LFS_STEP:
cands.append(path)
# the LFS book's script for this package, if one has been generated
cands.append(os.path.join("/usr/src/lfs-pkgusr/scripts", name + ".sh"))
cands.append(os.path.join(pkgusr_home(name), "install_" + name))
for c in cands:
try:
with open(c, encoding="utf-8", errors="replace") as f:
body = f.read()
except OSError:
continue
# the shim definition itself is not a use of systemctl
uses = [ln for ln in body.splitlines()
if "systemctl" in ln
and "command -v systemctl" not in ln
and not ln.strip().startswith("#")]
if uses:
# the PATH, not just the name: the next thing anyone does with
# this list is open the file and add the SysV equivalent
out.append((name, len(uses), c))
break
return out
def _machine_avoids():
"""Everything this machine never builds, from avoid.stack.
`avoid` entries were only consulted by `stack`, so `install` and
`update` walked the book's dependency graph with nothing excluded. The
exclusions describe the MACHINE -- that is why they were moved into
their own file in 1.14.224 -- so every plan should honour them.
"""
out = set()
d = STACK_DIR if "STACK_DIR" in globals() else "/etc/pkgusr/stacks"
for f in sorted(glob.glob(os.path.join(d, "*.stack"))):
try:
for r in _parse_stack_file(f):
if r["kind"] == "avoid":
out.add(r["target"])
except (ValueError, OSError):
continue
return out
def _stack_targets(name):
"""The packages a stack builds -- book/git/tar entries, not checks."""
path = name if os.path.sep in name else os.path.join(
STACK_DIR if "STACK_DIR" in globals() else "/etc/pkgusr/stacks",
name if name.endswith(".stack") else name + ".stack")
if not os.path.isfile(path):
printError("no such stack: %s (%s)" % (name, path))
return None
try:
rows = _parse_stack_file(path)
except ValueError as e:
printError(str(e))
return None
# a stack on another book: refresh it once, and make sure the entries
# read it even when this file is not the installed copy
_sb = {r["target"]: r["opts"]["from-book"] for r in rows
if r["kind"] == "book" and r["opts"].get("from-book")}
if _sb:
_stack_book_map().update({k.lower(): v for k, v in _sb.items()})
_BOOK_FOR_PKG.clear()
os.environ["PM_STACK_BOOKS"] = ",".join(
"%s=%s" % kv for kv in sorted(_sb.items()))
for _sel in sorted(set(_sb.values())):
_book_path_for(_sel, refresh=True)
printInfo(" %d package(s) of this stack read the %s book"
% (list(_sb.values()).count(_sel), _sel))
return [r["target"] for r in rows
if r["kind"] in ("book", "git", "tar")]
def cmd_update(args):
# "UPDATE THE MEDIA CHAIN" WITHOUT TYPING EIGHTEEN NAMES.
#
# `stack media --run` builds what is MISSING and never updates what is
# installed -- correct, and not what someone means by "update the media
# stack". The other end, `update webkitgtk`, is one package. Between
# them there was nothing but a list of names to type out.
if getattr(args, "stack", None):
_tg = _stack_targets(args.stack)
if _tg is None:
return 2
if args.packages:
printInfo(" --stack %s and named packages: using the named ones"
% args.stack)
else:
args.packages = _tg
printInfo(" --stack %s: %d package(s) considered -- %s"
% (args.stack, len(_tg), ", ".join(_tg[:8])
+ (" ..." if len(_tg) > 8 else "")))
printInfo(" only those the book has moved past will appear in "
"the plan.")
printInfo(" pick from them instead of taking all: --select")
# stack TARBALL packages (yt-dlp): their release, never the book
if not getattr(args, "local", None) and args.packages:
_tar_done = _update_tar_packages(args)
if _tar_done:
args.packages = [p for p in args.packages if p not in _tar_done]
if not args.packages:
return
# --local: update a single package from a custom/local install script
# (e.g. a git-based package).
if getattr(args, "local", None):
name = args.packages[0] if args.packages else None
if not name:
printError("update --local needs a package name.")
return
path = os.path.abspath(args.local)
install_local(name, path, args.run, args.yes, "update",
test=getattr(args, "test", False),
clean=getattr(args, "clean", False))
return
# git-repo packages (not from the book): update by re-running their own
# git-based install script (git fetch + rebuild), bypassing book versions.
if args.packages and all(_is_git_package(p) or book_version(p) is None
for p in args.packages) \
and any(_is_git_package(p) for p in args.packages):
for name in args.packages:
script = _canonical_script_path(name) \
or os.path.join(pkgusr_home(name), f"install_{name}")
if not _is_git_package(name):
printWarning(f"{name}: not a git package and not in the book -- "
f"skipping (check the name, or use --local).")
continue
# IS IT PHASED? ASK THE RIGHT QUESTION.
#
# This looked for `unpack_pkg`, which is the RARE OVERRIDE a script
# defines to replace the built-in unpack entirely. A correct,
# current script does not define it -- the runner's own unpack is
# what clones git -- so every properly phased package was told its
# script "isn't phased", with advice (`template --git`) that
# overwrites the file and takes any hand-tuned build_pkg with it:
# meson setup build ... -D c_link_args=-licuuc
# A phased script is one that ENDS IN THE RUNNER. That is the
# thing that makes fetch/rebuild work, and it is what to test.
_body = open(script, errors="replace").read()
if not _is_phased_script(_body):
printWarning(f"{name}: this script does not end in the phase "
f"runner, so `update` cannot fetch and rebuild it "
f"cleanly.")
printInfo(" a phased script ends with:")
printInfo(' . "${PKGUSR_LIB:-lfs-phases}" && pkgusr_run "$@"')
printInfo(" and puts its commands in build_pkg / install_pkg.")
printInfo(f" `packagemanager template {name} --git <url>` writes"
f" a fresh one -- to a NEW file first, then move your")
printInfo(" build commands across: it does not keep them.")
# SAY WHETHER THERE IS ANYTHING TO UPDATE TO.
#
# "git package -- updating from its repo" is true of a package
# pinned to the newest tag, where the rebuild produces the same
# commit it already has: the command works, does exactly what it
# says, and changes nothing -- and there is no way to tell that
# from the output. The remote knows; ask it.
_st = None if getattr(args, "no_git_check", False) \
else _git_pkg_state(name)
if _st:
if _st["kind"].endswith("branch") and _st["remote"]:
if _st["local"][:12] == _st["remote"][:12]:
printInfo(" %s: %s is at %s, the same commit as here "
"-- this rebuilds what is already installed"
% (name, _st["ref"], _st["remote"][:8]))
else:
printInfo(" %s: %s moved %s -> %s%s -- this brings it in"
% (name, _st["ref"], (_st["local"] or "?")[:8],
_st["remote"][:8],
(" (%d commit(s))" % _st["behind"])
if _st["behind"] else ""))
elif _st["kind"] == "tag":
if _st["newer_tags"]:
printInfo(" %s: pinned to tag %s; %s is out"
% (name, _st["ref"], _st["newer_tags"][-1]))
printInfo(" a rebuild stays on %s. To move it:"
% _st["ref"])
printInfo(" packagemanager git-ref %s %s --run"
% (name, _st["newer_tags"][-1]))
else:
printInfo(" %s: pinned to tag %s, which is the newest "
"released -- this rebuilds the same commit"
% (name, _st["ref"]))
printInfo(f"{name}: git package -- updating from its repo.")
install_local(name, script, args.run, args.yes, "update",
test=getattr(args, "test", False),
clean=getattr(args, "clean", False))
return
flag_sig = "".join([
"r" if getattr(args, "recursive", False) else "",
"R" if args.dependents else "",
"e" if args.ignore_recommended else "", "o" if args.optional else "",
"c" if getattr(args, "clean", False) else "",
"i" if getattr(args, "reinstall", False) else ""])
base = ",".join(sorted(args.packages)) if args.packages else "all"
plan_key = base + (("#" + flag_sig) if flag_sig else "")
kind_of = {} # anchor -> required/recommended/...
cached = use_stored_plan("update", plan_key, assume_yes=args.yes) if args.run else None
no_book, no_record, unjudged, uptodate = [], [], [], []
git_tracked = []
_checked = 0
if cached is not None:
plan = [tuple(x) for x in cached]
else:
if args.packages and not getattr(args, "recursive", False):
# DEFAULT: just the named packages, no dependency walk
vers = book_versions_all()
vlookup = {t: vers.get(t) for t in args.packages}
names = list(args.packages)
elif args.packages:
vlookup = {}
for target in args.packages:
order, _missing = blfs_order_anchors(
target, args.ignore_recommended, args.optional)
for anchor, nv, kind, parent in order:
vlookup.setdefault(anchor, nv)
if anchor not in kind_of:
if not parent: # the requested target
kind_of[anchor] = ("requested", target)
else:
kind_of[anchor] = (kind or "required", parent)
if args.dependents: # super-recursive: also rebuild
for anchor, nv in book_rdeps( # everything that depends on it
target, args.ignore_recommended, args.optional).items():
vlookup.setdefault(anchor, nv)
kind_of.setdefault(anchor, ("dependent-of", target))
names = list(vlookup.keys())
else:
vlookup = book_versions_all()
names = list_all_users()
# a walk from a stack-book target carries that book's versions for
# every dependency; the ones it does not own read their own book
_stack_walk = getattr(args, "recursive", False) and any(
_book_override(t) for t in (args.packages or []))
_bk = _current_blfs_book()
_checked = len(names)
# ONE HEADER, TWO BOOKS. "against the book (13.1-systemd)" was true
# for 8 of these and false for the 11 that read svn -- and the svn
# ones are exactly the reason the command was typed.
_other = {}
for _n in names:
_ob = _book_override(_n)
if _ob:
_k = os.path.basename(_ob)
_other[_k] = _other.get(_k, 0) + 1
_books_txt = (f" ({_bk})" if _bk else "")
if _other:
_books_txt += "; " + ", ".join("%d from %s" % (v, k)
for k, v in sorted(_other.items()))
printInfo(f"Checking {len(names)} package(s) against the book"
+ _books_txt + "...")
named = set(args.packages) if args.packages else set()
plan, not_installed = [], []
# Two silent exits used to swallow most of a real system. Every
# package now lands in a NAMED bucket and the counts add up, because
# "97 checked, nothing to do" while `lfs update glibc` says 2.42 ->
# 2.43 is the plan lying about its own scope.
no_record, unjudged = [], []
for name in names:
u = gather_user(name)
anchor = u.meta.get("name") or name
bv = vlookup.get(anchor)
# EACH PACKAGE AGAINST ITS OWN BOOK: a stack's from-book (or
# machine.conf's book=) -- the default map would "update" an
# svn-built mpv back to the stable page.
if _book_override(anchor):
bv = book_version(anchor) or bv
elif _stack_walk:
bv = default_book_version(anchor) or bv
if u.state != "installed":
if name in named or anchor in named:
not_installed.append(name)
else:
# NOT silent: a package built by lfs-helper has no
# install_last, so it reads as "not installed" here and
# simply disappeared from every update-all.
unjudged.append(name)
continue
if not bv:
# not silently: an update-all that never mentions these reads
# as "everything is covered", and the LFS-book packages are
# exactly the ones that are not
no_book.append(name)
continue
if not u.version:
# An UNKNOWN installed version used to be folded into the
# same test as the comparison, where it came out falsy --
# and the package was filed as up to date. Not knowing is
# not the same as being current.
#
# BUT --reinstall IS NOT A QUESTION ABOUT VERSIONS. It says
# "build this again at the book's version", and the answer
# does not depend on what is recorded: with the version
# missing, `update curl --reinstall` reported "nothing to
# update, 1 it cannot judge" and did nothing, 28 times in a
# row, for packages whose records predate the tool.
if getattr(args, "reinstall", False):
plan.append((anchor, name, "(reinstall)", bv))
else:
# ...AND SAY WHAT THE BOOK HAS. "no recorded version"
# answers half the question: the reader wants to know
# what they would get, and the plan already knows it.
no_record.append("%s (book: %s)" % (name, bv or "?"))
continue
# A GIT-TRACKED PACKAGE HAS NO VERSION TO COMPARE.
#
# gst-plugins-rs is a `git` stack entry following upstream, and
# its name_version is `gst-plugins-rs-git`. Against the book's
# 1.28.6 that read as "older", so an update plan offered to
# replace a cargo-built checkout with the book's page -- a
# different build entirely, silently. `-git` is not a version
# and cannot be behind one.
_is_git_tracked = (str(u.version or "").endswith("-git")
or (u.meta or {}).get("is_git") in ("1", 1))
if _is_git_tracked:
git_tracked.append((name, u.version, bv))
continue
outdated = not _same_version(bv, u.version)
if outdated:
plan.append((anchor, name, u.version, bv))
elif getattr(args, "reinstall", False):
plan.append((anchor, name, "(reinstall)", bv))
else:
uptodate.append((anchor, name, u.version, bv))
# tell the user plainly about named packages that aren't installed
for name in not_installed:
if book_version(name) is None:
printWarning(f"{name} is NOT in the book (check the name, e.g. "
f"'libjpeg-turbo' vs 'libjpeg') -- "
f"try: packagemanager search {name}")
else:
printWarning(f"{name} is NOT installed (no completed install "
f"found) -- use: packagemanager install {name}")
if git_tracked:
# "LEFT ALONE" IS NOT AN ANSWER TO "IS IT OUT OF DATE?"
#
# This said `-git` cannot be behind a book version, which is true
# and beside the point: a package that follows a BRANCH is behind
# whenever the branch moves, and the remote can be asked in a
# second. A package pinned to a TAG is not behind at all -- the
# question there is whether a newer tag exists, and taking it is a
# decision, not an update. Both are answered here rather than
# left to the reader.
print()
printInfo(" %d package(s) follow GIT, not the book:"
% len(git_tracked))
_moved, _tagged = [], []
for _n, _iv, _bv in git_tracked:
_st = _git_pkg_state(_n) if not getattr(args, "no_git_check",
False) else None
if _st:
print(_git_pkg_line(_st))
if _st["kind"].endswith("branch") and _st["remote"] \
and _st["local"][:12] != _st["remote"][:12]:
_moved.append(_n)
if _st["newer_tags"]:
_tagged.append(_st)
else:
printInfo(" %-22s tracked as %-20s (the book has %s)"
% (_n, _iv, _bv or "no page"))
if _moved:
printInfo(" behind their branch -- fetch, rebuild, install:")
printInfo(" packagemanager update %s --run" % " ".join(_moved))
for _st in _tagged:
printInfo(" %s is pinned to %s, and %s is out:"
% (_st["name"], _st["ref"], _st["newer_tags"][-1]))
printInfo(" packagemanager git-ref %s %s --run"
% (_st["name"], _st["newer_tags"][-1]))
if not _moved and not _tagged:
printInfo(" every one of them is where its ref says it"
" should be")
printInfo(" the whole picture: packagemanager git-status")
printInfo(" (taking the BOOK's page instead would be a"
" different build:")
printInfo(" `packagemanager update <pkg> --reinstall --run`.)")
# named packages that are already current: offer to re-apply
for anchor, name, iv, bv in uptodate:
if name not in named and anchor not in named:
continue # only prompt for named ones
if getattr(args, "no_reinstall", False):
printInfo(f"{name} is already up to date ({bv}); skipping "
f"(--no-reinstall).")
continue
# A DRY RUN ASKS NOTHING. `update --stack media` prompted four
# times -- once per already-current package -- before printing a
# plan it was never going to carry out. A preview that needs
# answers is not a preview.
if not getattr(args, "run", False):
printInfo(f" {name} is already current ({bv}) -- not in the "
f"plan. To rebuild it anyway:")
printInfo(f" packagemanager update {name} --reinstall --run")
continue
if args.yes or ask_yes_no(f"{name} is already up to date ({bv}). "
f"Re-apply the update anyway?"):
plan.append((anchor, name, "(reinstall)", bv))
if not plan:
_bk = _current_blfs_book()
printSuccess("Nothing to update from the BLFS book"
+ (f" ({_bk})" if _bk else "")
+ " -- every package it carries is current.")
# consulted on --run too: with no BLFS work to do, the run has time,
# and "check them with: lfs update" is a worse answer than the answer
_report_no_book(no_book, consult_lfs=True,
named=getattr(args, "packages", None) or [],
no_record=no_record, unjudged=unjudged,
uptodate_n=len(uptodate),
total=_checked or None)
clear_plan("update", plan_key)
return
def reason_str(anchor):
kp = kind_of.get(anchor)
if not kp:
return ""
kind, parent = kp
if kind == "requested":
return "requested"
if kind == "dependent-of":
return f"depends on {parent}"
return f"{kind} by {parent}"
if args.select:
plan = _select_plan_rows(
plan, lambda r: f"{r[1]:<24} {r[2]} -> {r[3]:<20} "
f"{reason_str(r[0])}")
if not plan:
printWarning("Nothing selected.")
return
save_plan("update", plan_key, plan)
script_dir = args.outdir or os.path.join(TMP_DIR, "install_files")
os.makedirs(script_dir, exist_ok=True)
printHeader(f"Update plan ({len(plan)} package(s))")
print()
steps = []
for anchor, name, iv, bv in plan:
path = resolve_script(name, bv, script_dir, assume_yes=args.yes,
force_regen=getattr(args, "regenerate", False))
if not path:
continue
steps.append((anchor, name, iv, bv, path))
reason = reason_str(anchor)
rc_col = f" {colors.dim}({reason}){colors.normal}" if reason else ""
print(f" {name:<22} {iv} -> {colors.cyan}{bv}{colors.normal}{rc_col}")
# loud warning if any package would build from an OLD-format script
if not getattr(args, "regenerate", False):
cur_v = _current_script_version()
outdated = [name for _a, name, _i, _b, _p in steps
if _package_script_outdated(name, cur_v)]
if outdated:
bar = colors.red + "!" * 58 + colors.normal
trunc = len(outdated) > 12
print(f"\n{bar}")
printWarning(f"!! {len(outdated)} package(s) still use an OLD-format "
f"install script:")
printWarning("!! " + ", ".join(outdated[:12]) + (" ..." if trunc else ""))
if trunc:
printWarning("!! Full list (space-separated, pipe-ready):")
printWarning("!! packagemanager script-status --fast -q")
printWarning("!! A reinstall/update may not run your edits correctly "
"and could lose them.")
printWarning("!! MIGRATE FIRST (merges your edits, backs up .bak) -- "
"these packages, or all at once:")
if trunc:
printWarning("!! packagemanager migrate-scripts "
"$(packagemanager script-status --fast -q) --run")
else:
printWarning("!! packagemanager migrate-scripts "
+ " ".join(outdated) + " --run")
printWarning("!! or add --regenerate here to rebuild scripts from the "
"book (discards edits).")
print(f"{bar}")
# ---- conflicts, LAST, where they are read ------------------------------
# A version bump generates a fresh script; local edits to the installed
# version's script never carry over on their own. Collected here and said
# at the END of the plan -- one block to work through -- and --run below
# STOPS at the first one rather than silently discarding the edits.
conflicts = []
if not getattr(args, "regenerate", False):
for anchor, name, iv, bv, path in steps:
if _pending_edit_conflict(pkgusr_home(name), iv, bv):
conflicts.append((name, iv, bv, path))
if conflicts:
print()
printWarning(f"!! {len(conflicts)} package(s) carry LOCAL EDITS the "
f"update would leave behind:")
for name, iv, bv, path in conflicts:
home = pkgusr_home(name)
printWarning(f"!! {name}: install_{iv} differs from install_last")
printInfo(f" your edits : {os.path.join(home, 'install_' + iv)}")
printInfo(f" fresh {bv}: {path}")
printInfo(f" carry them : merge into "
f"{os.path.join(home, 'install_' + bv)} -- the update "
f"then uses that file")
printWarning("!! --run stops at the first of these. Merge the edits "
"(above), or add --regenerate to discard them all.")
# ---- the LFS book covers what BLFS does not --------------------------
# BLFS is PREFERRED: anything it carries was planned above, from its own
# script. Everything else falls back to the LFS book, so one `update`
# covers the whole system instead of planning two packages and printing a
# command for the other twenty-nine.
_blfs_planned = {n.lower() for _a, n, _i, _b, _p in steps}
_lfs_steps = []
if not args.packages:
for _n, _have, _want in _lfs_stale_list(
include_current=getattr(args, "reinstall", False)):
if _n.lower() in _blfs_planned:
continue # BLFS has it: preferred, done
_lfs_steps.append((_n, _n, _have, _want, _LFS_STEP))
if _lfs_steps:
print()
printInfo("From the LFS book (BLFS does not carry these):")
for _a, _n, _iv, _bv, _p in _lfs_steps:
_risk = " (toolchain)" if _n.lower() in _TOOLCHAIN_RISK else ""
if _iv == _bv:
print(f" {_n:<22} {_iv} {colors.dim}(rebuild, same version)"
f"{colors.normal}{_risk}")
else:
print(f" {_n:<22} {_iv} -> {colors.cyan}{_bv}"
f"{colors.normal}{_risk}")
steps += _lfs_steps
total = len(steps)
# Rebuilding the toolchain on a running system is not the same act as
# rebuilding `less`: get it wrong and there is no compiler left to fix it
# with. They are planned and shown, but running them has to be asked for.
_risky = [n for _a, n, _i, _b, p in steps
if p == _LFS_STEP and n.lower() in _TOOLCHAIN_RISK]
if _risky:
print()
if args.run and not getattr(args, "toolchain", False):
printWarning("!! toolchain packages in this plan: %s"
% ", ".join(_risky))
printInfo(" Rebuilding those on the running system can leave it")
printInfo(" without a working compiler. SKIPPED unless asked:")
printInfo(" packagemanager update --run --toolchain")
steps = [s for s in steps if s[1] not in _risky]
total = len(steps)
elif not args.run:
printInfo("Toolchain packages above (%s) need --toolchain with "
"--run." % ", ".join(_risky))
# A systemd book on a SysV system: the scripts call systemctl, the shim
# turns each call into a visible skip, and the build succeeds -- but the
# service was not enabled. Worth knowing BEFORE the build, not as
# notices scrolling past during it.
_sysd = _systemctl_users(steps)
if _sysd:
print()
printWarning("%d package(s) in this plan run systemctl, which this "
"system does not have:" % len(_sysd))
for _n, _c, _path in _sysd[:12]:
printInfo(" %-22s %d line(s) %s" % (_n, _c, _path))
if len(_sysd) > 12:
printInfo(" ... and %d more" % (len(_sysd) - 12))
printInfo(" Those lines are SKIPPED (and say so), so the build")
printInfo(" succeeds -- but the service is not enabled. Edit the")
printInfo(" file above to add the SysV equivalent; it is the file")
printInfo(" the build runs, and an edited script is kept.")
# The report below covers what the plan does NOT: listing a package as
# "the LFS book answers for this" directly above the same package in the
# plan printed every stale package twice.
_planned_now = {n.lower() for _a, n, _i, _b, _p in steps}
_report_no_book([n for n in no_book if n.lower() not in _planned_now],
named=getattr(args, "packages", None) or [],
consult_lfs=False,
no_record=[n for n in no_record
if n.lower() not in _planned_now],
unjudged=[n for n in unjudged
if n.lower() not in _planned_now])
_conflict_names = {c[0] for c in conflicts}
if not args.run:
print()
if args.select:
printInfo(f"Dry run -- you narrowed this to {len(steps)} package(s). "
f"It's cached under key '{plan_key}'.\n"
f" Run the SAME set with: packagemanager update "
+ " ".join(list(args.packages) +
(["--reinstall"] if args.reinstall else []) +
["--run --yes"]) +
"\n (don't re-add --select unless you want to re-pick).")
else:
# THE COMMAND THAT WAS TYPED, NOT ITS EXPANSION -- `--stack media`
# rather than nineteen names, and WITH --regenerate when it was
# given, or the "exact" command brings the warning straight back.
_who = (["--stack", args.stack] if getattr(args, "stack", None)
else list(args.packages))
exact = "packagemanager update " + " ".join(
_who + [f for f in (
"-f" if args.force else "", "-R" if args.dependents else "",
"--recursive" if getattr(args, "recursive", False) else "",
"--reinstall" if args.reinstall else "",
"--regenerate" if getattr(args, "regenerate", False) else "",
"--clean" if getattr(args, "clean", False) else "",
"--test" if getattr(args, "test", False) else "",
"--keep-sources" if getattr(args, "keep_sources", False) else "",
"-e" if args.ignore_recommended else "",
"--optional" if args.optional else "") if f] + ["--run"])
_report_stale_requires([st[1] for st in steps],
_verbose=getattr(args, "verbose", False))
# WHERE THE WHOLE LIST IS. Asking about one package tells you
# nothing about the rest, and the command for "everything the
# book has moved on from" is this one with no name after it.
# WHAT WILL ACTUALLY CHANGE, in one line. The requirement
# report above lists a dozen packages and reads like a plan; it
# is not one, and saying so beats hoping the header is noticed.
printInfo(" this plan updates %d package(s): %s"
% (len(steps), ", ".join(st[1] for st in steps[:10])
+ (" ..." if len(steps) > 10 else "")))
printInfo(" nothing else is touched -- the requirements above"
" are a comparison, not a list of work.")
if args.packages and not getattr(args, "stack", None):
printInfo(" other ways to choose:")
printInfo(" packagemanager update #"
" everything the book has moved past")
printInfo(" packagemanager update --stack media #"
" just that stack's packages")
printInfo(" packagemanager update --stack media --select"
" --run # pick from them")
printInfo(f"Dry run -- this exact command runs THIS plan ({len(steps)} "
f"package(s)):\n {exact}")
printInfo(f"(cached under key '{plan_key}', so --run reuses/resumes it).")
if args.reinstall:
printInfo("Note: --reinstall reuses each package's edited "
"install_<ver> script if present; add --regenerate to "
"rebuild the scripts from the book instead.")
return
print()
if not args.yes and not ask_yes_no(f"Update {len(steps)} package(s) now?"):
printWarning("Aborted.")
return
pm = find_pm_install()
install_only = getattr(args, "install_only", False)
env = {**os.environ, "PM_MODE": "install" if install_only else "update",
"PM_YES": "1" if args.yes else "0"}
if getattr(args, "test", False):
env["PM_TEST"] = env["PKGUSR_TESTS"] = "1"
if getattr(args, "reinstall", False):
env["PM_REINSTALL"] = "1"
remaining = list(plan)
total = len(steps)
errors = 0
run_log = start_run_log("update", plan_key)
for i, (anchor, name, iv, bv, path) in enumerate(steps, 1):
if name in _conflict_names:
save_plan("update", plan_key, remaining) # resume-able
clear_status()
home = pkgusr_home(name)
printError(f"stopping BEFORE {name}: install_{iv} carries local "
f"edits the fresh {bv} script would leave behind "
f"({i - 1}/{total} done).")
printInfo(f" your edits : {os.path.join(home, 'install_' + iv)}")
printInfo(f" fresh {bv}: {path}")
printInfo(f" Merge them into {os.path.join(home, 'install_' + bv)}"
f" -- the update then uses that file -- and re-run to "
f"resume; or re-run with --regenerate to discard the "
f"edits.")
return
set_status(f"[{i}/{total}] updating {name} {iv} -> {bv}"
+ (f" ({errors} error(s) so far)" if errors else ""))
verb = "Installing" if install_only else "Updating"
printHeader(f"({i}/{total}) {verb} {name}: {iv} -> {bv}")
if path == _LFS_STEP:
# The LFS book owns this package, and `lfs` already knows how to
# build one of its packages -- fetching the tarball, generating
# the script, handing over to lfs-helper. Reimplementing that
# here would be a second builder for the same book, which is the
# whole disease this project keeps treating.
_lfs_bin = shutil.which("lfs") or "/usr/bin/lfs"
_cmd = [_lfs_bin, "install", "--reinstall", name]
if args.yes:
_cmd.append("--yes")
printInfo(" via the LFS book: %s" % " ".join(_cmd))
try:
_rc = subprocess.run(_cmd).returncode
except Exception as e:
printError(f" could not run lfs: {e}")
_rc = 1
if _rc != 0:
errors += 1
printError(f"{name}: the LFS build failed (exit {_rc}); "
f"stopping ({i - 1}/{total} done).")
save_plan("update", plan_key, remaining)
clear_status()
return
remaining = [p for p in remaining if p[0] != anchor]
save_plan("update", plan_key, remaining)
continue
cleaning = getattr(args, "clean", False)
old_paths = _pkglist_paths(name) if cleaning else None
t0 = time.time()
penv = {**env, "PM_NO_REFRESH": "1"} if cleaning else env
result = run_pm_install(name, path, penv)
add_run_entry(run_log, name, "update", result.returncode == 0,
result.returncode, _pkg_log_dir(name))
if result.returncode != 0:
errors += 1
save_plan("update", plan_key, remaining) # resume-able
clear_status()
printError(f"update failed for {name} (exit {result.returncode}); "
f"stopping ({i - 1}/{total} done, {errors} error(s)).")
printInfo(f" logs: {_pkg_log_dir(name)} "
f"(see all: packagemanager errors)")
base = ("packagemanager update " + " ".join(list(args.packages)))
printInfo(" The plan is saved. To resume after fixing the problem:\n"
f" - if the BUILD was fine and only install failed "
f"(perms, a missing dir): retry just install (no recompile):\n"
f" {base} --install-only --run\n"
f" (or for one package: packagemanager script install "
f"{name})\n"
f" - to REBUILD from scratch: {base} --run "
f"(add --regenerate to rebuild the script from the book)")
# Last, and the copy that survives -- see the install path above.
_canon = os.path.join(pkgusr_home(name), os.path.basename(path))
_edit = _canon if os.path.isfile(_canon) else path
print()
printInfo(" To change what it does, edit:")
printError(f" {_edit}")
sys.exit(result.returncode)
if getattr(args, "clean", False):
_clean_after_install(name, old_paths, t0)
_maybe_remove_old_sources(name, bv, getattr(args, "keep_sources", False))
remaining = [p for p in remaining if p[0] != anchor]
save_plan("update", plan_key, remaining)
clear_status()
clear_plan("update", plan_key)
printSuccess(f"Done -- {total} package(s) updated"
+ (f", {errors} error(s)." if errors else "."))
# THE PATTERN HAS TO BE A REAL PATH.
#
# This built the hint by hand -- os.path.join(BASE_DIR, ...) -- and got it
# wrong twice over: it printed "/usr/src/p_<pkg>/log", missing the pkgusr/
# component that every package home actually has. Someone following it
# found nothing, twice: the directory does not exist, and until 1.14.71
# the logs were not there either. _pkg_log_dir is the function that knows.
printInfo(f" results: packagemanager errors logs: "
f"{_pkg_log_dir('<name>')}")
# =========================================================================== #
# command: remove [IMPLEMENTED]
# =========================================================================== #
#
# Default `remove <pkg>` UNINSTALLS the package's files (everything it put on
# the system, from pkg.lst) but KEEPS the package-user and its /usr/src/<pkg>
# home -- so reinstalling is easy and anything special you keep in that home is
# preserved. The file removal runs AS THE PACKAGE USER (su -), so the kernel
# only lets it delete what that user is allowed to (its own files in the
# install-group dirs); it can never remove another package's or root's files by
# accident. `--purge` additionally deletes the user, its group and the home.
#
# (default) uninstall files, keep user + /usr/src/<pkg>
# --purge also remove the user/group and /usr/src/<pkg>
# --run actually do it (default is a dry-run plan); --yes skips prompt
#
# Later helpers noted for the nimgnu_ collector-group layer (not yet built):
# make a dir an install-group dir, fix install-group dir permissions,
# add a dir to a nimgnu_* group.
# --------------------------------------------------------------------------- #
# dry-run plan cache
#
# A dry run stores what it computed (with a timestamp) so the matching --run can
# reuse it instead of recomputing -- e.g. re-scanning the filesystem for a
# package's files. Before reusing a stored plan you are shown when it was made
# and asked to confirm.
# --------------------------------------------------------------------------- #
def _plans_dir():
d = os.path.join(TMP_DIR, "plans")
os.makedirs(d, exist_ok=True)
return d
def _run_log_path():
return os.path.join(TMP_DIR, "last_run.json")
def start_run_log(kind, key):
"""Begin recording a fresh install/update run (overwrites the previous one)."""
os.makedirs(TMP_DIR, exist_ok=True)
data = {"kind": kind, "key": key, "started": time.time(), "entries": []}
try:
with open(_run_log_path(), "w") as f:
json.dump(data, f)
except OSError:
pass
return data
def add_run_entry(log, pkg, action, ok, exit_code, log_dir):
log["entries"].append({"pkg": pkg, "action": action, "ok": ok,
"exit": exit_code, "log_dir": log_dir,
"time": time.time()})
try:
with open(_run_log_path(), "w") as f:
json.dump(log, f)
except OSError:
pass
def read_run_log():
try:
with open(_run_log_path()) as f:
return json.load(f)
except (OSError, ValueError):
return None
def cmd_plan(args):
"""List or clear cached dry-run plans (install/update/remove/fix-install)."""
d = _plans_dir()
files = sorted(f for f in os.listdir(d) if f.endswith(".json"))
if args.action == "clear":
targets = [f for f in files if (not args.key or args.key in f)]
for f in targets:
try:
os.remove(os.path.join(d, f))
except OSError:
pass
printSuccess(f"cleared {len(targets)} plan(s).")
return
if not files:
printInfo("No cached plans.")
return
printHeader("Cached plans")
print()
for f in files:
try:
data = json.load(open(os.path.join(d, f)))
n = len(data.get("data") or [])
when = _ago(data.get("time", 0))
kind = data.get("kind", "?")
key = data.get("key", "")
print(f" {kind:<12} {key:<24} {n:>4} item(s) {when} [{f}]")
except (OSError, ValueError):
print(f" {f} (unreadable)")
printInfo("\nclear one: packagemanager plan clear <file-or-key>\n"
"clear all: packagemanager plan clear")
# ---------------------------------------------------------------------------
# WHO WAS BUILT BEFORE WHAT.
#
# Every generated script records the page's own required/recommended/optional
# lists, and every package's pkg.lst is written when it finished installing.
# Those two facts together answer the question that cost this system its ALSA
# and its WebRTC: was anything a package can USE installed after that package
# was built? Nothing else has to be inferred, guessed or remembered.
# ---------------------------------------------------------------------------
def _pkg_dirs():
root = os.path.join(BASE_DIR, load_config().get("pkgusr_subdir", "pkgusr"))
if not os.path.isdir(root):
root = BASE_DIR
out = {}
for d in sorted(os.listdir(root)):
full = os.path.join(root, d)
if not os.path.isdir(full):
continue
if not os.path.isfile(os.path.join(full, "install_last")):
continue
out[re.sub(r"^" + re.escape(pkgusr_prefix()), "", d)] = full
return out
def _built_when(home):
"""When this package last finished installing.
pkg.lst is written after the install, so its mtime is the install time.
install_last is the fallback: generated just before the run.
"""
for f in ("pkg.lst", "install_last"):
p = os.path.join(home, f)
try:
return os.path.getmtime(p), f
except OSError:
continue
return None, ""
def _script_deps(home):
"""required/recommended/optional, as the page declared them."""
out = {}
try:
body = open(os.path.join(home, "install_last"),
encoding="utf-8", errors="replace").read()
except OSError:
return out
for kind in ("required", "recommended", "optional"):
m = re.search(r"^%s=\((.*?)\)$" % kind, body, re.M | re.S)
if not m:
continue
out[kind] = re.findall(r"'([^']*)'", m.group(1))
return out
def _norm(s):
return re.sub(r"[^a-z0-9]", "", s.lower())
_DEP_IDX = {}
def _dep_index(installed):
"""title -> package, built from the packages themselves.
A dep is written as the page's TITLE ('gst-plugins-base-1.28.1',
'FreeType-2.14.1'); a package directory is the page's id
('gst10-plugins-base', 'freetype2'). Normalising the strings cannot
bridge that, and 119 of them did not resolve -- which quietly broke the
REBUILD ORDER, because a dependency edge that does not resolve is an edge
that does not exist: gst10-plugins-bad sorted before gst10-plugins-base,
and fontconfig before freetype2.
Every script already records `name_version` -- the tarball base, which is
the title -- so the mapping is in the data and does not have to be
guessed.
"""
key = tuple(sorted(installed))
if _DEP_IDX.get("key") == key:
return _DEP_IDX["idx"]
idx = {}
for name, home in installed.items():
idx.setdefault(_norm(name), name)
try:
body = open(os.path.join(home, "install_last"),
encoding="utf-8", errors="replace").read(4096)
except OSError:
continue
m = re.search(r"^name_version='([^']*)'", body, re.M)
if not m:
continue
base = re.sub(r"-[0-9][0-9a-z.]*$", "", m.group(1))
idx.setdefault(_norm(base), name)
_DEP_IDX.update(key=key, idx=idx)
return idx
def _resolve_dep(dep, installed):
"""A dep is 'Little-CMS-2.18'; a package dir is 'lcms2'.
THE MAJOR VERSION IS PART OF THE NAME, sometimes. gtk3 and gtk4 both
record name_version 'GTK-<v>', so the base name 'GTK' matched whichever
was indexed first -- and `GTK-4.20.3` resolved to **gtk3**. webkitgtk
requires both, so it listed gtk3 twice and gtk4 not at all; the same
collision hides libsoup-3.x behind libsoup, sdl3 behind sdl, qt6 behind
qt.
So try base+major BEFORE base: `GTK-4.20.3` -> gtk4, `libsoup-3.6.6` ->
libsoup3, and `Opus-1.6.1` -> opus1 does not exist, which falls through
to opus as before.
"""
idx = _dep_index(installed)
base = re.sub(r"-[0-9][0-9a-z.]*$", "", dep)
m = re.search(r"-([0-9]+)", dep[len(base):]) if len(dep) > len(base) else None
major = m.group(1) if m else ""
cands = []
if major:
cands += [base + major, base.replace("-", "") + major,
base.split("-")[0] + major]
cands += [base, base.replace("-", ""), base.split("-")[0]]
for cand in cands:
n = _norm(cand)
if n and n in idx:
return idx[n]
return None
# DOCS AND TESTS ARE NOT FEATURES.
#
# graphviz appeared against wayland, llvm, ruby, libinput, libexif and
# libical; docutils against glib2 and libdrm. Rebuilding six packages to
# regenerate documentation nobody reads is not the point of this audit, and
# burying the real findings under it is how a report gets ignored.
_DOC_DEPS = {"graphviz", "docutils", "doxygen", "sphinx", "texlive", "asciidoc",
"asciidoctor", "dejagnu", "valgrind", "gtk-doc", "xmlto", "ruby",
"fop", "docbook-xsl", "docbook", "libxslt-docs", "dblatex"}
# ...and these are build-time conveniences, not linked features
_TOOL_DEPS = {"git", "cmake", "cython", "fcron", "meson", "ninja", "nasm",
"yasm", "expect", "which", "mako", "setuptools", "wheel",
"hatchling", "pytest", "gi-docgen", "gobject-introspection-docs",
"unifdef", "gperf"}
# INSTALL DOES NOT REBUILD. `packagemanager install <pkg> --run
# --regenerate` on an installed package answers "Nothing to do -- everything
# is installed and up to date" -- correctly, and 28 times in a row, so the
# whole rebuild pass did nothing and said so in a way that looked like
# success. The verb that rebuilds at the same version is `update
# --reinstall`.
def _rebuild_cmd(name):
return ["packagemanager", "update", name, "--reinstall", "--regenerate",
"--run", "--yes"]
def _required_graph(installed, kinds=("required",)):
"""package -> the packages it depends on, from its own script."""
g = {}
for name, home in installed.items():
deps, d = set(), _script_deps(home)
for kind in kinds:
for dep in d.get(kind, []):
got = _resolve_dep(dep, installed)
if got and got != name:
deps.add(got)
g[name] = deps
return g
def _depends_on(g, a, b, seen=None):
"""does a require b, directly or through anything?"""
if seen is None:
seen = set()
if a in seen:
return False
seen.add(a)
for d in g.get(a, ()):
if d == b or _depends_on(g, d, b, seen):
return True
return False
# A TIMESTAMP IS EVIDENCE; A PROBE IS AN ANSWER.
#
# Two packages built in the same session cannot be ordered by pkg.lst
# mtimes, and that is exactly where the interesting cases are (ffmpeg and
# alsa-lib, zero days apart). The stacks already carry `expect=` probes that
# read the built artifact -- run them.
def cmd_probe(args):
import glob as _glob # noqa: also used by the no-probes branch below
stacks = sorted(_glob.glob("/etc/pkgusr/stacks/*.stack"))
if not stacks:
printError("no stack files in /etc/pkgusr/stacks")
return 1
n = ok = bad = 0
for path in stacks:
for e in _parse_stack_file(path):
probe = e["opts"].get("expect")
if not probe:
continue
if args.package and e["target"] not in args.package:
continue
n += 1
r = subprocess.run(["/bin/bash", "-c", probe],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
stack_name = os.path.basename(path).replace(".stack", "")
if r.returncode == 0:
ok += 1
printSuccess("%-22s %s" % (e["target"], probe))
continue
bad += 1
printError("%-22s %s" % (e["target"], probe))
# A PROBE THAT FAILS BECAUSE THE PACKAGE IS NOT THERE IS NOT THE
# SAME FINDING. gst10-libav had never been built -- it was
# missing from the stack until now -- and the report said "the
# feature is NOT in the installed package", which sends the
# reader looking for a build option.
home = pkgusr_home(e["target"])
installed = os.path.exists(os.path.join(home, "pkg.lst"))
if not installed:
printInfo(" that package is NOT INSTALLED here -- nothing"
" was built without the feature,")
printInfo(" there is nothing built at all. Build it:")
printInfo(" packagemanager stack %s --only %s --run"
% (stack_name, e["target"]))
else:
printInfo(" the package is installed and the feature is"
" not in it. Rebuild:")
# not `stack --only`: that runs install, which no-ops on an
# installed package
printInfo(" %s" % " ".join(_rebuild_cmd(e["target"])))
# ...and what the probe itself said, because a probe can be wrong
# about a path as easily as a package can be wrong about a feature
first = (r.stdout or b"").decode("utf-8", "replace").strip()
if first:
printInfo(" the check said: %s" % first.splitlines()[0][:120])
if n == 0:
printInfo("no expect= probes in the stacks"
+ (" for %s" % ", ".join(args.package) if args.package else ""))
# SAY WHY THERE ARE NONE. A stack file you have edited is never
# overwritten -- the new one lands beside it as .stack.new -- so an
# installed tool can carry probes that no installed STACK has, and
# "no probes" reads as "nothing to check" rather than "your stacks
# predate them".
waiting = []
for newf in sorted(_glob.glob("/etc/pkgusr/stacks/*.stack.new")):
try:
if "expect=" in open(newf, encoding="utf-8",
errors="replace").read():
waiting.append(newf)
except OSError:
continue
if waiting:
print("")
printWarning("%d newer stack file(s) beside yours DO carry probes:"
% len(waiting))
for newf in waiting:
base = os.path.basename(newf).replace(".stack.new", "")
printInfo(" %s" % newf)
printInfo(" take it: packagemanager stack %s --take-new"
% base)
printInfo(" Your copy was edited, so make install left it alone"
" and put the new one beside it.")
else:
printInfo(" Probes live in the stack files as expect=\"...\"."
" These stacks have none,")
printInfo(" which is what an older stack looks like:"
" compare with the shipped one.")
return 0
print("")
if bad:
printWarning("%d of %d probe(s) failed -- see each line: a package that"
" is not installed and one" % (bad, n))
printWarning(" built without a feature are different problems.")
return 1
printSuccess("all %d probe(s) pass: the features they check are present." % ok)
return 0
def cmd_late_deps(args):
installed = _pkg_dirs()
if not installed:
printError("no package users with an install_last under %s" % BASE_DIR)
return 1
kinds = [k.strip() for k in (args.kind or "recommended,optional").split(",")
if k.strip()]
# WHY DID IT DECIDE THAT? The order is computed from edges, and an edge
# that does not resolve is invisible -- so when a position looks wrong,
# the useful thing is the graph it actually used for that package.
if getattr(args, "why", False):
for name in (args.package or sorted(installed)):
if name not in installed:
printWarning("%s is not installed here" % name)
continue
when, src = _built_when(installed[name])
print("")
printHeader(name)
print(" built: %s (%s)"
% (time.strftime("%Y-%m-%d %H:%M",
time.localtime(when)) if when else "unknown", src))
d = _script_deps(installed[name])
for kind in ("required", "recommended", "optional"):
for dep in d.get(kind, []):
got = _resolve_dep(dep, installed)
if got is None:
print(" %-12s %-28s -> no installed package of that name"
% (kind, dep))
continue
dw, _ = _built_when(installed[got])
if dw and when:
diff = dw - when
rel = ("built %+d day(s) relative to it" % (diff // 86400)
if abs(diff) >= 86400 else
"built %+d hour(s) relative to it" % (diff // 3600))
else:
rel = "no date"
print(" %-12s %-28s -> %s, %s" % (kind, dep, got, rel))
return 0
only = set(args.package or [])
problems, unresolved, nodate = [], set(), []
for name, home in installed.items():
if only and name not in only:
continue
when, src = _built_when(home)
if when is None:
nodate.append(name)
continue
late = []
for kind in kinds:
for dep in _script_deps(home).get(kind, []):
got = _resolve_dep(dep, installed)
if got is None:
unresolved.add(dep)
continue
if got == name:
continue
dwhen, _ = _built_when(installed[got])
if dwhen is None:
continue
# SLACK THAT HIDES THE CASE IS TOO MUCH SLACK.
#
# A day was chosen because pkg.lst is rewritten by
# reload-pkg-list and by --clean, so packages from one run
# differ by minutes. But `ffmpeg ... alsa-lib +0 day(s)` was
# then filed as fine, and alsa-lib a few HOURS after ffmpeg
# is the original bug exactly. An hour covers the rewrite
# noise; anything more is a gap.
if dwhen - when > 3600:
late.append((kind, got, dep, dwhen - when))
if late:
problems.append((name, when, src, late))
# THREE DIFFERENT ANSWERS, and the report was giving one.
#
# ORDERABLE the dependency does not need this package, so it can simply
# be built first -- a stack line moves and the problem is
# gone for the next machine.
# CIRCULAR the dependency REQUIRES this package (cairo needs glib2,
# mesa needs libxcb). No order exists. The only fix is a
# second pass: build, build the dependency, rebuild.
# COSMETIC documentation and build tools. Nothing to gain.
g = _required_graph(installed)
# RECOMMENDED IS A DEPENDENCY TOO, for the purpose of asking whether an
# order exists. With `required` alone the report said
# in the stack: put mesa before libxcb
# and mesa cannot precede libxcb -- it needs it. BLFS lists that edge as
# recommended, not required, and cairo needs glib2 the same way. Advice
# that cannot be taken is worse than no advice.
g_order = _required_graph(installed, ("required", "recommended"))
orderable, circular, cosmetic = [], [], []
for name, when, src, late in problems:
o, c, d = [], [], []
for item in late:
dep_pkg = item[1]
if dep_pkg in _DOC_DEPS or dep_pkg in _TOOL_DEPS:
d.append(item)
elif _depends_on(g_order, dep_pkg, name):
c.append(item)
else:
o.append(item)
if o:
orderable.append((name, when, src, o))
if c:
circular.append((name, when, src, c))
if d:
cosmetic.append((name, when, src, d))
def _show(rows, header, note):
if not rows:
return
print("")
printHeader(header)
printInfo(" " + note)
for name, when, src, late in sorted(rows, key=lambda r: -len(r[3])):
print("")
printWarning("%s (built %s)"
% (name, time.strftime("%Y-%m-%d", time.localtime(when))))
for kind, got, _dep, gap in late:
when_s = ("%d day(s)" % (gap // 86400) if gap >= 86400
else "%d hour(s)" % (gap // 3600))
print(" %-12s %-22s installed %s later%s"
% (kind, got, when_s,
" <- same session: only a probe can settle it"
if gap < 21600 else ""))
# SAY WHAT TO CHANGE, not just what is wrong. The orderable ones
# are a stack edit; without the line the reader has to work out
# which file and which position.
if rows is orderable:
print(" in the stack: put %s before %s"
% (", ".join(sorted({g for _k, g, _d, _x in late})), name))
if not problems:
printSuccess("no package was built before something it can use.")
_show(orderable, "built too early -- and an order exists",
"move these dependencies earlier in the stack, then rebuild these once.")
_show(circular, "built too early -- no order exists",
"the dependency requires this package, so it cannot go first."
" A second pass is the only fix.")
if args.include_docs:
_show(cosmetic, "documentation and build tools only",
"rebuilding gains you documentation, not features.")
elif cosmetic:
print("")
printInfo("%d package(s) differ only in documentation or build tools"
" (--include-docs to see them)" % len(cosmetic))
# THE REBUILD ORDER IS NOT THE REPORT ORDER. Rebuilding a library after
# the thing that links it means doing it twice.
todo = {r[0] for r in orderable} | {r[0] for r in circular}
if args.include_docs:
todo |= {r[0] for r in cosmetic}
if todo:
# ORDER BY EVERY EDGE THAT DECIDES A FEATURE, not just the required
# ones: mesa RECOMMENDS libxcb, so libxcb has to be rebuilt first or
# mesa is rebuilt against the old one. Cycles are expected here
# (cairo and glib2 recommend each other) and are broken below.
# ORDER THE CLUSTERS, NOT THE NODES.
#
# Picking one node out of a cycle and then going back to the ready
# scan strands the rest of the cluster at the end: freetype2 and
# harfbuzz (which recommend/require each other) came out LAST, after
# fontconfig and cairo -- both of which require freetype2. So the
# cycle has to be treated as one unit.
#
# Tarjan gives the strongly connected components; the components form
# a DAG by construction, so they sort cleanly. Inside a component
# there is no right answer, and the same rule as before applies:
# least dependent, most depended on.
nodes = set(todo)
# THE OPTIONAL EDGE IS THE WHOLE POINT OF THIS PASS.
#
# The order graph was required+recommended, so mpg123 was rebuilt
# BEFORE pulseaudio -- and mpg123's missing feature IS pulseaudio.
# Ordering a rebuild by the edges that do not decide features, while
# ignoring the ones that do, cannot be right.
#
# Optional edges make the graph much more cyclic; that is honest
# rather than inconvenient, and SCCs handle it. Inside a component
# the STRONG edges (required, recommended) still decide, because
# those are the ones that break a build rather than a feature.
g_all = _required_graph(installed,
("required", "recommended", "optional"))
adj = {n: (g_all.get(n, set()) & nodes) for n in nodes}
strong = {n: (g_order.get(n, set()) & nodes) for n in nodes}
index, low, on_stack, stack, sccs, counter = {}, {}, set(), [], [], [0]
def _strong(v):
# iterative: a 400-package graph would recurse past the limit
work = [(v, iter(sorted(adj[v])))]
index[v] = low[v] = counter[0]; counter[0] += 1
stack.append(v); on_stack.add(v)
while work:
node, it = work[-1]
advanced = False
for w in it:
if w not in index:
index[w] = low[w] = counter[0]; counter[0] += 1
stack.append(w); on_stack.add(w)
work.append((w, iter(sorted(adj[w]))))
advanced = True
break
if w in on_stack:
low[node] = min(low[node], index[w])
if advanced:
continue
work.pop()
if work:
low[work[-1][0]] = min(low[work[-1][0]], low[node])
if low[node] == index[node]:
comp = []
while True:
w = stack.pop(); on_stack.discard(w); comp.append(w)
if w == node:
break
sccs.append(comp)
for n in sorted(nodes):
if n not in index:
_strong(n)
# Tarjan emits each component only after everything it depends on,
# so this is already dependency order.
comp_of = {n: i for i, c in enumerate(sccs) for n in c}
ordered, cycles = [], []
def _inside(c):
cs = set(c)
return sorted(c, key=lambda n: (len(strong[n] & cs),
len(adj[n] & cs),
-sum(1 for m in cs if n in strong[m]),
-sum(1 for m in cs if n in adj[m]),
n))
for comp in sccs:
if len(comp) > 1:
cycles.extend(_inside(comp))
ordered.extend(_inside(comp))
else:
ordered.extend(comp)
print("")
printHeader("rebuild, in this order (%d)" % len(ordered))
printInfo(" dependencies first, so nothing is rebuilt twice.")
for n in ordered:
print(" %s" % " ".join(_rebuild_cmd(n)))
if cycles:
print("")
printInfo(" %d of these are in dependency cycles, where no order"
" is completely right:" % len(cycles))
printInfo(" " + ", ".join(cycles))
printInfo(" each was placed by fewest unmet dependencies, then by"
" how many wait for it.")
printInfo(" If a feature is still missing afterwards, run"
" late-deps again: a cycle")
printInfo(" can need two passes, and the second one is short.")
if args.rebuild:
print("")
if not args.run:
printInfo("that is the plan; add --run to do it.")
return 1
for i, n in enumerate(ordered, 1):
printHeader("rebuild %d/%d: %s" % (i, len(ordered), n))
rc = subprocess.run(_rebuild_cmd(n)).returncode
if rc != 0:
printError("%s failed (exit %d) -- stopping here, with"
" %d left." % (n, rc, len(ordered) - i))
printInfo(" fix it, then run late-deps --rebuild --run again:"
" what succeeded is recorded and will not repeat.")
return 1
printSuccess("rebuilt %d package(s)." % len(ordered))
return 0
if unresolved and args.verbose:
print("")
printInfo("could not match to an installed package (so not checked): %s"
% ", ".join(sorted(unresolved)))
elif unresolved:
print("")
printInfo("%d dependency name(s) match no INSTALLED package"
" (--verbose lists them)." % len(unresolved))
printInfo(" Mostly that means they are not installed, which is"
" nothing to fix -- a package")
printInfo(" that is not there cannot have been built after anything."
" A name you know IS")
printInfo(" installed appearing there is worth reporting: it is an"
" edge this cannot see.")
if nodate:
printInfo("%d package(s) have no pkg.lst and no install_last date"
% len(nodate))
return 1 if problems else 0
def cmd_errors(args):
"""Show which packages failed in the LAST install/update run, with their log
locations. With --all, show every package's result."""
log = read_run_log()
if not log:
printInfo("No install/update run recorded yet.")
return
entries = log.get("entries", [])
fails = [e for e in entries if not e["ok"]]
oks = [e for e in entries if e["ok"]]
printHeader(f"Last run: {log.get('kind')} {log.get('key', '')}")
print(f"\n {_ago(log.get('started', 0))} -- "
f"{len(oks)} ok, {colors.red}{len(fails)} failed{colors.normal}, "
f"{len(entries)} total\n")
show = entries if getattr(args, "all", False) else fails
if not show:
printSuccess("No errors in the last run.")
return
for e in show:
mark = (f"{colors.green}ok{colors.normal}" if e["ok"]
else f"{colors.red}FAILED (exit {e['exit']}){colors.normal}")
print(f" {e['pkg']:<26} {e['action']:<8} {mark}")
print(f" log: {e['log_dir']}")
if fails:
newest = max(fails, key=lambda e: e["time"])
printInfo(f"\n tip: view the newest failing log with:\n"
f" ls -t {newest['log_dir']} | head -1 # then less that file")
def set_status(msg):
"""Show a persistent one-line status at the bottom (progress across many
packages). No-op when stdout isn't a terminal so it can't corrupt logs."""
if sys.stdout.isatty():
sys.stdout.write("\r\033[K" + msg[:120])
sys.stdout.flush()
def clear_status():
if sys.stdout.isatty():
sys.stdout.write("\r\033[K")
sys.stdout.flush()
def _versioned_script_in(home):
"""The editable install_<name>-<version> in a package user's home.
That is the file a person is told to edit, and the one the copy that
actually runs is made from -- so a failure has to be able to name it.
"""
# THE ONE THAT WAS LAST INSTALLED, NOT THE FIRST IN THE ALPHABET.
#
# A home keeps every version's script: p_mesa has install_Mesa-25.3.5
# and install_Mesa-26.1.7. sorted() returned 25.3.5, so `script install
# mesa` ran the OLD version's install over a 26.1.7 build ("9" also sorts
# after "10"). install_last names the version last staged; take that
# file. Without it, the newest by modification time.
try:
cands = [f for f in os.listdir(home)
if f.startswith("install_") and not f.endswith((".edited", ".bak"))
and re.search(r"-\d[\w.]*$", f)]
except OSError:
return None
if not cands:
return None
try:
m = re.search(r"^name_version=['\"]([^'\"]+)['\"]",
open(os.path.join(home, "install_last"),
encoding="utf-8", errors="replace").read(), re.M)
if m and ("install_" + m.group(1)) in cands:
return os.path.join(home, "install_" + m.group(1))
except OSError:
pass
return max((os.path.join(home, f) for f in cands), key=os.path.getmtime)
def _unpacked_source_dir(home):
"""Where the package unpacked, when there is exactly one candidate."""
src = os.path.join(home, "src")
try:
dirs = [d for d in os.listdir(src) if os.path.isdir(os.path.join(src, d))]
except OSError:
return None
return os.path.join(src, dirs[0]) if len(dirs) == 1 else None
def _pkg_log_dir(user):
return os.path.join(pkgusr_home(user), "log")
def _maybe_remove_old_sources(user, keep_name_version, keep_sources=False):
"""After a successful (re)install, remove OLD unpacked source dirs and stale
downloaded archives in the package user's home, keeping the current version.
Removes automatically; pass keep_sources=True to skip."""
if keep_sources:
return
home = pkgusr_home(user)
keep = keep_name_version
victims = []
try:
for fn in os.listdir(home):
p = os.path.join(home, fn)
if fn.startswith(".") or fn.startswith("install_") or fn == "log":
continue
if fn == "pkg.lst" or fn == "pkg.lst.new":
continue
if fn == keep: # current source dir
continue
is_archive = fn.endswith((".tar", ".tgz", ".tar.gz", ".tar.bz2",
".tar.xz", ".zip"))
if (os.path.isdir(p) or is_archive) and keep.split("-")[0].lower() \
in fn.lower():
if fn.startswith(keep): # current version's archive
continue
victims.append(p)
except OSError:
return
if not victims:
return
for p in victims:
try:
if os.path.isdir(p):
shutil.rmtree(p)
else:
os.remove(p)
except OSError as e:
printWarning(f" could not remove {p}: {e}")
printInfo(f" removed {len(victims)} old source item(s) for {user} "
f"(keeping {keep}).")
def _plan_path(kind, key):
safe = re.sub(r"[^A-Za-z0-9_.-]+", "_", key)
return os.path.join(_plans_dir(), f"{kind}__{safe}.json")
def save_plan(kind, key, data):
try:
with open(_plan_path(kind, key), "w") as f:
json.dump({"plan_version": PLAN_VERSION, "time": time.time(),
"kind": kind, "key": key, "data": data}, f)
except OSError:
pass
def load_plan(kind, key):
try:
with open(_plan_path(kind, key)) as f:
plan = json.load(f)
if plan.get("plan_version") != PLAN_VERSION: # structure changed -> ignore
return None
return plan
except (OSError, ValueError):
return None
def clear_plan(kind, key):
try:
os.remove(_plan_path(kind, key))
except OSError:
pass
def _ago(t):
s = max(0, int(time.time() - t))
if s < 90:
return f"{s}s ago"
if s < 5400:
return f"{s // 60}m ago"
if s < 172800:
return f"{s // 3600}h ago"
return f"{s // 86400}d ago"
def use_stored_plan(kind, key, assume_yes=False):
"""Return the stored plan's data if one exists and the user confirms using
it (given its age), else None."""
plan = load_plan(kind, key)
if not plan:
return None
n = len(plan.get("data") or [])
when = _ago(plan.get("time", 0))
if assume_yes:
printInfo(f"Using stored dry-run from {when} ({n} item(s)).")
return plan["data"]
if ask_yes_no(f"Found a dry-run from {when} ({n} item(s)). Use it "
f"(skip re-scanning)?"):
return plan["data"]
return None
def find_forall():
return os.environ.get("FORALL_BIN", "forall_direntries_from").split()
def owned_entries(name):
"""All filesystem entries owned by user OR group `name`, scanned as root via
forall_direntries_from. None if the tool isn't available. Used for the
install-group and nimgnu-group scans."""
try:
r = subprocess.run(find_forall() + [name], capture_output=True, text=True)
except FileNotFoundError:
return None
return [ln for ln in r.stdout.splitlines() if ln.strip()]
def pkglist_entries(pkg_list):
try:
with open(pkg_list, encoding="utf-8", errors="replace") as f:
return [ln.strip() for ln in f if ln.strip()]
except OSError:
return []
def scan_owned_as_user(user):
"""Files/dirs owned by `user`, found by running forall_direntries_from AS
that user (su -), so the search only descends where the user can go instead
of walking the whole filesystem as root. Returns a list, or None if forall
is unavailable."""
forall = " ".join(shlex.quote(x) for x in find_forall())
inner = f"{forall} {shlex.quote(user)}"
cmd = ["su", "-", user, "-c", inner] if _can_su(user) else ["sh", "-c", inner]
try:
r = subprocess.run(cmd, capture_output=True, text=True)
except FileNotFoundError:
return None
return [ln for ln in r.stdout.splitlines() if ln.strip()]
def remove_paths_as_user(user, paths, as_user=True):
"""Delete `paths`: files/symlinks first, then now-empty directories deepest-
first, streaming a live progress line. By default runs AS the package user
(su -) so the kernel enforces what may be removed; with as_user=False it runs
as the current (root) process -- the thorough fallback for files the user
itself can't reach. Returns the number of entries removed."""
paths = [p for p in paths if p]
if not paths:
return 0
listfile = os.path.join(_plans_dir(),
re.sub(r"[^A-Za-z0-9_.-]+", "_", user) + ".rmlist")
with open(listfile, "w") as f:
f.write("\n".join(paths) + "\n")
os.chmod(listfile, 0o644)
lf = shlex.quote(listfile)
inner = (
f'while IFS= read -r p; do '
f'if [ -L "$p" ] || {{ [ -f "$p" ] && [ ! -d "$p" ]; }}; then '
f'rm -f -- "$p" 2>/dev/null && echo "$p"; fi; done < {lf}; '
f"awk '{{print length\"\\t\"$0}}' {lf} | sort -rn | cut -f2- | "
f'while IFS= read -r p; do [ -d "$p" ] && rmdir -- "$p" 2>/dev/null && echo "$p"; done'
)
if as_user and _can_su(user):
cmd = ["su", "-", user, "-c", inner]
else:
cmd = ["sh", "-c", inner]
count = 0
tty = sys.stdout.isatty()
try:
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
text=True, bufsize=1)
for line in p.stdout:
e = line.strip()
if not e:
continue
count += 1
if tty and (count % 5 == 0 or count == 1):
sys.stdout.write(f"\r removed {count}... {os.path.basename(e)[:40]:<40}")
sys.stdout.flush()
p.wait()
finally:
if tty:
sys.stdout.write("\r" + " " * 70 + "\r")
sys.stdout.flush()
try:
os.remove(listfile)
except OSError:
pass
return count
def user_supplementary_groups(name):
if grp is None:
return []
return sorted(g.gr_name for g in grp.getgrall() if name in g.gr_mem)
_STATE = {"verbose": False}
def _run(cmd, show_output=False):
"""Run a command, always echoing the command line so the user sees the
system change being made. With --verbose (or show_output=True) the command's
own output is shown too; otherwise it's captured and surfaced only on error."""
print(f" {colors.blue}$ "
f"{' '.join(shlex.quote(c) for c in cmd)}{colors.normal}")
try:
if _STATE["verbose"] or show_output:
r = subprocess.run(cmd)
else:
r = subprocess.run(cmd, capture_output=True, text=True)
if r.returncode != 0:
printError(f" {' '.join(cmd)} -> {r.stderr.strip() or r.returncode}")
except FileNotFoundError:
# a missing helper is an ordinary problem to report, not a stack trace
printError(f" {cmd[0]}: command not found")
return False
except PermissionError as e:
printError(f" {cmd[0]}: {e}")
return False
return r.returncode == 0
def remove_one(name, args):
u = gather_user(name)
home = pkgusr_home(name)
has_home = os.path.isdir(home)
if not u.has_user and not has_home:
printWarning(f"{name}: nothing to remove (no user, no home).")
return
# safety: for --purge, never remove a user whose home isn't under /usr/src
if args.purge and u.has_user:
# `u.has_user` was computed for the ACCOUNT; this looked the ANCHOR up
# in passwd, so `packagemanager remove cargo-c --purge` died with
# KeyError: getpwnam(): name not found: 'cargo-c'
# A missing account is a normal answer here, not an exception.
try:
pw_dir = pwd.getpwnam(pkgusr_name(name)).pw_dir if pwd else ""
except KeyError:
pw_dir = ""
if not pw_dir.startswith(BASE_DIR.rstrip("/") + "/"):
printError(f"{name}: user home {pw_dir} is not under {BASE_DIR}; "
f"refusing to purge (not a package-user).")
return
# THE ACCOUNT FOR EVERYTHING THAT RUNS AS IT. The file search and the
# delete ran `su - gitsync` / `forall_direntries_from gitsync` with the
# name as typed: there is no such account, so _can_su said no, the search
# ran as root for a user that does not exist, and found nothing --
# "uninstalled 0 files owned by gitsync", then the account and home went.
# On an installed package that orphans every file it ever installed.
acct = pkgusr_name(name)
groups = user_supplementary_groups(acct)
def header():
printHeader(f"{'Purge' if args.purge else 'Remove'} {name}")
print()
tag = " (will be removed)" if args.purge \
else " (kept -- use --purge to remove)"
if has_home:
print(f" home directory : {home}{tag}")
if u.has_user:
print(f" user/group : {acct} (uid {u.uid}, gid {u.gid}){tag}")
if groups:
print(f" member of : {', '.join(groups)}")
def do_scan():
if getattr(args, "root", False):
printInfo(f"Scanning the whole filesystem for files owned by {acct}, "
f"as root (thorough; may take a while)...")
return owned_entries(acct)
printInfo(f"Searching for files owned by {acct}, as {acct} "
f"(su - {acct}; may take a moment)...")
return scan_owned_as_user(acct)
# DRY RUN: search and store the result for a fast --run later
if not args.run:
entries = do_scan()
if entries is None:
printError("forall_direntries_from not found; cannot search.")
return
save_plan("remove", name, entries)
header()
print(f" installed files : {len(entries)} (owned by {acct})")
printInfo("\nDry run -- re-run with --run (it will offer to reuse this "
"search instead of scanning again).")
return
# RUN
header()
what = f"purge {name} (all its files + user + home)" if args.purge \
else f"uninstall all files owned by {name} (keep user + home)"
if not args.yes and not ask_yes_no(f"\nReally {what}?"):
printWarning("Aborted.")
return
# reuse a stored dry-run if the user confirms; otherwise search now
entries = use_stored_plan("remove", name, assume_yes=args.yes)
if entries is None:
entries = do_scan() or []
total = remove_paths_as_user(acct, entries,
as_user=not getattr(args, "root", False))
clear_plan("remove", name)
printInfo(f" uninstalled {total} files owned by {acct}")
if args.purge:
# THE ACCOUNT, NOT THE ANCHOR -- AND CHECK IT WORKED.
#
# `remove --purge mako` ran `userdel mako`. There is no such user;
# the account is p_mako, which stayed in /etc/passwd while the tool
# printed "mako purged." The files and the home were gone, so the
# system was left with an account owning nothing and a report saying
# otherwise.
_acct = pkgusr_name(name)
if u.has_user:
if not _run(["userdel", _acct]): # _run returns True on success
printError(f" userdel {_acct} did not succeed -- the account "
f"is still there")
printInfo(f" check with: grep {_acct} /etc/passwd")
if grp is not None:
try:
grp.getgrnam(_acct)
_run(["groupdel", _acct])
except KeyError:
pass
if has_home and os.path.isdir(home):
try:
shutil.rmtree(home)
except OSError as e:
printError(f" rmtree {home}: {e}")
# ...AND WHAT THE BUILD REMEMBERS ABOUT IT. lfs-helper's phase record
# survived, so the next build of the same name said "keeping the
# source tree: 'build' already finished here" about a tree that had
# just been deleted.
_ph = os.path.join(os.environ.get("LFS_PKGUSR_DIR", "/usr/src/lfs-pkgusr"),
"progress", "phases")
for _k in {name, _acct}:
try:
os.remove(os.path.join(_ph, _k))
except OSError:
pass
_left = []
if pwd is not None:
try:
pwd.getpwnam(_acct)
_left.append("the account " + _acct)
except KeyError:
pass
if os.path.isdir(home):
_left.append(home)
if _left:
printWarning(f"{name}: files removed, but {', '.join(_left)} "
f"remain -- NOT fully purged")
else:
printSuccess(f"{name} purged.")
else:
for fname in ("pkg.lst", "pkg.lst.new"): # manifest now meaningless
try:
os.remove(os.path.join(home, fname))
except OSError:
pass
printSuccess(f"{name} uninstalled (user kept" +
(f", {home} kept)." if has_home else ")."))
def cmd_remove(args):
for name in args.packages:
remove_one(name, args)
# =========================================================================== #
# command: reload-pkg-list [IMPLEMENTED]
# =========================================================================== #
#
# Regenerate /usr/src/<name>/pkg.lst from `list_package <name>`. By default the
# work runs as a DETACHED (disowned) background task -- new session, no
# controlling terminal -- so it survives this process exiting and can't be
# interrupted with Ctrl-C. Use --foreground to wait for it.
# LIST_PACKAGE_BIN overrides the `list_package` command (for testing).
def find_list_package():
return os.environ.get("LIST_PACKAGE_BIN", "list_package").split()
def _can_su(user):
"""We can su to the user only if we're root and the user exists."""
if os.geteuid() != 0 or pwd is None:
return False
try:
pwd.getpwnam(user)
return True
except KeyError:
return False
def _python_module_here(name):
"""Is a module of this name importable by the system python?"""
mod = name.replace("-", "_")
try:
return subprocess.run(
["python3", "-c",
"import importlib.metadata as m,sys;m.version(sys.argv[1])", mod],
capture_output=True, timeout=20).returncode == 0
except (OSError, subprocess.SubprocessError):
return False
# WHAT A MANIFEST SCAN MUST NOT WALK.
#
# `reload-pkg-list p_linux` took minutes, and the time went nowhere near the
# installed files: the scan walked /usr/src/pkgusr -- every package user's
# unpacked source tree, the kernel's own quarter-million files among them --
# plus /proc, /sys, the snapshots and the caches. None of it can contain an
# INSTALLED file by definition.
_SCAN_SKIP = ["/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp",
"/var/cache", "/mnt", "/media", "/lost+found",
"/.snapshots", "/snapshots", "/sources", "/build"]
def _scan_prune_args():
"""find(1) arguments that skip those directories (and stay on one fs)."""
skips = list(_SCAN_SKIP) + [d.rstrip("/") for d in pkgusr_roots()]
out = ["-xdev", "("]
for i, d in enumerate(skips):
if i:
out.append("-o")
out += ["-path", d]
out += [")", "-prune", "-o"]
return out
def write_pkg_list(user, background=True, as_root=False):
"""Regenerate /usr/src/<user>/pkg.lst. Runs list_package AS the package
user (su -) when possible, so it only scans directories that user owns --
much faster than scanning the whole filesystem as root."""
# THE ACCOUNT NAME, NOT THE PACKAGE NAME.
#
# `packagemanager reload-pkg-list mpv` passed "mpv" straight through.
# pkgusr_home() accepts either, so the home resolved; _can_su("mpv") did
# not, because the account is p_mpv -- so it took the root branch and ran
# `list_package mpv`, which found no such user, produced nothing, and left
# an empty root-owned pkg.lst.new behind. And it printed "regeneration
# started".
user = pkgusr_name(user)
home = pkgusr_home(user)
if not os.path.isdir(home):
printError(f"{user}: no package home at {home}")
return False
lp = " ".join(shlex.quote(x) for x in find_list_package())
quser = shlex.quote(user)
# THE HOME IS NOT INSTALLED FILES.
#
# list_package lists everything the account owns, and that includes its
# own home -- sources, build trees, the install script. For the kernel
# that is the whole tree:
# # recorded 266621 file(s) for p_linux
# A manifest is what the package put ON THE SYSTEM, which is what
# `verify` checks and what `remove` deletes; 266k source files in it
# make both meaningless. The build's own tracking has always excluded
# the home (it compares snapshots of the tree outside it); this is the
# same rule for the rescan. awk, not grep, because a filter that
# removes every line must still exit 0.
# ...AND NOT THROUGH A SYMLINK INTO IT.
#
# Excluding the home was not enough: modules_install leaves
# /lib/modules/<ver>/build as a SYMLINK to the source tree, the scan
# follows it, and the same quarter-million files come back recorded as
# /lib/modules/<ver>/build/... -- outside the home, so the home filter
# never saw them. The symlink itself belongs in the manifest; what is
# on the far side of it does not.
#
# The prefix match (index()==1) was too strict: it assumed every line is
# an absolute path starting exactly at the home, and the manifest stayed
# at 268k entries with the filter in place. Matching the home ANYWHERE
# in the line cannot be fooled by a leading ./ or a relative listing,
# and no installed file has a package home in its path.
_filter = ("awk -v h=%s -v m=/var/mail/ -v s=/var/spool/mail/ "
"'index($0,h) { next } "
" index($0,m)==1 { next } "
" index($0,s)==1 { next } "
" /\\/lib\\/modules\\/[^/]+\\/build\\// { next } "
" /\\/lib\\/modules\\/[^/]+\\/source\\// { next } "
" { print }'"
% shlex.quote(home.rstrip("/") + "/"))
# THE SCAN ITSELF, PRUNED -- as the package user (so unreadable
# directories are skipped), or as root with --root (thorough, slow).
_find = " ".join(shlex.quote(x) for x in
["find", "/"] + _scan_prune_args()
+ ["-user", user, "-print"])
_scan = f"{_find} 2>/dev/null | LC_ALL=C sort -u"
if _can_su(user) and not as_root:
# su - starts in the user's home, so pkg.lst.new/pkg.lst are ~/...
inner = (f"{_scan} | {_filter} > pkg.lst.new"
f" && mv pkg.lst.new pkg.lst")
cmd = ["su", "-", user, "-c", inner]
else:
tmp = shlex.quote(os.path.join(home, "pkg.lst.new"))
dst = shlex.quote(os.path.join(home, "pkg.lst"))
cmd = ["sh", "-c", f"{_scan} | {_filter} > {tmp}"
f" && mv {tmp} {dst}"
f" && chown {quser} {dst} 2>/dev/null; true"]
if background: # detached / disowned
subprocess.Popen(cmd, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL, start_new_session=True)
return True
return subprocess.run(cmd).returncode == 0
def cmd_reload_pkg_list(args):
# SAY WHAT HAPPENED, NOT THAT IT STARTED.
#
# A disowned task that dies in its first second reported "regeneration
# started" and left the user watching an empty pkg.lst.new and an empty
# process table. This command is asked for by a person, so it runs in the
# foreground by default and reports the RESULT; --background is there for
# anyone who wants the old behaviour.
rc = 0
for user in args.packages:
acct = pkgusr_name(user)
if getattr(args, "background", False):
if write_pkg_list(user, background=True):
printInfo(f"{acct}: pkg.lst regeneration started"
f" (disowned background task).")
continue
_as_root = getattr(args, "root", False)
printInfo(f"{acct}: scanning for files it owns "
+ ("as root (every directory) ..." if _as_root
else f"as {acct} ..."))
_t0 = time.time()
if not write_pkg_list(user, background=False, as_root=_as_root):
printError(f"{acct}: could not regenerate pkg.lst")
rc = 1
continue
manifest = os.path.join(pkgusr_home(acct), "pkg.lst")
try:
with open(manifest) as f:
n = sum(1 for line in f if line.strip())
except OSError:
n = 0
if n:
printSuccess(f"{acct}: {n} file(s) recorded in {manifest}"
f" ({time.time() - _t0:.0f}s)")
if not _as_root:
printInfo(f" scanned as {acct}: directories it cannot enter"
f" were skipped, and so were")
printInfo(f" {', '.join(_SCAN_SKIP[:6])} ... and the package"
f" source roots.")
printInfo(f" If something is missing: packagemanager"
f" reload-pkg-list {user} --root")
else:
printWarning(f"{acct}: no files found -- it owns nothing on disk.")
printInfo(" Either it was never installed, or its files ended up"
" owned by someone else.")
return rc
# =========================================================================== #
# command: add-user [IMPLEMENTED]
# =========================================================================== #
def report_created_user(name, kind="package-user"):
"""After creating a user, tell the person exactly what now exists / changed
on the system (uid/gid, home, and the databases that were written)."""
# THE ACCOUNT THAT WAS CREATED, NOT THE NAME IT WAS ASKED FOR.
#
# `add-user n76310` created p_n76310 and then reported on "n76310" --
# the HUMAN login of the same name: its uid, its /home/n76310, its
# audio/video/wheel groups. Every line was true of the wrong account,
# which is the most misleading kind of report there is.
try:
pw = pwd.getpwnam(name)
except (KeyError, TypeError):
try:
name = pkgusr_name(name)
pw = pwd.getpwnam(name)
except (KeyError, TypeError):
printWarning(f" {name}: created, but not found in the user database?")
return
printInfo(f" {kind} '{name}': uid {pw.pw_uid}, gid {pw.pw_gid}")
printInfo(f" recorded in: /etc/passwd, /etc/group, /etc/shadow")
home = pw.pw_dir
if os.path.isdir(home):
printInfo(f" home directory created: {home}")
else:
printInfo(f" home directory (not present): {home}")
groups = user_supplementary_groups(name)
if groups:
printInfo(f" supplementary groups: {', '.join(groups)}")
def _sanitise_user_name(mod):
"""A Python distribution name is not necessarily a valid user name.
"beautifulsoup4" is fine; "ruamel.yaml", "Jinja2" and "typing_extensions"
are not what we want as accounts. Lower-case and reduce to [a-z0-9_-]."""
n = re.sub(r"[^a-z0-9_-]+", "-", mod.lower()).strip("-")
return n[:32] or "pymodule"
def _pip_site_dir():
"""The system site-packages directory pip installs into."""
import glob as _glob
for pat in ("/usr/lib/python3*/site-packages",
"/usr/lib/python3*/dist-packages"):
hits = sorted(_glob.glob(pat))
if hits:
return hits[-1]
return None
def _pip_owner_of_python():
"""Which package owns Python's tree -- the collector group is named after it."""
site = _pip_site_dir()
if not site:
return "python"
d = site
while d not in ("/", ""):
try:
import pwd as _pwd
return _pwd.getpwuid(os.stat(d).st_uid).pw_name
except (KeyError, OSError):
d = os.path.dirname(d)
return "python"
# One module, installed the package-user way. Shared by `packagemanager pip`
# and `packagemanager setup`, because they want exactly the same thing and a
# second copy of it is how the two would drift apart.
def pip_install_module(mod, site, grp, find_links=None, upgrade=False,
no_deps=False, target=None):
"""Install ONE Python module as its own package user. True on success.
`target` is an explicit file to install (a wheel in the sources directory);
without it pip is given the module NAME and finds it however it can.
"""
# THE ACCOUNT NAME COMES FROM THE CHOKEPOINT.
#
# This read `_sanitise_user_name(mod)` and used the result for every call
# below. create_package_user normalises internally, so the account it made
# was `p_requests` -- and then usermod and su were handed the bare
# `requests`, which does not exist:
# usermod: user 'requests' does not exist
# The same species as `chown: invalid user: 'wget:wget'`: half the code
# asking for the name, half assuming it.
user = pkgusr_name(_sanitise_user_name(mod))
printHeader(f"{mod} -> package user '{user}'")
if not _user_exists(user):
if not create_package_user(user):
printError(f"could not create {user}")
return False
report_created_user(user)
# THE COLLECTOR GROUP IS GRANTED THROUGH ONE DOOR.
#
# lfs-helper owns the rules: join the group the directory already carries,
# ask which group should own it when there is none, never create one for
# root, never hand a package's tree to the install group. This did its own
# groupadd/chgrp -R/chmod -R instead -- a third implementation, after
# lfs-helper's and packagemanager_install's, and the only one that never
# asked what the directory already carried.
if shutil.which("lfs-helper"):
_run(["lfs-helper", "grant-dir", site, user, "--run"])
else:
if not _group_exists(grp):
printInfo(f"creating collector group {grp}")
_create_collector_group(grp)
_run([_cmd_path("usermod"), "-a", "-G", grp, user])
if _group_exists(grp):
_run(["chgrp", "-R", grp, site])
_run(["chmod", "-R", "g+w", site])
# PIP_USER=0 / PYTHONNOUSERSITE=1: without these pip quietly falls back
# to ~/.local when site-packages is not writable, and then reports
# "Requirement already satisfied" from there ever after -- so the module
# is installed for exactly one user and importable by nobody else.
env = ["env", "PIP_USER=0", "PYTHONNOUSERSITE=1"]
pip_args = ["pip3", "install"]
if upgrade:
pip_args.append("--upgrade")
if no_deps:
pip_args.append("--no-deps")
if find_links:
pip_args += ["--no-index", "--find-links", find_links]
pip_args.append(target or mod)
cmd = ["su", "-s", "/bin/bash", user, "-c",
" ".join(shlex.quote(x) for x in env + pip_args)]
printInfo(" ".join(pip_args) + f" (as {user})")
if _run(cmd, show_output=True):
# RECORD IT LIKE EVERY OTHER INSTALL.
#
# This installed the module and wrote nothing: no pkg.lst, no
# install_last. So beautifulsoup4, certifi, idna, requests, urllib3
# and the rest of the bootstrap set -- installed, importable, in daily
# use by these very tools -- read as
# not installed user + home exist, but nothing installed yet
# and were offered for rebuild by `verify`, with a command that could
# not work because they are not in the book either. An install that
# leaves no record is invisible to everything downstream of it.
_record_pip_install(user, mod, target)
printSuccess(f"{mod} installed as package-user {user}")
return True
printError(f"{mod} failed -- see the output above")
return False
def _record_pip_install(user, mod, target=None):
"""The manifest and the version stamp a pip install owes the system."""
home = pkgusr_home(user)
if not os.path.isdir(home):
return
ok = write_pkg_list(user, background=False)
# the version pip actually landed, asked of the module itself
ver = ""
try:
out = subprocess.run(
["python3", "-c",
"import importlib.metadata as m,sys;print(m.version(sys.argv[1]))",
mod], capture_output=True, text=True, timeout=30)
if out.returncode == 0:
ver = out.stdout.strip()
except (OSError, subprocess.SubprocessError):
pass
# install_last is what the tools read for name_version; a pip module has no
# script, so write the minimum that makes it a package like the others
try:
with open(os.path.join(home, "install_last"), "w") as f:
f.write("### installed by `packagemanager pip` -- no book page\n")
f.write("name=%s\n" % shlex.quote(mod))
f.write("name_version=%s\n" % shlex.quote(
"%s-%s" % (mod, ver) if ver else mod))
f.write("validate_cmd=%s\n" % shlex.quote(
"python3 -c 'import importlib.metadata as m;"
"print(m.version(\"%s\"))'" % mod))
os.chmod(os.path.join(home, "install_last"), 0o444)
except OSError:
pass
if not ok:
printWarning(" (could not record %s's file list; `packagemanager "
"verify` may report it as not installed)" % user)
def cmd_pip_record(names):
"""Give an already-installed pip module the record it never got.
Installs before 1.14.37 wrote neither pkg.lst nor install_last, so the
modules these tools themselves depend on -- beautifulsoup4, certifi, idna,
requests, urllib3 -- read as "not installed" forever. Reinstalling them
would work, and is the wrong answer: they ARE installed, and nothing but
the paperwork is missing. With no argument, every account in that state.
"""
if not names:
names = [n for n in list_all_users()
if not gather_user(n).meta.get("name_version")
and _python_module_here(n)]
if not names:
printSuccess("Nothing to record -- every python module has one.")
return 0
printInfo("Recording: " + " ".join(names))
rc = 0
for name in names:
user = pkgusr_name(name)
if not os.path.isdir(pkgusr_home(user)):
printError(f"{name}: no package home -- nothing to record.")
rc = 1
continue
if not _python_module_here(name):
printError(f"{name}: python cannot import it, so it is not "
f"installed -- record refused.")
rc = 1
continue
_record_pip_install(user, name.replace("-", "_"))
printSuccess(f"{name}: recorded.")
return rc
def cmd_pip(args):
"""Install or update a Python module AS A PACKAGE USER.
A bare `pip3 install beautifulsoup4` installs as whoever ran it -- usually
root -- so the files belong to nobody in particular and no package user can
ever update or remove them. This does it the package-user way: one user per
module, joined to the collector group that owns Python's tree, with pip kept
out of ~/.local so the module lands where every other package can import it.
"""
site = _pip_site_dir()
if not site:
printError("no system site-packages directory found -- is Python installed?")
return 1
grp = collector_prefix() + _pip_owner_of_python()
modules = args.modules
if not modules:
printError("usage: packagemanager pip install <module> [<module> ...]")
return 1
for mod in modules:
pip_install_module(mod, site, grp,
find_links=args.find_links,
upgrade=args.upgrade,
no_deps=getattr(args, "no_deps", False))
return 0
def cmd_add_user(args):
for name in args.packages:
# THE ACCOUNT, NOT THE PACKAGE NAME.
#
# `add-user n76310` asked whether "n76310" exists. It did -- it is
# the human's login account -- so this said "already exists" and
# created nothing, and the account p_n76310 was then made on the fly
# by the install runner, with the hint's default home
# (/usr/src/p_n76310, no pkgusr/ component). verify scans
# /usr/src/pkgusr and never saw it. Same fault as 1.14.85: a package
# name is not an account name.
acct = pkgusr_name(name)
if _user_exists(acct):
printInfo(f"{acct}: already exists.")
continue
if create_package_user(name):
printSuccess(f"Created package-user {acct}.")
report_created_user(acct)
# =========================================================================== #
# first-run setup + config command
# =========================================================================== #
def _adopt_existing_settings():
"""What can be answered without asking anybody.
Three sources, in order of authority:
1. the lfs config -- the interview's own answers, which is what the
system was actually built with;
2. the environment `lfs` exports into the chroot;
3. the system itself -- the collector groups and accounts that are
already there.
Returns {key: (value, where it came from)} for everything that is known.
"""
found = {}
shared = _shared_lfs_config()
for key in _SHARED_WITH_LFS + ("user_prefix",):
val = shared.get(key)
if val:
found[key] = (str(val).rstrip("_"), "the lfs config")
for key, env in (("collector_prefix", "LFS_COLLECTOR_PREFIX"),
("pkgusr_prefix", "LFS_PKGUSR_PREFIX"),
("cfguser_prefix", "LFS_CFGUSR_PREFIX"),
("main_user", "LFS_MAIN_USER")):
if key in found:
continue
val = os.environ.get(env)
if val:
found[key] = (val.rstrip("_"), "the chroot environment ($%s)" % env)
# THE SYSTEM ITSELF. Collector groups live at gid 90000+, and every one of
# them carries the prefix -- so on a built system the answer is written in
# /etc/group whether or not any config survived.
if "collector_prefix" not in found:
pfx = _prefix_from_groups()
if pfx:
found["collector_prefix"] = (pfx, "the collector groups on this system")
if "pkgusr_prefix" not in found:
pfx = _prefix_from_accounts(("p", "pkg"))
if pfx:
found["pkgusr_prefix"] = (pfx, "the package users on this system")
if "main_user" not in found:
mu = _guess_main_user()
if mu:
found["main_user"] = (mu, "the login account on this system")
return found
def _prefix_from_groups():
"""The prefix the collector groups already carry, if they agree on one."""
if grp is None:
return ""
seen = {}
try:
groups = grp.getgrall()
except Exception:
return ""
for g in groups:
if not (90000 <= g.gr_gid < 99000) or "_" not in g.gr_name:
continue
seen[g.gr_name.split("_", 1)[0]] = seen.get(g.gr_name.split("_", 1)[0], 0) + 1
if not seen:
return ""
# One prefix, or the commonest: a tree that changed prefix mid-life has
# both, and the one on most groups is the one in use.
return max(seen.items(), key=lambda kv: kv[1])[0]
def _prefix_from_accounts(candidates):
"""Which of these prefixes the package users actually wear."""
if pwd is None:
return ""
counts = {}
try:
users = pwd.getpwall()
except Exception:
return ""
for u in users:
if not (10000 <= u.pw_uid < 65000) or "_" not in u.pw_name:
continue
pfx = u.pw_name.split("_", 1)[0]
if pfx in candidates:
counts[pfx] = counts.get(pfx, 0) + 1
if not counts:
return ""
return max(counts.items(), key=lambda kv: kv[1])[0]
def first_run_setup():
"""Write packagemanager's config, asking only for what nothing knows.
This asked for all three settings every time the config file was absent --
on a system built by these tools, where `lfs build-system session` had
already collected every one of them, written them into config.json, and
copied that into the chroot. Being asked for a value the machine is
holding teaches you that the answer does not matter, which is the opposite
of true: a collector prefix that disagrees with the built system gives you
two parallel sets of groups over the same directories, with no error.
"""
cfg = load_config()
known = _adopt_existing_settings()
for key, (val, _src) in known.items():
cfg[key] = val
missing = [k for k in ("collector_prefix", "user_prefix")
if not cfg.get(k)]
if not missing:
printHeader("Settings")
print()
printInfo("Taken from what this system was built with -- nothing to answer:")
for key in ("collector_prefix", "pkgusr_prefix", "cfguser_prefix",
"user_prefix", "main_user"):
if not cfg.get(key):
continue
src = known.get(key, (None, "the default"))[1]
print(" %-18s %-12s (%s)" % (key, cfg[key], src))
if save_config(cfg):
printSuccess("Saved %s -- change any of it with: packagemanager config"
% CONFIG_PATH)
else:
printWarning("Could not save config; using these values for this run only.")
print()
return
printHeader("First-run setup")
print()
printInfo("packagemanager needs a few settings (saved so you're only asked once):")
print(" - collector groups own shared install directories, so several")
print(" package-users can install into the same place;")
print(" - shared users run apps under a separate account;")
print(" - the main user is your own human login account.")
print(" (Existing users: enter your current values, e.g. 'nimgnu', 'u'.)")
print()
guess = cfg.get("main_user") or _guess_main_user() or "none"
try:
cp = input(f" Collector-group prefix [{cfg['collector_prefix']}]: ").strip()
up = input(f" Application-user prefix [{cfg['user_prefix']}]: ").strip()
mu = input(f" Main (human) user name [{guess}]: ").strip()
except EOFError:
cp = up = mu = ""
cfg["collector_prefix"] = (cp or cfg["collector_prefix"]).rstrip("_")
cfg["user_prefix"] = (up or cfg["user_prefix"]).rstrip("_")
mu = mu or guess
if mu and mu.lower() != "none":
cfg["main_user"] = mu
if save_config(cfg):
printSuccess(f"Saved {CONFIG_PATH} (change later with: packagemanager config).")
else:
printWarning("Could not save config; using these values for this run only.")
print()
def _guess_main_user():
su = os.environ.get("SUDO_USER")
if su and su != "root":
return su
if pwd is not None:
for p in sorted(pwd.getpwall(), key=lambda x: x.pw_uid):
if 1000 <= p.pw_uid < 9999:
return p.pw_name
return ""
def _blfs_store_lines():
rc, out, _ = run_blfs(["books"])
store = default = None
for line in (out or "").splitlines():
if line.startswith("Store:"):
store = line.split(":", 1)[1].strip()
elif line.startswith("Default:"):
default = line.split(":", 1)[1].strip()
return store, default
def cmd_script(args):
"""Run one phase of a package's install script AS the package user, e.g.
'packagemanager script nano install' to re-run only the install step after
fixing permissions (no recompile)."""
# THE ACCOUNT, NOT THE ANCHOR. This took args.package verbatim, so
# packagemanager script install rust
# ran `su - rust -c 'bash ~/install_rust install'` -- an account that
# does not exist, and a script named after the anchor rather than the
# package user. Everything else in the tool goes through pkgusr_name.
# TWO NAMES, AND THEY ARE NOT INTERCHANGEABLE.
# anchor -- what the BOOK calls it (rust); blfs generates from this
# name -- the ACCOUNT (p_rust); su and the script path use this
# It used the anchor for both (su - rust: no such account), and the first
# fix used the account for both (blfs script failed for p_rust).
anchor = args.package
name = pkgusr_name(anchor)
phase = args.phase
# fetch: get the script, run NOTHING. It lands where edits live -- the
# package user's home when there is one, /tmp's script dir otherwise --
# which is exactly the file resolve_script prefers, so "fetch, edit,
# install" and the 1.11.31 conflict flow ("merge into install_<version>")
# are the same motion.
if phase == "fetch":
bv = book_version(name)
if not bv:
printError(f"{name} is not in the BLFS book.")
printInfo(f" LFS-book packages have their script fetched by: "
f"lfs script fetch {name}")
sys.exit(1)
home = pkgusr_home(name)
target = os.path.join(home, f"install_{bv}")
if os.path.isfile(target) and not args.regenerate:
printSuccess(f"already there: {target}")
printInfo(" edit that file -- install and update use it. "
"Rebuild it from the book with --regenerate.")
return
outdir = home if os.path.isdir(home) \
else os.path.join(TMP_DIR, "install_files")
os.makedirs(outdir, exist_ok=True)
path = resolve_script(name, bv, outdir, assume_yes=True,
force_regen=args.regenerate)
if not path:
sys.exit(1)
if os.path.isdir(home) and os.path.dirname(
os.path.abspath(path)) != os.path.abspath(home):
shutil.copy(path, target)
path = target
# the canonical script belongs to the package user, like every other
# file in its home; a not-yet-created account simply keeps root's copy
try:
import pwd
pw = pwd.getpwnam(pkgusr_name(name))
os.chown(path, pw.pw_uid, pw.pw_gid)
except (KeyError, OSError):
pass
printSuccess(f"script: {path}")
printInfo(f" edit it, then: packagemanager install {name} --run")
return
home = pkgusr_home(name)
if not _user_exists(name):
printError(f"{name}: no such package user.")
return
# THE CANONICAL FILE, NOT A COPY OF IT.
#
# This used install_<user> -- the redundant runtime copy 1.14.16 removed
# everywhere else -- so a failure reported "ran: install_p_glib2" while
# telling the person to edit install_GLib-2.86.4, and an edit to the file
# they were pointed at did not affect the file that ran.
home_script = (_versioned_script_in(home)
or os.path.join(home, f"install_{name}"))
# (re)generate if asked, missing, or the on-disk script predates the current
# phased format (old scripts can't find the build dir for install).
needs_regen = args.regenerate or not os.path.isfile(home_script)
if not needs_regen:
try:
with open(home_script) as f:
head = f.read(6000)
if not _script_is_current(head): # old/un-stamped format
needs_regen = True
printInfo("(install script is an older format -- regenerating)")
except OSError:
needs_regen = True
if needs_regen:
printInfo(f"Generating install script for {name}...")
script_dir = os.path.join(TMP_DIR, "install_files")
os.makedirs(script_dir, exist_ok=True)
rc, out, err = run_blfs(["script", anchor, "-o", script_dir])
m = re.search(r"^Created (.+)$", out, re.M)
if rc != 0 or not m:
printError(f"blfs script failed for {name} -- {(err or out).strip()}")
return
gen = m.group(1).strip()
# NEVER DESTROY AN EDIT IN SILENCE.
#
# This overwrites the script in the package's home whenever the tools
# move on -- and a user who had fixed something in it (a dead download
# URL, say) loses the fix with no message. Keep a copy and say so.
try:
_dest = os.path.join(home, os.path.basename(gen))
if (os.path.isfile(_dest)
and open(_dest).read() != open(gen).read()):
home_script = _dest
bak = home_script + ".edited"
shutil.copy(home_script, bak)
printWarning(" your version of %s differed -- saved as %s"
% (os.path.basename(home_script),
os.path.basename(bak)))
# Say what actually happens. This claimed the versioned file
# "is never regenerated" while regenerating it, which is how a
# person loses an edit twice: once to the overwrite, once to
# the sentence that said it could not happen.
printInfo(" (this file IS the one you edit; it was rewritten"
" because you asked for")
printInfo(" --regenerate or its format was too old."
" Your copy is the .edited file.)")
except OSError:
pass
# a regenerated script keeps the book's own name for the file
home_script = os.path.join(home, os.path.basename(gen))
try:
shutil.copy(gen, home_script)
os.chmod(home_script, 0o755)
# `pwd` is imported inside a try in ANOTHER branch, so this
# referenced a name that may never have been bound:
# UnboundLocalError: cannot access local variable 'pwd'
# Import it here, where it is used, and treat a missing account
# as "leave it owned by root" rather than an error.
try:
import pwd as _pw_mod
_pw = _pw_mod.getpwnam(name)
os.chown(home_script, _pw.pw_uid, _pw.pw_gid)
except (ImportError, KeyError, OSError):
pass
except OSError as e:
printError(f"could not place script in {home}: {e}")
return
printHeader(f"{name}: {phase}")
# THE BUILD ENVIRONMENT HAS TO REACH THE BUILD.
#
# `packagemanager env --set XORG_CONFIG=...` writes /etc/pkgusr/build.env,
# and lfs-helper's run_phase_as pastes it in front of the command it runs.
# This path built its own `su` and passed nothing, so the same package
# failed here with
# this package needs XORG_CONFIG to be set
# right after the person had set it and been shown it in the table.
# `su -` starts a login shell and drops the caller's environment, so the
# variables have to travel INSIDE the command, not in subprocess's env.
_bootenv = dict(_read_build_env())
if getattr(args, "test", False):
_bootenv["PM_TEST"] = _bootenv["PKGUSR_TESTS"] = "1"
_pairs = "".join("%s=%s " % (k, shlex.quote(v))
for k, v in sorted(_bootenv.items()))
# THE SAME DOOR AS EVERY OTHER BUILD.
#
# This ran the script itself, and so had none of what lfs-helper's
# cmd_build wraps around one: the compiler wrappers, the file tracking,
# the failure box -- and above all the GRANT-AND-RETRY. A glib2 install
# that stopped on
# PermissionError: [Errno 13] '/usr/share/gettext/its'
# was repaired automatically when the same script ran through lfs-helper,
# and repeated the identical traceback forever when it ran through here.
# `packagemanager script install X` is the command the failure box itself
# tells people to use, so it must be the same door.
_helper = shutil.which("lfs-helper")
if _helper and _can_su(name) and phase != "fetch":
cmd = [_helper, "build", name, "--script", home_script,
"--phase", phase, "--force"]
printInfo(" $ " + " ".join(cmd[1:]))
else:
_run = f"bash {shlex.quote(f'install_{name}')} {shlex.quote(phase)}"
inner = f"cd ~ && {('env ' + _pairs) if _pairs else ''}{_run}"
cmd = (["su", "-", name, "-c", inner] if _can_su(name)
else ["sh", "-c", inner])
printInfo(f" $ su - {name} -c 'bash ~/install_{name} {phase}'")
if _bootenv:
printInfo(" with: " + " ".join(sorted(_bootenv)) + " (from %s)"
% BUILD_ENV_FILE)
_records = phase in ("install", "all", "update")
env = {**os.environ}
old_paths = (_pkglist_paths(name)
if getattr(args, "clean", False) and phase in ("all", "install", "update")
else None)
t0 = time.time()
result = subprocess.run(cmd, env=env)
if result.returncode != 0:
# NAME THE FILES. This printed the exit code and a log directory and
# left the person to work out which of the three copies of the script
# had just run, where it built, and what to do next -- while the same
# failure through lfs-helper prints all of it. A failure report that
# does not say where is barely a report.
printError(f"phase '{phase}' failed for {name} (exit {result.returncode}).")
printInfo(f" ran : {home_script}")
_edit = _versioned_script_in(home)
if _edit:
printInfo(f" edit : {_edit}")
printInfo(" (that one is yours and is never regenerated;"
" the file above is a copy of it)")
_src = _unpacked_source_dir(home)
if _src:
printInfo(f" source : {_src}")
printInfo(f" logs : {_pkg_log_dir(name)}")
printInfo(f" retry : packagemanager script {phase} "
f"{getattr(args, 'package', name)}")
if phase in ("install", "configure", "test"):
# "No rule to make target 'install'" means the source is there and
# unconfigured -- the build phase never finished, and running one
# later phase cannot make up for it.
printInfo(" If the build phase never finished here, no later"
" phase can work.")
printInfo(f" packagemanager script build "
f"{getattr(args, 'package', name)}")
sys.exit(result.returncode)
if old_paths is not None:
_clean_after_install(name, old_paths, t0)
# A PHASE THAT INSTALLED MUST BE RECORDED.
#
# This runs the script directly, and only pm-install writes the manifest
# -- so after `packagemanager script install rust` succeeded, the planner
# still called Rustc "new" and the next stack run started building it
# again from scratch. An install nobody recorded did not happen, as far
# as every other command is concerned.
if _records:
try:
_rr = subprocess.run(["lfs-helper", "record-install", name],
capture_output=True, text=True)
for _l in (_rr.stdout or "").splitlines():
if _l.strip():
printInfo(" " + _l.strip())
if _rr.returncode != 0:
printWarning(" could not record the install -- the planner will"
" treat %s as not installed" % name)
except (OSError, subprocess.SubprocessError) as e:
printWarning(f" could not record the install ({e})")
printSuccess(f"{name}: {phase} done. logs: {_pkg_log_dir(name)}")
def cmd_paths(args):
printHeader("Where packagemanager keeps things")
print()
def row(label, path):
exists = os.path.exists(path)
mark = "" if exists else " (not present)"
print(f" {label:<22}: {path}{mark}")
row("config file", CONFIG_PATH)
row("package-user homes", BASE_DIR)
row("install scripts", INSTALL_SCRIPTS_DIR)
row("generated scripts", os.path.join(TMP_DIR, "install_files"))
row("dry-run plans", os.path.join(TMP_DIR, "plans"))
store, default = _blfs_store_lines()
print()
if store:
print(f" blfs store : {store}")
row(" book files", os.path.join(store, "books"))
row(" parsed cache", os.path.join(store, "cache"))
row(" blfs config", os.path.join(store, "config.json"))
if default:
print(f" blfs default book : {default}")
else:
printWarning(" blfs store : (could not query blfs)")
print()
print(f" blfs binary : {' '.join(find_blfs())}")
print(f" install engine : {' '.join(find_pm_install())}")
print(f" forall tool : {' '.join(find_forall())}")
print(f" list_package : {' '.join(find_list_package())}")
print(f" add_package_user : {' '.join(find_add_package_user())}")
print()
cfg = load_config()
print(f" collector prefix : {cfg['collector_prefix']}_")
print(f" shared-user prefix : {cfg['user_prefix']}_")
print(f" main user : {main_user() or '(unset)'}")
# Settings worth asking for when they are missing. Only ones that change
# behaviour: a blank main_user means "no human account", which is a decision,
# not an oversight, so it is asked once and then left alone.
_CONFIG_PROMPTS = [
("collector_prefix",
"prefix for collector groups (shared install-dir groups)",
"sysgroup"),
("user_prefix", "prefix for shared application users", "u"),
("main_user", "your human login account (blank if none)",
lambda: _guess_main_user() or ""),
]
def _book_in_use(tool):
"""Which book each tool would use -- the thing you actually want to know
when a package cannot be found."""
if tool == "blfs":
rc, out, _e = run_blfs(["books"])
if rc != 0:
return "(blfs not available)"
for ln in out.splitlines():
if "[" in ln and "]" in ln:
return ln.split("[", 1)[1].split("]", 1)[0]
return "(none cached -- blfs fetch, or blfs import <file>)"
# Read the config directly. Parsing `lfs config` needed a mounted LFS
# tree, which does not exist inside the chroot -- so the book the system
# was built from showed as "(unknown)" on the very system it built.
cfg = _shared_lfs_config()
ver = cfg.get("build_book") or cfg.get("default")
if not ver:
return "(no book configured)"
for cand in (os.environ.get("LFS_STORE"), "/usr/share/lfs"):
if not cand:
continue
p = os.path.join(cand, "books", f"LFS-BOOK-{ver}-NOCHUNKS.html")
if os.path.isfile(p):
return os.path.basename(p)
return f"{ver} (not cached here)"
def _accounts_with_prefix(prefix):
"""Package-user accounts currently using this prefix.
Used to refuse a prefix change that would orphan a built tree: the
accounts are not renamed by changing the setting.
"""
if pwd is None:
return []
pfx = (prefix.rstrip("_") + "_") if prefix else ""
out = []
for entry in pwd.getpwall():
if not pfx or entry.pw_name.startswith(pfx):
if entry.pw_dir.startswith(BASE_DIR.rstrip("/") + "/"):
out.append(entry.pw_name)
return out
def _config_sources():
"""Every prefix and directory, with the reason it has that value.
The user could not find where these live -- fairly: prefixes come from
/usr/share/lfs/config.json (authoritative), then
/etc/pkgusr/packagemanager.conf, then the environment, then a default;
and the DIRECTORIES were environment-only, documented nowhere.
"""
shared = _shared_lfs_config() or {}
conf = {}
try:
import configparser
cp = configparser.ConfigParser()
with open(CONFIG_PATH) as f:
cp.read_string("[c]\n" + f.read())
conf = dict(cp["c"])
except Exception:
conf = {}
def src(key, env, value):
if os.environ.get(env):
return "the environment ($%s)" % env
if key in shared:
return "the lfs config (%s)" % _lfs_config_path()
if key in conf:
return "packagemanager.conf"
return "the built-in default"
rows = [
("package-user prefix", pkgusr_prefix(),
src("pkgusr_prefix", "LFS_PKGUSR_PREFIX", None)),
("config-user prefix", cfguser_prefix(),
src("cfguser_prefix", "LFS_CFGUSR_PREFIX", None)),
("collector prefix", collector_prefix(),
src("collector_prefix", "LFS_COLLECTOR_PREFIX", None)),
("shared-user prefix", user_prefix(),
src("user_prefix", "LFS_USER_PREFIX", None)),
("base directory", BASE_DIR,
"the environment ($PKGUSR_BASE)" if os.environ.get("PKGUSR_BASE")
else "the built-in default"),
("package homes", os.path.join(BASE_DIR, PKGUSR_SUBDIR) if PKGUSR_SUBDIR
else BASE_DIR,
src("pkgusr_subdir", "PKGUSR_SUBDIR", None)),
("config-user homes", os.path.join(BASE_DIR, CFGUSR_SUBDIR) if CFGUSR_SUBDIR
else BASE_DIR,
src("cfguser_subdir", "CFGUSR_SUBDIR", None)),
]
return rows
def _lfs_config_path():
"""The lfs config actually in use -- not the default path.
The report named /usr/share/lfs/config.json whatever it had read, which
is wrong wherever LFS_STORE points somewhere else.
"""
for cand in (os.environ.get("LFS_STORE"), "/usr/share/lfs",
os.path.expanduser("~/.local/share/lfs")):
if cand and os.path.isfile(os.path.join(cand, "config.json")):
return os.path.join(cand, "config.json")
return "/usr/share/lfs/config.json"
_LFS_CONFIG_HINT = "/usr/share/lfs/config.json"
def cmd_config(args):
# --show: the whole picture, no questions. `config` on its own asks for
# the main user before it prints anything, which is not what somebody
# checking their prefixes wants.
if getattr(args, "show", False):
printHeader("Prefixes and directories")
print()
for label, value, source in _config_sources():
print(" %-20s %-28s %s" % (label, value or "(none)", source))
print()
# Say what the keys ARE. This told people to export variables, which
# was the only way until 1.13.69 -- and then kept telling them so,
# which is why the report looked unchanged after the keys were added.
printInfo(" every value below can be set in either file; the lfs "
"config wins for the")
printInfo(" prefixes, and the environment wins over both.")
print()
printInfo(" %s" % _lfs_config_path())
printInfo(' "pkgusr_prefix": "p" "cfguser_prefix": "cfg"')
printInfo(' "collector_prefix": "nimgnu"')
printInfo(' "pkgusr_subdir": "pkgusr" "cfguser_subdir": "cfg"')
print()
printInfo(" %s" % CONFIG_PATH)
printInfo(" pkgusr_home=/usr/src/pkgusr (the base; the "
"subdir is taken off)")
printInfo(" pkgusr_subdir=pkgusr cfguser_subdir=cfg")
printInfo(" user_prefix=u main_user=<login>")
printWarning(" a prefix is chosen BEFORE the first account exists; "
"changing it on a built")
printWarning(" tree makes every tool look for accounts that are not "
"there.")
return
return _cmd_config_interactive(args)
def _cmd_config_interactive(args):
cfg = load_config()
changed = False
# THE PACKAGE-USER PREFIX HAD NO FLAG AT ALL.
#
# It could only be changed by editing config.json by hand -- and it is
# the one prefix that renames every account the system has. So: a flag,
# and a refusal when accounts already exist under the current one.
for _flag, _key in (("pkgusr_prefix", "pkgusr_prefix"),
("cfguser_prefix", "cfguser_prefix")):
_new = getattr(args, _flag, None)
if _new is None:
continue
_new = _new.rstrip("_")
_old = cfg.get(_key, _DEFAULT_CONFIG.get(_key, ""))
if _new == _old:
printInfo(" %s is already %r" % (_key, _new or ""))
continue
_have = _accounts_with_prefix(_old)
if _have and not getattr(args, "force", False):
printError("%d account(s) already use the '%s_' prefix: %s%s"
% (len(_have), _old, ", ".join(sorted(_have)[:4]),
" ..." if len(_have) > 4 else ""))
printWarning(" Changing it now does NOT rename them. Every tool "
"would look for accounts")
printWarning(" named %r and find nothing: the packages would read "
"as not installed," % (_new + "_" if _new else "<none>"))
printWarning(" their manifests would orphan, and a rebuild would "
"make a second set.")
printWarning(" This belongs on a tree with no package users yet.")
printInfo(" If you are certain (a fresh tree, or you will rename "
"the accounts yourself):")
printInfo(" packagemanager config --%s %s --force"
% (_flag.replace("_", "-"), _new or "''"))
return
cfg[_key] = _new
changed = True
printWarning(" %s -> %r%s" % (_key, _new or "(no prefix)",
" [--force]" if _have else ""))
# ...and put it where it will actually be read
_save_shared_key(_key, _new)
# THE DIRECTORIES NEED FLAGS TOO.
#
# They became configurable in 1.13.52/53 and could still only be changed
# by editing a file. Same guard as the prefixes: moving the layout does
# not move the homes, so it belongs on a tree that has none yet.
for _flag, _key in (("base_dir", "pkgusr_home"),
("pkgusr_subdir", "pkgusr_subdir"),
("cfguser_subdir", "cfguser_subdir")):
_new = getattr(args, _flag, None)
if _new is None:
continue
_new = _new.rstrip("/")
_homes = [h for h in (os.path.join(BASE_DIR, PKGUSR_SUBDIR),
os.path.join(BASE_DIR, CFGUSR_SUBDIR))
if os.path.isdir(h) and os.listdir(h)]
if _homes and not getattr(args, "force", False):
printError("account homes already exist under the current layout:")
for _h in _homes:
printError(" %s (%d entries)" % (_h, len(os.listdir(_h))))
printWarning(" Changing the layout does NOT move them. Every "
"tool would look in the new")
printWarning(" place and find nothing. This belongs on a tree "
"with no accounts yet.")
printInfo(" If you are certain (or you will move the homes "
"yourself):")
printInfo(" packagemanager config --%s %s --force"
% (_flag.replace("_", "-"), _new or "''"))
return
cfg[_key] = _new
changed = True
printWarning(" %s -> %r" % (_key, _new or "(none)"))
_save_shared_key(_key, _new)
if args.collector_prefix:
cfg["collector_prefix"] = args.collector_prefix.rstrip("_")
changed = True
if args.user_prefix:
cfg["user_prefix"] = args.user_prefix.rstrip("_")
changed = True
if getattr(args, "editor", None):
cfg["editor"] = args.editor
changed = True
if getattr(args, "difftool", None):
cfg["difftool"] = args.difftool
changed = True
if args.main_user:
cfg["main_user"] = args.main_user
changed = True
if getattr(args, "shared_hub", None):
_h = os.path.abspath(args.shared_hub).rstrip("/")
if cfg.get("shared_hub") != _h:
cfg["shared_hub"] = _h
changed = True
printInfo(" the links move the next time the accounts are set up:")
printInfo(" packagemanager stack services --only app-users --run")
# Nothing given on the command line and something is unset? Ask. Leaving
# a setting blank and carrying on means it surfaces much later as a failed
# install with no obvious cause.
if not changed and not getattr(args, "show", False):
for key, prompt, default in _CONFIG_PROMPTS:
if cfg.get(key):
continue
d = default() if callable(default) else default
try:
ans = input(f" {prompt} [{d}]: ").strip()
except (EOFError, KeyboardInterrupt):
print()
break
cfg[key] = ans if ans else d
changed = True
if changed:
if save_config(cfg):
printSuccess(f"Saved {CONFIG_PATH}")
else:
printError("could not save config.")
printHeader("Configuration")
print()
where = CONFIG_PATH + ("" if config_exists() else " (not created yet -- defaults)")
print(f" config file : {where}")
print(f" collector prefix : {cfg['collector_prefix']}_ "
f"(shared install-dir groups)")
for label, tool, args_ in (("LFS book", "lfs", ["config"]),
("BLFS book", "blfs", ["books"])):
print(f" {label:<16} : {_book_in_use(tool)}")
print(f" user prefix : {cfg['user_prefix']}_ (shared users)")
print(f" main user : {cfg.get('main_user') or '(not set)'} "
f"(your human login account)")
print(f" shared hub : {shared_hub()} "
f"(links to every shared directory)")
print(f" editor : {editor()}")
print(f" difftool : {difftool()}")
# =========================================================================== #
# command: user (shared users -- the u_* accounts) [IMPLEMENTED]
# =========================================================================== #
#
# Application users run programs under a separate account (e.g. u_firefox). No
# install file is needed -- they install nothing; they exist to run an app and
# to carry supplementary groups (audio, video, u_xdg_runtime, ...).
# create : make the account (add_package_user)
# delete : remove account + group + home
# list : list shared users and their groups
# allow : add supplementary group(s) to a user (usermod -a -G)
# disallow: remove supplementary group(s) (gpasswd -d)
def _user_prefixed(app):
up = user_prefix()
return app if app.startswith(up) else up + app
def _add_groups(user, groups):
for g in groups:
if not _group_exists(g):
printWarning(f"{g}: no such group -- skipping "
f"(create a collector group with 'sysgroup create', or "
f"it must be an existing system group).")
continue
if _in_group(user, g):
printInfo(f"{user} is already in {g}.")
continue
if _run(["usermod", "-a", "-G", g, user]):
printSuccess(f"Added {user} to {g}.")
_APPLICATION_USER_HELP = """\
Create an application user and make it usable from your own account.
A package user owns the FILES a package installs. An application user is a
separate idea: an account a program RUNS as, so that a browser or a media
player cannot read your documents, your keys, or another application's data.
Creating the account is the easy part; what makes it usable is the rest:
* your home directory stays yours, and the account's stays its own: the
account's home is 0700
* your login joins the account's group, so a shared directory works
* `su - u_<name>` from your account without a password, where the hint's
/etc/pam.d/su_u_user is installed (skipped where it is not: su then
simply asks for a password)
* with --shared, the account joins the group on your XDG runtime directory
-- the display, the session bus and audio live there, so a graphical
program cannot draw anything without it. That group is read from the
directory, not assumed: it is set up once for you, and every shared
account afterwards joins whatever is there.
* with --share-dir, a directory both accounts can write, symlinked into
your home -- how you get files in and out
* with --launcher, a wrapper in ~/bin so running `<name>` starts the
program as that account
Your own login is the "main user" setting; without it there is nothing to
grant access to:
packagemanager config --main-user <your-login>
Everything here is idempotent, and can be applied to an account that already
exists -- including one made by hand:
packagemanager user setup firefox --shared --launcher --desktop
Examples:
packagemanager user create firefox --shared --share-dir --launcher --desktop
packagemanager user setup firefox --desktop # just add the menu entry
packagemanager user setup firefox --shared # just the session group
"""
def cmd_user_list(args):
up = user_prefix()
if pwd is None:
return
names = sorted(p.pw_name for p in pwd.getpwall() if p.pw_name.startswith(up))
if args.filter:
names = [n for n in names if args.filter.lower() in n.lower()]
printHeader("shared users")
print()
for n in names:
groups = user_supplementary_groups(n)
print(f" {n:<26} groups: {', '.join(groups) if groups else '-'}")
print(f"\n{len(names)} user(s).")
_SETUP_HELP = """\
Bootstrap the tooling on a freshly booted system.
LFS ships no download tool, so a new system cannot fetch even the sources for
its own next package. And `lfs` and `blfs` need `requests` and
`beautifulsoup4` to read the books, so without them the tools cannot run on the
system they just built.
packagemanager setup --sources what it needs downloaded
packagemanager setup --run install it
The tarballs must be on disk BEFORE the reboot, while the host still has a
network -- `lfs build-system get-sources` asks this command for the list.
"""
# What a freshly booted system needs before it can fetch anything itself.
#
# wget comes from the BLFS BOOK: the book already knows its URL and keeps it
# current, and a second list of download links is a second thing to go stale.
_SETUP_BLFS_PACKAGES = ["wget"]
# The Python modules `lfs` and `blfs` need to parse the books.
#
# WHEELS, not source tarballs, and not from the book. A modern sdist needs its
# build backend -- beautifulsoup4 wants hatchling, requests wants setuptools --
# and with no package index reachable pip cannot fetch one:
# BackendUnavailable: Cannot import 'hatchling.build'
# A wheel is already built, so pip only unpacks it. All of these are pure
# Python, so the "any" wheel works on every architecture.
#
# Dependencies first: setup installs with --no-deps, so pip will not pull them.
_SETUP_WHEELS = [
"https://files.pythonhosted.org/packages/py3/u/urllib3/urllib3-2.2.3-py3-none-any.whl",
"https://files.pythonhosted.org/packages/py3/c/charset_normalizer/charset_normalizer-3.4.0-py3-none-any.whl",
"https://files.pythonhosted.org/packages/py3/i/idna/idna-3.10-py3-none-any.whl",
"https://files.pythonhosted.org/packages/py3/c/certifi/certifi-2024.8.30-py3-none-any.whl",
"https://files.pythonhosted.org/packages/py3/s/soupsieve/soupsieve-2.6-py3-none-any.whl",
"https://files.pythonhosted.org/packages/py3/r/requests/requests-2.32.3-py3-none-any.whl",
"https://files.pythonhosted.org/packages/py3/b/beautifulsoup4/beautifulsoup4-4.12.3-py3-none-any.whl",
]
def setup_sources():
"""Every URL this command needs, resolved once."""
import subprocess
urls = []
for pkg in _SETUP_BLFS_PACKAGES:
try:
out = subprocess.run(["blfs", "sources", pkg],
capture_output=True, text=True, timeout=120)
except (OSError, subprocess.SubprocessError):
printWarning("could not run `blfs sources %s` -- is blfs installed?"
% pkg)
continue
got = [l.strip() for l in (out.stdout or "").splitlines()
if l.strip().startswith(("http://", "https://", "ftp://"))]
if not got:
printWarning("the BLFS book gave no download for '%s' "
"(try: blfs fetch)" % pkg)
continue
urls += got
return urls + _SETUP_WHEELS
# Where the tarballs and wheels are, on the booted system.
#
# Book 3.1's /sources, kept: `lfs build-system get-sources` put them there
# while the host still had a network, and that is the last moment there was
# one. Overridable, because a tree can be mounted anywhere.
def setup_sources_dir():
return os.environ.get("LFS_SOURCES_DIR") or "/sources"
def _wheel_dist(url):
"""(distribution, version, filename) from a wheel URL.
PEP 427 fixes the filename: {dist}-{version}-{python}-{abi}-{platform}.whl,
so the first two fields are exactly what pip records in site-packages as
<dist>-<version>.dist-info. That is what makes the check below an
identity, not a guess.
"""
fn = url.rsplit("/", 1)[-1]
parts = fn[:-4].split("-") if fn.endswith(".whl") else fn.split("-")
if len(parts) < 2:
return fn, "", fn
return parts[0], parts[1], fn
def _wheel_installed(site, dist, version):
return bool(site) and os.path.isdir(
os.path.join(site, "%s-%s.dist-info" % (dist, version)))
def _have_ca_bundle():
return any(os.path.isfile(c) and os.path.getsize(c) > 0 for c in _CA_BUNDLES)
def _make_ca_installed():
return bool(shutil.which("make-ca")) or os.path.isfile("/usr/sbin/make-ca")
def _bootstrap_state():
"""What is already true, so bootstrap can be run twice and do nothing twice.
Five stages, in the order they unblock each other. Each is a question this
tool already knew how to answer on its own; what was missing was the
SEQUENCE -- a freshly booted system where every check says something
different and nothing says what to do first.
"""
site = _pip_site_dir()
wheels = []
for url in _SETUP_WHEELS:
dist, ver, fn = _wheel_dist(url)
wheels.append((dist, ver, fn, url, _wheel_installed(site, dist, ver)))
return {
"site": site,
"wget": shutil.which("wget"),
"wheels": wheels,
"book": _blfs_book_available(),
"make_ca": _make_ca_installed(),
"ca": _have_ca_bundle(),
"unverified": _wgetrc_is_weakened(),
}
def _wgetrc_is_weakened():
"""Is certificate checking still turned off in /etc/wgetrc?"""
try:
return _WGETRC_MARK in open("/etc/wgetrc").read()
except OSError:
return False
def _setup_install_wget(yes):
"""Install wget through the ordinary BLFS path.
Invoked as a command rather than called as a function: `packagemanager
install wget --run` is a build plan, a script, a package user and a
manifest, and there is no version of "just the wget part" of that which is
not a second implementation of it.
"""
return _install_via_self("wget", yes)
def _install_via_self(pkg, yes, recursive=False, required_only=False, ignore=None,
ignore_prereqs=False):
cmd = [sys.executable, os.path.abspath(__file__), "install", pkg, "--run"]
if recursive:
cmd.append("--recursive")
if ignore_prereqs:
# a stack is a long unattended run: a decision to push past a missing
# tool has to reach the install it delegates to
cmd.append("--ignore-prereqs")
if required_only:
cmd.append("--ignore-recommended")
if ignore:
# ONE list, already joined by the caller. A second block here
# re-joined the STRING -- ",".join("texlive,...") is one comma per
# LETTER -- and the command line grew a second --ignore reading
# "t,e,x,l,i,v,e,...".
cmd += ["--ignore", ignore]
if yes:
cmd.append("--yes")
printInfo(" $ " + " ".join(cmd[2:]))
try:
return subprocess.run(cmd).returncode == 0
except (OSError, subprocess.SubprocessError) as e:
printError("could not install %s: %s" % (pkg, e))
return False
# The stages, in the order they unblock each other. Kept as data so the
# report, the work and the tests all read the same list.
_BOOTSTRAP_STAGES = [
# The modules come first: looking a package up in the book needs them, and
# installing the download tool IS a book lookup.
("wheels", "Python modules for reading the books",
"`blfs` and `packagemanager` cannot parse a book"),
("wget", "a download tool",
"nothing can be fetched at all"),
("book", "the BLFS book on disk",
"every package lookup fails the same way: not found"),
("make_ca", "make-ca, for certificates",
"downloads stay unverified"),
("ca", "a CA certificate bundle",
"make-ca is installed but has generated nothing"),
]
def cmd_setup(args):
if args.sources:
for u in setup_sources():
print(u)
return
st = _bootstrap_state()
srcdir = getattr(args, "sources_dir", None) or setup_sources_dir()
force = getattr(args, "force", False)
yes = getattr(args, "yes", False)
todo_wget = not st["wget"] or force
todo_wheels = [w for w in st["wheels"] if force or not w[4]]
n_wheels = len(st["wheels"]) - len(todo_wheels)
def mark(done):
return " ok " if done else " -- "
printHeader("packagemanager bootstrap")
print()
print(" A freshly booted LFS system has no download tool, no Python")
print(" modules to read a book with, no BLFS book and no certificates.")
print(" These are the five things between here and a working `blfs`:")
print()
print("%s1. book-parsing modules %d of %d installed"
% (mark(not todo_wheels), n_wheels, len(st["wheels"])))
print("%s2. a download tool %s" % (mark(st["wget"]),
st["wget"] or "wget is not installed"))
print("%s3. the BLFS book %s" % (mark(st["book"]),
"found" if st["book"] else "no book on disk"))
print("%s4. make-ca %s" % (mark(st["make_ca"]),
"installed" if st["make_ca"] else "not installed"))
print("%s5. CA certificates %s" % (mark(st["ca"]),
"present" if st["ca"] else "no bundle generated yet"))
print()
print(" site-packages : %s" % (st["site"] or "NOT FOUND -- is Python installed?"))
print(" sources : %s" % srcdir)
if st["unverified"]:
print()
printWarning(" Certificate checking is OFF in /etc/wgetrc so that")
printWarning(" make-ca can be fetched at all. It is turned back on")
printWarning(" automatically once stage 5 is done.")
if not st["site"]:
printError("\nNo system site-packages directory: nothing can be installed.")
sys.exit(1)
# Stages 1 and 2 need no network -- only the files get-sources already put
# on disk. Stages 3-5 need a working download, which is exactly what 1 and
# 2 buy you, so they run straight after in the same command.
if not todo_wget and not todo_wheels:
print()
printSuccess("Stages 1 and 2 are done -- `blfs` and `packagemanager` can read a book.")
if not args.run:
_bootstrap_next_steps(st)
printInfo("\nDry run -- to do these, re-run with --run.")
return
failed = _bootstrap_finish(st, yes)
_bootstrap_report(failed)
return
if not args.run:
print()
_bootstrap_next_steps(st)
printInfo("\nDry run -- to apply all of this, re-run with --run.")
return
# The wheels are already on disk and pip must not reach for an index: this
# system has no CA certificates yet, so anything that tries is a hang, not
# an error.
missing = [fn for _d, _v, fn, _u, _h in todo_wheels
if not os.path.isfile(os.path.join(srcdir, fn))]
if missing:
printError("\nThese wheels are not in %s:" % srcdir)
for fn in missing:
print(" %s" % fn)
printWarning("They had to be downloaded before the reboot:")
printWarning(" lfs build-system get-sources --run (on the host)")
sys.exit(1)
failed = []
# MODULES FIRST, THEN THE DOWNLOAD TOOL.
#
# Installing wget means looking it up in the BLFS book, and reading the
# book needs requests and beautifulsoup4. With the tool installed first,
# the very first run on a fresh system could not parse the book it was
# holding and said the package did not exist:
#
# $ install wget --run
# wget: not in the BLFS book.
# searching the book for wget Nothing similar found either.
#
# ...and it only ever succeeded on a SECOND run, once stage 2 had put the
# parsers in place -- which looked like flakiness rather than an order.
# The wheels need neither the book nor a network, so they come first and
# nothing is lost.
if todo_wheels:
printHeader("1. book-parsing modules")
grp = collector_prefix() + _pip_owner_of_python()
# IN ORDER, with --no-deps: pip cannot reach an index to resolve
# anything, so the order in _SETUP_WHEELS is the dependency order and
# each module must already have what it imports.
for dist, ver, fn, url, _have in todo_wheels:
if not pip_install_module(dist, st["site"], grp,
find_links=srcdir, no_deps=True,
target=os.path.join(srcdir, fn)):
failed.append(dist)
if todo_wget:
printHeader("2. a download tool")
if failed:
printWarning("The book parsers did not install, so the book cannot")
printWarning("be read and the download tool cannot be looked up.")
printWarning("Fix those first; this stage would fail for that reason")
printWarning("rather than for one of its own.")
elif not _setup_install_wget(yes):
failed.append("wget")
else:
# A system with no certificates cannot verify anything it fetches.
# This is the same call the tool makes on every invocation; doing it
# here means the very next download works.
ensure_wget_workaround()
print()
if failed:
# Stop here: everything after this point needs what just failed.
_bootstrap_report(failed)
printSuccess("Stages 1 and 2 done.")
failed = _bootstrap_finish(_bootstrap_state(), yes)
_bootstrap_report(failed)
def _bootstrap_report(failed):
print()
if failed:
printError("bootstrap stopped with %d failure(s): %s"
% (len(failed), ", ".join(failed)))
printInfo("Fix them and run it again -- it skips what is already done.")
sys.exit(1)
st = _bootstrap_state()
if st["book"] and st["make_ca"] and st["ca"]:
printSuccess("Bootstrap complete -- all five stages done.")
# THE STARTUP HELPERS BELONG TO THE BOOTSTRAP. BLFS packages drop
# /etc/profile.d snippets that call pathprepend, which LFS does not
# provide -- rust's snippet failed on every package-user login.
# lfs-helper writes them; this is the command people run on a fresh
# system, so it asks for them here.
try:
_pr = subprocess.run(["lfs-helper", "install-profile"],
capture_output=True, text=True)
for _l in (_pr.stdout or "").splitlines():
if _l.strip():
printInfo(" " + _l.strip())
except (OSError, subprocess.SubprocessError):
printWarning(" could not run `lfs-helper install-profile`; BLFS"
" profile.d snippets may fail with"
" 'pathprepend: command not found'")
printInfo("Try it: blfs order <pkg> / packagemanager install <pkg>")
else:
printSuccess("Nothing left that this command can do.")
def _bootstrap_finish(st, yes):
"""Do stages 3, 4 and 5 -- the ones that need a working download.
These used to be PRINTED, on the reasoning that each has to be seen to work
before the next is worth trying. That was a poor trade: `bootstrap --run`
is a person saying "get this system ready", and answering with three
commands to type is refusing the request while looking like an answer.
Each stage still reports what it did, and a failure still stops the ones
that depend on it -- which is the part that actually mattered.
"""
failed = []
# 3. the book. Only reachable now: fetching it needs the download tool
# that stage 2 just installed.
if not st["book"]:
printHeader("3. the BLFS book")
blfs = _find_blfs_cmd()
if not blfs:
printError(" blfs is not installed, so the book cannot be fetched.")
failed.append("the BLFS book")
else:
printInfo(" $ blfs fetch")
try:
rc = subprocess.run(blfs + ["fetch"]).returncode
except (OSError, subprocess.SubprocessError) as e:
printError(" could not run blfs: %s" % e)
rc = 1
if rc != 0 or not _blfs_book_available():
printError(" no book was fetched -- this needs a network.")
printInfo(" Or copy one in: blfs import /path/to/BLFS-BOOK-...html")
failed.append("the BLFS book")
# 4. make-ca, through the ordinary install path so BLFS resolves what it
# needs. Without a book there is nothing to look it up in, so this is
# skipped rather than attempted and blamed on itself.
if not st["make_ca"]:
printHeader("4. make-ca")
if "the BLFS book" in failed:
printWarning(" Skipped: there is no book to look make-ca up in.")
failed.append("make-ca")
else:
printInfo(" This fetches over an UNVERIFIED connection -- that is")
printInfo(" what the /etc/wgetrc marker is for, and the last time")
printInfo(" it happens.")
if not _install_via_self("make-ca", yes, recursive=True):
failed.append("make-ca")
# 5. the bundle. `make-ca -g` alone reports "No update required!" and
# writes nothing, because its copy of certdata.txt looks current right
# after installing -- so the install looks fine while the system still has
# no certificates.
# THE TRIGGER MUST MATCH WHAT COUNTS AS DONE.
#
# This ran only when there was NO bundle at all, while 1.11.19 made the
# finishing condition "a bundle AND no empty store". So on a tree with the
# GnuTLS bundle written and /etc/ssl/certs empty, stage 5 was skipped
# entirely and then the check below reported the empty store as a failure
# -- correctly, and without ever having tried to fix it. Reported twice,
# attempted never.
if not _have_ca_bundle() or _empty_cert_stores():
printHeader("5. the certificate bundle")
if "make-ca" in failed:
printWarning(" Skipped: make-ca is not installed.")
failed.append("certificates")
else:
if not _run_make_ca():
failed.append("certificates")
# Turn verification back on only when there is something that can actually
# verify. make-ca writes several stores and EXITS 0 having failed some of
# them, so "a bundle exists" is not the same as "verification works".
if _have_ca_bundle():
empty = _empty_cert_stores()
if empty:
printWarning("")
printWarning(" Some certificate stores were NOT written, even after")
printWarning(" granting and generating again:")
for d, why in empty:
printWarning(" %-28s %s" % (d, why))
printWarning(" A tool that reads one of those still cannot verify")
printWarning(" anything. OpenSSL reads /etc/ssl/certs by default,")
printWarning(" so this matters to wget itself.")
printWarning(" Grant it by hand and generate once more:")
printWarning(" lfs-helper grant-dir %s %s --run"
% (_OPENSSL_CERT_DIR, pkgusr_name("make-ca")))
printWarning(" /usr/sbin/make-ca -g --force")
if "certificates" not in failed:
failed.append("certificates")
else:
restore_wget_verification()
return failed
# OpenSSL's default CApath, and the file make-ca writes beside it. These are
# what `wget` itself consults, so an empty one means verification fails even
# though _have_ca_bundle found a GnuTLS bundle elsewhere and said yes.
_OPENSSL_CERT_DIR = "/etc/ssl/certs"
def _empty_cert_stores():
"""Stores that exist but hold nothing, with why -- best guess."""
out = []
d = _OPENSSL_CERT_DIR
if os.path.isdir(d):
try:
n = len([f for f in os.listdir(d)
if f.endswith((".pem", ".crt", ".0"))])
except OSError:
n = 0
if n == 0:
why = "empty"
try:
owner = pwd.getpwuid(os.stat(d).st_uid).pw_name
if owner != "root":
why = "empty -- owned by %s" % owner
except (KeyError, OSError):
pass
out.append((d, why))
return out
def _run_make_ca(retry=True):
"""Generate the certificate stores, granting access if that is what stops it.
make-ca writes into several directories owned by other packages --
/etc/ssl/certs belongs to p_openssl -- and it EXITS 0 having failed some of
them:
p11-kit: couldn't create file: /etc/ssl/certs/...pem: Unknown error 13
Failed!!!
...
Extracting GNUTLS server auth certificates to: ...Done!
A zero exit and one bundle written is what made bootstrap report success
with an empty OpenSSL trust store. So: read the output, grant what the
errors name, and run it again.
"""
mk = shutil.which("make-ca") or "/usr/sbin/make-ca"
printInfo(" $ %s -g --force" % mk)
try:
r = subprocess.run([mk, "-g", "--force"], capture_output=True, text=True)
except (OSError, subprocess.SubprocessError) as e:
printError(" could not run make-ca: %s" % e)
return False
out = (r.stdout or "") + (r.returncode and (r.stderr or "") or (r.stderr or ""))
sys.stdout.write(r.stdout or "")
sys.stderr.write(r.stderr or "")
if r.returncode == 0 and not _empty_cert_stores():
return _have_ca_bundle()
if not retry:
if not _have_ca_bundle():
printError(" no certificate bundle was generated.")
return False
# Which directories did it fail on? Written to a file so the one parser
# both this and an install failure go through can read it.
tmp = os.path.join(TMP_DIR, "make-ca.log")
try:
os.makedirs(TMP_DIR, exist_ok=True)
with open(tmp, "w") as f:
f.write(out)
except OSError:
return False
account = pkgusr_name("make-ca")
if not _user_exists(account):
return _have_ca_bundle()
granted = auto_grant_from_log(account, tmp)
if not granted:
printWarning(" make-ca could not write every store, and nothing in its")
printWarning(" output names a directory to grant. Check by hand:")
printWarning(" ls -ald %s" % _OPENSSL_CERT_DIR)
return False
printInfo(" granted; generating again ...")
return _run_make_ca(retry=False)
def _find_blfs_cmd():
if shutil.which("blfs"):
return ["blfs"]
here = os.path.join(os.path.dirname(os.path.abspath(__file__)), "blfs")
if os.path.isfile(here):
return [sys.executable, here]
return None
def _bootstrap_next_steps(st):
"""What is left, and the command for it -- for a DRY RUN.
`--run` does these (see _bootstrap_finish); this is what to expect.
"""
if st["book"] and st["make_ca"] and st["ca"]:
print()
printSuccess("Bootstrap complete -- nothing is left to do.")
printInfo("Try it: blfs order <pkg> / packagemanager install <pkg>")
return
print()
printHeader("what --run would do next")
n = 3
if not st["book"]:
print()
print(" %d. The BLFS book. Without it every lookup fails as" % n)
print(" 'not found in book', whatever you ask for.")
print(" blfs books --discover")
print(" blfs fetch <version>")
print(" Or copy a book you already have into /usr/share/blfs/books/.")
n += 1
if not st["make_ca"]:
print()
print(" %d. make-ca. Until this is installed every download is" % n)
print(" unverified -- which is why /etc/wgetrc has verification")
print(" turned off: make-ca itself has to be fetched somehow.")
print(" packagemanager install make-ca --run")
n += 1
if not st["ca"]:
print()
print(" %d. The certificate bundle. `make-ca -g` generates nothing" % n)
print(" when it thinks its copy of certdata.txt is current, which")
print(" it is right after installing -- so force it once:")
print(" /usr/sbin/make-ca -g --force")
print(" The tools re-enable certificate checking on their own once")
print(" a bundle exists.")
print()
pass
def cmd_user_create(args):
if _resolve_shared_level(args) is None:
return 2
u = _user_prefixed(args.name)
if _user_exists(u):
printInfo(f"{u}: already exists.")
else:
printHeader(f"Create shared user {u}")
print()
cmd = find_add_package_user() + [u, u, str(APP_UID_MIN), str(APP_UID_MAX),
u, str(APP_UID_MIN), str(APP_UID_MAX)]
printInfo(f"Creating shared user: {' '.join(cmd)}")
if not _run(cmd):
return
printSuccess(f"{u} created.")
report_created_user(u, kind="application user")
if args.groups:
_add_groups(u, [g for g in re.split(r"[,\s]+", args.groups) if g])
# AN APPLICATION USER IS NOT A PACKAGE USER, AND ITS HOME IS NOT IN
# /usr/src.
#
# `add_package_user` is what creates the account -- it handles the uid
# range and the matching group -- and it puts the home where PACKAGE
# homes go:
# /usr/src/u_claude, /usr/src/u_browser-mini, ...
# A package user's home is a build tree. An application user's home is
# a home: it holds a browser profile, a .config, a Downloads directory.
# It belongs in /home, which is also where the shared-directory links
# and the XDG skeleton expect it.
_relocate_app_home(u)
# An account on its own is not usable: you cannot become it, its home is
# world-readable, and nothing launches the program. Do the rest.
_setup_application_user(u, args)
def _relocate_app_home(u):
"""Move an application user's home from the package tree to /home.
Only from the package-user base, only when the target does not already
exist, and with `usermod -m` so the contents come along. Anything else
is left exactly as it is: a home someone has already set up elsewhere
is not this function's business.
"""
cur = _home_of(u) or ""
root = appuser_home()
want = os.path.join(root, u)
if not cur or os.path.realpath(cur) == os.path.realpath(want):
return
# Moved from where these tools put it before, and nowhere else: the
# package tree, or /home/<user> (the old hard-coded place). A home
# someone chose elsewhere is not this function's business.
base = os.path.realpath(BASE_DIR)
rc = os.path.realpath(cur)
if not (rc.startswith(base + os.sep)
or rc == os.path.realpath(os.path.join("/home", u))):
printInfo(" %s's home is %s -- left alone (not a place these tools"
" put it)" % (u, cur))
return
if os.path.exists(want):
printWarning(" %s already exists -- not moving %s's home there"
% (want, u))
printInfo(" check it, then: usermod -d %s -m %s" % (want, u))
return
if not os.path.isdir(root):
_run(["install", "-d", "-o", "root", "-g", "root", "-m", "0755", root])
printInfo(" %s's home is %s -- application users live in %s"
% (u, cur, root))
if _run(["usermod", "-d", want, "-m", u]):
_run(["chmod", "0750", want])
printSuccess(" moved to %s (0750)" % want)
_repoint_shared_link(u, cur, want)
else:
printWarning(" could not move it (is %s logged in or running"
" something?); by hand:" % u)
printInfo(" usermod -d %s -m %s && chmod 0750 %s" % (want, u, want))
def _repoint_shared_link(u, old_home, new_home):
"""~/Shared/<Name> of the main user pointed into the old home."""
hub = shared_hub()
if not hub or not os.path.isdir(hub):
return
for n in os.listdir(hub):
p = os.path.join(hub, n)
if not os.path.islink(p):
continue
t = os.readlink(p)
if t.startswith(old_home.rstrip("/") + "/"):
nt = new_home.rstrip("/") + t[len(old_home.rstrip("/")):]
os.unlink(p)
os.symlink(nt, p)
printInfo(" %s -> %s" % (p, nt))
def _xdg_runtime_dir_of(user):
"""The main user's XDG runtime directory, however this system names it."""
try:
import pwd
uid = pwd.getpwnam(user).pw_uid
except (KeyError, ImportError):
return None
for cand in (os.environ.get("XDG_RUNTIME_DIR") if user == os.environ.get("USER") else None,
"/run/user/%d" % uid,
"/tmp/xdg-%s" % user,
"/tmp/xdg-%d" % uid):
if cand and os.path.isdir(cand):
return cand
return None
def xdg_runtime_group():
"""The one group that opens the main user's session to shared accounts."""
return collector_prefix() + "xdg-runtime"
def _join_xdg_runtime_group(main, u):
"""Put an application user -- and the main user -- in the session group.
A "shared user" is one that may reach the main user's XDG runtime
directory: the display socket (and pipewire's) live there, so without it
a graphical program starts and then cannot draw anything.
THE GROUP IS NAMED, NOT READ OFF THE DIRECTORY. This used to take
whatever group /tmp/xdg-<main> carried -- which is the main user's own
private group -- refuse, and say "set up a runtime group first", which
nothing did. <prefix>_xdg-runtime is created here (collector range), the
main user joins it (a user may only chgrp to a group it is in) and so does
every shared account. The directory is recreated at every boot, so its
group is applied by the session itself: /usr/bin/xdg-runtime-share and
/etc/profile.d/xdg-runtime-share.sh, written by `cfg app-users`.
"""
g = xdg_runtime_group()
if not _create_collector_group(g):
printWarning(f" could not create the session group {g} -- {u} will "
f"not reach the session")
return
for who in (main, u):
_run(["usermod", "-a", "-G", g, who])
printInfo(f" {main} and {u} are in '{g}' (the session group)")
rt = _xdg_runtime_dir_of(main)
if not rt:
printInfo(f" no runtime directory for {main} right now -- the session "
f"applies the group when it starts")
return
try:
import grp
cur = grp.getgrgid(os.stat(rt).st_gid).gr_name
except (KeyError, ImportError, OSError):
cur = ""
if cur != g:
# root may do it now; the session does it on every later login
_run(["chgrp", g, rt])
_run(["chmod", "0750", rt])
printInfo(f" {rt}: group {g}, 0750 (was {cur or '?'})")
XDG_SKEL = "/etc/pkgusr/skel-u_xdg/.bash_profile"
def _link_xdg_profile(main, u):
"""Point a shared user's .bash_profile at the XDG skeleton.
Being in the session group is necessary but not sufficient: the account
also has to KNOW where the session is. XDG_RUNTIME_DIR, WAYLAND_DISPLAY
and the package-user PATH come from this one file, so a shared user
without it can reach the socket and still not find it.
It is a symlink, not a copy: change the file once and every shared account
picks it up."""
if not os.path.isfile(XDG_SKEL):
printWarning(f" {XDG_SKEL} is missing -- {u} will not find the session")
printInfo(" It ships with these tools; reinstall them, or write it "
"by hand.")
return
# The skeleton is installed with a placeholder, because the runtime
# directory belongs to whoever is logged in. Fill it in once.
try:
text = open(XDG_SKEL).read()
except OSError as e:
printWarning(f" could not read {XDG_SKEL}: {e}")
return
if "@XDG_RUNTIME_DIR@" in text:
rt = _xdg_runtime_dir_of(main)
if not rt:
printWarning(f" {XDG_SKEL} still has a placeholder and no runtime "
f"directory was found for {main}")
printInfo(f" Set it by hand: XDG_RUNTIME_DIR in {XDG_SKEL}")
return
try:
with open(XDG_SKEL, "w") as f:
f.write(text.replace("@XDG_RUNTIME_DIR@", rt))
printInfo(f" {XDG_SKEL}: XDG_RUNTIME_DIR set to {rt}")
except OSError as e:
printWarning(f" could not write {XDG_SKEL}: {e}")
return
# the ACCOUNT's home, not a package home: this linked
# /usr/src/u_claude/.bash_profile
home = _app_home(u)
target = os.path.join(home, ".bash_profile")
if os.path.islink(target) and os.path.realpath(target) == \
os.path.realpath(XDG_SKEL):
printInfo(f" {target} already points at the XDG profile")
return
try:
if os.path.exists(target) or os.path.islink(target):
os.replace(target, target + ".pkgusr")
printInfo(f" kept the old one as {target}.pkgusr")
os.symlink(XDG_SKEL, target)
_run(["chown", "-h", f"{u}:{u}", target])
printInfo(f" {target} -> {XDG_SKEL}")
except OSError as e:
printWarning(f" could not link {target}: {e}")
def _setup_application_user(u, args):
"""Make an application user actually usable from your own account.
Creating the account is the easy part. What makes it useful:
* your account may `su - <user>` without a password (via PAM)
* the home is 0700, so no other account can read it
* your account joins the user's group, for the shared directory
* an optional shared directory both accounts can write
* an optional launcher in ~/bin so the program runs as that user
"""
home = _app_home(u)
main = main_user()
if not main:
printWarning("\nNo main user is set, so this account cannot be reached "
"from your own.")
printWarning("Set it, then run this again:")
printWarning(" packagemanager config --main-user <your-login>")
return
# 1. the home is private -- 0700, NOT 0750 (1.14.315)
#
# 0750 let the account's GROUP in, and your login is in that group (for
# the shared directory). So "a browser cannot read your documents" held,
# and the reverse did not: you could read its profile and cookies, and
# so could anything running as you. The shared directory lives in the
# hub now, so nothing needs to pass through the home.
if os.path.isdir(home):
_run(["chmod", "0700", home])
printInfo(f" home {home} is 0700 (private to {u})")
# 2. your account joins the user's group -- this is what makes a shared
# directory work, and what lets PAM identify you as entitled
_run(["usermod", "-a", "-G", u, main])
printInfo(f" {main} joined group '{u}'")
# 3. su without a password, if this system uses the hint's PAM file
_allow_su_via_pam(u, main)
# 4. the session: a shared user is one that may reach the main user's
# XDG runtime directory (the display, the session bus, the audio
# socket). The group is whatever that directory carries -- reading it
# rather than assuming a name, since the prefix is configurable.
if getattr(args, "shared", False):
_join_xdg_runtime_group(main, u)
_link_xdg_profile(main, u)
# 5. a shared directory, when asked for
#
# THE DIRECTORY LIVES IN THE HUB, THE LINK LIVES IN THE HOME (1.14.315).
# It was the other way round: the data in <account home>/Shared and a
# link to it in the hub -- so reaching it meant passing through the
# account's home, and the home had to be group-readable (0750) for
# that. Your login is in the account's group, so it could read the
# browser profile, the cookies, everything. Now the data is
# /home/Shared/<Name> (2770 u_<name>:u_<name>), the home is 0700, and
# the account's own ~/Shared is a link into the hub -- which keeps
# `--download-dir ~/Shared` in app-users.conf meaning the same thing.
if getattr(args, "share_dir", False):
_hub = _ensure_shared_hub(main)
share = os.path.join(_hub, args.name[:1].upper() + args.name[1:])
if os.path.islink(share):
printWarning(f" {share} is a link (to {os.readlink(share)}) -- the "
f"layout before 1.14.315; left alone")
printInfo(" move the data in, as the 1.14.315 notes say, then "
"run this again")
else:
_run(["mkdir", "-p", share])
_run(["chown", f"{u}:{u}", share])
_run(["chmod", "2770", share]) # setgid: new files keep the group
printInfo(f" shared directory: {share} (2770 {u}:{u})")
# ...and the account's ~/Shared points at it
_in_home = os.path.join(home, "Shared")
if os.path.islink(_in_home):
if os.path.realpath(_in_home) != os.path.realpath(share):
printWarning(f" {_in_home} points at "
f"{os.readlink(_in_home)} -- left alone")
elif os.path.isdir(_in_home):
if not os.listdir(_in_home):
os.rmdir(_in_home)
_run(["ln", "-s", share, _in_home])
_run(["chown", "-h", f"{u}:{u}", _in_home])
else:
# DATA IS NEVER MOVED BY THIS. It was asked for as a
# hand step; the command is printed, not run.
printWarning(f" {_in_home} still holds files -- not "
f"moved (that is yours to do)")
printInfo(f" mv {_in_home}/* {_in_home}/.[!.]* {share}/ "
f"&& rmdir {_in_home} && ln -s {share} {_in_home}")
elif not os.path.lexists(_in_home):
_run(["ln", "-s", share, _in_home])
_run(["chown", "-h", f"{u}:{u}", _in_home])
# 6. a launcher and a menu entry, when the program is really there
app = getattr(args, "app", None) or getattr(args, "name", None) \
or u[len(user_prefix()):]
import shutil as _sh
# AN APP CAN BE A COMMAND LINE. `--app "/bin/browser-mini
# https://claude.ai/new"` is the whole point of a per-account browser,
# and `which` on that string finds nothing -- so the launcher and the
# menu entry were silently skipped for exactly the accounts that needed
# them most. Look up the FIRST word; run the whole line.
_app_argv = app.split()
_app_cmd = _app_argv[0] if _app_argv else app
_app_base = os.path.basename(_app_cmd)
want_launcher = getattr(args, "launcher", False)
want_desktop = getattr(args, "desktop", False)
if want_launcher or want_desktop:
# Their original checked `which $app` first and skipped both when it
# was missing -- a launcher for a program that is not installed is
# just a broken command in your PATH.
if not (_sh.which(_app_cmd) or os.path.isfile(_app_cmd)):
printWarning(f" no '{_app_cmd}' on PATH -- no launcher or menu "
f"entry")
printInfo(f" (install it first, then: packagemanager user "
f"setup {args.name} --launcher --desktop)")
else:
# the launcher is NAMED after the account, not after the command:
# two accounts can run the same browser with different arguments,
# and ~/bin/browser-mini cannot be both.
if want_launcher:
_write_launcher(main, u, app, name=args.name)
if want_desktop:
_write_desktop_entry(main, u, app, name=args.name)
def _home_of(user):
try:
import pwd
return pwd.getpwnam(user).pw_dir
except (KeyError, ImportError):
return None
def _allow_su_via_pam(u, main):
"""Let <main> su to <u> without a password, if the PAM file is present.
The hint ships /etc/pam.d/su_u_user with a case statement listing which
accounts each user may become. It is not part of stock LFS, so this is a
no-op when the file does not exist -- `su` then simply asks for a
password, which still works."""
pam = "/etc/pam.d/su_u_user"
if not os.path.isfile(pam):
printInfo(f" ({pam} not present -- `su - {u}` will ask for a password)")
return
try:
text = open(pam).read()
except OSError as e:
printWarning(f" could not read {pam}: {e}")
return
if re.search(r"^\s*%s\)" % re.escape(u), text, re.M):
printInfo(f" {pam} already allows it")
return
marker = 'case "$PAM_USER" in'
if marker not in text:
printWarning(f" {pam} has no '{marker}' block -- not editing it")
return
lines = text.splitlines(True)
out = []
for line in lines:
out.append(line)
if marker in line:
out.append(' %s) USERS="%s %s"; ;;\n' % (u, u, main))
try:
with open(pam, "w") as f:
f.writelines(out)
# IT IS A SCRIPT pam EXECUTES. Rewriting it must not drop the exec
# bit, and a file written 644 makes `su` ask for a password with
# nothing in the logs to say why.
_m = os.stat(pam).st_mode & 0o777
if not _m & 0o111:
os.chmod(pam, 0o755)
printInfo(f" {pam} was not executable ({_m:o}) -- set to 755, "
f"pam_exec.so runs it")
printInfo(f" {main} may now `su - {u}` without a password")
# ...AND pam ONLY READS IT IF /etc/pam.d/su SAYS SO.
_su = "/etc/pam.d/su"
_line = "auth sufficient pam_exec.so quiet %s" % pam
try:
_has = os.path.isfile(_su) and "su_u_user" in open(_su).read()
except OSError:
_has = False
if not _has:
printWarning(f" {_su} does not call it, so su will still ask for"
f" a password.")
printInfo(" add, above the other auth lines:")
printInfo(" %s" % _line)
printInfo(" `packagemanager stack services --only app-users "
"--run` does it for you.")
except OSError as e:
printWarning(f" could not write {pam}: {e}")
def _resolve_shared_level(args):
"""--shared-slim / --shared-full on top of the original --shared.
`--shared` joins the group on the XDG runtime directory -- the display,
the session bus and audio sockets live there, so without it a graphical
program cannot draw. Audio and video are SEPARATE groups, and were only
reachable by spelling them out in --groups.
An account that runs one browser window does not need a microphone. So
the level is named: slim is the desktop alone, full adds audio and video.
"""
slim = getattr(args, "shared_slim", False)
full = getattr(args, "shared_full", False)
if slim and full:
printError("--shared-slim and --shared-full are opposites; pick one")
return None
if slim or full:
args.shared = True
if full:
have = [g for g in (getattr(args, "groups", "") or "").split(",") if g]
for g in ("audio", "video"):
if g not in have:
have.append(g)
args.groups = ",".join(have)
printInfo(" --shared-full: desktop + groups %s" % ", ".join(have))
elif slim:
printInfo(" --shared-slim: desktop only (no audio, no video)")
return args
def cmd_user_setup(args):
"""Apply the application-user setup to an account that already exists.
Everything `user create` does after making the account, so a user made
earlier -- or made by hand -- can be brought up to the same state. Each
step is idempotent, so running it again is safe and only fills in what is
missing."""
if _resolve_shared_level(args) is None:
return 2
u = _user_prefixed(args.name)
if not _user_exists(u):
printError(f"{u}: no such user.")
printInfo(f" Create it first: packagemanager user create {args.name}")
sys.exit(1)
printHeader(f"Set up {u}")
print()
# an account made before its home had a place is brought there too
_relocate_app_home(u)
if args.groups:
_add_groups(u, [g for g in re.split(r"[,\s]+", args.groups) if g])
_setup_application_user(u, args)
# BROWSERS THAT TAKE --app-id. A window's app id is what sway matches rules
# and the taskbar groups on; without one every account's browser is the same
# "browser-mini" window. A line that does not give one gets its launcher name.
_APP_ID_PROGRAMS = ("browser-mini", "browser-big")
# WHAT YOU TYPE, AND WHAT HAPPENS WHEN YOU TYPE NOTHING.
#
# {} the word is the arguments / a search term
# {https://site/} ...and this page when there are no arguments
#
# The braces are the placeholder; anything INSIDE them is the fallback. It
# cannot be spelled `{|url}`: `|` is app-users.conf's field separator.
_TEMPLATE_RE = re.compile(r"\{[^{}]*\}")
def _is_template(word):
"""Does this word carry a {} placeholder (with or without a fallback)?"""
return bool(_TEMPLATE_RE.search(word))
def _launcher_words(u, app, who):
"""The command line as words, with ~/ made the ACCOUNT's home, and an
--app-id added for a browser that has none."""
import shlex as _shlex
try:
words = _shlex.split(app)
except ValueError:
words = app.split()
home = _app_home(u) or ""
words = [(home + w[1:]) if (w.startswith("~/") and home) else w for w in words]
if words and os.path.basename(words[0]) in _APP_ID_PROGRAMS \
and "--app-id" not in words:
words[1:1] = ["--app-id", who]
return words
def _recolor_svg(path, color):
"""Paint a one-colour SVG in `color`.
A brand icon is a single path with one fill (or none at all, inheriting
black). simpleicons.org used to do this server-side -- /openai/38BDF8 --
but only for the icons it still carries, and it dropped `openai` and
`grok`. Doing it here works on any monochrome SVG, wherever it came
from. `fill="none"` is left alone: that is a hole, not a colour. A
multi-colour logo comes out flat, which is why this only happens when a
colour is asked for.
"""
try:
t = open(path, encoding="utf-8", errors="replace").read()
except OSError:
return False
if "<svg" not in t:
return False
new = re.sub(r'fill="(?!none\b)[^"]*"', 'fill="%s"' % color, t)
new = re.sub(r'fill:\s*(?!none\b)[^;"\']+', 'fill:%s' % color, new)
if not re.search(r'<svg\b[^>]*\bfill=', new):
new = re.sub(r'<svg\b', '<svg fill="%s"' % color, new, count=1)
if new == t:
return False
try:
open(path, "w").write(new)
except OSError:
return False
return True
def _fetch_icon(main, who, icon):
"""The menu icon, fetched once into ~/.local/share/icons/lfs-apps.
icon: an http(s) URL -> that file; a word -> the simpleicons.org slug of
that name; empty -> the launcher name as the slug; "-" -> no icon. A
trailing #RRGGBB paints a monochrome SVG that colour after it is
fetched. A failed download is not an error: the entry falls back to the
program's own icon name.
"""
if icon == "-":
return None
home = _home_of(main)
if not home:
return None
spec = icon or ""
# #RRGGBB AT THE END IS A COLOUR, NOT PART OF THE URL. No URL these
# icons come from uses a fragment, and a colour has to be expressible for
# a plain slug too (`openai#38BDF8`), not just for a URL.
color = ""
m = re.search(r"#([0-9A-Fa-f]{3}|[0-9A-Fa-f]{6})$", spec)
if m:
color = "#" + m.group(1)
spec = spec[:m.start()]
url = spec if re.match(r"^https?://", spec) else \
"https://cdn.simpleicons.org/%s" % ((spec or who).lower())
ext = os.path.splitext(url.split("?")[0])[1].lower()
ext = ext if ext in (".svg", ".png", ".xpm") else ".svg"
d = os.path.join(home, ".local", "share", "icons", "lfs-apps")
path = os.path.join(d, who + ext)
stamp = path + ".from"
# the stamp records the WHOLE spec, so changing only the colour refetches
want = url + color
if os.path.isfile(path) and os.path.getsize(path) > 0:
try:
if open(stamp).read().strip() == want:
return path
except OSError:
pass
os.makedirs(d, exist_ok=True)
tmp = path + ".part"
r = subprocess.run(["curl", "-fsSL", "--max-time", "20", url, "-o", tmp],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
if r.returncode != 0 or not os.path.isfile(tmp) or os.path.getsize(tmp) == 0:
try:
os.unlink(tmp)
except OSError:
pass
printWarning(f" no icon for {who} from {url} -- the menu uses the program's")
if "cdn.simpleicons.org" in url:
printInfo(" (simpleicons.org has dropped several brands, `openai` "
"and `grok` among them -- give the icon column a URL)")
return None
os.replace(tmp, path)
if color and ext == ".svg":
printInfo(f" icon {who}: painted {color}" if _recolor_svg(path, color)
else f" icon {who}: {color} changed nothing (not a one-colour SVG)")
open(stamp, "w").write(want + "\n")
for f in (d, path, stamp):
_run(["chown", f"{main}:{main}", f])
printInfo(f" icon: {path} ({url})")
return path
def _write_desktop_entry(main, u, app, name=None, icon=""):
"""A menu entry that runs the launcher -- named after the LAUNCHER, so
one account can have several (google and duckduckgo run as u_browser)."""
home = _home_of(main)
if not home:
return
who = os.path.basename(name or app.split()[0])
appdir = os.path.join(home, ".local", "share", "applications")
os.makedirs(appdir, exist_ok=True)
path = os.path.join(appdir, f"{who}.desktop")
launcher = os.path.join(home, "bin", who)
words = _launcher_words(u, app, who)
prog = os.path.basename(words[0]) if words else who
icon_path = _fetch_icon(main, who, icon)
takes_args = any(_is_template(w) for w in words)
# A MENU NEEDS A CATEGORY. An entry with none is "Other" at best and
# missing at worst, depending on the menu -- and these are the entries a
# person actually clicks. Taken from the program, since that is what is
# known here.
if prog in _APP_ID_PROGRAMS:
cats = "Network;WebBrowser;"
elif prog in ("mpv", "vlc", "ffplay"):
cats = "AudioVideo;Video;Player;"
else:
cats = "Utility;"
text = ("[Desktop Entry]\n"
f"Name={who.capitalize()}\n"
f"Comment=Runs as the application user {u}\n"
f"Exec={launcher}{' %U' if takes_args else ''}\n"
"Terminal=false\n"
"Type=Application\n"
f"Categories={cats}\n"
f"Icon={icon_path or prog}\n"
f"StartupWMClass={who if prog in _APP_ID_PROGRAMS else prog.capitalize()}\n")
if os.path.exists(path):
try:
cur = open(path, encoding="utf-8", errors="replace").read()
except OSError:
cur = ""
if cur == text:
return
if "Comment=Runs as the application user " not in cur:
printInfo(f" {path} was not written by these tools -- left alone")
return
printInfo(f" menu entry {path} was stale -- rewritten")
with open(path, "w") as f:
f.write(text)
os.chmod(path, 0o644)
_run(["chown", f"{main}:{main}", path])
printInfo(f" menu entry: {path}")
_LAUNCHER_TEMPLATE = r'''#!/bin/bash
# Runs @APP@ as the application user @USER@.
# written by packagemanager user launcher -- rewritten when stale
T=(@ARR@)
enc() { # URL-encode, byte by byte; a space becomes +
local LC_ALL=C s="$*" o="" c i
for ((i = 0; i < ${#s}; i++)); do
c="${s:i:1}"
case "$c" in
[a-zA-Z0-9._~-]) o+="$c" ;;
" ") o+="+" ;;
*) printf -v c "%%%02X" "'$c"; o+="$c" ;;
esac
done
printf "%s" "$o"
}
cmd=()
for w in "${T[@]}"; do
if [ "$w" = "{}" ]; then
cmd+=("$@") # the arguments, one by one
elif [[ "$w" == *"{"*"}"* ]]; then
pre="${w%%\{*}"; rest="${w#*\{}" # ...{ inner }...
inner="${rest%%\}*}"; post="${rest#*\}}"
if [ $# -gt 0 ]; then
cmd+=("$pre$(enc "$*")$post") # what you typed, URL-encoded
elif [ -n "$inner" ]; then
cmd+=("$inner") # nothing typed: the default page
else
p="$pre"; case "$p" in *\?*) p="${p%\?*}" ;; esac
cmd+=("$p") # nothing typed, no default: cut at ?
fi
else
cmd+=("$w")
fi
done
# A MENU ENTRY HAS NO TERMINAL, AND su WANTS ONE.
#
# From a terminal this is the whole story: su authenticates through
# /etc/pam.d/su_u_user and execs the program. Started from a .desktop entry,
# a hotkey or a launcher menu there is no tty at all -- su fails, its message
# goes to a stderr nobody is reading, and the entry looks like it "does
# nothing". script(1) gives it a pty and nothing else; the typescript goes
# to /dev/null and -e returns the program's own exit status.
if [ -t 0 ] && [ -t 1 ]; then
exec su - @QUSER@ -c 'exec "$0" "$@"' -- "${cmd[@]}"
fi
script -qec "$(printf '%q ' su - @QUSER@ -c 'exec "$0" "$@"' -- "${cmd[@]}")" /dev/null
rc=$?
# ...and say so where it can be seen, since there is no terminal to print to
if [ "$rc" != 0 ] && command -v notify-send >/dev/null 2>&1; then
notify-send "@WHO@" "did not start (exit $rc) -- run @WHO@ in a terminal to see why"
fi
exit $rc
'''
def _write_launcher(main, u, app, name=None):
"""A wrapper in ~/bin, named after the LAUNCHER, that runs the command
line as the application account.
{} IN THE COMMAND LINE IS WHAT YOU TYPE.
a word that IS {} -> the arguments, one by one ("browser-mini {}")
{} inside a word -> the arguments as a URL-encoded search term
("https://www.youtube.com/results?search_query={}")
and with no arguments the word is cut before
its ? (or before {}): the site's front page
{https://site/} in one -> the same, but with nothing typed the word
becomes exactly what is inside the braces.
Cutting at the ? gives /search/ -- a search
page for a search nobody made. This gives
the page you actually want to land on, and
it is per line: reddit's is the front page
today and may be your own profile tomorrow.
no {} at all -> arguments are NOT passed: `claude foo` must
not hand browser-mini a second address
THE ARGUMENTS GO THROUGH AS ARGUMENTS. The old writer built
su -c "app $(printf %q-each "$@")", which with no arguments produced one
EMPTY argument ("a second address:"). Here the final list is built in
the launcher and handed to su after --: bash gets $0 and "$@" exactly.
"""
home = _home_of(main)
if not home:
return
bindir = os.path.join(home, "bin")
os.makedirs(bindir, exist_ok=True)
who = os.path.basename(name or app.split()[0])
path = os.path.join(bindir, who)
import shlex as _shlex
words = _launcher_words(u, app, who)
arr = " ".join(_shlex.quote(w) for w in words)
# A PLAIN TEMPLATE, NOT %-FORMATTING. This body is bash full of `%02X`
# and `${w%%...}`, and every one of them had to be written `%%%%` here.
# One miscount and the launcher is subtly wrong, which is how the empty
# second address survived so long. Three named slots, .replace(), done.
text = _LAUNCHER_TEMPLATE.replace("@APP@", app).replace("@USER@", u) \
.replace("@QUSER@", _shlex.quote(u)) \
.replace("@WHO@", who) \
.replace("@ARR@", arr)
if os.path.exists(path):
try:
cur = open(path, encoding="utf-8", errors="replace").read()
except OSError:
cur = ""
# OURS AND STALE IS REWRITTEN; ANYTHING ELSE IS YOURS.
if cur == text:
printInfo(f" launcher {path} is current")
return
if not re.search(r"^# Runs .* as the application user \S+\.$", cur, re.M):
# SAY WHAT TO DO ABOUT IT. A launcher from an older version of
# these tools is not recognised either, so "left alone" quietly
# kept the OLD behaviour -- no default page, no --app-id -- on
# exactly the launchers the conf had just changed.
printInfo(f" {path} exists and was not written by these tools -- left alone")
printInfo(f" to take the generated one: rm {path} and run this again")
return
_was = re.search(r"su - (\S+)", cur)
printInfo(f" launcher {path} was stale (ran as "
f"{_was.group(1) if _was else '?'}) -- rewritten")
with open(path, "w") as f:
f.write(text)
os.chmod(path, 0o755)
_run(["chown", f"{main}:{main}", path])
printInfo(f" launcher: {path}")
def cmd_user_launcher(args):
"""One launcher (and menu entry) for an EXISTING application account.
Separate from `user setup` because an account can have several: the
`browser` account runs browser-mini, browser-big, google and duckduckgo.
"""
u = _user_prefixed(args.account)
if not _user_exists(u):
printError(f"{u}: no such account -- `user create {args.account}` first")
return 1
main = main_user()
if not main:
printError("no main user set (packagemanager config --main-user <login>)")
return 1
who = args.name
import shutil as _sh
words = _launcher_words(u, args.app, who)
if not words or not (_sh.which(words[0]) or os.path.isfile(words[0])):
printWarning(f" {who}: no '{words[0] if words else args.app}' on PATH "
f"-- no launcher or menu entry")
return 0
_write_launcher(main, u, args.app, name=who)
if not args.no_desktop:
_write_desktop_entry(main, u, args.app, name=who, icon=args.icon or "")
return 0
def _ours_for(path, marker):
try:
return marker in open(path, encoding="utf-8", errors="replace").read()
except OSError:
return False
def cmd_user_remove_launcher(args):
"""Remove a launcher, its menu entry and icon -- only files these tools
wrote (a hand-made ~/bin/<name> is never touched)."""
main = main_user()
home = _home_of(main) if main else None
if not home:
printError("no main user set")
return 1
who = os.path.basename(args.name)
files = []
lp = os.path.join(home, "bin", who)
if os.path.isfile(lp) and _ours_for(lp, "as the application user"):
files.append(lp)
dp = os.path.join(home, ".local", "share", "applications", who + ".desktop")
if os.path.isfile(dp) and _ours_for(dp, "Comment=Runs as the application user "):
files.append(dp)
idir = os.path.join(home, ".local", "share", "icons", "lfs-apps")
if os.path.isdir(idir):
files += [os.path.join(idir, f) for f in os.listdir(idir)
if f.startswith(who + ".")]
if not files:
printInfo(f" {who}: no launcher, menu entry or icon of these tools")
return 0
for f in files:
printInfo((" remove " if args.run else " would remove ") + f)
if args.run:
for f in files:
os.unlink(f)
else:
printInfo(" dry run -- add --run")
printInfo(f" and take its line out of /etc/pkgusr/stacks/app-users.conf, "
f"or the next `cfg app-users` writes it again")
return 0
def cmd_user_allow(args):
user = args.user
up = user_prefix()
if not user.startswith(up):
printWarning(f"'{user}' has no '{up}' prefix -- shared users are "
f"usually named '{up}{user}'.")
if not _user_exists(user) and _user_exists(up + user):
printInfo(f"Using '{up}{user}' instead.")
user = up + user
if not _user_exists(user):
printError(f"{user}: no such user.")
return
_add_groups(user, args.groups)
def cmd_user_disallow(args):
for g in args.groups:
if _in_group(args.user, g):
if _run(["gpasswd", "-d", args.user, g]):
printSuccess(f"Removed {args.user} from {g}.")
else:
printInfo(f"{args.user} is not in {g}.")
# WHAT AN ACCOUNT'S HOME HOLDS WHEN IT HAS NEVER RUN ANYTHING.
# `user create` puts a .bash_profile symlink there (and keeps whatever was
# there as .bash_profile.pkgusr), plus an empty Shared if the line asked for
# one. Nothing else is ours.
_HOME_SCAFFOLD = {".bash_profile", ".bash_profile.pkgusr", ".bashrc",
".bash_logout", ".bash_history", ".profile", ".dircolors",
"pkg.lst"}
def _home_is_untouched(home):
"""Is there anything in this home worth keeping?
The archive exists because a home holds a browser profile: logins,
cookies, history, downloads -- months of state that `userdel -r` would
take with it and nothing could give back. A home that was created and
never used holds none of that, and a tarball of it is 200 bytes of
nothing in /root, one per account, fourteen more on the next purge.
"""
try:
entries = os.listdir(home)
except OSError:
return False
for n in entries:
if n in _HOME_SCAFFOLD:
continue
p = os.path.join(home, n)
if n == "Shared" and os.path.isdir(p) and not os.path.islink(p) \
and not os.listdir(p):
continue
return False
return True
def _shared_hub(main_home=""):
"""Where the per-account links live -- the system hub, not a home
(1.14.312; the argument is kept for the callers and ignored)."""
return shared_hub()
def _ensure_shared_hub(main):
"""Create the hub, and make the main user's ~/Shared a link to it.
MOVING, NOT COPYING, AND ONLY OUR LINKS. A ~/Shared that is already a
directory has the links earlier releases made in it; each one moves to
the hub (same name, same target). Anything else in it -- a file, a
directory, a link of your own that clashes with one in the hub -- is
yours, so the directory stays, and says so. Only an EMPTY ~/Shared is
replaced by the link.
"""
hub = shared_hub()
if not os.path.isdir(hub):
_run(["mkdir", "-p", hub])
# root owns it, your login's group may add a link by hand (0775): the
# hub is a system path that YOU curate, and nobody else may
grp_name = main if (main and _group_exists(main)) else "root"
_run(["chown", f"root:{grp_name}", hub])
_run(["chmod", "0775" if grp_name != "root" else "0755", hub])
mh = _home_of(main) if main else None
if not mh:
return hub
home_link = os.path.join(mh, "Shared")
if os.path.islink(home_link):
if os.path.realpath(home_link) != os.path.realpath(hub):
printWarning(f" {home_link} points at {os.readlink(home_link)}, "
f"not the hub {hub} -- left alone")
return hub
if os.path.isdir(home_link):
moved, kept = 0, []
for n in sorted(os.listdir(home_link)):
p = os.path.join(home_link, n)
dest = os.path.join(hub, n)
if not os.path.islink(p):
kept.append(n)
continue
t = os.readlink(p)
if os.path.lexists(dest):
if os.path.islink(dest) and os.readlink(dest) == t:
os.unlink(p) # already moved on an earlier run
continue
kept.append(n)
continue
os.symlink(t, dest)
os.lchown(dest, 0, 0)
os.unlink(p)
moved += 1
if moved:
printInfo(f" moved {moved} link(s) from {home_link} to {hub}")
if kept:
printWarning(f" {home_link} still holds {', '.join(kept[:5])}"
f"{' ...' if len(kept) > 5 else ''} -- yours, so it "
f"stays a directory")
printInfo(f" the shared links are in {hub}; move those by hand, "
f"then run this again")
return hub
try:
os.rmdir(home_link)
except OSError:
return hub
os.symlink(hub, home_link)
try:
_pw = pwd.getpwnam(main)
os.lchown(home_link, _pw.pw_uid, _pw.pw_gid)
except (KeyError, OSError):
pass
printInfo(f" {home_link} -> {hub}")
return hub
def _remove_empty_hub(mh):
"""The hub goes when nothing is left in it -- and ~/Shared, the link to
it, goes with it, or it dangles."""
hub = shared_hub()
if not (hub and os.path.isdir(hub) and not os.listdir(hub)):
return
try:
os.rmdir(hub)
printInfo(f" removed {hub} (empty)")
except OSError:
return
hl = os.path.join(mh, "Shared") if mh else ""
if hl and os.path.islink(hl) and not os.path.exists(hl):
try:
os.unlink(hl)
printInfo(f" removed {hl} (it pointed at the hub)")
except OSError:
pass
def _stale_shared_links(main_home, home=""):
"""Links in ~/Shared that point into `home`, plus every link there that
leads nowhere at all. A dangling link is what keeps a SHARED_* variable
alive after the account behind it is gone."""
hub = _shared_hub(main_home)
out = []
if not hub or not os.path.isdir(hub):
return out
for f in sorted(os.listdir(hub)):
fp = os.path.join(hub, f)
if not os.path.islink(fp):
continue
if not os.path.exists(fp): # dangling: nothing to keep
out.append(fp)
elif home and os.readlink(fp).startswith(home.rstrip("/") + "/"):
out.append(fp)
return out
def _shared_vars_note(paths):
"""Say what an open shell still thinks, and how to correct it.
/etc/profile.d/shared-dirs.sh runs at LOGIN. Removing the link removes
tomorrow's variable; the shell you are typing in keeps today's until you
log in again -- so `cd $SHARED_BROWSER_MINI` still completes and still
fails, which reads like the removal did not work."""
names = [os.path.basename(p).upper().replace("-", "_") for p in paths]
if not names:
return
printInfo(" shells that are already open still export "
+ ", ".join("SHARED_" + n for n in names))
printInfo(" they are read at login: log in again, or right now:")
printInfo(" unset " + " ".join("SHARED_" + n for n in names))
def cmd_user_delete(args):
"""Remove an application account SAFELY.
Only an application account: the user prefix, and a home under the
application-user root or the package tree. Refused while it runs
anything. Without --no-archive its home is packed first into
/root/<account>-home-<date>.tar.gz (0600). Then: its launchers, menu
entries and icons (the ones these tools wrote that run as it), the
~/Shared/<Name> link, its line in /etc/pam.d/su_u_user, the account, its
group, and the home (kept with --keep-home).
"""
u = _user_prefixed(args.name)
if not u.startswith(user_prefix()):
printError(f"{u}: not an application account ({user_prefix()}...) -- refused")
return 1
exists = _user_exists(u)
home = (pwd.getpwnam(u).pw_dir if (exists and pwd) else "") or _app_home(u)
roots = [os.path.realpath(appuser_home()), os.path.realpath(BASE_DIR)]
if home and not any(os.path.realpath(home).startswith(r.rstrip("/") + "/") for r in roots):
printError(f"{u}: home {home} is not under {' or '.join(roots)} -- refused")
return 1
main = main_user()
mh = _home_of(main) if main else None
# what goes
launchers = []
if mh:
for d, marker in ((os.path.join(mh, "bin"), f"as the application user {u}."),
(os.path.join(mh, ".local", "share", "applications"),
f"Comment=Runs as the application user {u}\n")):
if os.path.isdir(d):
for f in sorted(os.listdir(d)):
fp = os.path.join(d, f)
if os.path.isfile(fp) and _ours_for(fp, marker):
launchers.append(fp)
icons = []
if mh:
idir = os.path.join(mh, ".local", "share", "icons", "lfs-apps")
names = {os.path.splitext(os.path.basename(f))[0] for f in launchers}
if os.path.isdir(idir):
icons = [os.path.join(idir, f) for f in sorted(os.listdir(idir))
if f.split(".")[0] in names]
# THE LINK, AND ANY LINK THAT NO LONGER LEADS ANYWHERE.
#
# A link was only removed when it pointed inside THIS home, so a home
# moved after the link was made (usermod -d, or the move to
# /home/shared_user) left ~/Shared/<Name> behind, dangling. Nothing
# then removed it -- and /etc/profile.d/shared-dirs.sh enumerates that
# directory, which is why $SHARED_BROWSER_MINI outlived the account.
links = _stale_shared_links(mh, home)
printHeader(f"Remove application account {u}")
print(f" account : {u}" + ("" if exists else " (already gone)"))
if home:
print(f" home : {home}" + (" (kept: --keep-home)" if args.keep_home else ""))
if os.path.isdir(home) and not args.no_archive:
if _home_is_untouched(home):
print(" archive : none -- nothing in this home but the files "
"these tools put there")
else:
print(f" archive : /root/{u}-home-<date>.tar.gz first (0600)")
for f in launchers + icons + links:
print(f" remove : {f}")
print(" pam : its line in /etc/pam.d/su_u_user")
if exists:
busy = subprocess.run(["pgrep", "-u", u], stdout=subprocess.DEVNULL).returncode == 0
if busy:
printError(f" {u} is running something -- close it (pkill -u {u}) and try again")
return 1
if not args.run:
printInfo("\n dry run -- nothing removed. Remove: "
f"packagemanager user delete {args.name} --run")
printInfo(" and take its lines out of /etc/pkgusr/stacks/app-users.conf, "
"or the next `cfg app-users` creates it again")
return 0
if not args.yes and not ask_yes_no(f"\nReally remove {u}?"):
printWarning("Aborted.")
return 1
if home and os.path.isdir(home) and not args.no_archive \
and not _home_is_untouched(home):
arch = f"/root/{u}-home-{time.strftime('%Y%m%d-%H%M%S')}.tar.gz"
old_umask = os.umask(0o077)
ok_arch = _run(["tar", "-C", os.path.dirname(home), "-czf", arch,
os.path.basename(home)])
os.umask(old_umask)
if not ok_arch:
printError(f" could not archive {home} -- nothing removed")
return 1
_run(["chmod", "600", arch])
printInfo(f" archived {home} -> {arch}")
for f in launchers + icons + links:
try:
os.unlink(f)
printInfo(f" removed {f}")
except OSError as e:
printWarning(f" {f}: {e}")
pam = "/etc/pam.d/su_u_user"
if os.path.isfile(pam):
try:
txt = open(pam).read()
new = re.sub(r"^\s*%s\)[^\n]*\n" % re.escape(u), "", txt, flags=re.M)
if new != txt:
open(pam, "w").write(new)
printInfo(f" removed {u} from {pam}")
except OSError as e:
printWarning(f" {pam}: {e}")
if exists:
_run(["userdel", u])
if _group_exists(u):
_run(["groupdel", u])
if home and os.path.isdir(home) and not args.keep_home:
try:
shutil.rmtree(home)
printInfo(f" removed {home}")
except OSError as e:
printError(f" rmtree {home}: {e}")
# ITS SHARED DIRECTORY IS NOT IN ITS HOME ANY MORE (1.14.315), so the
# archive above does not carry it. Empty: removed. With files: kept,
# and said -- those are files you put there, not the account's.
_bare = u[len(user_prefix()):]
_sd = os.path.join(shared_hub(), _bare[:1].upper() + _bare[1:])
if os.path.isdir(_sd) and not os.path.islink(_sd):
if not os.listdir(_sd):
try:
os.rmdir(_sd)
printInfo(f" removed {_sd} (empty)")
except OSError:
pass
else:
printWarning(f" kept {_sd} -- it has files in it; look, then:"
f" rm -r {_sd}")
# the hub is ours only while it holds our links: an empty hub is
# left-over furniture, one with something else in it is yours.
_remove_empty_hub(mh)
printSuccess(f"{u} removed.")
_shared_vars_note(links)
printInfo(" take its lines out of /etc/pkgusr/stacks/app-users.conf, "
"or the next `cfg app-users` creates it again")
return 0
# =========================================================================== #
# command: user purge -- undo the whole app-users setup [IMPLEMENTED]
# =========================================================================== #
#
# `user delete` removes ONE account. What it cannot remove is everything the
# accounts share, because the next account still needs it: the session group,
# the two profile fragments, /usr/bin/xdg-runtime-share, the pam file and the
# line in /etc/pam.d/su that calls it. Left behind, those are what makes a
# "deleted" setup still visible -- a group you are in, variables in every new
# shell, an auth line pointing at a file that is gone.
#
# NOTHING HERE TOUCHES A FILE THESE TOOLS DID NOT WRITE. Each file is read
# first and only removed if it carries the marker `cfg app-users` writes.
# The home archives in /root are never removed: they are the undo of the undo.
APP_USERS_MARKER = "# written by stacks/app-users.sh -- safe to delete, regenerated"
APP_USERS_CONF = "/etc/pkgusr/stacks/app-users.conf"
def _conf_accounts(path):
"""The account names in app-users.conf: column 1, both file formats."""
out = []
try:
lines = open(path, encoding="utf-8", errors="replace").read().splitlines()
except OSError:
return out
for raw in lines:
s = raw.strip()
if not s or s.startswith("#"):
continue
a = s.split("|")[0].strip()
if a in ("", "name", "user", "account") or re.search(r"[^a-z0-9_-]", a):
continue
if a not in out:
out.append(a)
return out
def _ours(path, marker):
"""A file that exists and says these tools wrote it."""
if not os.path.isfile(path):
return False
try:
return marker in open(path, encoding="utf-8", errors="replace").read()
except OSError:
return False
def cmd_user_purge(args):
"""Undo the application-account setup: the accounts, then what they share.
Dry run unless --run. Homes are archived to /root first, exactly as
`user delete` does, and those archives stay.
"""
conf = args.conf or APP_USERS_CONF
main = main_user()
mh = _home_of(main) if main else ""
accounts = _conf_accounts(conf)
if args.accounts:
accounts = [a for a in re.split(r"[,\s]+", args.accounts) if a]
# ACCOUNTS THE FILE NO LONGER MENTIONS ARE STILL OURS TO REMOVE. That is
# exactly the browser-mini/browser-big case: the line was rewritten, the
# account stayed. Found in passwd, not by listing a directory, so an
# account whose home is already gone is found too.
strays = []
if pwd:
root = os.path.realpath(appuser_home())
for e in pwd.getpwall():
if not e.pw_name.startswith(user_prefix()):
continue
a = e.pw_name[len(user_prefix()):]
if a in accounts or a in strays:
continue
if os.path.realpath(e.pw_dir or "").startswith(root.rstrip("/") + "/"):
strays.append(a)
strays.sort()
todo = [a for a in accounts + strays if _user_exists(_user_prefixed(a))]
printHeader("Undo the application-account setup")
print()
print(" accounts : " + (", ".join(user_prefix() + a for a in todo) or "none left"))
shared = []
for p in ("/etc/profile.d/shared-dirs.sh", "/etc/profile.d/xdg-runtime-share.sh",
"/usr/bin/xdg-runtime-share"):
if _ours(p, APP_USERS_MARKER):
shared.append(p)
elif os.path.exists(p):
printWarning(f" {p} is not ours -- left alone")
pam_file = "/etc/pam.d/su_u_user"
pam_su = "/etc/pam.d/su"
# an ACTIVE auth line, not a comment that happens to name the file
pam_line = bool(os.path.isfile(pam_su) and re.search(
r"^\s*auth\s+sufficient\s+pam_exec\.so.*su_u_user",
open(pam_su, encoding="utf-8", errors="replace").read(), re.M))
for p in shared:
print(f" remove : {p}")
if os.path.isfile(pam_file):
print(f" remove : {pam_file}")
if pam_line:
print(f" edit : {pam_su} (drop the pam_exec line; backup kept)")
g = xdg_runtime_group()
if _group_exists(g):
print(f" group : {g} (removed; {main or 'you'} leaves it)")
print(" kept : /root/" + user_prefix() + "*-home-*.tar.gz (the archives)")
print(f" kept : {conf} -- delete the lines you no longer want, or the")
print(" next `cfg app-users` builds all of this again")
if not args.run:
printInfo("\n dry run -- nothing removed. Undo it: "
"packagemanager user purge --run")
return 0
if not args.yes and not ask_yes_no("\nReally undo all of it?"):
printWarning("Aborted.")
return 1
# 1. the accounts, through the command that already knows how
import argparse as _ap
gone = 0
for a in todo:
rc = cmd_user_delete(_ap.Namespace(
name=a, run=True, yes=True,
keep_home=args.keep_home, no_archive=args.no_archive))
if rc:
printWarning(f" {a}: not removed -- stopping before the shared parts")
printInfo(" (close what it is running, then run this again)")
return 1
gone += 1
# 2. anything left in ~/Shared that leads nowhere, then the hub itself
dead = _stale_shared_links(mh)
for p in dead:
try:
os.unlink(p)
printInfo(f" removed {p}")
except OSError as e:
printWarning(f" {p}: {e}")
_remove_empty_hub(mh)
idir = os.path.join(mh, ".local", "share", "icons", "lfs-apps") if mh else ""
if idir and os.path.isdir(idir) and not os.listdir(idir):
try:
os.rmdir(idir)
printInfo(f" removed {idir} (empty)")
except OSError:
pass
# 3. the files every account shared
for p in shared:
try:
os.unlink(p)
printInfo(f" removed {p}")
except OSError as e:
printWarning(f" {p}: {e}")
if os.path.isfile(pam_file):
try:
os.unlink(pam_file)
printInfo(f" removed {pam_file}")
except OSError as e:
printWarning(f" {pam_file}: {e}")
# THE LINE IN /etc/pam.d/su GOES LAST AND WITH A BACKUP. It is the one
# edit here that can cost you `su` if it goes wrong.
if pam_line:
bak = f"{pam_su}.bak-{time.strftime('%Y%m%d-%H%M%S')}"
try:
txt = open(pam_su).read()
new = "\n".join(l for l in txt.splitlines()
if not re.match(r"^\s*auth\s+sufficient\s+pam_exec\.so.*su_u_user",
l)) + "\n"
if new != txt:
shutil.copy2(pam_su, bak)
open(pam_su, "w").write(new)
printInfo(f" {pam_su}: pam_exec line removed (backup {bak})")
except OSError as e:
printWarning(f" {pam_su}: {e}")
# 3b. the directory the accounts lived in, if it is now empty
root = appuser_home()
if os.path.isdir(root) and not os.listdir(root):
try:
os.rmdir(root)
printInfo(f" removed {root} (empty)")
except OSError:
pass
# 4. the session group -- last, when nothing is in it any more
if _group_exists(g):
_run(["groupdel", g])
printInfo(f" removed group {g}")
printSuccess(f"undone: {gone} account(s) and everything they shared.")
_shared_vars_note(dead)
printInfo(" your session still has the old group until you log in again")
# THE ARCHIVES ARE THE ONE THING THIS DOES NOT REMOVE, so say what they
# are and how to be rid of them. They only exist for a home that had
# something in it -- a browser profile, its logins and its downloads.
import glob as _glob
arch = sorted(_glob.glob(f"/root/{user_prefix()}*-home-*.tar.gz"))
if arch:
printInfo(f" {len(arch)} home archive(s) kept in /root -- browser profiles, "
f"nothing else can give them back")
printInfo(f" look: tar tzf {arch[-1]} | head")
printInfo(f" or drop: rm /root/{user_prefix()}*-home-*.tar.gz")
if mh and os.path.isfile(os.path.join(mh, ".config", "sway", "config")):
printInfo(" and `exec xdg-runtime-share` is still in your sway config "
"-- yours to remove")
return 0
# =========================================================================== #
# command: network [IMPLEMENTED]
# =========================================================================== #
#
# "dhcpcd is not started automatically" is four questions wearing one coat:
# is there an ifconfig file for that interface, does the interface exist,
# does something run at boot that reads those files, and did the thing it
# hands off to (wpa, dhcpcd) get what it needs. Each has its own command and
# none of them says the others' answers, so the person ends up reading a boot
# log line by line -- which is how an ifconfig.wlan0 whose wpa config the
# service could not find (1.14.333) survived three sessions.
#
# This is READ-ONLY. It says what is there and what would happen at the next
# boot; `stack network --only net-wifi --run` is what changes anything.
def _net_ifaces():
"""Every interface the kernel has: name -> (state, addresses, wireless)."""
out = {}
try:
names = sorted(os.listdir("/sys/class/net"))
except OSError:
return out
for n in names:
d = os.path.join("/sys/class/net", n)
try:
with open(os.path.join(d, "operstate")) as f:
state = f.read().strip()
except OSError:
state = "?"
wireless = os.path.exists(os.path.join(d, "wireless")) \
or os.path.exists(os.path.join(d, "phy80211"))
out[n] = {"state": state, "addrs": [], "wireless": wireless}
# addresses, from `ip` when it is here (parsing /proc/net/fib_trie is not
# a thing to do twice)
rc, txt = 0, ""
try:
r = subprocess.run(["ip", "-o", "-4", "addr", "show"],
capture_output=True, text=True, timeout=10)
rc, txt = r.returncode, r.stdout
except (OSError, subprocess.SubprocessError):
rc = 1
if rc == 0:
for ln in txt.splitlines():
f = ln.split()
if len(f) >= 4 and f[1] in out:
out[f[1]]["addrs"].append(f[3])
return out
def _net_procs(name):
"""Which interfaces a running daemon was started for: pid -> args."""
found = []
for pid in os.listdir("/proc"):
if not pid.isdigit():
continue
try:
with open("/proc/%s/cmdline" % pid, "rb") as f:
argv = f.read().decode("utf-8", "replace").split("\0")
except OSError:
continue
argv = [a for a in argv if a]
if not argv:
continue
base = os.path.basename(argv[0])
# dhcpcd renames itself: "dhcpcd: wlan0 [ip4]" / "dhcpcd: [manager]"
if base.startswith(name) or argv[0].startswith(name + ":"):
found.append(" ".join(argv))
return found
def _net_default_route():
try:
with open("/proc/net/route") as f:
for ln in f.readlines()[1:]:
p = ln.split()
if len(p) > 2 and p[1] == "00000000":
gw = int(p[2], 16)
return "%d.%d.%d.%d dev %s" % (gw & 0xff, (gw >> 8) & 0xff,
(gw >> 16) & 0xff,
(gw >> 24) & 0xff, p[0])
except (OSError, ValueError):
pass
return ""
def _net_wpa_conf_dir():
"""Where the installed wpa service looks for its configuration."""
for d in ("/lib/services", "/usr/lib/services"):
p = os.path.join(d, "wpa")
if os.path.isfile(p):
try:
txt = open(p, encoding="utf-8", errors="replace").read()
except OSError:
continue
m = re.search(r"(/[A-Za-z0-9_./${}-]*wpa_supplicant[A-Za-z0-9_.${}-]*\.conf)", txt)
if m:
return os.path.dirname(m.group(1))
return "/etc/sysconfig"
def cmd_network(args):
printHeader("network")
ifcfg_dir = "/etc/sysconfig"
files = {}
for f in sorted(glob.glob(os.path.join(ifcfg_dir, "ifconfig.*"))):
if ".bak-" in f:
continue
iface = os.path.basename(f)[len("ifconfig."):]
try:
body = open(f, encoding="utf-8", errors="replace").read()
except OSError:
continue
files[iface] = {
"path": f,
"onboot": bool(re.search(r'^\s*ONBOOT=["\']?yes', body, re.M)),
"service": (re.search(r'^\s*SERVICE=["\']?([a-z0-9_-]+)', body, re.M)
or [None, "-"])[1],
"metric": (re.search(r'-m\s*(\d+)', body) or [None, ""])[1],
"wpa_service": (re.search(r'^\s*WPA_SERVICE=["\']?([a-z0-9_-]+)',
body, re.M) or [None, ""])[1],
"dhcp_start": (re.search(r'^\s*DHCP_START=["\']?([^"\'\n]*)',
body, re.M) or [None, None])[1],
}
ifaces = _net_ifaces()
_dhcp = _net_procs("dhcpcd")
_wpa = _net_procs("wpa_supplicant")
_manager = any("[manager]" in p for p in _dhcp)
print(" %-16s %-8s %-19s %-22s %s"
% ("interface", "state", "address", "at boot", "note"))
_rows = sorted(set(list(ifaces) + list(files)))
_problems = []
for n in _rows:
if n == "lo":
continue
i = ifaces.get(n)
c = files.get(n)
state = i["state"] if i else "ABSENT"
addr = ", ".join(i["addrs"]) if i and i["addrs"] else "-"
if c:
boot = ("yes, SERVICE=%s" % c["service"]) if c["onboot"] \
else "no (ONBOOT)"
else:
boot = "-- no ifconfig file"
note = ""
if c and not i:
note = "file for an interface that is not here -> FAIL at boot"
_problems.append((n, note))
elif i and not c and i["state"] == "up":
note = "up, and nothing brings it up at boot"
_problems.append((n, note))
elif i and c and c["onboot"] and not i["addrs"] and i["state"] == "up":
note = "up with no address"
_problems.append((n, note))
print(" %-16s %-8s %-19s %-22s %s" % (n, state, addr, boot, note))
print()
print(" default route : %s" % (_net_default_route() or "NONE"))
print(" dhcpcd : %s" % (", ".join(_dhcp) if _dhcp else "not running"))
if _dhcp and not _manager:
print(" (one per interface, started by ifup --"
" no manager instance)")
print(" wpa_supplicant : %s" % (", ".join(_wpa) if _wpa else "not running"))
# the wpa configuration, per interface that asks for it
_wdir = _net_wpa_conf_dir()
print(" wpa configs : %s/wpa_supplicant-<iface>.conf (what the"
" service reads)" % _wdir)
for n, c in sorted(files.items()):
if c["service"] != "wpa":
continue
_p = os.path.join(_wdir, "wpa_supplicant-%s.conf" % n)
if os.path.isfile(_p):
print(" %-14s %s" % (n, os.path.basename(_p)))
else:
print(" %-14s MISSING %s" % (n, _p))
_problems.append((n, "no wpa config where the service looks --"
" it cannot associate, so dhcpcd gets nothing"))
# THE HAND-OFF. services/wpa starts wpa_supplicant and then runs
# /lib/services/${WPA_SERVICE} -- ONLY if WPA_SERVICE is set. Without it
# the antenna associates and nobody asks for an address, which looks
# exactly like "dhcpcd does not start at boot".
_sdir = next((d for d in ("/lib/services", "/usr/lib/services")
if os.path.isdir(d)), "/lib/services")
for n, c in sorted(files.items()):
if c["service"] == "wpa":
if c["onboot"] and not c["wpa_service"]:
_problems.append((n, "SERVICE=wpa without WPA_SERVICE=dhcpcd --"
" it associates and never asks for an address"))
elif c["wpa_service"] and not os.path.isfile(
os.path.join(_sdir, c["wpa_service"])):
_problems.append((n, "WPA_SERVICE=%s, but %s/%s does not exist"
% (c["wpa_service"], _sdir, c["wpa_service"])))
_to_dhcpcd = c["service"] == "dhcpcd" or c["wpa_service"] == "dhcpcd"
if _to_dhcpcd and c["onboot"] and c["dhcp_start"] is not None \
and not re.search(r"(^|\s)(-b|--background)(\s|$)", c["dhcp_start"]):
_problems.append((n, "DHCP_START has no -b -- boot waits for a lease"
" and gives up if none comes in time"))
_uses = set()
for c in files.values():
if c["onboot"]:
_uses.add(c["service"])
if c["wpa_service"]:
_uses.add(c["wpa_service"])
for prog, svc in (("/sbin/dhcpcd", "dhcpcd"), ("/sbin/wpa_supplicant", "wpa")):
if svc in _uses and not os.access(prog, os.X_OK):
_problems.append((os.path.basename(prog),
"%s is not executable (or not there) -- the service"
" script calls exactly that path" % prog))
for n, c in sorted(files.items()):
if c["service"] == "wpa":
print(" hand-off : %-14s wpa -> %s" % (
n, c["wpa_service"] or "NOTHING (no WPA_SERVICE)"))
# what runs at boot -- per runlevel, because two S*network links in one
# runlevel run the whole thing twice, and a link in 5 does nothing when
# inittab boots into 3
_initd, _rcroot = _svc_dirs()
_dflt = _svc_default_runlevel()
_by_lvl = {}
if _rcroot:
for lvl in "0123456":
_l = sorted(os.path.basename(x) for x in glob.glob(
os.path.join(_rcroot, "rc%s.d" % lvl, "S*network")))
if _l:
_by_lvl[lvl] = _l
_net_links = sorted({x for v in _by_lvl.values() for x in v})
if _by_lvl:
for lvl, _l in sorted(_by_lvl.items()):
_mark = " <- default runlevel, this is what runs" if lvl == _dflt else ""
if len(_l) > 1:
_mark = (" <- default runlevel, BOTH run" if lvl == _dflt
else " <- BOTH run in this runlevel")
_problems.append(("rc%s.d" % lvl, "%d network links (%s) -- the"
" network is brought up twice"
% (len(_l), ", ".join(_l))))
print(" boot rc%s.d : %s%s" % (lvl, ", ".join(_l), _mark))
if _dflt not in _by_lvl:
_problems.append(("runlevel %s" % _dflt, "the default runlevel has no"
" S*network link -- nothing reads the files at boot"))
else:
print(" boot : NO S*network link -- nothing reads the files"
" above at boot")
_problems.append(("network", "the network service is not linked into"
" any runlevel"))
print(" default level : %s (from /etc/inittab)" % _dflt)
print(" the `network` script runs ifup for every"
" ONBOOT=yes file above;")
print(" dhcpcd has no init script of its own (BLFS ships"
" none), it is")
print(" started BY ifup through /lib/services/dhcpcd.")
if _problems:
print()
printWarning(" %d thing(s) to look at:" % len(_problems))
for n, why in _problems:
printWarning(" %-16s %s" % (n, why))
printInfo(" write the missing pieces: packagemanager stack network"
" --only net-wifi --run")
printInfo(" (it writes ifconfig files and copies a wpa config to"
" where the service reads it)")
else:
printSuccess(" nothing here looks wrong.")
return 0
# =========================================================================== #
# command: books -- which book each tool reads, and what can go [IMPLEMENTED]
# =========================================================================== #
#
# Three tools read books and each keeps its own list: `lfs books`, `blfs
# books`, and the stack files' from-book lines, which neither of those knows
# about. `blfs books` showed four svn tarballs and a 13.0 book as "?" with
# no way to tell which of them anything still used. This asks every reader
# and says, per file, who reads it -- and removes the ones nobody does.
def _blfs_path_offline(sel):
"""A selector -> its cached file, never touching the network."""
if not sel:
return ""
if os.path.isfile(sel):
return os.path.abspath(sel)
try:
r = subprocess.run(find_blfs() + ["path", sel, "--offline"],
capture_output=True, text=True, timeout=60)
except (OSError, subprocess.SubprocessError):
return ""
out = (r.stdout or "").strip().splitlines()
return out[-1] if r.returncode == 0 and out and os.path.isfile(out[-1]) else ""
def _human_size(n):
for unit in ("B", "K", "M", "G"):
if n < 1024 or unit == "G":
return ("%d%s" % (n, unit)) if unit == "B" else ("%.0f%s" % (n, unit))
n /= 1024.0
def _book_users():
"""{realpath: [who reads it]} for the BLFS books, plus the books dir."""
users = {}
def add(path, why):
if path:
users.setdefault(os.path.realpath(path), []).append(why)
dflt = _default_book_file()
add(dflt, "default -- every package without a book of its own")
# stacks: from-book, per selector, with how many entries read it
_sd = STACK_DIR if "STACK_DIR" in globals() else "/etc/pkgusr/stacks"
for f in sorted(glob.glob(os.path.join(_sd, "*.stack"))):
try:
ents = _parse_stack_file(f)
except (ValueError, OSError):
continue
per = {}
for r in ents:
sel = r["opts"].get("from-book")
if sel and r["kind"] == "book":
per[sel] = per.get(sel, 0) + 1
for sel, n in per.items():
add(_blfs_path_offline(sel) or ("(not cached) " + sel),
"stack %s (%d entr%s, from-book %s)"
% (os.path.basename(f)[:-6], n, "y" if n == 1 else "ies", sel))
# machine.conf / package-fixes.conf: book = per package
import configparser
for cf in (os.environ.get("PM_MACHINE_CONF", MACHINE_CONF),
os.environ.get("PM_PACKAGE_FIXES", PACKAGE_FIXES)):
if not os.path.isfile(cf):
continue
cp = configparser.ConfigParser(strict=False)
cp.optionxform = str
try:
cp.read(cf)
except configparser.Error:
continue
for sect in cp.sections():
sel = cp[sect].get("book")
if sel:
add(_blfs_path_offline(sel.strip()) or ("(not cached) " + sel),
"%s [%s] book =" % (os.path.basename(cf), sect))
return users, dflt
def cmd_books(args):
"""Every cached LFS/BLFS book, who reads it, and --prune for the rest."""
unused = [] # (path, size) -- files nobody reads
printHeader("books")
# ---- LFS: the lfs tool's own two settings
lstore = os.environ.get("LFS_STORE") or "/usr/share/lfs"
try:
lcfg = json.load(open(os.path.join(lstore, "config.json")))
except (OSError, ValueError):
lcfg = {}
ldef = lcfg.get("default")
lbuild = lcfg.get("build_book") or ldef
lbooks = sorted(glob.glob(os.path.join(lstore, "books", "LFS-BOOK-*-NOCHUNKS.html")))
print("\n LFS (lfs, %s/books)" % lstore)
if not lbooks:
print(" (none cached)")
for b in lbooks:
v = _book_label(b)
why = []
if v == ldef:
why.append("default (lfs-helper on this system)")
if v == lbuild and v != ldef:
why.append("build book (chapters 5-9)")
elif v == lbuild:
why[-1] += ", build book"
sz = os.path.getsize(b)
wl = os.path.join(lstore, "books", "wget-list-%s.txt" % v)
if why:
print(" %-16s %6s %s" % (v, _human_size(sz), "; ".join(why)))
else:
print(" %-16s %6s %snot used%s" % (v, _human_size(sz), colors.yellow,
colors.normal))
unused.append((b, sz))
if os.path.isfile(wl):
unused.append((wl, os.path.getsize(wl)))
# ---- BLFS
users, dflt = _book_users()
bdir = os.path.dirname(dflt) if dflt else "/usr/share/blfs/books"
print("\n BLFS (blfs, %s)" % bdir)
try:
names = sorted(os.listdir(bdir))
except OSError:
names = []
files = [os.path.join(bdir, n) for n in names
if not n.endswith((".meta", ".part"))
and os.path.isfile(os.path.join(bdir, n))]
# svn arrives as a dated tarball that is stitched into the combined
# book; the newest one is what the next refresh compares against, every
# older one is only the source of a book that no longer exists
tars = sorted(f for f in files if re.search(r"\.tar\.(xz|gz|bz2)$", f))
newest_tar = {}
for t in tars:
key = re.sub(r"-\d{4}-\d{2}-\d{2}\.tar\.\w+$", "", os.path.basename(t))
newest_tar[key] = t # ISO dates sort, last wins
for f in files:
real = os.path.realpath(f)
sz = os.path.getsize(f)
if f in tars:
m = re.search(r"(\d{4}-\d{2}-\d{2})", f)
lab = "svn %s" % (m.group(1) if m else "tarball")
key = re.sub(r"-\d{4}-\d{2}-\d{2}\.tar\.\w+$", "", os.path.basename(f))
if newest_tar.get(key) == f:
print(" %-16s %6s source of the svn book (newest download)"
% (lab, _human_size(sz)))
else:
print(" %-16s %6s %sold svn download -- not used%s"
% (lab, _human_size(sz), colors.yellow, colors.normal))
unused.append((f, sz))
continue
lab = _book_label(f)
if lab == f.rsplit("/", 1)[-1] and not f.endswith(".html"):
continue # not a book (config, notes)
why = users.get(real)
if why:
print(" %-16s %6s %s" % (lab, _human_size(sz), why[0]))
for w in why[1:]:
print(" %-16s %6s %s" % ("", "", w))
else:
print(" %-16s %6s %snot used%s" % (lab, _human_size(sz),
colors.yellow, colors.normal))
unused.append((f, sz))
for p, why in users.items():
if p.startswith("(not cached)"):
printWarning(" %-16s %6s %s -- NOT CACHED (blfs fetch %s)"
% (p.split()[-1], "", why[0], p.split()[-1]))
print()
if not unused:
printSuccess(" every cached book is in use.")
return 0
total = sum(sz for _p, sz in unused)
if not getattr(args, "prune", False):
printInfo(" %d file(s), %s, not used by anything: "
"packagemanager books --prune --run" % (len(unused), _human_size(total)))
return 0
if not getattr(args, "run", False):
printInfo(" would remove (dry run -- add --run):")
for p, sz in unused:
printInfo(" %6s %s" % (_human_size(sz), p))
return 0
if os.geteuid() != 0:
printError("removing books needs root")
return 1
n = 0
for p, _sz in unused:
# the book, its .meta, and blfs's parsed index of it
extra = [p + ".meta"] + glob.glob(os.path.join(
os.path.dirname(os.path.dirname(p)), "cache",
os.path.basename(p) + ".*.json"))
for q in [p] + extra:
try:
if os.path.lexists(q):
os.remove(q)
if q == p:
n += 1
except OSError as e:
printWarning(" could not remove %s (%s)" % (q, e))
printSuccess(" removed %d book file(s), %s." % (n, _human_size(total)))
return 0
# =========================================================================== #
# command: service -- the SysV services, without remembering the layout
# =========================================================================== #
#
# A SysV service is two things in two places: /etc/rc.d/init.d/<name>, which
# starts it, and the S/K symlinks in /etc/rc.d/rc<N>.d, which decide whether
# boot does. Everything here is `ls`, `ln -s` and `rm` on those -- no daemon,
# no database, nothing this tool has to keep in step with the system.
#
# START IS NOT ENABLE. systemd taught everyone that they are different, and
# SysV has always had them separate; `service sshd start` runs the script now
# whatever the links say, and `enable` only touches the links.
SVC_STATE_DIR = "/var/lib/pkgusr/services"
SVC_ACTIONS = ("start", "stop", "restart", "reload", "force-reload", "status",
"enable", "disable", "list")
def _svc_dirs():
"""(init.d, the directory holding rc<N>.d) -- LFS's layout, then the
plain one, because only one of them exists on a given machine."""
for initd, rcroot in (("/etc/rc.d/init.d", "/etc/rc.d"), ("/etc/init.d", "/etc")):
if os.path.isdir(initd):
return initd, rcroot
return None, None
# NOT EVERY FILE IN init.d IS A SERVICE. `functions` is a library the others
# source, `rc` is what init runs, `template` is a skeleton to copy. Listing
# them invites starting them.
_SVC_NOT_SERVICES = {"functions", "rc", "rcS", "template", "README", "Makefile"}
def _svc_all(initd):
out = []
try:
names = sorted(os.listdir(initd))
except OSError:
return out
for n in names:
if n in _SVC_NOT_SERVICES or n.endswith((".bak", "~", ".orig", ".new")):
continue
p = os.path.join(initd, n)
if os.path.isfile(p) and os.access(p, os.X_OK):
out.append(n)
return out
def _svc_links(rcroot, name):
"""Every rc<N>.d link for this service: [(level, 'S'|'K', num, path)]."""
out = []
try:
entries = sorted(os.listdir(rcroot))
except OSError:
return out
for d in entries:
m = re.match(r"^rc([0-6S])\.d$", d)
if not m:
continue
full = os.path.join(rcroot, d)
try:
for f in sorted(os.listdir(full)):
lm = re.match(r"^([SK])(\d{2})" + re.escape(name) + r"$", f)
if lm:
out.append((m.group(1), lm.group(1), lm.group(2),
os.path.join(full, f)))
except OSError:
continue
return out
def _svc_default_runlevel():
try:
for line in open("/etc/inittab"):
m = re.match(r"^\s*id:([0-6]):initdefault:", line)
if m:
return m.group(1)
except OSError:
pass
return "3"
def _svc_header_levels(initd, name):
"""Default-Start / Default-Stop from the script's INIT INFO block.
The book's scripts carry it, so the runlevels a service belongs in are
written down on the machine -- no need to invent them here."""
start, stop = "", ""
try:
for line in open(os.path.join(initd, name), errors="replace"):
m = re.match(r"^#\s*Default-Start:\s*(.*)$", line)
if m:
start = "".join(re.findall(r"[0-6S]", m.group(1)))
m = re.match(r"^#\s*Default-Stop:\s*(.*)$", line)
if m:
stop = "".join(re.findall(r"[0-6S]", m.group(1)))
if line.startswith("### END INIT INFO"):
break
except OSError:
pass
return start, stop
def _svc_enabled_levels(rcroot, name):
return sorted({lv for lv, kind, _n, _p in _svc_links(rcroot, name) if kind == "S"})
def _svc_running(initd, name, quick=True):
"""'yes' / 'no' / '?' -- asked of the script itself.
The LSB convention is 0 for running and 1-4 for not, which every script
from the book follows through statusproc. A script with no `status` says
so by failing differently, and '?' is an honest answer."""
p = os.path.join(initd, name)
try:
r = subprocess.run([p, "status"], stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, timeout=10 if quick else 60)
except (OSError, subprocess.SubprocessError):
return "?", ""
out = (r.stdout or b"").decode("utf-8", "replace").strip()
if r.returncode == 0:
return "yes", out
if r.returncode in (1, 2, 3, 4):
return "no", out
return "?", out
def _svc_state_file(name):
return os.path.join(SVC_STATE_DIR, name + ".links")
def cmd_service(args):
initd, rcroot = _svc_dirs()
if not initd:
printError("no /etc/rc.d/init.d (nor /etc/init.d) -- no SysV services here")
return 1
# `service sshd start` and `service start sshd` both read naturally;
# whichever word is an action is the action.
name, action = args.name, args.action
if name in SVC_ACTIONS and (action is None or action not in SVC_ACTIONS):
name, action = action, name
if not name and not action:
action = "list"
if name and not action:
action = "status"
if action not in SVC_ACTIONS:
printError("%s: not one of %s" % (action, ", ".join(SVC_ACTIONS)))
return 2
if action == "list":
return _svc_cmd_list(initd, rcroot, args)
if not name:
printError("which service? packagemanager service list")
return 2
script = os.path.join(initd, name)
if not os.path.isfile(script):
printError("%s: no such service" % name)
near = [s for s in _svc_all(initd) if name in s]
if near:
printInfo(" did you mean: %s" % ", ".join(near))
printInfo(" what there is: packagemanager service list")
return 1
if action in ("start", "stop", "restart", "reload", "force-reload", "status"):
# STARTING IS NOT ENABLING, and it does not ask the links for
# permission: this is the whole point of being able to run sshd for an
# afternoon without it coming back at the next boot.
if action == "start" and not _svc_enabled_levels(rcroot, name):
printInfo(" %s is not enabled at boot -- starting it anyway" % name)
if not os.access(script, os.X_OK):
printWarning(" %s is not executable -- fixing that first" % script)
_run(["chmod", "755", script])
return subprocess.run([script, action]).returncode
# ---- enable / disable: the links, and only the links -------------------
links = _svc_links(rcroot, name)
if action == "disable":
gone = [p for _lv, kind, _n, p in links if kind == "S"]
if not gone:
printInfo("%s is already not started at boot" % name)
return 0
# REMEMBER EXACTLY WHAT WAS THERE. The priority number is the only
# thing that cannot be worked out again -- it lives in the
# blfs-bootscripts Makefile, which may be long gone by then.
try:
os.makedirs(SVC_STATE_DIR, exist_ok=True)
with open(_svc_state_file(name), "w") as f:
f.write("\n".join(gone) + "\n")
except OSError as e:
printWarning(" could not record the links (%s) -- enable will "
"rebuild them from the script's header" % e)
for p in gone:
try:
os.unlink(p)
printInfo(" removed %s" % p)
except OSError as e:
printWarning(" %s: %s" % (p, e))
printSuccess("%s: disabled (it is not stopped -- `service %s stop` does that)"
% (name, name))
return 0
# enable
if _svc_enabled_levels(rcroot, name):
printInfo("%s is already started at boot (runlevels %s)"
% (name, ",".join(_svc_enabled_levels(rcroot, name))))
return 0
# 1. the exact links we removed, if we removed them
restored = []
try:
saved = [l.strip() for l in open(_svc_state_file(name)) if l.strip()]
except OSError:
saved = []
for p in saved:
d = os.path.dirname(p)
if not os.path.isdir(d):
continue
try:
os.symlink("../init.d/" + name, p)
restored.append(p)
except OSError as e:
printWarning(" %s: %s" % (p, e))
if restored:
for p in restored:
printInfo(" %s -> ../init.d/%s" % (p, name))
try:
os.unlink(_svc_state_file(name))
except OSError:
pass
printSuccess("%s: enabled (restored as it was)" % name)
return 0
# 2. otherwise build them: the number from a link that survived, the
# runlevels from the script's own header.
num = args.priority or next((n for _lv, _k, n, _p in links), None) or "50"
start, stop = _svc_header_levels(initd, name)
if args.levels:
start = "".join(re.findall(r"[0-6S]", args.levels))
start = start or "2345"
stop = stop or "016"
made = []
for lv in start:
d = os.path.join(rcroot, "rc%s.d" % lv)
if not os.path.isdir(d):
continue
p = os.path.join(d, "S%s%s" % (num, name))
if not os.path.lexists(p):
try:
os.symlink("../init.d/" + name, p)
made.append(p)
except OSError as e:
printWarning(" %s: %s" % (p, e))
for lv in stop:
d = os.path.join(rcroot, "rc%s.d" % lv)
if not os.path.isdir(d):
continue
p = os.path.join(d, "K%s%s" % (num, name))
if not os.path.lexists(p):
try:
os.symlink("../init.d/" + name, p)
made.append(p)
except OSError as e:
printWarning(" %s: %s" % (p, e))
if not made:
printWarning("%s: nothing to link (no rc<N>.d directories?)" % name)
return 1
for p in made:
printInfo(" %s -> ../init.d/%s" % (p, name))
printSuccess("%s: enabled at runlevels %s (priority %s)" % (name, start, num))
printInfo(" it is not started yet: packagemanager service %s start" % name)
return 0
def _svc_cmd_list(initd, rcroot, args):
names = _svc_all(initd)
if not names:
printInfo("no services in %s" % initd)
return 0
default_lv = _svc_default_runlevel()
printHeader("services (init scripts in %s, boot links in %s/rc<N>.d)"
% (initd, rcroot))
print(" default runlevel: %s\n" % default_lv)
print(" %-22s %-10s %-9s %s" % ("service", "at boot", "running", "runlevels"))
print(" " + "-" * 60)
for n in names:
lv = _svc_enabled_levels(rcroot, n)
boot = "yes" if default_lv in lv else ("other" if lv else "no")
run = "-"
if not args.quick:
run, _out = _svc_running(initd, n)
print(" %-22s %-10s %-9s %s" % (n, boot, run, ",".join(lv) or "-"))
print()
printInfo(" packagemanager service <name> start|stop|restart|status")
printInfo(" packagemanager service <name> enable|disable (boot only)")
if args.quick:
printInfo(" (--quick: 'running' not asked; drop it to ask each script)")
# THE SERVICES THAT ARE NOT INIT SCRIPTS. /lib/services/* are what `ifup`
# hands an interface to; they never appear in rc<N>.d and cannot be
# started, and not saying so is how "dhcpcd is installed but nothing runs
# it" happens.
for sd in ("/lib/services", "/usr/lib/services"):
if os.path.isdir(sd):
have = sorted(os.listdir(sd))
if have:
print()
printInfo(" network services in %s (run by ifup, never on their "
"own): %s" % (sd, ", ".join(have)))
printInfo(" an interface uses one by naming it: SERVICE=\"<name>\""
" in /etc/sysconfig/ifconfig.<iface>")
break
return 0
# =========================================================================== #
# command: template [IMPLEMENTED]
# =========================================================================== #
#
# Write an install-script skeleton for a package that isn't in the BLFS book,
# in the exact shape lfs-helper pm-install expects (header vars + install_pkg()
# [+ configure_pkg()] + the sourced-safe install/update dispatcher).
_TEMPLATE = '''\
#!/bin/bash
############################################
### {name_version}
### script format v17 origin=packagemanager
### phases: all (default) | unpack | build | test | install | configure | update
url="" # optional: where this came from
name="{name}"
name_version="{name_version}"
link="{link}" # tarball URL, OR a git URL (see is_git below)
pkg="{pkg}" # tarball filename (leave empty for git)
is_git="{is_git}" # "1" to clone $link with git instead of fetching a tarball
git_ref="" # optional git tag/branch/commit to check out
md5_sum="" # optional md5 of $pkg (tarball only)
additional_links=()
info=""
required=() # e.g. ('GTK-3.24.51')
recommended=()
optional=()
installed_content=()
# --- how packagemanager confirms this is really installed (optional) ---
installed_program="" # e.g. "ffmpeg" (checked on PATH)
installed_programs=() # e.g. ('gst-launch-1.0' 'gst-inspect-1.0')
installed_libraries=() # e.g. ('libnice.so') (checked in /usr/lib,...)
installed_directory="" # e.g. "/usr/lib/ffmpeg-7.1" (version in the name is checked)
installed_directories=()
# best signal: a command that prints the installed version. If its output
# contains the book version, packagemanager marks the package "installed";
# otherwise "unvalidated". Example for ffmpeg:
# validate_cmd="ffmpeg -version | grep -oP '(?<=version ).*(?= Copyright)'"
validate_cmd=""
# install_groups=('{collector}_...') # if it installs into collector dirs
build_pkg() {{
# >>> your CONFIGURE + COMPILE commands go here (no install yet) <<<
# meson setup build --prefix=/usr && ninja -C build
# ./configure --prefix=/usr && make
:
}}
install_pkg() {{
# >>> your INSTALL commands go here (the step that writes into system dirs) <<<
# ninja -C build install / make install
:
}}
configure_pkg() {{
# >>> optional post-install configuration goes here <<<
:
}}
# fetch, unpack, run each phase, resume in the build dir: all in the runner
. "${{PKGUSR_LIB:-lfs-phases}}" && pkgusr_run "$@"
'''
_FN_RE = r"^%s\(\)\s*\{\n(.*?)^\}[ \t]*$"
def _phase_body(text, fn):
"""The body of a phase function (build_pkg, install_pkg, ...) in a script
of the shared-runner format, None when the script does not define it."""
m = re.search(_FN_RE % re.escape(fn), text, re.S | re.M)
return m.group(1) if m else None
def _set_phase_body(text, fn, body):
"""Replace a phase function's body in a template-shaped script."""
return re.sub(_FN_RE % re.escape(fn), lambda m: "%s() {\n%s\n}" % (fn, body),
text, count=1, flags=re.S | re.M)
def _between(text, a, b):
m = re.search(re.escape(a) + r"\n?(.*?)" + re.escape(b), text, re.S)
return m.group(1) if m else None
def _strip_lines(region, prefixes):
out = []
for ln in region.splitlines():
s = ln.strip()
if any(s.startswith(p) for p in prefixes):
continue
out.append(ln)
return "\n".join(out).strip("\n")
_BUILD_SCAFFOLD = ("_enter_build", "# clean any stale",
"for _d in build builddir _build; do [ -d",
"# remember where the build", "# install phase can resume",
'pwd > "$BUILD_ROOT/.pm_build_cwd"',
": # (no separate build step)")
_INSTALL_SCAFFOLD = ("_enter_build", "# resume where", '# "ninja install"',
"# tree was built by an older script.",
": # (this package installs")
_CONFIG_SCAFFOLD = ('echo "## Configuration"', "# >>> optional", ": ;")
_AUTODETECT_RE = re.compile(
r'[ \t]*if \[ -f "\$BUILD_ROOT/\.pm_build_cwd".*?\n[ \t]*fi\n', re.S)
def extract_phase_cmds(text, phase):
"""Pull the user's actual commands (scaffolding removed) out of a phased
install script for build|install|configure. None if that phase isn't in
this script format."""
if phase in ("build", "install", "configure"):
# shared-runner format: the function IS the user's commands
r = _phase_body(text, phase + "_pkg")
if r is not None:
return _strip_lines(r, _CONFIG_SCAFFOLD if phase == "configure" else ())
if phase == "build":
r = _between(text, "#### BUILD ####", "#### BUILD DONE ####")
return _strip_lines(r, _BUILD_SCAFFOLD) if r is not None else None
if phase == "install":
r = _between(text, "#### INSTALL ####", "#### INSTALL DONE ####")
if r is None:
return None
r = _AUTODETECT_RE.sub("", r)
return _strip_lines(r, _INSTALL_SCAFFOLD)
if phase == "configure":
r = (_between(text, "#### CONFIGURE ####", "#### CONFIGURE DONE ####")
or _between(text, "#### CONFIGURATION ####",
"#### CONFIGURATION DONE ####"))
return _strip_lines(r, _CONFIG_SCAFFOLD) if r is not None else None
return None
def _current_script_version():
"""Ask blfs for the current generated-script format version (int)."""
rc, out, _ = run_blfs(["--version"])
m = re.search(r"script v(\d+)", out or "")
return int(m.group(1)) if m else None
def _script_format_version(text):
m = re.search(r"script format v(\d+)", text or "")
return int(m.group(1)) if m else None
def _script_is_current(text, current=None):
"""Current if the stamped format version matches blfs's. Un-stamped
(very old / hand-written) scripts are treated as NOT current."""
fv = _script_format_version(text)
if fv is None:
return False
if current is None:
current = _current_script_version()
return current is not None and fv >= current
def script_edited(text):
"""Has the user edited a blfs-generated script since it was created?
Compares the embedded '### pm_md5=' checksum to the script's actual content.
Returns True (edited), False (pristine), or None (no checksum -- unknown,
e.g. a very old or hand-written script)."""
m = re.search(r'^### pm_md5=([0-9a-f]+)\s*$', text, re.M)
if not m:
return None
stored = m.group(1)
body = re.sub(r'\n### pm_md5=[0-9a-f]+', '', text, count=1)
return hashlib.md5(body.encode()).hexdigest() != stored
# old non-phased scripts fetched+unpacked+built+installed all inside install_pkg();
# this drops the fetch/unpack preamble (the new unpack_pkg handles that) and keeps
# the user's real build+install commands.
_OLD_FETCH_RE = re.compile(
r'\s*if \[ -n "\$link" \].*?\n\s*fi\n', re.S)
def old_install_body(text):
"""Extract the user's commands from an OLD non-phased script -- either the
'#### INSTALLATION ####' block or the install_pkg() function body -- with the
fetch/unpack preamble removed. None if neither is present."""
r = _between(text, "#### INSTALLATION ####", "#### INSTALLATION DONE ####")
if r is None:
m = re.search(r'install_pkg\s*\(\)\s*\{(.*?)\n\}', text, re.S)
r = m.group(1) if m else None
if r is None:
return None
r = _OLD_FETCH_RE.sub("\n", r)
# drop a leading `cd "$pkg_dir"` left over from the old unpack preamble
r = re.sub(r'^\s*cd "\$pkg_dir"\s*\n', "", r, count=1)
return r.strip("\n")
def _migration_log_path():
"""Persistent, human-readable record of script migrations."""
for cand in (os.path.dirname(CONFIG_PATH), TMP_DIR):
try:
os.makedirs(cand, exist_ok=True)
return os.path.join(cand, "migrations.log")
except OSError:
continue
return os.path.join(TMP_DIR, "migrations.log")
def log_migration(pkg, nv, action, kept, backup):
line = (f"{time.strftime('%Y-%m-%d %H:%M:%S')} {action:<9} {pkg:<24} "
f"{(nv or ''):<20} kept={','.join(kept) or '-'} backup={backup}\n")
try:
with open(_migration_log_path(), "a") as f:
f.write(line)
except OSError:
pass
def _canonical_script_path(name):
"""The canonical, editable install script for a package -- the SAME file
resolve_script/update use: /usr/src/<name>/install_<name_version>. Prefers
the file for the currently-installed version; otherwise the versioned file
whose name matches its own name_version (not the runtime 'install_<user>'
copy or 'install_last')."""
home = pkgusr_home(name)
if not os.path.isdir(home):
return None
u = gather_user(name)
if u.version:
p = os.path.join(home, f"install_{u.version}")
if os.path.isfile(p):
return p
try:
files = [f for f in os.listdir(home)
if f.startswith("install_") and f != "install_last"
and not f.endswith(".bak") and not f.endswith(".book")]
except OSError:
return None
for f in sorted(files): # versioned canonical
p = os.path.join(home, f)
if os.path.isfile(p):
nv = _script_name_version(p)
if nv and f == f"install_{nv}":
return p
for f in sorted(files): # else any with a name_version
p = os.path.join(home, f)
if os.path.isfile(p) and _script_name_version(p):
return p
return None
def _restamp_script(path):
"""After a hand-merge, set the current format stamp + a fresh checksum so the
script is recognized as CURRENT and becomes the new pristine baseline (so it
stops triggering migrate warnings)."""
cur = _current_script_version()
if cur is None:
return False
try:
text = open(path, encoding="utf-8", errors="replace").read()
except OSError:
return False
text = re.sub(r'\n### pm_md5=[0-9a-f]+', '', text) # drop old checksum
if re.search(r'### generated by blfs.*script format v\d+', text):
text = re.sub(r'(### generated by blfs.*script format v)\d+',
lambda m: m.group(1) + str(cur), text, count=1)
else:
text = text.replace(
'#!/bin/bash',
f'#!/bin/bash\n### generated by blfs (hand-merged) -- '
f'script format v{cur}', 1)
digest = hashlib.md5(text.encode()).hexdigest()
text = re.sub(r'(### generated by blfs[^\n]*script format v\d+)',
lambda m: m.group(1) + f'\n### pm_md5={digest}', text, count=1)
try:
with open(path, "w") as f:
f.write(text)
return True
except OSError:
return False
def _script_origin(text):
"""Which generator wrote this script, per its own stamp ('' if unstamped).
The shared contract (item 6, step 3): every generated script carries
`script format vN origin=<tool>`. Before it, this tool knew only blfs's
wording, so an LFS-book script failed the format test and was reported as
OLD-format -- with advice that would have rewritten it into a BLFS one.
Recognising the other dialect was the containment; asking is the fix."""
m = re.search(r"script format v\d+\s+origin=(\w+)", text or "")
if m:
return m.group(1)
# Legacy, before the shared stamp: infer from each tool's own mark.
if "### lfs-crosschain-script v" in (text or ""):
return "lfs"
if "generated by blfs" in (text or ""):
return "blfs"
return ""
def _is_lfs_book_script(text):
"""Was this script generated by `lfs` from the LFS book?"""
return _script_origin(text) == "lfs"
def _package_script_outdated(name, current_ver):
"""True if this package's canonical install_<ver> script predates the current
format (so a reinstall would reuse an old script / risk losing edits).
Only scripts THIS tool's generator owns are judged. The format counters
are per generator and independent -- comparing blfs's v4 with lfs's v17
would be a number against an unrelated number -- so a script from another
origin is left to the tool that writes it."""
p = _canonical_script_path(name)
if not p:
return False
try:
txt = open(p, encoding="utf-8", errors="replace").read(6000)
except OSError:
return False
origin = _script_origin(txt)
if origin and origin != "blfs":
return False
return not _script_is_current(txt, current_ver)
def _script_norm(text):
"""Script content for comparison: drop the derived checksum line."""
return re.sub(r'\n### pm_md5=[0-9a-f]+', '', text or '')
def classify_script(name, script_dir):
"""Compare a package's canonical install script to a freshly-generated book
script. Returns (status, cand_path, book_path, book_nv):
up-to-date -- matches the book exactly
edited -- you changed it (differs, and it's not pristine)
outdated -- book has newer commands, but you made no edits (safe to adopt)
no-baseline -- differs and has no checksum (can't tell -- treat like edited)
no-script / not-in-book -- nothing to compare
"""
cand = _canonical_script_path(name)
if not cand:
return ("no-script", None, None, None)
old = open(cand, encoding="utf-8", errors="replace").read()
rc, out, err = run_blfs(["script", name, "-o", script_dir])
m = re.search(r"^Created (.+)$", out, re.M)
if rc != 0 or not m:
return ("not-in-book", cand, None, None)
book_path = m.group(1).strip()
book = open(book_path, encoding="utf-8", errors="replace").read()
book_nv = _script_name_version(book_path) or _script_name_version(cand)
if _script_norm(old) == _script_norm(book):
return ("up-to-date", cand, book_path, book_nv)
ed = script_edited(old)
if ed is False:
return ("outdated", cand, book_path, book_nv) # book changed, no edits
# WAS IT ONLY machine.conf? Scripts written before apply_machine_opts
# re-stamped its merge carry a checksum from BEFORE the merge, so they
# read as edited. Build what this machine would get from the book --
# the book's script with machine.conf applied -- and compare with the
# version number and checksums normalised away. If that is the whole
# difference, nobody touched the script. (Conservative: a page whose
# commands changed between versions still reads as edited.)
if ed is True and _only_machine_opts_differ(name, old, cand, book_path, book_nv):
return ("machine.conf", cand, book_path, book_nv)
# ...AGAINST THE BOOK IT CAME FROM. media reads svn now, so the book
# script above is gst-plugins-bad 1.28.7 from the SysV edition, while the
# script on disk is 1.28.6 from the default book. Two pages, two
# wordings: that comparison can never succeed, and all five
# machine.conf-only scripts read "edited". Find a cached book that has
# the script's own version and compare with that page.
if ed is True and _machine_opts_for(name):
_on = _script_name_version(cand) or ""
_obp = _origin_book_script(name, _on, book_path, script_dir)
if _obp and _only_machine_opts_differ(name, old, cand, _obp, _on):
return ("machine.conf", cand, book_path, book_nv)
if ed is None:
return ("no-baseline", cand, book_path, book_nv)
return ("edited", cand, book_path, book_nv)
def _origin_book_script(name, old_nv, book_path, script_dir):
"""The book script for the version `old_nv`, from whichever cached book
has that version -- or None. Skipped when book_path already is it."""
if not old_nv or _script_name_version(book_path) == old_nv:
return None
books = []
for bf in [_default_book_file(), _book_override(name)] + \
sorted(glob.glob("/usr/share/blfs/books/*.html")):
if bf and os.path.isfile(bf) and bf not in books:
books.append(bf)
for bf in books:
if _map_lookup(_versions_for(bf), [name]) != old_nv:
continue
import tempfile as _tf
d = _tf.mkdtemp(prefix="origin-", dir=script_dir)
rc, out, _ = run_blfs(["script", name, "-o", d], book=bf)
m = re.search(r"^Created (.+)$", out, re.M)
if rc == 0 and m:
return m.group(1).strip()
return None
def _only_machine_opts_differ(name, old, cand, book_path, book_nv):
"""True when `old` equals the book's script + machine.conf, modulo version."""
if not _machine_opts_for(name):
return False
try:
import tempfile as _tf
with _tf.NamedTemporaryFile("w", delete=False, suffix=".sh") as tmp:
tmp.write(open(book_path, encoding="utf-8", errors="replace").read())
tpath = tmp.name
# quietly: this is a comparison, not a merge -- the "machine.conf
# [x]: ..." lines it prints belong to a real generation
import contextlib as _cl, io as _io
with _cl.redirect_stdout(_io.StringIO()), _cl.redirect_stderr(_io.StringIO()):
_apply_machine_opts_inner(name, tpath)
mine = open(tpath, encoding="utf-8", errors="replace").read()
os.unlink(tpath)
except OSError:
return False
old_nv = _script_name_version(cand) or ""
def _ver(nv):
m = re.search(r"-(\d[\w.+~]*)$", nv or "")
return m.group(1) if m else ""
ov, nv = _ver(old_nv), _ver(book_nv)
def norm(t, frm="", to=""):
if frm and to and frm != to:
t = t.replace(frm, to)
t = re.sub(r"\n### pm_md5=[0-9a-f]+", "", t)
t = re.sub(r"(?m)^### generated by blfs[^\n]*$", "", t)
t = re.sub(r"\b[0-9a-f]{32}\b", "MD5", t) # tarball checksums
return "\n".join(l.rstrip() for l in t.splitlines() if l.strip())
return norm(old, ov, nv) == norm(mine)
_SCRIPT_STATUS_COLOR = {"up-to-date": colors.green, "outdated": colors.cyan,
"machine.conf": colors.cyan,
"edited": colors.yellow, "no-baseline": colors.yellow,
"no-script": colors.dim, "not-in-book": colors.dim}
def cmd_script_status(args):
"""Report whether each package's install script has been changed from the
book's version (edited), is behind the book (outdated), or matches it."""
names = args.packages if args.packages else list_all_users()
# --fast: format-stamp check only (no book generation) -- quick way to get
# the full list of scripts the update warning flags as OLD-format.
if getattr(args, "fast", False):
cur = _current_script_version()
outdated = [n for n in sorted(set(names))
if _package_script_outdated(n, cur)]
if getattr(args, "quiet", False):
print(" ".join(outdated))
return
printHeader("Old-format install scripts")
for n in outdated:
print(f" {n}")
print(f"\n {len(outdated)} of {len(set(names))} need migration.")
if outdated:
printInfo("Migrate them (merges your edits): packagemanager "
"migrate-scripts " + " ".join(outdated) + " --run")
return
script_dir = os.path.join(TMP_DIR, "install_files")
os.makedirs(script_dir, exist_ok=True)
quiet = getattr(args, "quiet", False)
changed_names, counts, _states = [], {}, {}
rows = []
for name in sorted(set(names)):
st, cand, _bp, _nv = classify_script(name, script_dir)
if st == "no-script":
continue
counts[st] = counts.get(st, 0) + 1
_states[name] = st
rows.append((name, st, cand))
if st in ("edited", "no-baseline", "outdated", "machine.conf"):
changed_names.append(name)
if quiet: # names only, for piping
print(" ".join(changed_names))
return
printHeader("Install-script status")
print(f"\n {colors.dim}up-to-date=matches book edited=you changed it "
f"outdated=book newer (no edits) no-baseline=can't tell{colors.normal}")
print(f" {colors.dim}machine.conf=differs only by machine.conf's options, "
f"which are re-applied: safe to --regenerate{colors.normal}\n")
for name, st, cand in rows:
color = _SCRIPT_STATUS_COLOR.get(st, "")
print(f" {name:<24} {color}{st:<12}{colors.normal} {cand or ''}")
print("\n " + " ".join(f"{k}: {v}" for k, v in sorted(counts.items())))
# LEAD WITH WHAT NEEDS DOING.
#
# `unvalidated` is the normal state for most of a system -- it means only
# that the page named no file to check, not that anything is wrong -- and
# on a real tree it is 226 of 242 lines. The dozen that need action
# scroll past inside it. Repeat them at the end, where the eye lands,
# with the command that fixes them.
# ONE PASS, NOT TWO. This re-assessed every package to find the ones to
# repeat -- a second validate command per package on a 242-package tree.
# The states were already computed in the loop above.
_act = [n for n, st in _states.items() if st in ("not installed", "broken")]
if _act:
print()
printWarning("%d package(s) need attention (everything else is fine "
"-- 'unvalidated' just means" % len(_act))
printWarning("the book named no file to check):")
for n in _act:
u = gather_user(n)
st, why = assess(u, None, run_cmd=False)
printWarning(" %-24s %-14s %s" % (n, st, why))
# ONLY WHAT THIS COMMAND CAN ACTUALLY BUILD.
#
# The list was pasted straight into an install command that could not
# work: collector groups and a stray `root` are not packages at all,
# and pkgusr / linux-headers are LFS steps with no BLFS page. The one
# bad name then killed the whole run (fixed in 1.14.36), but even
# without that, offering a command that cannot succeed is worse than
# offering none.
_vers = book_versions_all() or {}
_book = [n for n in _act if _vers.get(n) or _vers.get(n.lower())]
_other = [n for n in _act if n not in _book]
if _book:
printInfo(" Rebuild the BLFS ones:")
printInfo(" packagemanager install %s --run --recursive"
% " ".join(_book))
if _other:
printInfo(" Not BLFS packages, and not rebuilt by that command:")
printInfo(" " + ", ".join(_other))
printInfo(" LFS steps (linux-headers, pkgusr, ...) belong to the"
" chroot build: lfs-helper build <name> --force")
# NOT `packagemanager remove root`. For a directory with no
# account the fix is to delete the DIRECTORY, and suggesting a
# remove command against a name like `root` is an invitation to
# an accident.
printInfo(" Anything else is a directory or account with no"
" package behind it -- the line")
printInfo(" above names it; a leftover directory can simply be"
" removed.")
# WHAT TO DO, BY SCRIPT STATE. The block above was copied from `verify`,
# whose states are "not installed"/"broken" -- a script is never either,
# so it always ended in "nothing needs attention", directly under five
# scripts marked "edited".
_hand = sorted(n for n, st in _states.items() if st in ("edited", "no-baseline"))
_safe = sorted(n for n, st in _states.items() if st in ("outdated", "machine.conf"))
if _hand:
print()
printWarning(" %d script(s) carry edits of yours -- migrating MERGES them"
" (opens %s):" % (len(_hand), difftool()))
printWarning(" " + ", ".join(_hand))
printInfo(" never edited them by hand? Then regenerating loses nothing"
" (machine.conf is re-applied):")
printInfo(" packagemanager update %s --regenerate --run"
% " ".join(_hand))
if _safe:
print()
printInfo(" %d script(s) have no edits of yours -- regenerating them from"
" the book loses nothing" % len(_safe))
printInfo(" (machine.conf's options are applied again on every"
" generation):")
printInfo(" packagemanager update %s --regenerate --run"
% " ".join(_safe))
if not _act and not _hand and not _safe:
printSuccess(" every script matches the book.")
if changed_names:
printInfo("Migrate the changed/outdated ones: packagemanager "
"migrate-scripts " + " ".join(changed_names) + " --run")
def cmd_regenerate_script(args):
"""Generate fresh install script(s) from the book and place them in each
package-user's home as the canonical install_<book_nv> (what the next
install/update uses). Existing scripts are kept unless --overwrite."""
script_dir = os.path.join(TMP_DIR, "install_files")
os.makedirs(script_dir, exist_ok=True)
written, skipped, failed = 0, 0, 0
for name in sorted(set(args.packages)):
home = pkgusr_home(name)
if not os.path.isdir(home):
printWarning(f"{name}: no package-user home at {home} -- skipping "
f"(create it first with: packagemanager add-user {name}).")
failed += 1
continue
rc, out, err = run_blfs(["script", name, "-o", script_dir])
m = re.search(r"^Created (.+)$", out, re.M)
if rc != 0 or not m:
printWarning(f"{name}: not in the book / can't generate "
f"({(err or out).strip()[:60]}).")
failed += 1
continue
gen = m.group(1).strip()
nv = _script_name_version(gen) or name
dest = os.path.join(home, f"install_{nv}")
if os.path.exists(dest) and not args.overwrite:
printWarning(f"{name}: install_{nv} already exists -- pass --overwrite "
f"to replace it (a .bak is kept).")
skipped += 1
continue
backed = False
try:
if os.path.exists(dest):
shutil.copy(dest, dest + ".bak")
backed = True
shutil.copy(gen, dest)
except OSError as e:
printError(f"{name}: could not write {dest} -- {e}")
failed += 1
continue
_chown_to(dest, name)
printSuccess(f" {name:<24} wrote install_{nv}"
+ (f" (backup: {dest}.bak)" if backed else ""))
written += 1
printHeader("regenerate-script summary")
print(f"\n {written} written, {skipped} kept (exists, no --overwrite), "
f"{failed} failed.")
if skipped:
printInfo("Re-run with --overwrite to replace the existing scripts.")
def cmd_migrate_scripts(args):
"""Upgrade install scripts written by an OLDER packagemanager to the current
phased format, MERGING in your edited build/install/configure commands so
they aren't lost. Dry-run by default; --run rewrites (backing up the old
script as install_<ver>.bak). --deps also migrates each package's book
dependencies. --vimdiff opens every script that couldn't be auto-merged in
vimdiff (old vs freshly-generated) so you can finish it by hand. --log shows
the history of past migrations instead of migrating."""
if getattr(args, "log", False):
path = _migration_log_path()
printHeader("Migration history")
try:
with open(path) as f:
lines = f.read().splitlines()
except OSError:
lines = []
if not lines:
printInfo(f"No migrations recorded yet ({path}).")
return
tail = lines if getattr(args, "all", False) else lines[-40:]
print()
for ln in tail:
print(" " + ln)
print(f"\n {len(lines)} migration(s) total -- log: {path}")
return
names = list(args.packages) if args.packages else list_all_users()
if getattr(args, "deps", False) and args.packages:
expanded = set(names)
for target in args.packages:
order, _missing = blfs_order_anchors(target, False, False)
for anchor, _nv, _kind, _parent in order:
expanded.add(anchor)
names = sorted(expanded)
script_dir = os.path.join(TMP_DIR, "install_files")
os.makedirs(script_dir, exist_ok=True)
current_ver = _current_script_version()
pristine, edited, already, cannot = [], [], 0, 0
for name in sorted(set(names)):
if not os.path.isdir(pkgusr_home(name)):
continue
st, cand, book_path, book_nv = classify_script(name, script_dir)
if st == "no-script":
continue
if st == "not-in-book":
cannot += 1
continue
if st == "up-to-date":
already += 1
continue
# where the migrated script should live: named for the BOOK version, so
# it's the default script the next update/reinstall picks up.
out_path = os.path.join(pkgusr_home(name), f"install_{book_nv}")
# "machine.conf" is outdated too: its only difference from the book is
# machine.conf's options, which every generation re-applies
if st in ("outdated", "machine.conf"): # no edits of yours -> safe
pristine.append((name, cand, book_path, book_nv, out_path))
else: # edited / no-baseline -> hand-merge
edited.append((name, cand, book_path, book_nv, out_path, st))
# ---- compact report (no per-package warning spam) ----
printHeader("migrate-scripts")
print(f"\n {len(pristine)} to auto-migrate (book changed, no local edits)")
print(f" {len(edited)} to hand-merge in {difftool()} (you edited them)")
print(f" {already} up to date" + (f" {cannot} not in book" if cannot else ""))
for name, _c, _b, _nv, _o, reason in edited:
print(f" - {name:<22} ({'you edited it' if reason == 'edited' else 'no baseline'})")
if not args.run and not args.vimdiff:
print()
printInfo("Dry run -- re-run with --run to auto-migrate the unedited ones "
"and open the edited ones in " + difftool() + ".")
return
def _finalize(name, cand, out_path, source, backup_note):
"""Back up the old script, install `source` as install_<book_nv>, restamp."""
try:
shutil.copy(cand, cand + ".bak")
if out_path != cand:
shutil.copy(cand, out_path + ".bak") if os.path.exists(out_path) \
else None
shutil.copy(source, out_path)
except OSError as e:
printError(f" {name}: could not write -- {e}")
return None
_restamp_script(out_path)
_chown_to(out_path, name)
# if the canonical was under a different (old-version) name, drop it so the
# book-versioned one is unambiguously the default.
if out_path != cand:
try:
os.remove(cand)
except OSError:
pass
return out_path
# ---- auto-migrate the unedited-but-outdated ones (only with --run) ----
if args.run:
for name, cand, book_path, book_nv, out_path in pristine:
res = _finalize(name, cand, out_path, book_path, "book")
if res:
printSuccess(f" {name:<22} migrated -> {os.path.basename(res)}"
f" (backup: {cand}.bak)")
log_migration(name, book_nv, "migrated", ["book"], cand + ".bak")
# ---- hand-merge the edited ones in vimdiff (NEW on left, OLD on right) ----
if edited and (args.run or args.vimdiff):
tool = difftool()
mtmp = os.path.join(TMP_DIR, "migrate")
os.makedirs(mtmp, exist_ok=True)
print()
printInfo(f"Opening {len(edited)} script(s) in {tool}: the NEW book "
f"version is on the LEFT (keep it) -- pull your changes in from "
f"your OLD version on the RIGHT, then save the LEFT (:wqa).")
for i, (name, cand, book_path, book_nv, out_path, _st) in enumerate(edited, 1):
work = os.path.join(mtmp, f"install_{name}.merge")
try:
shutil.copy(book_path, work) # LEFT = new book (editable)
except OSError as e:
printError(f" {name}: {e}")
continue
printInfo(f"[{i}/{len(edited)}] {name}: LEFT=new book RIGHT={cand}")
try:
subprocess.run(tool.split() + [work, cand])
except OSError as e:
printError(f"could not launch {tool}: {e} "
f"(set one: packagemanager config --difftool ...)")
break
if args.yes or ask_yes_no(f" adopt the merged (LEFT) version for "
f"{name} as install_{book_nv}?"):
res = _finalize(name, cand, out_path, work, "merge")
if res:
printSuccess(f" {name}: merged -> {os.path.basename(res)}"
f" (backup: {cand}.bak)")
log_migration(name, book_nv, "merged", ["hand-merge"],
cand + ".bak")
else:
printInfo(f" {name}: kept your old script (skipped)")
printHeader("migrate-scripts summary")
print(f"\n {len(pristine) if args.run else 0} auto-migrated, "
f"{len(edited) if (args.run or args.vimdiff) else 0} hand-merged, "
f"{already} up to date.")
if args.run or args.vimdiff:
printInfo(f"logged to {_migration_log_path()} "
f"(view: packagemanager migrate-scripts --log)")
def _chown_to(path, user):
if pwd is None:
return
try:
pw = pwd.getpwnam(user)
os.chown(path, pw.pw_uid, pw.pw_gid)
except (KeyError, OSError):
pass
def cmd_template(args):
name = args.name
version = args.version or "1.0"
name_version = f"{name}-{version}"
is_git = "1" if args.git else ""
link = args.git or args.link or ""
pkg = "" if args.git else (args.pkg or f"{name}-{version}.tar.xz")
text = _TEMPLATE.format(name=name, version=version, name_version=name_version,
link=link, pkg=pkg, is_git=is_git,
collector=collector_prefix().rstrip("_"))
outdir = args.outdir or "."
os.makedirs(outdir, exist_ok=True)
path = os.path.join(outdir, f"install_{name_version}")
with open(path, "w") as f:
f.write(text)
os.chmod(path, 0o755)
printSuccess(f"Wrote template {path}")
if args.git:
printInfo(f"git-based: it will clone {args.git}\n"
f" set git_ref= in the file to pin a tag/branch/commit.")
printInfo("Edit the build/install commands, then:\n"
f" packagemanager install {name} --local {path} --run\n"
f" (or: packagemanager add-user {name}; "
f"lfs-helper pm-install {name} {path})")
# =========================================================================== #
# command: env [IMPLEMENTED]
# =========================================================================== #
#
# The book asks for a handful of exported variables (TEXLIVE_PREFIX,
# XORG_CONFIG, ...). Exporting them in a shell does not help: builds run as
# a package user through `su -`, which drops the environment. They live in
# one file instead, and lfs-helper passes it to every phase.
BUILD_ENV_FILE = os.environ.get("LFS_BUILD_ENV", "/etc/pkgusr/build.env")
def _read_build_env():
out = {}
try:
with open(BUILD_ENV_FILE) as f:
for line in f:
line = line.strip()
if line and not line.startswith("#") and "=" in line:
k, v = line.split("=", 1)
out[k.strip()] = v.strip()
except OSError:
pass
return out
def _write_build_env(vals):
os.makedirs(os.path.dirname(BUILD_ENV_FILE), exist_ok=True)
with open(BUILD_ENV_FILE, "w") as f:
f.write("# Build-time variables the BLFS book expects to be exported.\n")
f.write("# Written by `packagemanager env`; read by every build.\n")
for k in sorted(vals):
f.write("%s=%s\n" % (k, vals[k]))
# Values the book states plainly but that the generator cannot lift safely:
# XORG_CONFIG is a quoted, multi-line flag list built from XORG_PREFIX, and a
# half-captured fragment would be worse than asking. Offered as a SUGGESTION
# at the prompt -- the user presses Enter or types their own.
_ENV_SUGGESTIONS = {
"XORG_PREFIX": "/usr",
"KF6_PREFIX": "/usr",
"QT6PREFIX": "/opt/qt6",
"TEXLIVE_PREFIX": "/opt/texlive/2025",
"XORG_CONFIG": "--prefix=/usr --sysconfdir=/etc "
"--localstatedir=/var --disable-static",
}
def _env_wanted_by_scripts():
"""Which variables do the generated scripts ask for, and does the book
answer them itself? Returns {name: suggested_value_or_empty}."""
wanted, stale, asked_by = {}, set(), {}
dirs = [os.path.join(TMP_DIR, "install_files")] if TMP_DIR else []
dirs += [os.path.join(pkgusr_root(), d, "")
for d in ()] # package homes are scanned below
for d in dirs:
if not d or not os.path.isdir(d):
continue
for fn in os.listdir(d):
try:
body = open(os.path.join(d, fn), encoding="utf-8",
errors="replace").read()
except OSError:
continue
# A SCRIPT FROM AN OLDER GENERATOR ASKS OLDER QUESTIONS.
# These files are whatever the last run produced; after the guard
# was narrowed (1.12.56) the stale ones still listed EUID,
# DOCNAME and the rest, so the fix looked like it had done
# nothing. Only scripts from the CURRENT generator are consulted.
gen = _script_gen_version(os.path.join(d, fn))
cur = _blfs_gen_version() or TOOL_VERSION
if gen and cur and gen != cur:
stale.add(fn)
continue
for n in _ENV_GUARD_RE.findall(body):
wanted.setdefault(n, "")
asked_by.setdefault(n, set()).add(fn.replace("install_", ""))
# the book's own answer, if the generator found one
for n, v in re.findall(r'^\t([A-Z][A-Z0-9_]{2,})="([^"]*)"\s+# set by',
body, re.M):
wanted[n] = v
_env_wanted_by_scripts.asked_by = asked_by
if stale and not wanted:
# EVERY script was stale, so this returned nothing and `env --ask`
# had nothing to ask about -- which is how XORG_PREFIX went unset
# while XORG_CONFIG was being set from the very same book page.
printWarning("all %d cached script(s) are from an older blfs, so"
" nothing here can say which variables your packages"
" need." % len(stale))
printWarning(" They are regenerated on the next install; the common"
" ones are:")
for n in sorted(_ENV_SUGGESTIONS):
printWarning(" packagemanager env --set %s=%s"
% (n, _ENV_SUGGESTIONS[n]))
elif stale:
printWarning("ignored %d script(s) from an older blfs -- they ask"
" questions the current one no longer asks; they are"
" regenerated on the next install" % len(stale))
return wanted
# The book's Xorg page sets these TOGETHER:
# export XORG_PREFIX="/usr"
# export XORG_CONFIG="--prefix=$XORG_PREFIX --sysconfdir=/etc ..."
# so someone who sets XORG_CONFIG has read that page and means both. Setting
# only one leaves the other packages to fail later, one at a time -- which is
# what happened: XORG_CONFIG was set, and seventeen packages later libevdev
# stopped on XORG_PREFIX.
_ENV_COMPANIONS = {
"XORG_CONFIG": ["XORG_PREFIX"],
"XORG_PREFIX": ["XORG_CONFIG"],
}
def _warn_env_companions(vals):
for name in sorted(vals):
for other in _ENV_COMPANIONS.get(name, ()):
if other in vals:
continue
printWarning("%s is set but %s is not, and the book's page sets"
" both." % (name, other))
printWarning(" packagemanager env --set %s=%s"
% (other, _ENV_SUGGESTIONS.get(other, "...")))
SYSTEM_ENV_FILE = "/etc/profile.d/pkgusr-system.sh"
SYSTEM_ENV_MARKER = ("# written by `packagemanager env --system` -- "
"regenerate, do not edit")
def _system_env_values():
"""This system's own facts, for anything that needs to ask.
Scripts, launchers and one-off commands keep rediscovering the same few
things -- who the human is, what prefixes this machine uses, where the
shared directories are -- and each rediscovery is a different guess:
`cat /etc/pkgusr/desktop-user` in one place, a hard-coded `u_` in
another, `$HOME/Shared` in a third.
NOT the PKGUSR_ namespace. The tools READ PKGUSR_BASE, PKGUSR_CONFIG
and PKGUSR_LIB as overrides, so exporting that namespace system-wide
would reconfigure every tool in every shell -- a login that quietly
points the package manager at another tree. PKGUSR_SYS_ is inert:
nothing consumes it, so writing it cannot change what anything does.
"""
main = main_user() or ""
vals = {
"PKGUSR_SYS_MAIN_USER": main,
"PKGUSR_SYS_USER_PREFIX": user_prefix(),
"PKGUSR_SYS_PKG_PREFIX": pkgusr_prefix(),
"PKGUSR_SYS_COLLECTOR_PREFIX": collector_prefix(),
}
home = _home_of(main) if main else None
if home:
vals["PKGUSR_SYS_HOME"] = home
# the hub, whoever asks -- not a path inside one person's home
vals["PKGUSR_SYS_SHARED"] = shared_hub()
return vals
def cmd_env_system(args):
"""Write, or show, the system-fact variables as a profile fragment."""
vals = _system_env_values()
missing = [k for k, v in vals.items() if not v]
body = [SYSTEM_ENV_MARKER,
"# This system's own facts. Read them; do not set them here --",
"# `packagemanager config` is where they live and this file is",
"# generated from it.",
""]
for k in sorted(vals):
if vals[k]:
body.append('export %s="%s"' % (k, vals[k]))
text = "\n".join(body) + "\n"
if not getattr(args, "run", False):
print(text, end="")
printInfo(" (dry run: --run writes %s)" % SYSTEM_ENV_FILE)
if missing:
printWarning(" not known yet, so not exported: %s"
% ", ".join(missing))
printInfo(" packagemanager config --main-user <your-login>")
return 0
d = os.path.dirname(SYSTEM_ENV_FILE)
if not os.path.isdir(d):
printError("no %s -- nowhere to put a login-shell fragment" % d)
printInfo(" the values are above; export them from your own shell")
return 1
# NEVER OVERWRITE A FILE WE DID NOT WRITE.
if os.path.exists(SYSTEM_ENV_FILE):
try:
had = open(SYSTEM_ENV_FILE, encoding="utf-8",
errors="replace").read()
except OSError:
had = ""
if SYSTEM_ENV_MARKER not in had:
printError("%s exists and is not ours -- left alone"
% SYSTEM_ENV_FILE)
printInfo(" move it aside, or export these yourself:")
print(text, end="")
return 1
with open(SYSTEM_ENV_FILE, "w") as f:
f.write(text)
os.chmod(SYSTEM_ENV_FILE, 0o644)
# ROOT WROTE IT, SO ROOT OWNS IT. Writing into an existing file keeps
# that file's owner, and this one was first created inside a package
# user's build window -- so it stayed p_gstreamer10's, a package owning a
# file no package installed, in a directory every login shell sources.
# Re-asserted on every write; a chown that cannot run is not fatal.
try:
if os.stat(SYSTEM_ENV_FILE).st_uid != 0:
_run(["chown", "root:root", SYSTEM_ENV_FILE])
except OSError:
pass
printSuccess("wrote %s" % SYSTEM_ENV_FILE)
for k in sorted(vals):
if vals[k]:
printInfo(" %-28s %s" % (k, vals[k]))
if missing:
printWarning(" not known yet: %s" % ", ".join(missing))
printInfo(" login shells read /etc/profile, which reads that directory.")
printInfo(" in this shell, now: . %s" % SYSTEM_ENV_FILE)
return 0
def cmd_env(args):
if getattr(args, "system", False):
return cmd_env_system(args)
vals = _read_build_env()
if args.set:
for pair in args.set:
if "=" not in pair:
printError("not NAME=VALUE: %s" % pair)
sys.exit(1)
k, v = pair.split("=", 1)
vals[k.strip()] = v.strip()
_write_build_env(vals)
printSuccess("wrote %s" % BUILD_ENV_FILE)
_warn_env_companions(vals)
wanted = _env_wanted_by_scripts()
if args.ask:
for name in sorted(wanted):
if name in vals:
continue
suggested = wanted[name] or _ENV_SUGGESTIONS.get(name, "")
prompt = " %s" % name
if suggested:
prompt += " [%s]" % suggested
who = sorted(getattr(_env_wanted_by_scripts, "asked_by", {}).get(name, ()))
printWarning("%s is needed by: %s" % (name, ", ".join(who) or "a generated script"))
if who:
printInfo(" the book's page for that package explains it:"
" blfs debug %s" % re.sub(r"-[0-9].*$", "", who[0]))
if not suggested:
printInfo(" press Enter to skip -- the build stops at that"
" package rather than using an empty value")
try:
got = input(prompt + ": ").strip()
except EOFError:
got = ""
vals[name] = got or suggested
if not vals[name]:
del vals[name]
_write_build_env(vals)
printSuccess("wrote %s" % BUILD_ENV_FILE)
if args.refresh:
# STALE SCRIPTS HIDE THE QUESTIONS. They are regenerated per
# install anyway, so deleting them costs nothing and lets the next
# `env` (or the next build) see the current generator's guards.
removed = 0
d = os.path.join(TMP_DIR, "install_files") if TMP_DIR else None
cur = _blfs_gen_version() or TOOL_VERSION
if d and os.path.isdir(d):
for fn in os.listdir(d):
fp = os.path.join(d, fn)
gen = _script_gen_version(fp)
if gen and cur and gen != cur:
try:
os.unlink(fp)
removed += 1
except OSError:
pass
printSuccess("removed %d stale script(s); they regenerate on demand"
% removed)
wanted = _env_wanted_by_scripts()
printHeader("build environment (%s)" % BUILD_ENV_FILE)
if not vals and not wanted:
print(" nothing set, and no generated script asks for anything")
for k in sorted(set(vals) | set(wanted)):
state = vals.get(k)
if state:
print(" %-20s = %s" % (k, state))
else:
_sugg = wanted.get(k) or _ENV_SUGGESTIONS.get(k, "")
_who = sorted(getattr(_env_wanted_by_scripts, "asked_by", {}).get(k, ()))
print(" %-20s NOT SET (%s%s)"
% (k,
"wanted by " + ", ".join(_who[:2]) if _who else "a script needs it",
"; the book says %s" % _sugg if _sugg else ""))
if any(k not in vals for k in wanted):
printInfo("\n set them with: packagemanager env --ask")
printInfo(" or one at a time: packagemanager env --set NAME=VALUE")
elif not vals:
# An empty list is not a promise. A variable only becomes visible
# once the package's script has been generated, which happens during
# the run -- so say what will happen rather than implying there is
# nothing to do.
printInfo("\n A variable becomes visible when its package's script is")
printInfo(" generated, which happens during the build. The build warns")
printInfo(" before that package starts, and stops rather than passing an")
printInfo(" empty value. Set one now if you already know it:")
printInfo(" packagemanager env --set XORG_CONFIG='--prefix=/usr "
"--sysconfdir=/etc --localstatedir=/var --disable-static'")
# =========================================================================== #
# command: stack [IMPLEMENTED]
# =========================================================================== #
#
# One command, one environment. A stack file is the PLAN -- an ordered list
# of targets that together make a working environment (sway + a webkit
# browser, say) -- and `packagemanager stack <name> --run` replays it through
# the exact machinery a full fresh build proved: book targets go through the
# ordinary install planner (BLFS resolves the dependency chains, installed
# packages are skipped), git targets are built from an auto-generated local
# template, and check/cfg targets are root scripts shipped beside the stack.
# It stops at the first failure and skips what is done, so re-running after a
# fix continues instead of starting over.
#
# File format, one entry per line:
# book <target> [key=value ...] a BLFS book package (deps resolved)
# <any> ... when="<shell>" only when that command succeeds --
# a package this machine needs only
# sometimes (btrfs-progs on a btrfs
# system). It is asked every run.
# git <name> url=<repo> [ref=<ref>] [prog=<installed-program>]
# tar <name> url=<tarball-url> pkg=<filename> [prog=|have=] [build=|install=]
# check <name> root script <stackdir>/<name>.sh;
# a non-zero exit STOPS the stack
# cfg <name> root script <stackdir>/<name>.sh
# '#' comments and blank lines are ignored. Entries run top to bottom.
#
# Stack files are looked for in: the path as given, /etc/pkgusr/stacks/,
# and stacks/ beside this tool.
def _find_stack_file(name):
cands = [name]
if not name.endswith(".stack"):
cands.append(name + ".stack")
out = []
for c in cands:
out.append(c)
out.append(os.path.join("/etc/pkgusr/stacks", c))
out.append(os.path.join(os.path.dirname(os.path.abspath(__file__)),
"stacks", c))
for c in out:
if os.path.isfile(c):
found = os.path.abspath(c)
_warn_shadowed_stack(found)
return found
return None
def _warn_shadowed_stack(found):
"""A stack file found BEFORE the installed one, that differs from it.
The path as given is tried first, so `packagemanager stack services` run
from ~/lfs read ~/lfs/services.stack -- and a `cfg` entry runs
<that directory>/<name>.sh. An old shadow-pam.sh there ran instead of
the 1.14.262 one in /etc/pkgusr/stacks, printed the old "su and
system-auth are in place", and nothing said which file had run. Say it
when the two differ; a checkout that matches is not worth a word."""
etc = "/etc/pkgusr/stacks"
if os.path.dirname(found) == etc:
return
inst = os.path.join(etc, os.path.basename(found))
if not os.path.isfile(inst):
return
try:
import filecmp
differ = not filecmp.cmp(found, inst, shallow=False)
# the cfg scripts beside it are what actually run
d = os.path.dirname(found)
scripts = [f for f in sorted(os.listdir(d)) if f.endswith(".sh")
and os.path.isfile(os.path.join(etc, f))
and not filecmp.cmp(os.path.join(d, f),
os.path.join(etc, f), shallow=False)]
except OSError:
return
if not differ and not scripts:
return
printWarning(" NOTE: using %s, not the installed %s" % (found, inst))
if scripts:
printWarning(" cfg scripts that differ from the installed ones and "
"will run from %s: %s" % (d, ", ".join(scripts)))
printWarning(" run from another directory, or give the full path, "
"to use the installed stack")
def _parse_stack_file(path):
"""-> list of {kind, target, opts{}} in file order. Raises ValueError
with a line number on anything malformed -- a plan half-understood is
worse than one refused."""
import shlex
entries = []
kinds = ("book", "git", "tar", "check", "cfg", "avoid", "include",
"from-book")
# `from-book svn`: the book entries of THIS file below it read that book
# (an entry's own from-book= wins; `from-book default` ends it). Not
# inherited by an included file, and not passed up to the includer.
_file_book = None
# THE "INCLUDE ONCE" SET BELONGS TO THE WHOLE PARSE, not to one include
# line. Owning it per line meant sway.stack's `include avoid` finished
# and cleared it, so `include media` started fresh and media's own
# `include avoid` spliced the 87 exclusions a second time -- and a third
# for the nesting: 332 entries where there are 158.
_top_parse = getattr(_parse_stack_file, "_spliced", None) is None
if _top_parse:
_parse_stack_file._spliced = {os.path.abspath(path)}
# A LINE MAY CONTINUE.
#
# Every entry had to be one line, so sway.stack's are 300 characters wide
# and nobody can read the build= command in them. A trailing backslash
# joins, as it does everywhere else in a shell-shaped file; the line
# NUMBER reported for an error stays the one the entry started on.
_joined, _buf, _start = [], "", 0
with open(path) as f:
for n, raw in enumerate(f, 1):
_s = raw.rstrip("\n")
if _buf:
_s = _buf + " " + _s.strip()
else:
_start = n
if _s.rstrip().endswith("\\"):
_buf = _s.rstrip()[:-1].rstrip()
continue
_buf = ""
_joined.append((_start, _s))
if _buf:
raise ValueError("%s:%d: line ends with a backslash and nothing follows"
% (path, _start))
try:
for n, raw in _joined:
line = raw.strip()
if not line or line.startswith("#"):
continue
# shlex, not split(): build= commands carry spaces, so values are
# quoted -- build="meson setup build; ninja -C build"
try:
parts = shlex.split(line, comments=True)
except ValueError as e:
raise ValueError("%s:%d: %s" % (path, n, e))
if not parts:
continue
kind = parts[0]
if kind not in kinds:
raise ValueError("%s:%d: unknown entry kind %r (one of %s)"
% (path, n, kind, "/".join(kinds)))
# ONE LIST, TWO ENTRY POINTS.
#
# The media chain -- codecs, gstreamer, the WebRTC pieces,
# webkitgtk -- is worth updating on its own without replaying a
# whole desktop. Copying those entries into a second stack file
# would give two lists of the same packages, and two lists drift:
# gst10-libav was added to one of them for three releases before
# anyone noticed the other.
#
# `include <name>` splices another stack's entries in at this
# point, in order. The included file is a stack in its own right,
# so `packagemanager stack media --run` builds just that part.
if kind == "from-book":
if len(parts) != 2:
raise ValueError("%s:%d: from-book takes one book selector"
" (svn, 13.0-systemd, a file)" % (path, n))
_file_book = None if parts[1] in ("default", "-") else parts[1]
continue
if kind == "include":
if len(parts) != 2:
raise ValueError("%s:%d: include takes one stack name"
% (path, n))
_inc = parts[1]
_p = _inc if os.path.sep in _inc else os.path.join(
os.path.dirname(os.path.abspath(path)),
_inc if _inc.endswith(".stack") else _inc + ".stack")
if not os.path.isfile(_p):
raise ValueError("%s:%d: include %s: no such stack (%s)"
% (path, n, _inc, _p))
if os.path.abspath(_p) == os.path.abspath(path):
raise ValueError("%s:%d: include %s includes itself"
% (path, n, _inc))
# A CYCLE WOULD RECURSE FOR EVER, and the error a reader gets
# from a RecursionError names none of the files involved.
_seen = getattr(_parse_stack_file, "_including", set())
if os.path.abspath(_p) in _seen:
raise ValueError("%s:%d: include %s: circular include"
% (path, n, _inc))
# INCLUDE ONCE PER COMPOSITION.
#
# sway.stack includes `avoid` AND `media`, and media.stack
# includes `avoid` too -- so the 87 exclusions were spliced
# three times: 332 entries where there are 158, and every
# avoided package listed three times in the plan. A second
# include of the same file adds nothing by definition, so
# skip it.
_sp = _parse_stack_file._spliced
if os.path.abspath(_p) in _sp:
continue # already in this plan
_sp.add(os.path.abspath(_p))
_seen = _seen | {os.path.abspath(path)}
_parse_stack_file._including = _seen
try:
entries.extend(_parse_stack_file(_p))
finally:
_parse_stack_file._including = \
_seen - {os.path.abspath(path)}
continue
if len(parts) < 2:
raise ValueError("%s:%d: %r needs a target" % (path, n, kind))
target = parts[1]
opts = {}
for kv in parts[2:]:
if "=" not in kv:
raise ValueError("%s:%d: option %r is not key=value"
% (path, n, kv))
k, v = kv.split("=", 1)
opts[k] = v
if kind == "avoid":
# a package this stack must never build, however it is
# reached -- texlive and LibreOffice arrive as RECOMMENDED
# dependencies of harfbuzz, three layers down, and a desktop
# does not want either
entries.append({"kind": "avoid", "target": target, "opts": opts})
continue
if kind == "git" and "url" not in opts:
raise ValueError("%s:%d: a git entry needs url=" % (path, n))
if kind == "tar" and ("url" not in opts or "pkg" not in opts):
raise ValueError("%s:%d: a tar entry needs url= and pkg=" % (path, n))
if kind == "book" and _file_book:
opts.setdefault("from-book", _file_book)
if opts.get("from-book") in ("default", "-"):
del opts["from-book"]
entries.append({"kind": kind, "target": target, "opts": opts})
if not entries:
raise ValueError("%s: the stack file has no entries" % path)
finally:
if _top_parse:
_parse_stack_file._spliced = None
return entries
def _stack_git_script(entry, script_dir):
"""Generate the local install script for a git OR tar entry, in the
template's shape, with the family's build convention: make all && make
PREFIX=/usr install. Overridable per entry with build= / install=
(';'-separated)."""
name = entry["target"]
opts = entry["opts"]
is_git = "1" if entry["kind"] == "git" else ""
build_cmds = opts.get("build", "make all").replace(";", "\n")
install_cmds = opts.get("install", "make PREFIX=/usr install").replace(";", "\n")
# `prog=` is the BINARY this entry installs; it is not always the entry's
# own name. Defaulting it to the name made verify check libsrtp for a
# program called `libsrtp`, and report "missing: program libsrtp" for
# every library and data package in the stack -- twelve of them.
prog = opts.get("prog", "")
text = _TEMPLATE.format(name=name, version=opts.get("version", "git"),
name_version="%s-%s" % (name, opts.get("version", "git")),
link=opts["url"], pkg=opts.get("pkg", ""),
is_git=is_git,
collector=collector_prefix().rstrip("_"))
# pin the ref and the doneness signal into the generated header
text = text.replace('git_ref=""', 'git_ref="%s"' % opts.get("ref", ""), 1)
# SUBMODULES ARE THE PROJECT'S SPARE COPIES, NOT ALWAYS ITS NEEDS.
# SDL_mixer registers eleven (flac, opus, vorbis, mpg123, ...) for builds
# that vendor them -- Android, MSVC -- and on this system every one of
# them is cloned and then ignored. `submodules=no` clones without them;
# wl-mirror, whose protocols ARE its submodules, keeps the default.
if str(opts.get("submodules", "")).lower() in ("no", "0", "off", "false"):
text = re.sub(r'(?m)^(git_ref="[^"\n]*".*)$',
lambda m: m.group(1) + '\ngit_submodules="0" # stack: submodules=no',
text, count=1)
text = text.replace('installed_program=""', 'installed_program="%s"' % prog, 1)
# ...and `have=` is the entry's OWN statement of what proves it installed
# -- the stack already uses it to skip an entry that is done. It is the
# right validation target for a package with no binary: a .pc file, a
# header directory, a font directory.
_have = opts.get("have", "")
if _have:
text = text.replace('installed_directory=""',
'installed_directory="%s"' % _have, 1)
# the template ships with placeholder build/install bodies; put the
# family's convention in
text = _set_phase_body(text, "build_pkg", build_cmds)
text = _set_phase_body(text, "install_pkg", install_cmds)
os.makedirs(script_dir, exist_ok=True)
path = os.path.join(script_dir, "install_%s" % name)
with open(path, "w") as f:
f.write(text)
os.chmod(path, 0o755)
return path, (prog or name)
def _stack_root_script(entry, stack_dir):
return os.path.join(stack_dir, entry["target"] + ".sh")
def _stack_state_file(stack_path):
d = os.path.join(os.environ.get("LFS_PKGUSR_DIR", "/usr/src/lfs-pkgusr"),
"progress", "stacks")
base = os.path.basename(stack_path)
if base.endswith(".stack"):
base = base[:-6]
return os.path.join(d, base + ".done")
def _stack_done_set(stack_path):
try:
return set(l.strip() for l in open(_stack_state_file(stack_path))
if l.strip())
except OSError:
return set()
def _stack_mark_done(stack_path, key):
f = _stack_state_file(stack_path)
try:
os.makedirs(os.path.dirname(f), exist_ok=True)
with open(f, "a") as fh:
fh.write(key + "\n")
except OSError as e:
printWarning("could not record stack progress (%s) -- the next run "
"re-judges this entry" % e)
def _check_stack_refs(rows):
"""Ask every git remote whether the ref this stack pins still exists.
Two entries in one stack had rotted: usrsctp ref=1.0.0 (the tags are
0.9.3.0, 0.9.4.0, 0.9.5.0) and foot ref=1.24.1 (1.24.0 exists, 1.24.1
never did). Each cost a clone, a build attempt and a round trip to find
out. A pinned ref is data that rots, and only the remote can check it --
so check them all at once, before anything is built.
"""
git = shutil.which("git")
if not git:
printError("git is not installed -- cannot check any ref.")
return 1
bad = unreachable = 0
# (unreachable is counted for no-ref entries too, so the summary cannot
# claim an all-clear over a remote nobody asked)
for r in rows:
if r.get("kind") != "git":
continue
url = (r["opts"].get("url") or "").strip()
ref = (r["opts"].get("ref") or "").strip()
target = r.get("target")
if not url:
continue
if not ref:
# AN ENTRY WITH NO REF STILL HAS A URL.
#
# Four entries tracked their default branch and were passed over
# untouched, so the check said "every pinned ref exists" without
# having contacted them at all. A repository that has been
# renamed, made private or moved fails identically to a bad ref --
# at clone time, after everything before it has built. Ask.
try:
out = subprocess.run([git, "ls-remote", "--heads", url],
capture_output=True, text=True, timeout=120)
except (OSError, subprocess.SubprocessError):
out = None
if out is None or out.returncode != 0:
printWarning(" %-22s no ref=, and %s could not be reached"
% (target, url))
unreachable += 1
elif not out.stdout.strip():
bad += 1
printError(" %-22s no ref=, and the remote has NO branches"
% target)
printInfo(" %s -- moved, renamed or empty?" % url)
else:
printSuccess(" %-22s no ref= (default branch, reachable)"
% target)
continue
if re.fullmatch(r"[0-9a-f]{7,40}", ref):
printInfo(" %-22s %s (a commit id -- not checked)" % (target, ref))
continue
try:
out = subprocess.run([git, "ls-remote", "--tags", "--heads", url],
capture_output=True, text=True, timeout=120)
except (OSError, subprocess.SubprocessError) as e:
printWarning(" %-22s could not reach %s (%s)" % (target, url, e))
unreachable += 1
continue
if out.returncode != 0:
printWarning(" %-22s could not reach %s" % (target, url))
unreachable += 1
continue
names = {l.rsplit("/", 1)[-1] for l in out.stdout.splitlines()}
names = {n[:-3] if n.endswith("^{}") else n for n in names}
if ref in names:
printSuccess(" %-22s %s" % (target, ref))
continue
bad += 1
printError(" %-22s %s DOES NOT EXIST" % (target, ref))
# SORT TAGS AS VERSIONS, NOT AS STRINGS.
#
# foot's real tags include 1.28.0, and a reverse string sort offered
# "1.9.2, 1.9.1, 1.9.0, 1.9, 1.8.2" -- because "9" > "2" one character
# at a time. A suggestion that names an ancient release as the newest
# is worse than no suggestion: it invites a downgrade.
def _vkey(n):
base = n[1:] if n[:1] == "v" else n
parts = re.split(r"[.\-_]", base)
out = []
for x in parts:
out.append((0, int(x), "") if x.isdigit() else (1, 0, x))
return out
near = sorted((n for n in names if n[:1].isdigit() or n[:1] == "v"),
key=_vkey, reverse=True)[:5]
if near:
printInfo(" remote has: " + ", ".join(near))
if bad:
print()
printError("%d entr%s a ref the remote does not have."
% (bad, "y pins" if bad == 1 else "ies pin"))
printInfo(" Fix ref= in the stack file before building.")
else:
print()
# "every pinned ref exists" while five remotes were unreachable is a
# false all-clear: the ones that could not be asked are exactly the
# ones that might be wrong.
if unreachable:
printSuccess("every ref that COULD be checked exists.")
else:
printSuccess("every pinned ref exists.")
if unreachable:
printWarning(" %d remote(s) could not be reached -- those refs are unverified."
% unreachable)
return 1 if bad else 0
def _list_stacks():
"""`packagemanager stack` with no name: what stacks there are.
Every *.stack in /etc/pkgusr/stacks (and stacks/ beside this tool, when
that is a different directory), with its one-line description, how many
entries it has and how many the cache records as done, and whether a
newer shipped copy (.new) is waiting. avoid.stack is a list of what is
never built, not something to run, and says so.
"""
dirs = []
for d in ("/etc/pkgusr/stacks",
os.path.join(os.path.dirname(os.path.abspath(__file__)), "stacks")):
if os.path.isdir(d) and os.path.realpath(d) not in [os.path.realpath(x) for x in dirs]:
dirs.append(d)
if not dirs:
printError("no stack directory (/etc/pkgusr/stacks) -- `make install` puts them there")
return 1
any_behind = []
# DONE IS WHAT ANY STACK RECORDED. sway includes media, so media's
# packages were built -- and recorded -- under sway, and `media` said
# "1/19 done" with all of it installed.
done_all = set()
for d in dirs:
for f in os.listdir(d):
if f.endswith(".stack"):
done_all |= _stack_done_set(os.path.join(d, f))
# INSTALLED IS NOT CURRENT. The cache says an entry was built once; it
# never looks again. Compare each book entry's installed version with
# the book, so "79/79 done" cannot hide a sway that is three releases
# behind. One map for the whole book (cached); no book, no judgement.
try:
_bv = book_versions_all() or {}
except Exception:
_bv = {}
for d in dirs:
files = sorted(f for f in os.listdir(d) if f.endswith(".stack"))
printHeader("stacks in %s" % d)
if not files:
printInfo(" (none)")
continue
for f in files:
path = os.path.join(d, f)
name = f[:-len(".stack")]
desc = ""
try:
first = open(path, encoding="utf-8", errors="replace").readline()
if " -- " in first:
desc = first.split(" -- ", 1)[1].strip()
except OSError:
pass
try:
# what it BUILDS: the included avoid list is not an entry
entries = [e for e in _parse_stack_file(path) if e["kind"] != "avoid"]
except (OSError, ValueError) as ex:
print(" %-10s (does not parse: %s)" % (name, ex))
continue
behind = ""
if name == "avoid":
state = "never built"
else:
done = len(done_all
& {"%s %s" % (e["kind"], e["target"]) for e in entries})
state = "%d/%d done" % (done, len(entries))
if _bv:
old_n = 0
for e in entries:
if e["kind"] != "book":
continue
want = _bv.get(e["target"]) or _bv.get(e["target"].lower())
have = gather_user(e["target"]).version if want else None
if want and have and not _same_version(want, have):
old_n += 1
if old_n:
behind = " %d entr%s behind the book" % (old_n, "y" if old_n == 1 else "ies")
any_behind.append(name)
new = " (.new waiting: --take-new)" if os.path.isfile(path + ".new") else ""
print(" %-10s %-15s %s%s%s" % (name, state, desc, behind, new))
print()
printInfo(" plan one: packagemanager stack <name>")
printInfo(" where it is, and what is behind (dependencies too):"
" packagemanager stack <name> --status")
if not _bv:
printInfo(" (no book on disk: versions not compared)")
printInfo(" (\"done\" means built once -- a stack run never updates; "
"packagemanager update --stack <name> does)")
return 0
def _book_label(path):
"""A book file -> the selector people use: 13.1-systemd, 12.4, svn."""
b = os.path.basename(path or "")
for pat in (r"BLFS-BOOK-(.+?)-nochunks\.html$",
r"(.+?)-combined-nochunks\.html$",
r"LFS-BOOK-(.+?)-NOCHUNKS\.html$"):
m = re.match(pat, b)
if m:
return m.group(1)
return b or "?"
def _nv_version(nv):
"""gst-plugins-base-1.28.7 -> 1.28.7 (the whole string if no version)."""
m = re.search(r"-(v?\d[\w.+~]*)$", nv or "")
return m.group(1) if m else (nv or "")
def _stack_row_view(e, state):
"""(installed, book, from, verdict, color) for one stack line.
THE LINE SAID `book WebKitGTK-2.54.0` FOR A PACKAGE THAT IS 2.54.0.
"book X" read as "will build X", for something already current, and
nothing said which book X came from. A row now says what is installed,
what the book has, which book that is, and what --run or update would
do about it."""
kind, target = e["kind"], e["target"]
if kind in ("check", "cfg"):
if state == "MISSING SCRIPT":
return ("", "", "script", state, colors.red)
return ("", "", "script", "done (stack cache)" if "done" in state
else "runs with --run", colors.dim if "done" in state else "")
u = gather_user(target)
inst = _nv_version(u.version) if u.state == "installed" and u.version \
else ("yes" if u.state == "installed" else "-")
if kind in ("git", "tar"):
if state.startswith("done") or state == "installed":
# a git build's "version" is a name, not a number
return (inst if re.search(r"\d", inst) else "yes", "", kind,
"installed", colors.green)
return (inst, "", kind, "not installed", colors.yellow)
if not state.startswith(("book ", "done")):
return (inst, "", "", state, colors.yellow) # NOT IN BOOK, unjudged ...
try:
bnv = book_version(target)
except Exception:
bnv = None
over = _book_override(target)
src = e["opts"].get("from-book") or (
_book_label(over) if over else _book_label(_default_book_file()))
bv = _nv_version(bnv) if bnv else "?"
if u.state != "installed":
return (inst, bv, src, "not installed", colors.yellow)
if not bnv or not u.version or u.version == bnv:
return (inst, bv, src, "current", colors.green)
if _vsort_key(_nv_version(u.version)) > _vsort_key(bv):
return (inst, bv, src, "newer than this book", colors.dim)
return (inst, bv, src, "update available", colors.cyan)
_STACK_ROW_FMT = " %-6s %-26s %-12s %-12s %-14s %s"
def cmd_stack(args):
if not args.name:
return _list_stacks()
path = _find_stack_file(args.name)
if not path:
printError("no stack file for %r (looked in ., /etc/pkgusr/stacks, "
"stacks/ beside this tool)" % args.name)
sys.exit(1)
try:
entries = _parse_stack_file(path)
# --yes REACHES THE SCRIPTS THIS RUNS. A cfg step calls `lfs-helper
# pm-install`, which asks "Press enter to continue" unless PM_YES is
# set -- so an unattended stack run stopped at a prompt nobody was
# there to answer.
if getattr(args, "yes", False):
os.environ["PM_YES"] = "1"
os.environ["LFS_ASSUME_YES"] = "1"
# A NEWER VERSION OF THIS FILE IS SITTING BESIDE IT.
#
# `make install` leaves the shipped copy as <stack>.new when the
# user's own differs (1.14.58/1.14.98), and running the stack said
# nothing about it: "3 entries" while the .new had six, twice in a
# row, with no hint that a merge was owed. The moment to mention it
# is the moment someone runs the file.
_new = path + ".new"
if os.path.isfile(_new):
try:
_n_entries = len(_parse_stack_file(_new))
except (OSError, ValueError):
_n_entries = None
if (getattr(args, "take_new", False) and _n_entries is not None
and _n_entries < len(entries) and not args.force):
# A .new WITH FEWER ENTRIES IS A DOWNGRADE.
#
# The chroot refresh writes <stack>.new from the HOST's copy
# when the chroot's differs (1.14.105). If the host is the
# stale one -- it often is, since `make install` keeps a file
# it thinks the user edited -- then --take-new would replace
# a 6-entry stack with a 3-entry one, and the next refresh
# would offer it again. Refuse the shrink; --force overrides.
printError("that .new file has FEWER entries (%d) than this one"
" (%d) -- not taking it." % (_n_entries, len(entries)))
printInfo(" It is probably the host's older copy, brought in by"
" the chroot refresh.")
printInfo(" Update the host first:")
printInfo(" cp /etc/pkgusr/stacks/%s{.new,} # on the HOST"
% os.path.basename(path))
printInfo(" ...or take it anyway with --take-new --force.")
sys.exit(1)
if getattr(args, "take_new", False):
# TAKE IT, rather than describe it again. Three runs in a
# row ended with the user reading a note and doing nothing:
# the note was right and the work it asked for was mine to
# do. Theirs is kept as .bak, so nothing is lost.
_bak = path + ".bak"
try:
shutil.copy2(path, _bak)
shutil.copy2(_new, path)
os.remove(_new)
except OSError as e:
printError("could not take the new stack file: %s" % e)
sys.exit(1)
printSuccess("took the shipped stack file; yours is %s" % _bak)
entries = _parse_stack_file(path)
# THE SAME MEASURE THE HEADER USES. This said "149 entries
# now" while the header two lines later said "(62 entries)":
# one counts every line of the file, the other the buildable
# plan after `avoid`. Two numbers for one file, both called
# entries.
printInfo(" %d line(s) in the file; the plan below is what"
" will be built." % len(entries))
print()
_new = None
if _new and os.path.isfile(_new):
# WHAT DIFFERS, NOT HOW MANY. The notice printed
# (149 entries, this file has 149)
# for two files that differ in every entry's options: the counts
# were equal, so its own evidence argued there was nothing to
# take. An entry count cannot see a changed expect= or a
# reordered library.
printWarning("a newer version of this stack ships beside it:")
printWarning(" %s" % _new)
try:
import difflib as _dl
a = open(path, encoding="utf-8", errors="replace").readlines()
b = open(_new, encoding="utf-8", errors="replace").readlines()
added = sum(1 for l in _dl.unified_diff(a, b, n=0)
if l.startswith("+") and not l.startswith("+++"))
removed = sum(1 for l in _dl.unified_diff(a, b, n=0)
if l.startswith("-") and not l.startswith("---"))
_same = (_n_entries == len(entries))
printWarning(" %d line(s) added, %d removed%s"
% (added, removed,
"; the entry count is the same, so the change"
" is in the entries themselves"
if _same and (added or removed) else ""))
except OSError:
pass
printInfo(" diff %s %s" % (path, _new))
printInfo(" ...and take what you want; this run uses YOUR file.")
print()
if getattr(args, "check_refs", False):
printHeader("Checking pinned git refs")
sys.exit(_check_stack_refs(entries))
except ValueError as e:
printError(str(e))
sys.exit(1)
stack_dir = os.path.dirname(path)
script_dir = os.path.join(TMP_DIR, "stack_files")
avoid = [e["target"] for e in entries if e["kind"] == "avoid"]
entries = [e for e in entries if e["kind"] != "avoid"]
if getattr(args, "avoided", False):
# the list the header only counts
printHeader("never built by stack %s (%d)" % (args.name, len(avoid)))
for a in sorted(set(avoid), key=str.lower):
print(" " + a)
return
# THE BOOKS THIS STACK READS. Its from-book entries become the
# packages' book for this run and for every install it starts; the
# book itself is refreshed once here (svn moves daily), and every
# lookup after that reads the file on disk.
_sb = {e["target"]: e["opts"]["from-book"] for e in entries
if e["kind"] == "book" and e["opts"].get("from-book")}
if _sb:
_stack_book_map().update({k.lower(): v for k, v in _sb.items()})
_BOOK_FOR_PKG.clear()
os.environ["PM_STACK_BOOKS"] = ",".join(
"%s=%s" % kv for kv in sorted(_sb.items()))
for _sel in sorted(set(_sb.values())):
_bp = _book_path_for(_sel, refresh=not getattr(args, "status",
False))
if not getattr(args, "status", False):
printInfo(" book %s: %d entr%s (%s)"
% (_sel, sum(1 for v in _sb.values() if v == _sel),
"y" if list(_sb.values()).count(_sel) == 1
else "ies",
os.path.basename(_bp) if _bp else "NOT ON DISK"))
if getattr(args, "no_git", False):
# git and tar entries are the extras -- cloned tools, fonts. Skipping
# them finishes the book part of the desktop first, and matters while
# git itself is not built yet ("git: command not found").
_n = len(entries)
_skipped_git = [e for e in entries if e["kind"] in ("git", "tar")]
entries = [e for e in entries if e["kind"] not in ("git", "tar")]
if not getattr(args, "status", False):
printInfo(" --no-git: skipping %d cloned/tarball entries\n"
% (_n - len(entries)))
# SOME CLONED ENTRIES ARE DEPENDENCIES, NOT EXTRAS.
#
# libnice and friends (1.13.69) are git entries that book
# packages LINK: gst-plugins-bad compiles webrtcbin only if
# libnice is present. Skipping them with --no-git builds those
# packages without WebRTC and says nothing -- and finding out
# later costs a webkitgtk rebuild.
_webrtc = [e["target"] for e in _skipped_git
if e["target"] in ("libnice", "libsrtp", "usrsctp",
"webrtc-audio-processing")]
if _webrtc:
printWarning(" NOTE: %s are skipped, and gst-plugins-bad / "
"webkitgtk link them." % ", ".join(_webrtc))
printWarning(" Built without them there is no WebRTC (no "
"camera or screen sharing in the browser),")
printWarning(" and adding them later means recompiling both. "
" Drop --no-git to include them.")
if not getattr(args, "status", False):
printHeader("stack: %s (%d entries)" % (os.path.basename(path), len(entries)))
if avoid:
# AVOID WAS ONLY A MESSAGE. It printed "never built by this
# stack" and then passed nothing to the installer, so the
# packages were built anyway. It is the stack's exclusion list
# now, handed to every entry's resolve.
# A COUNT, NOT THE LIST: eighty names above every plan pushed
# the plan itself off the screen.
print(" %d package(s) never built by this stack "
"(list: packagemanager stack %s --avoided)\n"
% (len(set(avoid)), args.name))
print(" file: %s\n" % path)
print(colors.dim + _STACK_ROW_FMT % ("", "entry", "installed", "book",
"from", "") + colors.normal)
import shutil as _sh
def _git_installed(e):
"""prog= on PATH, or have= (a file that only exists once it is
installed -- the signal for libraries, which put nothing on PATH)."""
have = e["opts"].get("have")
if have and os.path.exists(have):
return True
prog = e["opts"].get("prog", e["target"])
return bool(prog and prog != "-" and _sh.which(prog))
# Entries that finished in an earlier run are recorded and SKIPPED, so a
# resume does not re-plan a 200-package dependency tree for a stage that
# is already done. --force ignores the cache. Recorded per stack file
# under $LFS_PKGUSR_DIR/progress/stacks/.
# RUN ONE ENTRY, OR A FEW.
#
# A stack is a list, and after the first pass the only thing anyone wants
# is to redo ONE line of it -- a config step that failed, a package to
# rebuild. --force re-ran all 56. --only names entries (repeatable, or
# comma-separated); it implies --force for those and skips the rest.
donekeys = set() if args.force else _stack_done_set(path)
# RUN JUST ONE PART, AND REALLY RUN IT.
#
# A stack is long and its entries fail one at a time; re-running the
# whole thing to retry `cfg mytools` is a poor answer, and --force
# re-runs everything. --only names entries: they are exempt from the
# cache (so a "done" one is rebuilt) and everything else is skipped.
# (--only was declared long ago and never implemented -- the help
# promised "repeatable or comma-separated"; both work now.)
_only = {x.strip()
for spec in (getattr(args, "only", None) or [])
for x in str(spec).split(",") if x.strip()}
if _only:
_known = {e["target"] for e in entries}
_unknown = sorted(_only - _known)
if _unknown:
printError("not in this stack: %s" % ", ".join(_unknown))
printInfo(" entries: %s" % ", ".join(sorted(_known)))
# ...BUT IT MAY BE IN THE FILE WAITING BESIDE THIS ONE.
#
# The user asked for `--only mytools` against a stack whose
# newer copy has exactly that entry. "not in this stack" was
# true and useless: the answer was one flag away and the message
# did not say so.
_nf = path + ".new"
if os.path.isfile(_nf):
try:
_nnames = {x["target"] for x in _parse_stack_file(_nf)}
except (OSError, ValueError):
_nnames = set()
_there = sorted(set(_unknown) & _nnames)
if _there:
print()
printWarning("%s IS in the newer file beside this one:"
% ", ".join(_there))
printInfo(" %s" % _nf)
printInfo(" Take it and run the same command:")
printInfo(" packagemanager stack %s --take-new %s"
% (args.name, " ".join(
"--only " + x for x in sorted(_only))))
sys.exit(1)
donekeys = {k for k in donekeys
if k.split(" ", 1)[-1] not in _only}
# ...and SKIP the rest. Exempting the named entries from the cache is
# only half of it: without this the other 55 were still planned and run.
if _only:
entries = [e for e in entries if e["target"] in _only]
printInfo("running only: %s" % ", ".join(e["target"] for e in entries))
print()
# WHEN=: AN ENTRY THIS MACHINE DOES NOT NEED.
#
# btrfs-progs belongs in the stack of a btrfs machine and nowhere else,
# and a second stack file per filesystem is not an answer. The test is
# a shell command, run here, every time -- so a disk added later is
# picked up without editing anything.
_when_out = []
_kept = []
for e in entries:
_w = e["opts"].get("when")
if _w and subprocess.run(["/bin/bash", "-c", _w],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL).returncode != 0:
_when_out.append((e["target"], _w))
continue
_kept.append(e)
entries = _kept
if _when_out and not getattr(args, "status", False):
for _t, _w in _when_out:
printInfo(" %s: not for this machine -- when=%s" % (_t, _w))
print()
rows = []
for e in entries:
e["_key"] = "%s %s" % (e["kind"], e["target"])
# (entries are already filtered to --only above; the old `only`
# variable was removed with the duplicate implementation, and this
# reference outlived it -- NameError on every stack run.)
# A CACHED cfg ENTRY IS RE-CHECKED WHEN IT CAN BE.
#
# app-users puts the pam_exec line into /etc/pam.d/su; shadow's
# configure phase later rewrote su without it, and every stack run
# after that said `cfg app-users done (stack cache)` -- so
# `su - u_browser-mini` asked for a password and nothing re-ran the
# step that fixes it. A cfg entry's expect= states what it keeps
# true; when that no longer holds, the cache is wrong, not the check.
if (e["_key"] in donekeys and e["kind"] == "cfg"
and e["opts"].get("expect")
and subprocess.run(["/bin/bash", "-c", e["opts"]["expect"]],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL).returncode != 0):
donekeys.discard(e["_key"])
if not getattr(args, "status", False):
printWarning(" cfg %s is cached as done, but its check fails"
" now -- running it again" % e["target"])
if e["_key"] in donekeys:
rows.append((e, "done (stack cache)"))
if not getattr(args, "status", False):
_i, _b, _f, _v, _c = _stack_row_view(e, "done (stack cache)")
e["_verdict"] = _v
print(_STACK_ROW_FMT % (e["kind"], e["target"], _i, _b, _f,
_c + _v + colors.normal))
continue
if e["kind"] in ("git", "tar"):
state = "installed" if _git_installed(e) else "to build"
elif e["kind"] in ("check", "cfg"):
sp = _stack_root_script(e, stack_dir)
state = "script" if os.path.isfile(sp) else "MISSING SCRIPT"
else:
# ASK THE BOOK. A dry run that says "book" without looking is a
# plan half-judged: sway and wlroots sat in a stack as book
# entries, and BLFS carries neither -- the first --run would have
# discovered it one llvm build too late.
# book_version cannot tell "anchor absent" from "no book on
# disk", so probe availability ONCE with an anchor every BLFS
# has; without a book, every entry is honestly unjudged.
if not hasattr(cmd_stack, "_book_ok"):
try:
cmd_stack._book_ok = book_version("dbus") is not None \
or book_version("DBus") is not None
except Exception:
cmd_stack._book_ok = False
if not cmd_stack._book_ok:
state = "book (no book on disk -- unjudged)"
else:
try:
bv = book_version(e["target"])
state = ("book %s" % bv) if bv else "NOT IN BOOK"
except Exception:
state = "book (lookup failed -- unjudged)"
# A book entry carrying url= FALLS BACK to git where the book
# does not know it -- one stack file serves a system whose book
# flavour has the package and one whose book does not.
if state == "NOT IN BOOK" and "url" in e["opts"]:
e["kind"] = "git"
state = ("installed" if _git_installed(e)
else "not in book -> git fallback")
rows.append((e, state))
if not getattr(args, "status", False):
_i, _b, _f, _v, _c = _stack_row_view(e, state)
e["_verdict"] = _v
print(_STACK_ROW_FMT % (e["kind"], e["target"], _i, _b, _f,
_c + _v + colors.normal))
notinbook = [e["target"] for e, st in rows if st == "NOT IN BOOK"]
if notinbook:
printWarning("\nnot in the book: " + ", ".join(notinbook)
+ "\n WHY is the first question -- the page may be there under"
"\n another anchor, or this book may be a different one than"
"\n you think:"
+ "".join("\n blfs debug %s" % t for t in notinbook)
+ "\n then: rename the entry to the book's anchor, or make it a"
"\n tar/git entry with build=/install= commands.")
if args.run and not getattr(args, "skip_unknown", False):
# ONE BAD LINE MUST NOT COST THE OTHER EIGHT. Refusing the whole
# plan is right by default -- a plan that silently drops entries
# is how a stack looks finished while a package is missing -- but
# an entry the book has no page for blocked the eight that build
# and the checks that report, with no way past it but editing the
# file.
printError("refusing to run a plan with unknown targets.")
printInfo(" to run the rest and come back to it:"
"\n packagemanager stack %s --run --skip-unknown"
% os.path.basename(path))
sys.exit(1)
if args.run:
printWarning(" --skip-unknown: leaving %d entr%s out of this run"
% (len(notinbook), "y" if len(notinbook) == 1 else "ies"))
entries = [e for e, st in rows if st != "NOT IN BOOK"]
rows = [(e, st) for e, st in rows if st != "NOT IN BOOK"]
missing = [e["target"] for e, st in rows if st == "MISSING SCRIPT"]
if missing:
printError("\nmissing root script(s) beside the stack: "
+ ", ".join("%s.sh" % m for m in missing))
sys.exit(1)
if getattr(args, "status", False):
# WHERE ARE WE IN THE TREE?
#
# Entry counts answered the wrong question: "1 of 39" says nothing
# about the hundreds of packages those entries pull in. Resolve each
# book entry's dependency order and report the PACKAGES -- installed,
# remaining, and what comes next -- using install_tag, the same
# judgement `search` and the install plan use, so all three agree.
seen, pkgs, skipped_by_avoid = set(), [], set()
for e, _st in rows:
if e["kind"] != "book":
continue
try:
# THE SAME GRAPH THE BUILD WILL USE. Asking without the
# entry's deps= showed 272 packages for a plan that builds
# 99 -- a status that does not match the run is worse than
# no status.
order, _missing = blfs_order_anchors(
e["target"],
no_recommended=(e["opts"].get("deps") == "required"))
except SystemExit:
continue
for anchor, nv, _kind, _parent in order:
# THE STATUS MUST NOT COUNT WHAT THE RUN WILL SKIP. The avoid
# list reaches the installer (1.12.78) and never reached this
# display, so it reported 273 packages for a run that skips
# qt6, okular, LibreOffice, CUPS and the rest by name.
if any(anchor.lower() == a.lower() for a in avoid):
skipped_by_avoid.add(anchor)
continue
if anchor not in seen:
seen.add(anchor)
pkgs.append((anchor, nv))
done_pkgs = [(a, v) for a, v in pkgs if "not installed" not in install_tag(a, v)]
todo = [(a, v) for a, v in pkgs if (a, v) not in set(done_pkgs)]
# DONE BY ANY STACK: media's entries are built through sway, which
# includes it, and recorded in sway's cache -- "1 of 19 done" with
# 189 of 215 packages installed.
_done_any = set()
for _sd in {os.path.dirname(path), "/etc/pkgusr/stacks"}:
if os.path.isdir(_sd):
for _sf in os.listdir(_sd):
if _sf.endswith(".stack"):
_done_any |= _stack_done_set(os.path.join(_sd, _sf))
done_n = sum(1 for e, st in rows
if st == "done (stack cache)" or e.get("_key") in _done_any
or ("%s %s" % (e["kind"], e["target"])) in _done_any)
_tags = {(a, v): install_tag(a, v) for a, v in done_pkgs}
behind = [a for (a, v), t in _tags.items() if "book is newer" in t]
printHeader("stack %s" % args.name)
print(" entries : %d of %d done" % (done_n, len(rows)))
if pkgs:
print(" packages: %d of %d installed" % (len(done_pkgs), len(pkgs)))
if behind:
# INSTALLED IS NOT CURRENT, and --run never updates. Say how many,
# and the one command that does.
print(" behind : %d installed older than the book (%s%s)"
% (len(behind), ", ".join(behind[:5]), " ..." if len(behind) > 5 else ""))
print(" update them, with what depends on them: "
"packagemanager update --stack %s" % args.name)
if skipped_by_avoid:
print(" avoided : %d named in the stack (%s%s)"
% (len(skipped_by_avoid),
", ".join(sorted(skipped_by_avoid)[:5]),
" ..." if len(skipped_by_avoid) > 5 else ""))
nxt_entry = next((e["target"] for e, st in rows if st != "done (stack cache)"), None)
print(" next : %s" % (nxt_entry or "nothing left to do"))
if todo:
# SAY WHAT A RUN WOULD DO, because the list above reads like a
# plan and is not one. `--run` builds what is MISSING; it never
# updates a package that is installed, however far behind the
# book it is. On a system built from an older book that is half
# the tree marked "(update available)", and it looked like
# `stack media --run` was about to rebuild glib2.
print(" to build (missing): " + ", ".join(a for a, _ in todo[:6])
+ (" ..." if len(todo) > 6 else ""))
print(" %d missing in total. Installed packages are NOT"
" touched by --run," % len(todo))
print(" whatever the book now carries: \"book is newer\" is a"
" report, not a plan.")
print(" one package, deliberately: packagemanager update"
" <name> --run")
if getattr(args, "full", False):
print()
for a, v in pkgs:
tag = install_tag(a, v)
print(" %-30s %-26s %s" % (a, v, tag))
else:
print(" (full tree: --status --full)")
print(" state : %s" % _stack_state_file(path))
return
if not args.run:
# WHAT EACH COMMAND WOULD DO. --run builds what is missing and never
# touches an installed package; moving those to the book's version is
# `update`. Two commands, so two lines -- each with its names.
_miss = [e["target"] for e, _st in rows
if e.get("_verdict", "") == "not installed"]
_upd = [e["target"] for e, _st in rows
if e.get("_verdict", "") == "update available"]
_books = {}
for e, _st in rows:
if e["kind"] == "book":
_sel = e["opts"].get("from-book") or "default"
_books[_sel] = _books.get(_sel, 0) + 1
print()
print(" books : " + ", ".join(
"%s (%d)" % ((_book_label(_default_book_file()) + ", default")
if k == "default" else k, n)
for k, n in sorted(_books.items())) +
" all books: packagemanager books")
if _miss:
print(" missing : %d (%s) -> packagemanager stack %s --run"
% (len(_miss), ", ".join(_miss[:5]) + (" ..." if len(_miss) > 5 else ""),
args.name))
if _upd:
print(" behind : %d (%s) -> packagemanager update --stack %s --run"
% (len(_upd), ", ".join(_upd[:5]) + (" ..." if len(_upd) > 5 else ""),
args.name))
if not _miss and not _upd:
printSuccess(" every entry is installed and current.")
printInfo("\nDry run -- nothing was built.")
return
print()
done = 0
for e, state in rows:
kind, target = e["kind"], e["target"]
if state == "done (stack cache)":
done += 1
continue
printHeader("stack (%d/%d): %s %s" % (done + 1, len(rows), kind, target))
# A BOOK ENTRY CAN BECOME A CLONE. elogind is `book ... url=...`,
# and its kind only turns into "git" when the judging finds it absent
# from the book -- after the early --no-git filter has run. So the
# decision is taken here too, where the kind is final.
if getattr(args, "no_git", False) and kind in ("git", "tar"):
printInfo(" %s: skipped (--no-git)" % target)
continue
if kind == "book":
# NEVER drop a whole CLASS of dependencies.
#
# `required-only=1` (--ignore-recommended) was an attempt to keep
# texlive and LibreOffice out of a sway build. It removed far
# more: texlive's OWN dependencies -- Cairo, libpaper, the PATH
# page -- are all "recommended", so texlive was left with none,
# scheduled first, and built against a freetype that did not
# exist yet. A recommendation is still a dependency; the book
# knows why it is there. If a package is unwanted, NAME IT.
# deps=required -- the book's REQUIRED graph only.
#
# A first attempt at this (1.12.47) went badly and was reverted:
# texlive's own dependencies are all recommended, so dropping the
# class left it with none and it was scheduled FIRST. That was an
# ORDERING bug, fixed since (1.12.69 is a real topological sort),
# not an argument against choosing the graph you want. The
# recommendations worth keeping are named as entries of their own,
# which is what a stack file is for.
if not _install_via_self(target, args.yes, recursive=True,
required_only=(e["opts"].get("deps") == "required"),
ignore=",".join(
[x for x in [e["opts"].get("ignore")] if x]
+ avoid) or None,
ignore_prereqs=getattr(args, "ignore_prereqs",
False)):
printError("stack stopped at %r (%d of %d done). Fix it and "
"re-run -- finished entries are skipped."
% (target, done, len(rows)))
sys.exit(1)
elif kind in ("git", "tar"):
if _git_installed(e) and not args.force:
printSuccess(" %s: already installed (--force rebuilds)"
% target)
else:
spath, _ = _stack_git_script(e, script_dir)
# THE SAME CHECKS AS A BOOK PACKAGE.
#
# A git entry never goes through the planner, so nothing had
# looked at what its commands need: elogind's clone died on
# `git: command not found` after the account was made, the
# source directory created and the phase started. The check
# is a dictionary lookup; run it here too.
# A GIT ENTRY CAN NAME ITS DEPENDENCIES.
#
# sway is built from git, so nothing in the book orders
# anything before it, and it stopped on
# ERROR: Dependency "json-c" not found
# json-c IS a BLFS package (JSON-C-0.18); it was simply never
# built, because a git entry had no way to say it needed one.
# `needs=` closes that: the named packages are installed --
# through the same door as everything else -- before the entry
# runs.
for _need in [_resolve_need(x.strip()) for x in
(e["opts"].get("needs") or "").replace(",", " ").split()
if x.strip()]:
if gather_user(_need).state == "installed":
continue
printInfo(" %s needs %s -- installing it first"
% (target, _need))
if not _install_via_self(_need, args.yes, recursive=True):
printError("stack stopped at %s %r: could not install"
" %s (%d of %d done)."
% (kind, target, _need, done, len(rows)))
sys.exit(1)
# MACHINE.CONF REACHES A GIT ENTRY TOO.
#
# apply_machine_opts was called only from resolve_script, the
# BOOK path. So the box could tell someone to put
# [webrtc-audio-processing]
# cpp_args = -include cstdint
# in machine.conf for a git stack entry, and nothing would
# read it -- the advice was correct and inert. A script is a
# script; the options belong to the package, not to where its
# commands came from.
apply_machine_opts(target, spath)
_MISSING_TOOLS.clear()
_check_script_prereqs(target, spath)
if _MISSING_TOOLS and not getattr(args, "ignore_prereqs", False):
# BUILD IT, DO NOT JUST NAME IT.
#
# Stopping with "packagemanager install git --run
# --recursive" asks the person to type what the stack is
# for. A plan does this itself (1.14.14: a missing
# provider joins the plan and it is rebuilt); a git entry
# is not a plan, so it stopped instead. If the book has
# the tool, install it and carry on -- that is what an
# unattended run is.
_fixable = sorted({_provider_of(t) for t in _MISSING_TOOLS
if _provider_of(t)})
_stuck = sorted(t for t in _MISSING_TOOLS
if not _provider_of(t))
for _prov in _fixable:
printInfo(" %s needs %s -- installing it first"
% (target, _prov))
if not _install_via_self(_prov, args.yes, recursive=True):
printError("stack stopped at %s %r: could not"
" install %s (%d of %d done)."
% (kind, target, _prov, done, len(rows)))
sys.exit(1)
if _stuck:
_report_missing_tools()
printError("stack stopped at %s %r: it needs %s, which"
" no BLFS page provides (%d of %d done)."
% (kind, target, ", ".join(_stuck),
done, len(rows)))
printInfo(" Those come from LFS -- check they were"
" built and are on PATH.")
printInfo(" To start regardless: --ignore-prereqs")
sys.exit(1)
install_local(target, spath, True, args.yes, "install")
else: # check / cfg -- a root script through the one runner
sp = _stack_root_script(e, stack_dir)
# THE SETTINGS COME FROM THE RESOLVER, NOT FROM EACH SCRIPT.
# power.sh parsed packagemanager.conf itself, with the lfs
# config's JSON syntax, on a key=value file -- so on the booted
# machine it found nothing and stopped the stack. The resolver
# knows every source (lfs config, this conf, the collector
# groups on the system); a cfg script reads the environment.
# Order: environment, lfs config, packagemanager.conf (both via
# load_config, which lets the lfs config win), then what the
# system shows. A built-in default is never exported.
_cenv = dict(os.environ)
_known = _adopt_existing_settings()
_file_keys = set()
try:
with open(CONFIG_PATH) as _cf:
_file_keys = {l.split("=", 1)[0].strip() for l in _cf
if "=" in l and not l.lstrip().startswith("#")}
except OSError:
pass
for _key, _var in (("collector_prefix", "LFS_COLLECTOR_PREFIX"),
("main_user", "LFS_MAIN_USER")):
if _cenv.get(_var):
continue
if _key in _known and _known[_key][1] == "the lfs config":
_cenv[_var] = _known[_key][0]
elif _key in _file_keys and load_config().get(_key):
_cenv[_var] = str(load_config()[_key]).rstrip("_")
elif _key in _known:
_cenv[_var] = _known[_key][0]
rc = subprocess.run(["lfs-helper", "run-script", "--user", "root",
"--as-root", "--script", sp,
"--phase", "all", "--name", ""],
env=_cenv).returncode
if rc != 0:
printError("stack stopped at %s %r (exit %d, %d of %d done)."
% (kind, target, rc, done, len(rows)))
sys.exit(1)
# DO NOT RECORD A GIT ENTRY THAT IS NOT THERE.
#
# The cache makes an entry disappear from every later run, so a false
# "done" is permanent. A git/tar entry declares what proves it
# installed (prog=/have=); if that is still absent after a build the
# tools said succeeded, say so and leave the entry unrecorded rather
# than hiding it forever.
# EXPECT: A FEATURE IS NOT PROVED BY A FILE EXISTING.
#
# ffmpeg and SDL3 autodetect ALSA. Built before alsa-lib existed --
# which is what happens when a library is listed AFTER a package
# whose dependency plan pulls its consumers in -- they configure
# cleanly, install cleanly, pass have=, and have no sound. Nothing
# in a build log says so, because nothing went wrong.
#
# expect="<shell>" runs after the entry and must succeed. It is the
# only thing here that checks what was built rather than whether a
# build happened.
_expect = e["opts"].get("expect")
_expect_ok = True
if _expect:
printInfo(" checking what %s was built with: %s" % (target, _expect))
_erc = subprocess.run(["/bin/bash", "-c", _expect],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL).returncode
if _erc != 0:
_expect_ok = False
printError("%s installed, but the feature check failed:" % target)
printError(" %s" % _expect)
if e["kind"] == "cfg":
# A cfg STEP HAS NO OPTIONAL DEPENDENCIES. Its check is
# what the step keeps true; after running it, that is
# still false -- the step's own output above says why.
printWarning(" This is a configuration step: it ran, and what it"
" keeps true is still not true.")
printWarning(" Read its output above for the part it could not"
" do, then run it again:")
printInfo(" packagemanager stack %s --only %s --run"
% (os.path.basename(path).replace(".stack", ""),
target))
_expect_msg_done = True
else:
_expect_msg_done = False
if _erc != 0 and not _expect_msg_done:
printWarning(" The package is built and installed -- it is"
" MISSING A FEATURE, which almost always means")
printWarning(" its optional dependency was not installed yet"
" when it configured.")
printInfo(" What the check says is missing is usually fixed by:")
printInfo(" packagemanager install <the dependency> --run")
# A STACK ENTRY DOES NOT REBUILD AN INSTALLED PACKAGE.
#
# This used to suggest `stack <file>.stack --only <pkg> --run`
# -- with the .stack suffix, which is not a stack name -- and
# that command runs `install`, which answers "everything is
# installed and up to date" and changes nothing. Re-running
# it can never fix the feature, so recommending it was a loop.
# ...AND THE RIGHT COMMAND DEPENDS ON THE KIND OF ENTRY.
#
# `update --reinstall --regenerate` regenerates from the
# BOOK, and a git or tar entry has no book page -- for
# gst-plugins-rs that advice could not work at all. Those
# are rebuilt by the stack itself, with --force.
if e["kind"] == "book":
printInfo(" %s" % " ".join(_rebuild_cmd(target)))
printInfo(" Re-running this stack entry will NOT rebuild"
" it: the package is installed,")
printInfo(" so the entry's install step has nothing to"
" do. The line above does.")
else:
printInfo(" packagemanager stack %s --only %s --run"
" --force"
% (os.path.basename(path).replace(".stack", ""),
target))
printInfo(" This is a %s entry, not a book page: there is"
" nothing to regenerate" % e["kind"])
printInfo(" from the book, and --force is what rebuilds"
" it. (`update --reinstall`")
printInfo(" would look for a book page that does not"
" exist.)")
# AND SAY WHAT have= PROVED, WHICH IS LESS THAN IT LOOKS.
# A part-installed package satisfies it: gst-plugins-rs's
# first run installed libgstrswebrtc.so and then died on the
# next plugin, so have= said "installed" and audiornnoise
# was never there.
_have_f = e["opts"].get("have")
if _have_f:
printWarning(" NOTE: have=%s exists, which is why this"
" was skipped as installed --" % _have_f)
printWarning(" but one file is not the whole package."
" The probe is the better answer.")
# A BOOK ENTRY WAS RECORDED DONE WITHOUT BEING CHECKED AT ALL.
#
# The guard below covered git and tar only. poppler -- a `book`
# entry -- came out `done (stack cache)` while `pkg-config
# --list-all | grep poppler` found nothing: not installed, and the
# stack skipped it on every later run, so zathura-pdf-poppler kept
# failing on a dependency the plan believed it had.
#
# Every kind gets the same question now: is it there AFTERWARDS?
# For a book package that is the package state, which is what
# `verify` reads -- an empty manifest is not an install.
_installed_now = True
_missing_what = ""
if e["kind"] in ("git", "tar"):
_installed_now = _git_installed(e)
_missing_what = (e["opts"].get("have")
or e["opts"].get("prog") or target)
elif e["kind"] == "book":
_u_now = gather_user(target)
_st_now, _ = assess(_u_now, None, run_cmd=False)
_installed_now = _st_now in ("installed", "unvalidated")
_missing_what = "%s (%s)" % (target, _st_now)
if not _expect_ok:
pass
elif not _installed_now:
printWarning("%s finished, but %s is still missing -- NOT recording"
" it as done, so the next run tries again."
% (target, _missing_what))
else:
_stack_mark_done(path, e["_key"])
done += 1
# A SECOND PASS IS PART OF THE JOB, not an afterthought.
#
# Optional features are decided at configure time, and a stack builds
# things in one order -- so anything whose optional dependency came later
# in the same run is installed without it. Nothing fails, so this is the
# only place the question gets asked.
if done and not getattr(args, "status", False):
printInfo("")
printInfo("Optional features are decided when a package configures, so"
" anything built")
printInfo("before a dependency it can use was built WITHOUT it. "
" Check, and fix:")
printInfo(" packagemanager late-deps")
printInfo(" packagemanager late-deps --rebuild --run")
# A RUN THAT LEFT SOMETHING OUT DID NOT FINISH. "all N entries done" with
# a skipped entry is exactly the sentence that makes a missing package
# invisible for a month.
if notinbook and getattr(args, "skip_unknown", False):
printWarning("\nstack %s: %d entries done, %d SKIPPED (not in the book): %s"
% (args.name, done, len(notinbook), ", ".join(notinbook)))
printInfo(" the stack is not complete. Why each one is missing:"
+ "".join("\n blfs debug %s" % t for t in notinbook))
sys.exit(1)
printSuccess("\nstack %s: all %d entries done." % (args.name, done))
# =========================================================================== #
# command: add-dir-to-nimgnu [IMPLEMENTED]
# =========================================================================== #
def cmd_add_dir_to_nimgnu(args):
args.group = nimgnu_name(args.group) # force a nimgnu_ group
cmd_make_group_dir(args)
# =========================================================================== #
# command: list [IMPLEMENTED]
# =========================================================================== #
def cmd_list(args):
names = list_all_users()
if not names:
printInfo("No package-users found.")
return
width = max(len(n) for n in names) + 2
per_row = max(1, 78 // width)
printHeader("Package-users")
print()
for i in range(0, len(names), per_row):
print(" " + "".join(n.ljust(width) for n in names[i:i + per_row]))
print(f"\n{len(names)} package-user(s).")
# =========================================================================== #
# command: files_with_broken_id [IMPLEMENTED]
# =========================================================================== #
#
# DIAGNOSTIC (finds, does not change anything): lists files whose owning uid or
# gid no longer maps to a name -- i.e. orphaned files left behind when a user or
# group was deleted while its files still existed (e.g. a --purge without
# removing the files first, or an old manual userdel). Writes the paths to a
# file so you can review and decide what to do (reassign, remove, or ignore).
def cmd_reclaim(args):
"""Give a package back the files something else took from it.
The ownership rule -- the installer owns what it installs -- is right for
packages that legitimately overlap, and wrong when one package installs a
BUNDLED COPY of another (1.14.116). By the time that is noticed, the
victim's files belong to the wrong account: /usr/bin/dvbv5-scan owned by
p_libdisplay-info, listed in p_v4l-utils's manifest. Only root can put
them back, and only the manifest knows which they are.
"""
if os.geteuid() != 0:
printError("reclaim changes ownership; run it as root.")
return 1
if getattr(args, "all", False):
# ONE ACCIDENT, EVERY PACKAGE. A tree-wide chown -- lfs-kernel's
# headers step gave p_linux-headers all of /usr/include -- is not a
# per-package problem, and naming the victims one at a time means
# finding them first.
args.packages = sorted(_pkg_dirs())
printInfo("checking %d installed package(s) against their manifests"
% len(args.packages))
elif not args.packages:
printError("name a package, or --all")
return 2
rc = 0
for name in args.packages:
acct = pkgusr_name(name)
manifest = os.path.join(pkgusr_home(acct), "pkg.lst")
try:
paths = [l.strip() for l in open(manifest) if l.strip()]
except OSError as e:
# under --all a package with no manifest is not an error, it is
# a package that has not recorded one -- saying so 90 times
# buries the packages that DID lose files
if getattr(args, "all", False):
continue
printError("%s: cannot read %s (%s)" % (acct, manifest, e))
rc = 1
continue
wrong = []
for path in paths:
try:
st = os.lstat(path)
except OSError:
continue
try:
owner = pwd.getpwuid(st.st_uid).pw_name
except (KeyError, AttributeError):
owner = str(st.st_uid)
if owner != acct:
wrong.append((path, owner))
if not wrong:
printSuccess("%s: every recorded file is already its own (%d checked)"
% (acct, len(paths)))
continue
printWarning("%s: %d recorded file(s) belong to someone else"
% (acct, len(wrong)))
for path, owner in wrong[:20]:
printInfo(" %-52s now %s" % (path, owner))
if len(wrong) > 20:
printInfo(" ... and %d more" % (len(wrong) - 20))
if not args.run:
printInfo(" Dry run -- re-run with --run to give them back.")
continue
back = 0
for path, _owner in wrong:
try:
os.lchown(path, pwd.getpwnam(acct).pw_uid,
grp.getgrnam(acct).gr_gid)
back += 1
except (OSError, KeyError) as e:
printError(" could not return %s (%s)" % (path, e))
rc = 1
printSuccess("%s: %d file(s) returned" % (acct, back))
return rc
def _setid_table():
"""setid.conf as {path: (mode, owner, group, package)}.
The 5th column (package) is the one this command is here for: it is the
only written-down statement of WHICH package installs a setuid file, and
a setuid file held by any other package is damage, not history.
"""
out = {}
for d in (os.environ.get("PM_STACK_DIR") or "/etc/pkgusr/stacks",
os.path.join(os.path.dirname(os.path.abspath(__file__)), "stacks")):
conf = os.path.join(d, "setid.conf")
if not os.path.isfile(conf):
continue
try:
with open(conf, errors="replace") as f:
for line in f:
line = line.strip()
if not line or line.startswith("#"):
continue
col = [c.strip() for c in line.split("|")]
if len(col) < 2 or not col[0].startswith("/"):
continue
col += [""] * (5 - len(col))
out[col[0]] = (col[1], col[2] or "root", col[3] or "-", col[4])
except OSError:
continue
break
return out
def cmd_disown(args):
"""Take back the files a package cannot have installed.
`reclaim` is the other direction: it pushes ownership TOWARDS pkg.lst,
which is right when a package lost files and wrong when the record is
what went bad. This one distrusts the record, and it exists because the
record did go bad -- three times, the same way:
-rwx--x--x 1 p_webkitgtk p_webkitgtk /usr/bin/su
lrwxrwxrwx 1 p_webkitgtk p_webkitgtk /usr/sbin/iptables -> xtables-nft-multi
Neither is webkitgtk's. They were swept into its manifest because
attribution is by TIME WINDOW and a webkitgtk build runs for hours, and
the lease then acted on that manifest -- clearing su's setuid bit on the
way, because `chown` does that and says nothing. 1.14.310 shuts both
doors; this repairs machines that already went through them.
Two rules, and nothing is guessed from either:
* setid.conf names the package that installs each setuid file. Anyone
else holding it gives it back to root, with the mode from the table.
* a symlink is installed by whoever installed its target. A link whose
target belongs to another package goes back to that package.
Whatever is handed back is struck from the package's pkg.lst as well:
an ownership fix that leaves the record claiming the file is a fix that
undoes itself on the next `--clean`, which would DELETE these paths.
"""
if args.run and os.geteuid() != 0:
printError("disown changes ownership; run it as root.")
return 1
setid = _setid_table()
if not setid:
printWarning("no setid.conf found -- only the symlink rule can run")
if getattr(args, "all", False):
names = sorted(_pkg_dirs())
elif args.packages:
names = args.packages
else:
printError("name a package, or --all")
return 2
accounts = set()
for root in pkgusr_roots():
try:
accounts.update(os.listdir(root))
except OSError:
pass
def owner_of(path, follow=False):
try:
st = os.stat(path) if follow else os.lstat(path)
return pwd.getpwuid(st.st_uid).pw_name
except (KeyError, OSError):
return None
rc, total = 0, 0
for name in names:
acct = pkgusr_name(name)
pkg = re.sub(r"^" + re.escape(pkgusr_prefix()), "", acct)
lst = os.path.join(pkgusr_home(acct), "pkg.lst")
try:
paths = [l.strip() for l in open(lst, errors="replace") if l.strip()]
except OSError as e:
if getattr(args, "all", False):
continue
printError("%s: cannot read %s (%s)" % (acct, lst, e))
rc = 1
continue
wrong = [] # (path, reason, give-to, mode-or-None)
for path in paths:
if owner_of(path) != acct:
continue
pol = setid.get(path)
if pol and pol[3] != pkg:
who = pol[3] or "no package"
wrong.append((path, "setid.conf says %s installs it" % who,
"root", pol[0]))
continue
if os.path.islink(path):
tgt = os.path.realpath(path)
to = owner_of(tgt, follow=True)
if to and to != acct and to != "root" and to in accounts:
wrong.append((path, "points at %s's %s" % (to, tgt), to, None))
if not wrong:
if not getattr(args, "all", False):
printSuccess("%s: nothing it could not have installed (%d checked)"
% (acct, len(paths)))
continue
total += len(wrong)
printWarning("%s: %d recorded path(s) belong somewhere else" % (acct, len(wrong)))
for path, why, to, _m in wrong:
printInfo(" %-46s -> %-12s %s" % (path, to, why))
if not args.run:
printInfo(" Dry run -- re-run with --run to hand them back.")
continue
done = []
for path, _why, to, mode in wrong:
try:
if to == "root":
os.lchown(path, 0, 0)
# THE MODE COMES BACK TOO. The chown that took the file
# cleared its setuid bit; handing it to root without the
# bit leaves `su` exactly as broken as it was.
if mode and mode != "-" and not os.path.islink(path):
os.chmod(path, int(mode, 8))
else:
os.lchown(path, pwd.getpwnam(to).pw_uid, grp.getgrnam(to).gr_gid)
done.append(path)
except (OSError, KeyError) as e:
printError(" could not hand back %s (%s)" % (path, e))
rc = 1
# ...AND OUT OF THE RECORD, or `update <pkg> --clean` deletes them.
if done:
keep = [p for p in paths if p not in set(done)]
try:
with open(lst + ".new", "w") as f:
f.write("\n".join(keep) + ("\n" if keep else ""))
os.replace(lst + ".new", lst)
except OSError as e:
printError(" %s: could not rewrite pkg.lst (%s)" % (acct, e))
rc = 1
# lfs-helper's build manifests: $LFS_PKGUSR_DIR/manifests/<pkg>.files,
# which defaults to /usr/src/lfs-pkgusr. It is the list the LEASE
# reads, so leaving the path in it re-arms the whole thing.
_state = (os.environ.get("LFS_PKGUSR_DIR")
or os.path.join(os.path.dirname(BASE_DIR.rstrip("/")) or "/usr/src",
"lfs-pkgusr"))
for man in dict.fromkeys(
[os.path.join(_state, "manifests", pkg + ".files"),
os.path.join("/usr/src/lfs-pkgusr/manifests", pkg + ".files")]):
if not os.path.isfile(man):
continue
try:
lines = [l.rstrip("\n") for l in open(man, errors="replace")]
left = [l for l in lines if l.strip() not in set(done)]
if len(left) != len(lines):
with open(man + ".new", "w") as f:
f.write("\n".join(left) + ("\n" if left else ""))
os.replace(man + ".new", man)
printInfo(" %s: %d line(s) removed" % (man, len(lines) - len(left)))
except OSError as e:
printError(" %s (%s)" % (man, e))
rc = 1
printSuccess("%s: %d path(s) handed back" % (acct, len(done)))
if total == 0:
printSuccess("nothing to hand back")
elif not args.run:
printInfo("")
printInfo(" After --run, put the setuid bits back and check them:")
printInfo(" lfs-helper setid --run")
return rc
def cmd_which_package(args):
"""Which package installed this path.
The README has promised this command for as long as it has existed and
packagemanager never had it:
packagemanager: error: argument <command>: invalid choice: 'which-package'
`lfs-helper which-package` does exist, but it reads the BUILD manifests --
the chapter 5-9 steps -- so for anything installed afterwards it answers
Nothing recorded means it was installed before file tracking existed,
or outside these tools entirely.
which is wrong twice over: these tools installed it, and they did record
it, in the package user's own pkg.lst. Two file registries, one question,
and each tool could only see its own.
"""
rc = 0
for target in args.paths:
path = os.path.abspath(os.path.expanduser(target))
printHeader(path)
owner = None
try:
owner = pwd.getpwuid(os.stat(path).st_uid).pw_name
except (KeyError, OSError):
pass
# The manifests: which package RECORDS having installed it.
hits = []
for root in pkgusr_roots():
try:
names = sorted(os.listdir(root))
except OSError:
continue
for n in names:
lst = os.path.join(root, n, "pkg.lst")
if not os.path.isfile(lst):
continue
try:
with open(lst, errors="replace") as f:
for line in f:
if line.strip() == path:
hits.append(n)
break
except OSError:
continue
if hits:
for n in hits:
printSuccess(f" recorded by: {n} ({os.path.join(pkgusr_home(n), 'pkg.lst')})")
else:
printWarning(" no pkg.lst records this path")
rc = 1
# ...and who owns it now, which is the other half of the answer and
# can disagree with the manifest.
if owner:
printInfo(f" owned on disk by: {owner}")
if hits and owner not in hits:
printWarning(f" the owner is not among the packages that "
f"recorded it -- one of the two is wrong")
else:
printWarning(" not on disk, or its owner does not resolve")
if not hits:
printInfo(" The build steps have their own manifests; for anything")
printInfo(" installed during chapters 5-9 ask:")
printInfo(f" lfs-helper which-package {path}")
return rc
def cmd_files_with_broken_id(args):
out_file = args.outfile or "/tmp/invalid_files.txt"
printInfo("Scanning for orphaned files (uid/gid with no matching name); "
"this runs 'find /' and may take a while. Nothing is changed.")
cmd = ["find", "/",
"(", "-path", "/proc", "-o", "-path", "/sys", "-o", "-path", "/dev",
"-o", "-path", "/run", "-o", "-path", "/mnt", ")", "-prune", "-o",
"(", "-nouser", "-o", "-nogroup", ")", "-print"]
try:
with open(out_file, "w") as f:
subprocess.run(cmd, stdout=f, stderr=subprocess.DEVNULL)
n = sum(1 for _ in open(out_file))
except OSError as e:
printError(f"scan failed: {e}")
return
if n:
printWarning(f"Found {n} orphaned file(s) (unresolved uid/gid). "
f"List written to {out_file} for you to review.")
else:
printSuccess("No orphaned files found (all uid/gid resolve).")
# =========================================================================== #
# command: init [IMPLEMENTED]
# =========================================================================== #
#
# Turn an existing system into a package-user system: create the `install`
# group (gid 9999 by default, per the hint) and turn the given directories into
# install dirs (group install, group-writable, sticky). Every command is shown
# and confirmed before running (y = yes, n = skip, a = yes-to-all, q = quit).
#
# Per the hint, the sticky bit should be OFF while a temporary owner user still
# needs to overwrite files -- use --no-sticky for that phase, then run
# 'fix-install' afterwards to set sticky everywhere.
_CONFIRM = {"all": False}
def confirm_run(display, func):
if _CONFIRM["all"]:
print(f" + {display}")
return func()
while True:
ans = input(f" run: {display} [y/N/a/q]? ").strip().lower()
if ans in ("y", "yes"):
return func()
if ans in ("", "n", "no"):
print(" skipped")
return None
if ans in ("a", "all"):
_CONFIRM["all"] = True
return func()
if ans in ("q", "quit"):
printWarning("quit.")
sys.exit(0)
def cmd_init(args):
gid = args.gid
mode = "g+w" if args.no_sticky else "g+w,o-t" # install dirs: no sticky
# build the list of commands this would run
steps = [] # (display, argv)
if not _group_exists("install"):
steps.append((f"groupadd -g {gid} install",
["groupadd", "-g", str(gid), "install"]))
dirs = list(args.dirs)
if args.from_file:
try:
with open(args.from_file) as f:
dirs += [ln.strip() for ln in f
if ln.strip() and not ln.lstrip().startswith("#")]
except OSError as e:
printError(f"could not read {args.from_file}: {e}")
for d in dirs:
if not os.path.isdir(d):
printWarning(f" {d}: not a directory -- skipping.")
continue
steps.append((f"chgrp install {d} && chmod {mode} {d}",
("compound", d)))
printHeader("Initialise package-user system")
print()
if _group_exists("install"):
printInfo("install group already exists.")
if not steps:
printInfo("Nothing to do. Pass dirs (or --from-file installdirs.lst) "
"to make install dirs.")
return
# DRY RUN (default): just show the commands
if not args.run:
print("Would run:")
for display, _ in steps:
print(f" {display}")
printInfo("\nDry run -- re-run with --run to execute "
"(you'll confirm each command, or use --yes).")
return
# RUN: confirm each command (y = yes, n = skip, a = all, q = quit)
if args.yes:
_CONFIRM["all"] = True
done = 0
for display, action in steps:
if isinstance(action, tuple) and action[0] == "compound":
d = action[1]
def do(d=d):
return _run(["chgrp", "install", d]) and _run(["chmod", mode, d])
if confirm_run(display, do):
done += 1
else:
if confirm_run(display, lambda a=action: _run(a)):
done += 1
printSuccess(f"init done ({done} command(s) run).")
# =========================================================================== #
# command: fix-install [IMPLEMENTED]
# =========================================================================== #
#
# Scan the whole filesystem for directories owned by group 'install' and make
# sure each is group-writable + sticky (the correct install-dir permissions).
# Dry-run by default (lists what it would change); --run applies the fix.
# Scan the whole filesystem for directories owned by group 'install' and make
# sure each is correctly group-writable and NOT sticky. Dry-run by default
# (lists what it would change and stores the list); --run applies the fix and
# can reuse the stored dry-run instead of re-scanning.
def cmd_fix_install(args):
if not _group_exists("install"):
printError("no 'install' group -- run 'packagemanager init' first.")
return
want_sticky = args.sticky
mode = "g+w,o+t" if want_sticky else "g+w,o-t"
def scan_wrong():
# EVERY DIRECTORY WITH GROUP install, FROM find(1).
#
# This asked forall_direntries_from -- the C helper from the
# package-users hint -- and on an older system that helper is an
# older build: it may stop at mount points or list by user only, and
# on a machine in daily use the install group is on hundreds of
# directories, not the twenty-two seed ones. "Wherever the install
# group is set, the directory needs g+w and the sticky bit" is a
# one-line find, so ask find, and prune only the virtual filesystems.
roots = [r for r in (getattr(args, "roots", None) or ["/"])]
# /snapshots holds btrfs/rsync copies of the whole tree: every
# install directory again, once per snapshot. Fixing those is
# pointless (a snapshot is read-only, or a record) and slow (it
# multiplies the scan by the snapshot count), so it is pruned by
# default; --exclude adds more, and --no-default-excludes puts
# /snapshots back if someone really wants it.
prune = ["/proc", "/sys", "/dev", "/run", "/tmp"]
if not getattr(args, "no_default_excludes", False):
prune += ["/snapshots", "/.snapshots"]
prune += [x.rstrip("/") or "/" for x in (getattr(args, "exclude", None) or [])]
printInfo("Scanning %s for directories owned by group 'install'"
" (pruning %s) ..." % (", ".join(roots), ", ".join(prune)))
cmd = ["find"] + roots + ["("] + \
sum([["-path", x, "-o"] for x in prune], [])[:-1] + \
[")", "-prune", "-o", "-type", "d", "-group", "install", "-print0"]
try:
r = subprocess.run(cmd, capture_output=True, timeout=1800)
entries = [e.decode("utf-8", "replace")
for e in r.stdout.split(b"\0") if e]
except (OSError, subprocess.SubprocessError) as e:
printError("find failed (%s); falling back to forall_direntries_from" % e)
entries = owned_entries("install")
if entries is None:
printError("forall_direntries_from not found either; cannot scan.")
return None
printInfo(" %d director%s group 'install'"
% (len(entries), "y carries" if len(entries) == 1 else "ies carry"))
wrong = []
for path in entries:
try:
st = os.stat(path)
except OSError:
continue
if not stat.S_ISDIR(st.st_mode):
continue
gw = bool(st.st_mode & stat.S_IWGRP)
sticky = bool(st.st_mode & stat.S_ISVTX)
if (not gw) or (want_sticky and not sticky) or \
((not want_sticky) and sticky):
wrong.append(path)
return wrong
if not args.run:
wrong = scan_wrong()
if wrong is None:
return
save_plan("fix-install", "install", wrong)
for path in wrong:
print(f" would fix {path}")
printInfo(f"\n{len(wrong)} install dir(s) need fixing. Re-run with --run "
f"(it can reuse this scan).")
return
wrong = use_stored_plan("fix-install", "install", assume_yes=args.yes)
if wrong is None:
wrong = scan_wrong()
if wrong is None:
return
for path in wrong:
_run(["chmod", mode, path])
clear_plan("fix-install", "install")
printSuccess(f"Fixed {len(wrong)} install dir(s).")
# --------------------------------------------------------------------------- #
# cli
# --------------------------------------------------------------------------- #
PM_HELP_EPILOG = """\
commands, by what you are doing:
finding out what is installed
list every package user on the system
info <pkg> what a package installed, and where it came from
search <pkg> find a package in the book
check <pkg> is it installed, and does it look right
verify [--fix] check every package, optionally repairing
files_with_broken_id files owned by a user that no longer exists
installing and updating
install <pkg> build and install as a package user
update [<pkg>] rebuild what the book has a newer version of
remove <pkg> uninstall and delete the package user
pip install <module> install a Python module as a package user
dependencies <pkg> what it needs, in build order
install scripts
script <phase> <pkg> run one phase of a package's script
regenerate-script <pkg> rewrite it from the book
script-status which scripts are edited, current, or outdated
migrate-scripts move scripts to a newer book, keeping edits
template <pkg> skeleton for a package not in the book
users, groups and permissions
add-user <pkg> create a package user
user / shared-user application users
sysgroup collector groups
make-group-dir <dir> hand a directory to a collector group
add-dir-to-sysgroup <dir> share an existing directory
fix-group-dir / fix-install
repair permissions
a whole environment at once
stack <name> replay a stack file: packages, order, options
stack <name> --only <pkg> one entry of it
stack <name> --take-new take the shipped stack beside your edited one
bootstrap get a freshly booted system to where this works
env build-time variables the book expects exported
features, not files
probe [<pkg>] run the stacks' expect= checks against what is
installed: is the FEATURE there, not the file
late-deps what was built before a dependency it can use
late-deps --rebuild --run rebuild those, in dependency order
late-deps <pkg> --why the edges it used, and which names resolved
settings and housekeeping
config show or change settings
paths where everything lives
init first-run setup
errors what went wrong in the last run
plan what a run would do, before doing it
reload-pkg-list refresh the cached package list
reclaim <pkg> | --all give a package back files another took from it
disown <pkg> | --all the other direction: take back what a package
cannot have installed (setuid files, other
packages' symlinks) and clean its pkg.lst
which-package <path> which package installed it
blfs ... pass a command straight to blfs
LFS base packages are the `lfs` tool's job; BLFS packages are handled here.
"""
# --------------------------------------------------------------------------- #
# housekeeping run on every invocation: cheap checks that stop a temporary
# workaround from quietly becoming permanent
# --------------------------------------------------------------------------- #
_CA_BUNDLES = ["/etc/ssl/certs/ca-bundle.crt", "/etc/ssl/ca-bundle.crt",
"/etc/pki/tls/certs/ca-bundle.crt",
"/etc/ssl/certs/ca-certificates.crt"]
_WGETRC_MARK = "lfs-temporary-no-verify"
def restore_wget_verification():
"""Re-enable certificate checking once CA certificates exist.
The build disables it in /etc/wgetrc so a certificate-less system can fetch
make-ca at all. Leaving that in place afterwards would mean every download
on this system stays unverified forever, so undo it the moment it is no
longer needed."""
rc = "/etc/wgetrc"
if not os.path.isfile(rc):
return
try:
text = open(rc).read()
except OSError:
return
if _WGETRC_MARK not in text:
return
have_ca = next((c for c in _CA_BUNDLES if os.path.isfile(c)
and os.path.getsize(c) > 0), None)
if not have_ca:
return
# keep everything outside the marked block, drop the block itself
out, inside = [], False
for line in text.splitlines(True):
if _WGETRC_MARK in line:
inside = "end " not in line
continue
if not inside:
out.append(line)
try:
with open(rc, "w") as f:
f.writelines(out)
except OSError as e:
printWarning(f"could not restore certificate checking in {rc}: {e}")
return
printInfo(f"CA certificates found ({have_ca}) -- certificate checking "
f"re-enabled in {rc}")
def check_make_ca_generated():
"""make-ca installed but no certificate bundle? Say what finishes the job.
`make-ca -g` reports "No update required!" and generates nothing when it
thinks its copy of certdata.txt is current -- which it is, straight after
install, even though no bundle has ever been written. The install then
looks successful while the system still has no certificates, and every
HTTPS fetch keeps warning."""
import shutil as _sh
if not _sh.which("make-ca") and not os.path.isfile("/usr/sbin/make-ca"):
return
if any(os.path.isfile(c) and os.path.getsize(c) > 0 for c in _CA_BUNDLES):
return
printWarning("\nmake-ca is installed but this system still has no CA "
"certificate bundle.")
printWarning("`make-ca -g` skips generating when it considers its copy of")
printWarning("certdata.txt current, which it is right after installing.")
printWarning("Force it once:")
printWarning(" /usr/sbin/make-ca -g --force")
printWarning("Then HTTPS verification works and the tools re-enable it "
"automatically.\n")
def ensure_wget_workaround():
"""Let wget fetch anything at all on a system with no CA certificates.
This is the other half of restore_wget_verification(). Without it the
first thing anyone tries -- installing make-ca, so that certificates exist
-- fails, because fetching make-ca itself needs a verified connection:
ERROR: cannot verify ftpmirror.gnu.org's certificate ...
ERROR: md5sum check failed
/etc/wgetrc is used rather than a $PATH wrapper because builds run through
`su -` with the package user's own environment, where a wrapper directory
on PATH is never seen.
"""
if any(os.path.isfile(c) and os.path.getsize(c) > 0 for c in _CA_BUNDLES):
return # certificates exist: nothing to do
if not shutil.which("wget"):
return # no wget yet, nothing to configure
rc = "/etc/wgetrc"
try:
text = open(rc).read() if os.path.isfile(rc) else ""
except OSError:
return
if _WGETRC_MARK in text:
return # already done
try:
with open(rc, "a") as f:
f.write(
"\n# --- %s ----------------------------------------\n"
"# This system has no CA certificates yet, so wget cannot\n"
"# verify anything. Downloads are UNVERIFIED until make-ca\n"
"# is installed; this block is removed automatically then.\n"
"check_certificate = off\n"
"# --- end %s ------------------------------------\n"
% (_WGETRC_MARK, _WGETRC_MARK))
except OSError as e:
printWarning(f"could not write {rc}: {e}")
return
printWarning(
"No CA certificates on this system yet, so wget could not verify\n"
"anything and every download would fail. Certificate checking has\n"
"been turned OFF in /etc/wgetrc so that make-ca can be fetched.\n"
"\n"
" Downloads are UNVERIFIED until make-ca is installed.\n"
" This is undone automatically as soon as certificates exist.\n")
def _current_blfs_book():
"""The book blfs would use right now ('' when unknown).
Asks `blfs books --brief` -- default + cached list, NO network -- and
reads the Default line. The old parser took the first [bracketed]
filename from the full listing, which is the alphabetically-first cached
book: right only while the default happened to sort first, and 12.4 sorts
before 13.0. An older blfs without --brief gets the plain listing, whose
Default line reads the same."""
rc, out, _e = run_blfs(["books", "--brief"])
if rc != 0:
rc, out, _e = run_blfs(["books"])
if rc != 0:
return ""
for ln in out.splitlines():
if ln.startswith("Default:"):
return ln.split(":", 1)[1].strip()
return ""
def _warn_init_mismatch():
"""The BLFS book's init flavour should match the system's.
A systemd book emits `systemctl` calls in its install scripts; on a SysV
system those fail with "command not found" in the middle of an otherwise
successful install. Since the generated scripts now shim systemctl into
a visible no-op where the binary is absent, USING the other flavour is a
reasonable choice -- the SysV book is no longer maintained -- so this is
asked ONCE and the answer is recorded. A different mismatch (the pair
changing direction, or a different book) asks again."""
have_systemd = (os.path.isdir("/usr/lib/systemd/system")
or bool(shutil.which("systemctl")))
book = _current_blfs_book()
if not book:
return
book_systemd = "systemd" in book.lower()
if book_systemd == have_systemd:
return
pair = "%s-on-%s" % ("systemd" if book_systemd else "sysv",
"systemd" if have_systemd else "sysv")
if load_config().get("init_mismatch_ok") == pair:
return
printWarning("\nThe BLFS book in use does not match this system's init:")
printInfo(f" book : {book} ({'systemd' if book_systemd else 'SysV'})")
printInfo(f" system : {'systemd' if have_systemd else 'SysV'}")
printInfo("")
printInfo("Install scripts from the wrong book carry commands this system")
printInfo("does not have. Generated scripts turn a missing systemctl into")
printInfo("a visible skip, so using this book anyway is workable. Or")
printInfo("switch with:")
printInfo("")
printInfo(" blfs fetch %s" % ("stable" if have_systemd is False
else "stable-systemd"))
printInfo("")
if sys.stdin.isatty():
if ask_yes_no("Keep using this book, and stop this warning?"):
cfg = load_config()
cfg["init_mismatch_ok"] = pair
if save_config(cfg):
printInfo(" recorded -- undo with: packagemanager config "
"(remove init_mismatch_ok)")
else:
printWarning(" could not save the answer; it will ask again.")
def _hint_generate_ca_bundle():
"""make-ca is installed but the certificate store is empty.
Installing make-ca does not create any certificates -- it installs the
tool that generates them. Until it is run there is still no CA bundle,
wget still cannot verify anything, and the /etc/wgetrc workaround stays in
place. The step is easy to miss because the install reports success."""
if any(os.path.isfile(c) and os.path.getsize(c) > 0 for c in _CA_BUNDLES):
return # certificates exist already
if not (os.path.isfile("/usr/sbin/make-ca")
or shutil.which("make-ca")):
return # make-ca not installed yet
printWarning("\nmake-ca is installed but no CA certificates exist yet.")
printInfo("Installing it does not create them -- it installs the tool.")
printInfo("Generate the certificate store now:")
printInfo("")
printInfo(" make-ca -g")
printInfo("")
printInfo("Then HTTPS verification works, and the temporary")
printInfo("'check_certificate = off' in /etc/wgetrc is removed "
"automatically.\n")
def _file_md5(path):
try:
with open(path, "rb") as f:
return hashlib.md5(f.read()).hexdigest()
except OSError:
return None
def _in_chroot():
"""Is there a host at all?
The whole point of the staleness warning is "the copy outside is newer
than the copy in here", and outside stops existing the moment the system
boots on its own. /proc/1/root is pid 1's root: in a chroot it is not
this root.
"""
try:
return os.stat("/proc/1/root/.").st_ino != os.stat("/").st_ino \
or os.stat("/proc/1/root/.").st_dev != os.stat("/").st_dev
except OSError:
return False
def warn_if_tools_are_stale():
"""Tell the user when the CHROOT is running older tools than the host.
The tools are copied into the tree; edit them outside and the chroot keeps
running the old copy until they are copied again. That is a genuinely
confusing failure mode -- behaviour that was fixed "yesterday" reappears --
so say so plainly.
ON A BOOTED SYSTEM THERE IS NO HOST. The stamp then records what was
copied in during the build, so every later `make install` on the machine
itself made all seven tools "differ" and this printed, before every
command, an instruction to run sync-tools on a host that no longer
exists. `make install` writes the stamp now; this says nothing unless
there is an outside to be behind.
"""
if not _in_chroot():
return
stamp = "/usr/share/lfs/tool-stamps.json"
if not os.path.isfile(stamp):
return
try:
stamps = json.load(open(stamp))
except (OSError, ValueError):
return
stale = []
for name, want in stamps.items():
path = shutil.which(name) or os.path.join("/usr/bin", name)
have = _file_md5(path)
if have and want and have != want:
stale.append(name)
if stale:
printWarning(
"\nThese tools differ from the copies on the host: "
+ ", ".join(sorted(stale)))
printWarning(" You are probably running an older version in here. "
"On the HOST, run:")
printWarning(" lfs build-system sync-tools --run")
printWarning(" lfs build-system install-tools --run --force\n")
def main():
ap = argparse.ArgumentParser(prog="packagemanager",
description="pkgusr-model package manager")
ap.add_argument("-V", "--version", action="version",
version=f"packagemanager {TOOL_VERSION} (build {_build_id()}, "
f"plan cache v{PLAN_VERSION})")
ap.add_argument("-v", "--verbose", action="store_true",
help="show the system commands being run and their output")
ap.epilog = PM_HELP_EPILOG
ap.formatter_class = argparse.RawDescriptionHelpFormatter
sub = ap.add_subparsers(dest="command", required=True, metavar="<command>")
# ---- info (implemented) ----
p = sub.add_parser("verify", help="recheck install state of all/named "
"package-users (shows WHY; --fix regenerates pkg.lst)")
p.add_argument("packages", nargs="*", help="default: all package-users")
p.add_argument("--fix", action="store_true",
help="regenerate pkg.lst for users that own files but have no "
"manifest (can flip them to 'installed')")
p.add_argument("--set-validate-command", action="store_true",
help="create/edit a package's persistent 'validate' script "
"(in /usr/src/<pkg>/validate) in your editor")
p.set_defaults(func=cmd_verify)
p = sub.add_parser("info", help="package-user info (short for 'all', "
"detailed for named packages)")
p.add_argument("packages", nargs="*", default=["all"],
help="package-user name(s), or 'all'")
p.add_argument("-s", "--short", action="store_true",
help="force the short one-line-per-package view")
p.add_argument("-l", "--long", action="store_true",
help="force the detailed view (link, md5, info, deps, groups)")
p.add_argument("--no-book", action="store_true",
help="don't look up book details for not-installed packages")
p.set_defaults(func=cmd_info)
# ---- the rest (stubbed for now) ----
p = sub.add_parser("check", help="report install state of a package")
p.add_argument("packages", nargs="+")
p.add_argument("--no-book", action="store_true",
help="skip the book version comparison (offline, faster)")
p.set_defaults(func=cmd_check)
p = sub.add_parser("network", help="every interface, its ifconfig file, "
"what runs at boot, and what is missing")
p.set_defaults(func=cmd_network)
p = sub.add_parser("rename", help="rename a package: account, group, home,"
" script, stack entry and records (dry run without --run)")
p.add_argument("old")
p.add_argument("new")
p.add_argument("--run", action="store_true")
p.set_defaults(func=cmd_rename)
p = sub.add_parser("dev", help="build a git package from your own local "
"checkout (a program you develop here)")
p.add_argument("name", nargs="?")
p.add_argument("path", nargs="?", help="the checkout, e.g. ~/dev/SwAS")
p.add_argument("--off", action="store_true", help="back to its url=")
p.add_argument("--hub", metavar="CHECKOUT",
help="your checkout; a bare copy in /srv/git becomes the source,"
" fed by `git push local`")
p.add_argument("--run", action="store_true")
p.set_defaults(func=cmd_dev)
p = sub.add_parser("books", help="every cached LFS/BLFS book and who reads"
" it (default, stacks, machine.conf); --prune the rest")
p.add_argument("--prune", action="store_true",
help="remove the books nothing uses (with --run)")
p.add_argument("--run", action="store_true")
p.set_defaults(func=cmd_books)
p = sub.add_parser("provider", help="which package provides a program "
"(itstool -> itstool); prints nothing if the book has none")
p.add_argument("tools", nargs="+")
p.set_defaults(func=cmd_provider)
p = sub.add_parser("search", help="find a package in the book")
p.add_argument("packages", nargs="+")
p.set_defaults(func=cmd_search)
p = sub.add_parser("dependencies", aliases=["deps"], help="recursive deps")
p.add_argument("packages", nargs="+")
p.add_argument("--tree", action="store_true", help="show blfs dependency tree")
p.add_argument("--no-recommended", action="store_true",
help="required deps only (skip recommended)")
p.add_argument("--optional", action="store_true",
help="also follow optional dependencies")
p.set_defaults(func=cmd_dependencies)
p = sub.add_parser("install", help="resolve deps + build/install missing")
p.add_argument("packages", nargs="+")
p.add_argument("-r", "--reinstall", action="store_true",
help="rebuild even packages already up to date")
p.add_argument("-i", "--ignore", help="comma-separated anchors to skip")
p.add_argument("-e", "--ignore-recommended", action="store_true",
help="required deps only (skip recommended)")
p.add_argument("--optional", action="store_true",
help="also pull in optional deps")
p.add_argument("--recursive", action="store_true",
help="also resolve and build the package's dependencies "
"(default: just the named package(s))")
p.add_argument("--update-deps", action="store_true",
help="also rebuild dependencies that are installed but "
"older than the book (default: leave them alone)")
p.add_argument("-f", "--force", action="store_true",
help="(deprecated: non-recursive is now the default; kept for "
"compatibility)")
p.add_argument("--keep-sources", action="store_true",
help="keep old unpacked source dirs/archives (default: remove "
"them after a successful install)")
p.add_argument("--local", metavar="SCRIPT",
help="install a custom package from a local install script "
"(made with 'template'); skips BLFS")
p.add_argument("--select", action="store_true",
help="interactively pick which packages of the plan to build "
"(all selected by default; use with --run to build the "
"chosen set, or on a dry-run to preview/cache it)")
p.add_argument("--test", action="store_true",
help="also run the package's test suite (between build and install)")
p.add_argument("--clean", action="store_true",
help="after install, remove stale files from a previous "
"version (configs and the source tree kept)")
p.add_argument("--no-git-check", action="store_true",
help="do not ask the git remotes whether a branch moved "
"(no network)")
p.add_argument("--regenerate", action="store_true",
help="rebuild scripts from the book, ignoring any edited "
"script in the package user's home")
p.add_argument("--ignore-prereqs", action="store_true",
help="start even when a build tool or variable the plan's "
"scripts need is missing (it will fail there)")
p.add_argument("--run", action="store_true",
help="actually build/install (default is a dry-run plan)")
p.add_argument("--yes", action="store_true",
help="skip the confirmation prompt when using --run")
p.add_argument("-o", "--outdir", metavar="DIR",
help="where to write generated scripts "
"(default: /tmp/packagemanager/install_files)")
p.set_defaults(func=cmd_install)
p = sub.add_parser("update", help="update installed packages to current book "
"versions. 'installed' = package-user + /usr/src/<name> + "
"(install_last version OR an existing pkg.lst path). "
"Not-installed named packages are reported, not silently "
"skipped; use 'verify' to recheck all.")
p.add_argument("packages", nargs="*",
help="packages to update (with deps); default: all installed")
p.add_argument("--toolchain", action="store_true",
help="also rebuild toolchain packages (glibc, binutils, "
"gcc) from the LFS book -- risky on a running system")
p.add_argument("--recursive", action="store_true",
help="also update the named package(s)' dependencies "
"(default: just the named package(s))")
p.add_argument("-f", "--force", action="store_true",
help="(deprecated: non-recursive is now the default; kept for "
"compatibility)")
p.add_argument("-R", "--dependents", action="store_true",
help="also update everything that (recursively) depends on the "
"named package(s) -- rebuild reverse deps too")
p.add_argument("-v", "--verbose", action="store_true",
help="show every requirement that differs, not the first 12")
p.add_argument("--stack", metavar="NAME",
help="only the packages of that stack (media, sway, "
"network, services, vision) -- 'what in the media "
"chain has the book moved past', without typing the "
"names. Combine with --select to pick from them.")
p.add_argument("--select", action="store_true",
help="interactively pick which packages of the plan to update "
"(all selected by default; combine with --run to update the "
"chosen set; on a dry-run it previews and caches the choice)")
p.add_argument("--regenerate", action="store_true",
help="rebuild scripts from the book, ignoring any edited "
"script in the package user's home")
p.add_argument("--ignore-prereqs", action="store_true",
help="start even when a build tool or variable the plan's "
"scripts need is missing (it will fail there)")
p.add_argument("--clean", action="store_true",
help="before installing each new version, remove the old "
"files (except configs under /etc,/var,... and the source "
"tree) so no stale files are left behind")
p.add_argument("--test", action="store_true",
help="also run the package's test suite (between build and install)")
p.add_argument("--local", metavar="SCRIPT",
help="update one package from a local install script "
"(custom or git-based); skips BLFS")
p.add_argument("--reinstall", action="store_true",
help="rebuild even packages that are already up to date")
p.add_argument("--install-only", action="store_true",
help="run only the install phase (no recompile) -- use to "
"retry after an install-step failure without rebuilding")
p.add_argument("--no-reinstall", action="store_true",
help="with --yes, do NOT re-apply already-up-to-date packages "
"(otherwise --yes accepts the re-apply prompt)")
p.add_argument("--keep-sources", action="store_true",
help="keep old unpacked source dirs/archives (default: remove "
"them after a successful update)")
p.add_argument("-e", "--ignore-recommended", action="store_true",
help="required deps only (skip recommended)")
p.add_argument("--optional", action="store_true", help="also follow optional deps")
p.add_argument("--run", action="store_true",
help="actually update (default is a dry-run plan)")
p.add_argument("--yes", action="store_true", help="skip prompts / reuse cache")
p.add_argument("-o", "--outdir", metavar="DIR",
help="where to write generated scripts "
"(default: /tmp/packagemanager/install_files)")
p.set_defaults(func=cmd_update)
p = sub.add_parser("remove", help="uninstall a package's files "
"(keep user+home; --purge to remove those too)")
p.add_argument("packages", nargs="+")
p.add_argument("--purge", action="store_true",
help="also remove the package-user, its group and /usr/src/<name>")
p.add_argument("--root", action="store_true",
help="scan/remove as root (thorough: finds files the user "
"itself can't reach) instead of as the package user")
p.add_argument("--run", action="store_true",
help="actually do it (default is a dry-run plan)")
p.add_argument("--yes", action="store_true",
help="skip the confirmation prompt when using --run")
p.set_defaults(func=cmd_remove)
p = sub.add_parser("reload-pkg-list", help="regenerate /usr/src/<name>/pkg.lst")
p.add_argument("--root", action="store_true",
help="scan as root: slower, but reaches directories the "
"package user cannot enter")
p.add_argument("packages", nargs="+")
p.add_argument("--background", action="store_true",
help="run detached and return at once (the old default)")
p.add_argument("--foreground", action="store_true",
help="wait for it instead of running as a disowned task")
p.set_defaults(func=cmd_reload_pkg_list)
# `bootstrap`, not `setup`.
#
# Every other command here is a verb naming what it does to PACKAGES --
# install, update, remove, verify. "setup" named neither the thing nor the
# action, and the vocabulary already used the word for something else:
# `packagemanager user setup` applies the shared-account treatment to an
# existing application user. Two unrelated commands, one word.
#
# `setup` stays as a hidden alias: an older installed `lfs` calls
# `packagemanager setup --sources` from get-sources, and a rename that
# breaks the tool that downloads the sources is a rename that costs a
# reboot to discover.
p = sub.add_parser(
"bootstrap", aliases=["setup"],
help="get a freshly booted system to where blfs and packagemanager work",
description=_SETUP_HELP,
formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument("--sources", action="store_true",
help="print the download URLs this needs, one per line, "
"and exit (used by `lfs build-system get-sources`)")
p.add_argument("--run", action="store_true", help="apply (default: dry run)")
p.add_argument("--force", action="store_true",
help="re-install everything, including what is already there")
p.add_argument("--sources-dir", dest="sources_dir",
help="where the downloaded tarballs and wheels are "
"(default: /sources)")
p.add_argument("--yes", action="store_true",
help="answer yes to the questions the wget build asks")
p.set_defaults(func=cmd_setup)
p = sub.add_parser(
"env",
help="build-time variables the book expects to be exported")
p.add_argument("--set", action="append", metavar="NAME=VALUE",
help="set one (repeatable)")
p.add_argument("--ask", action="store_true",
help="ask for every variable a generated script needs")
p.add_argument("--refresh", action="store_true",
help="drop scripts from an older blfs so the current "
"questions can be seen")
p.add_argument("--system", action="store_true",
help="this system's own facts -- main user, the three "
"prefixes, the Shared directory -- as PKGUSR_SYS_* "
"in /etc/profile.d/pkgusr-system.sh, so a script can "
"ask instead of guessing")
p.add_argument("--run", action="store_true",
help="with --system: write the file (default: print it)")
p.set_defaults(func=cmd_env)
p = sub.add_parser(
"stack",
help="replay a stack file: one command, one whole environment")
p.add_argument("name", nargs="?",
help="stack name or path (e.g. sway, ./sway.stack); "
"without one, list the stacks there are")
p.add_argument("--run", action="store_true",
help="actually build (default: print the plan)")
p.add_argument("--yes", "-y", action="store_true",
help="answer yes to every prompt (PM_YES for the installs)")
p.add_argument("--force", action="store_true",
help="rebuild git entries even when their program is on PATH")
p.add_argument("--status", action="store_true",
help="how far this stack has got, and what is next")
p.add_argument("--no-git", action="store_true",
help="skip git and tarball entries (the extras); useful "
"before git itself is built")
p.add_argument("--full", action="store_true",
help="with --status: every package in the tree and its state")
p.add_argument("--ignore-prereqs", action="store_true",
help="start entries even when a build tool or variable "
"their scripts need is missing")
p.add_argument("--skip-unknown", action="store_true",
help="run the entries the book does have, and report the "
"others (exits non-zero: the stack is NOT complete)")
p.add_argument("--only", action="append", metavar="ENTRY",
help="run just these entries (repeatable or comma-separated); "
"they are rebuilt even if the cache says done")
p.add_argument("--take-new", action="store_true",
help="replace this stack file with the shipped .new one "
"(yours is kept as .bak) and run that")
p.add_argument("--check-refs", action="store_true",
help="ask each git entry's remote whether its ref= still "
"exists, and stop -- builds nothing")
p.add_argument("--avoided", action="store_true",
help="list the packages this stack never builds, and stop")
p.set_defaults(func=cmd_stack)
p = sub.add_parser("reclaim",
help="give a package back files another package took "
"from it (ownership only; nothing is rebuilt)")
p.add_argument("packages", nargs="*")
# ONE ACCIDENT, EVERY PACKAGE. A tree-wide chown (lfs-kernel's headers
# step gave p_linux-headers all of /usr/include) is not a per-package
# problem, and naming the victims one at a time means finding them first.
p.add_argument("--all", action="store_true",
help="every installed package, not a named list")
p.add_argument("--run", action="store_true",
help="actually change ownership (default: dry run)")
p.set_defaults(func=cmd_reclaim)
p = sub.add_parser("disown",
help="hand back the files a package cannot have "
"installed (setuid files and other packages' "
"symlinks), and strike them from its pkg.lst")
p.add_argument("packages", nargs="*")
p.add_argument("--all", action="store_true",
help="every installed package, not a named list")
p.add_argument("--run", action="store_true",
help="actually change ownership (default: dry run)")
p.set_defaults(func=cmd_disown)
p = sub.add_parser("which-package", aliases=["which-pkg"],
help="which package installed a path (reads pkg.lst)")
p.add_argument("paths", nargs="+", metavar="PATH")
p.set_defaults(func=cmd_which_package)
p = sub.add_parser("pip", help="install or update a Python module as a "
"package user (not as root)")
p.add_argument("action", choices=["install", "update", "upgrade", "record"],
help="install, update, or record (write the manifest and "
"version for a module installed before 1.14.37, which "
"wrote no record at all)")
p.add_argument("modules", nargs="*", help="module names, e.g. beautifulsoup4")
p.add_argument("--find-links", metavar="DIR",
help="install from local wheels in DIR instead of the network "
"(there is no network inside the chroot)")
p.add_argument("--no-deps", action="store_true",
help="do not let pip pull dependencies -- name them yourself, "
"in order (what `setup` does, offline)")
p.set_defaults(func=lambda a: cmd_pip_record(a.modules) if a.action == "record"
else cmd_pip(
argparse.Namespace(modules=a.modules,
upgrade=(a.action in ("update", "upgrade")),
no_deps=a.no_deps,
find_links=a.find_links)))
p = sub.add_parser("add-user",
help="create a PACKAGE user (installs files; home /usr/src/<n>)")
p.add_argument("packages", nargs="+")
p.set_defaults(func=cmd_add_user)
p = sub.add_parser("script", help="run one phase of a package's install "
"script as the package user (unpack/build/install/...), "
"or 'fetch' to just get the script for editing")
p.add_argument("phase", choices=["all", "unpack", "build", "test", "install",
"configure", "update", "fetch"],
help="which phase to run ('fetch' gets the script, runs "
"nothing)")
p.add_argument("package")
p.add_argument("--regenerate", action="store_true",
help="regenerate the script from the book before running")
p.add_argument("--test", action="store_true",
help="also run the test suite (for the 'all'/'update' phase)")
p.add_argument("--clean", action="store_true",
help="after install, remove stale files from the old version "
"(configs and source tree kept)")
p.set_defaults(func=cmd_script)
p = sub.add_parser("probe", help="run the stacks' expect= feature checks "
"against what is installed")
p.add_argument("package", nargs="*", help="only these (default: all)")
p.set_defaults(func=cmd_probe)
p = sub.add_parser("late-deps", help="packages built BEFORE a dependency "
"they can use (so its support was compiled out)")
p.add_argument("package", nargs="*", help="only these (default: all)")
p.add_argument("--kind", help="which dependency kinds to check "
"(default recommended,optional; add required to include it)")
p.add_argument("--verbose", action="store_true",
help="also list dependency names that could not be matched")
p.add_argument("--include-docs", action="store_true",
help="include dependencies that only affect documentation "
"or are build tools (graphviz, docutils, git ...)")
p.add_argument("--why", action="store_true",
help="show the dependency edges actually used for the named "
"package(s), and which names did not resolve")
p.add_argument("--rebuild", action="store_true",
help="print the rebuild plan in dependency order; with "
"--run, carry it out")
p.add_argument("--run", action="store_true",
help="with --rebuild: actually rebuild")
p.set_defaults(func=cmd_late_deps)
p = sub.add_parser("errors", help="show which packages failed in the last "
"install/update run (with log locations)")
p.add_argument("--all", action="store_true",
help="show every package's result, not just failures")
p.set_defaults(func=cmd_errors)
p = sub.add_parser("plan", help="list or clear cached dry-run plans")
p.add_argument("action", nargs="?", default="list",
choices=["list", "clear"], help="list (default) or clear")
p.add_argument("key", nargs="?", help="for clear: a specific plan file name")
p.set_defaults(func=cmd_plan)
p = sub.add_parser("paths", help="show where everything is stored "
"(config, books, scripts, cache, plans, tools)")
p.set_defaults(func=cmd_paths)
p = sub.add_parser("config", help="show or change the prefixes / main user")
p.add_argument("--base-dir",
help="where account homes live (default /usr/src)")
p.add_argument("--pkgusr-subdir",
help="folder inside the base for PACKAGE users "
"(default 'pkgusr'; empty puts them in the base)")
p.add_argument("--cfguser-subdir",
help="folder inside the base for config users (default 'cfg')")
p.add_argument("--pkgusr-prefix",
help="prefix for PACKAGE users (empty string for none); "
"refuses if accounts already use the current one")
p.add_argument("--cfguser-prefix",
help="prefix for config users")
p.add_argument("--force", action="store_true",
help="change a prefix even though accounts already use it")
p.add_argument("--collector-prefix", help="prefix for shared install-dir groups")
p.add_argument("--user-prefix", help="prefix for shared users")
p.add_argument("--main-user", help="your human login account (e.g. n76310)")
p.add_argument("--shared-hub", metavar="DIR",
help="where the links to the shared directories live "
"(default /home/Shared); your ~/Shared and each "
"account's ~/Shared link into it")
p.add_argument("--editor", help="editor for 'verify --set-validate-command' "
"(default: $EDITOR or vim)")
p.add_argument("--difftool", help="diff tool for 'migrate-scripts --vimdiff' "
"(default: vimdiff)")
p.add_argument("--show", action="store_true",
help="just print the settings; do not ask for missing ones")
p.set_defaults(func=cmd_config)
p = sub.add_parser("blfs", help="run the blfs book tool "
"(books, set-default, search, ...)")
p.add_argument("args", nargs=argparse.REMAINDER,
help="arguments passed straight to blfs")
p.set_defaults(func=cmd_blfs)
# ---- user: shared users (u_*) ----
p = sub.add_parser("user", aliases=["shared-user"],
help="manage SHARED users (u_* accounts) that run applications")
usub = p.add_subparsers(dest="uaction", required=True)
q = usub.add_parser("list", help="list shared users and their groups")
q.add_argument("filter", nargs="?")
q.set_defaults(func=cmd_user_list)
q = usub.add_parser("create",
help="create an application user (u_<name>) and make "
"it reachable from your own account",
description=_APPLICATION_USER_HELP,
formatter_class=argparse.RawDescriptionHelpFormatter)
q.add_argument("name")
q.add_argument("--groups", help="also add these groups, e.g. audio,video")
q.add_argument("--shared-slim", dest="shared_slim", action="store_true",
help="share the DESKTOP only: the account joins the group "
"on your XDG runtime directory, so it can draw -- and "
"gets no audio and no camera. For an account that "
"only ever runs a browser window.")
q.add_argument("--shared-full", dest="shared_full", action="store_true",
help="the desktop AND audio and video: --shared-slim plus "
"the audio and video groups, for anything that plays "
"sound or uses a camera.")
q.add_argument("--shared", action="store_true",
help="a SHARED user: joins the group on your XDG runtime "
"directory, so it can reach the display, the session "
"bus and audio")
q.add_argument("--share-dir", action="store_true",
help="also make a directory both accounts can write, "
"linked into your home")
q.add_argument("--launcher", action="store_true",
help="a wrapper in ~/bin that runs the program as this user")
q.add_argument("--desktop", action="store_true",
help="also a menu entry, so it launches from the desktop")
q.add_argument("--app", help="program the launcher runs (default: <name>)")
q.set_defaults(func=cmd_user_create)
q = usub.add_parser("setup",
help="apply the same setup to a user that already "
"exists",
description=_APPLICATION_USER_HELP,
formatter_class=argparse.RawDescriptionHelpFormatter)
q.add_argument("name")
q.add_argument("--groups", help="also add these groups, e.g. audio,video")
q.add_argument("--shared-slim", dest="shared_slim", action="store_true",
help="share the DESKTOP only: the account joins the group "
"on your XDG runtime directory, so it can draw -- and "
"gets no audio and no camera. For an account that "
"only ever runs a browser window.")
q.add_argument("--shared-full", dest="shared_full", action="store_true",
help="the desktop AND audio and video: --shared-slim plus "
"the audio and video groups, for anything that plays "
"sound or uses a camera.")
q.add_argument("--shared", action="store_true",
help="join the group on your XDG runtime directory")
q.add_argument("--share-dir", action="store_true",
help="a directory both accounts can write")
q.add_argument("--launcher", action="store_true",
help="a wrapper in ~/bin")
q.add_argument("--desktop", action="store_true", help="a menu entry")
q.add_argument("--app", help="program to launch (default: <name>)")
q.set_defaults(func=cmd_user_setup)
q = usub.add_parser("delete", help="remove an application account safely: its "
"home archived first, launchers, pam line, account, group")
q.add_argument("name")
q.add_argument("--run", action="store_true", help="actually remove (default dry-run)")
q.add_argument("--yes", action="store_true", help="skip the confirmation prompt")
q.add_argument("--keep-home", action="store_true", help="leave the home directory")
q.add_argument("--no-archive", action="store_true", help="do not archive the home first")
q.set_defaults(func=cmd_user_delete)
q = usub.add_parser("purge", help="undo the whole application-account setup: "
"every account, then the session group, the profile "
"fragments and the pam wiring they shared")
q.add_argument("--run", action="store_true", help="actually remove (default dry-run)")
q.add_argument("--yes", action="store_true", help="skip the confirmation prompt")
q.add_argument("--keep-home", action="store_true", help="leave the home directories")
q.add_argument("--no-archive", action="store_true", help="do not archive the homes")
q.add_argument("--accounts", help="only these accounts (comma separated), "
"instead of every one in app-users.conf")
q.add_argument("--conf", help=f"read the account list here (default {APP_USERS_CONF})")
q.set_defaults(func=cmd_user_purge)
q = usub.add_parser("launcher", help="a launcher + menu entry for an existing "
"application account (one account can have several)")
q.add_argument("account")
q.add_argument("name", help="the launcher's name: ~/bin/<name>, <name>.desktop")
q.add_argument("--app", required=True, help="command line; {} = what you type")
q.add_argument("--icon", default="", help="URL, simpleicons.org slug, or - for none "
"(default: the launcher name as the slug)")
q.add_argument("--no-desktop", action="store_true", help="no menu entry")
q.set_defaults(func=cmd_user_launcher)
q = usub.add_parser("remove-launcher", help="remove a launcher, its menu entry and "
"icon (only ones these tools wrote)")
q.add_argument("name")
q.add_argument("--run", action="store_true")
q.set_defaults(func=cmd_user_remove_launcher)
q = usub.add_parser("allow", help="add supplementary group(s) to a user "
"(e.g. audio video)")
q.add_argument("user")
q.add_argument("groups", nargs="+")
q.set_defaults(func=cmd_user_allow)
q = usub.add_parser("disallow", help="remove supplementary group(s) from a user")
q.add_argument("user")
q.add_argument("groups", nargs="+")
q.set_defaults(func=cmd_user_disallow)
p = sub.add_parser("git-status", aliases=["git"],
help="what each git package follows (branch or tag) and "
"whether the remote has moved past it")
p.add_argument("packages", nargs="*", help="default: every git package here")
p.add_argument("--local-only", action="store_true",
help="do not ask the remotes (no network)")
p.set_defaults(func=cmd_git_status)
p = sub.add_parser("git-ref",
help="show the branches and tags a git package could "
"follow, or move it to one")
p.add_argument("package")
p.add_argument("ref", nargs="?", help="omit to list what the remote has")
p.add_argument("--run", action="store_true",
help="actually change git_ref= (default dry-run)")
p.set_defaults(func=cmd_git_ref)
p = sub.add_parser("used-by", aliases=["needed-by"],
help="what still links against a library, or against "
"this package's libraries (DT_NEEDED, not the "
"book's dependency list)")
p.add_argument("name", help="a package name, a soname (libfoo.so.4) or a "
"path to a library")
p.set_defaults(func=cmd_used_by)
p = sub.add_parser("service", help="the SysV services: list them, start/stop "
"one, enable/disable it at boot (start does not need "
"enable)")
p.add_argument("name", nargs="?", help="the service, as in /etc/rc.d/init.d/<name>")
p.add_argument("action", nargs="?",
help="list | status | start | stop | restart | reload | "
"enable | disable (either order: `service sshd start` "
"or `service start sshd`; no action means status)")
p.add_argument("--levels", help="enable: the runlevels to start in "
"(default: the script's own Default-Start, else 2345)")
p.add_argument("--priority", help="enable: the two-digit order number "
"(default: the one its other links already use)")
p.add_argument("--quick", action="store_true",
help="list: do not ask each script whether it is running")
p.set_defaults(func=cmd_service)
p = sub.add_parser("init", help="set up the install group + make install dirs "
"(dry-run unless --run; shows/confirms each command)")
p.add_argument("dirs", nargs="*", help="directories to turn into install dirs")
p.add_argument("--from-file", help="also read dir list from this file "
"(e.g. installdirs.lst)")
p.add_argument("--gid", type=int, default=9999, help="install group gid (default 9999)")
p.add_argument("--no-sticky", action="store_true",
help="plain group-writable (this is the default now)")
p.add_argument("--run", action="store_true",
help="execute (default just lists what it would do)")
p.add_argument("--yes", action="store_true", help="answer yes to every command")
p.set_defaults(func=cmd_init)
p = sub.add_parser("fix-install",
help="check/fix permissions on all group-'install' dirs")
p.add_argument("--run", action="store_true",
help="apply the fixes (default lists what would change)")
p.add_argument("roots", nargs="*",
help="directories to scan (default: / with /proc, /sys, "
"/dev, /run, /tmp and /snapshots pruned)")
p.add_argument("--exclude", action="append", metavar="DIR",
help="prune this directory too (repeatable)")
p.add_argument("--no-default-excludes", action="store_true",
help="scan /snapshots and /.snapshots as well")
p.add_argument("--sticky", action="store_true",
help="also set the sticky bit (default: group-writable, no sticky)")
p.add_argument("--yes", action="store_true",
help="reuse a stored dry-run without asking")
p.set_defaults(func=cmd_fix_install)
p = sub.add_parser("regenerate-script", aliases=["regen-script"],
help="write fresh install script(s) from the book into the "
"package home as install_<book_nv> (for the next "
"install/update); keeps existing unless --overwrite")
p.add_argument("packages", nargs="+", help="one or more package names")
p.add_argument("--overwrite", action="store_true",
help="replace an existing script (a .bak is kept)")
p.set_defaults(func=cmd_regenerate_script)
p = sub.add_parser("script-status", help="check whether each package's install "
"script is changed from the book (edited), behind the book "
"(outdated), or matches it")
p.add_argument("packages", nargs="*", help="default: all package-users")
p.add_argument("--fast", action="store_true",
help="format-stamp check only (no book comparison) -- the quick "
"way to list every OLD-format script")
p.add_argument("-q", "--quiet", action="store_true",
help="print only the package names needing migration, space-"
"separated (pipe into migrate-scripts)")
p.set_defaults(func=cmd_script_status)
p = sub.add_parser("migrate-scripts", help="upgrade older install scripts to "
"the current format, merging in your edited build/install "
"commands (dry-run unless --run; backs up as .bak)")
p.add_argument("packages", nargs="*", help="default: all package-users")
p.add_argument("--deps", action="store_true",
help="also migrate each named package's book dependencies "
"(the same set an update would touch)")
p.add_argument("--run", action="store_true",
help="rewrite the scripts (default lists what would change)")
p.add_argument("--vimdiff", action="store_true",
help="open every script that couldn't be auto-merged in "
"vimdiff (old vs new) to finish by hand, one at a time")
p.add_argument("--yes", action="store_true",
help="auto-adopt each merged (LEFT) version without prompting")
p.add_argument("--log", action="store_true",
help="show the history of past migrations (what/when) and exit")
p.add_argument("--all", action="store_true",
help="with --log, show the full history (default: last 40)")
p.set_defaults(func=cmd_migrate_scripts)
p = sub.add_parser("template", help="write an install-script skeleton for a "
"package not in the BLFS book (tarball or git)")
p.add_argument("name")
p.add_argument("--version", help="version string (default 1.0)")
p.add_argument("--git", metavar="URL",
help="make a git-based script that clones this repo "
"(e.g. https://gitlab.freedesktop.org/libnice/libnice.git)")
p.add_argument("--link", metavar="URL", help="tarball URL (non-git)")
p.add_argument("--pkg", metavar="FILE", help="tarball filename (non-git)")
p.add_argument("-o", "--outdir", help="where to write it (default .)")
p.set_defaults(func=cmd_template)
p = sub.add_parser("make-group-dir",
help="turn a directory into an install/sysgroup group dir")
p.add_argument("dirs", nargs="+")
p.add_argument("--group", default="install",
help="'install' (default) or a nimgnu name (group rwx, no setgid)")
p.add_argument("-R", "--recursive", action="store_true")
p.set_defaults(func=cmd_make_group_dir)
p = sub.add_parser("fix-group-dir",
help="re-apply correct permissions to group dir(s)")
p.add_argument("dirs", nargs="+")
p.add_argument("--group", help="force this group (else keep each dir's group)")
p.add_argument("-R", "--recursive", action="store_true")
p.set_defaults(func=cmd_fix_group_dir)
p = sub.add_parser("add-dir-to-sysgroup", aliases=["add-dir-to-nimgnu"],
help="assign directory(ies) to a sysgroup collector group")
p.add_argument("group")
p.add_argument("dirs", nargs="+")
p.add_argument("-R", "--recursive", action="store_true")
p.set_defaults(func=cmd_add_dir_to_nimgnu)
# ---- sysgroup collector groups (formerly 'nimgnu') ----
p = sub.add_parser("sysgroup", aliases=["nimgnu"],
help="manage sysgroup collector groups (shared install dirs)")
nsub = p.add_subparsers(dest="action", required=True)
q = nsub.add_parser("list", help="list collector groups and members")
q.add_argument("filter", nargs="?", help="only groups whose name contains this")
q.add_argument("-l", "--long", action="store_true",
help="also list each group's owned dirs/files (its manifest)")
q.set_defaults(func=cmd_nimgnu_list)
q = nsub.add_parser("create", help="create a nimgnu_<name> group + collector user")
q.add_argument("name")
q.set_defaults(func=cmd_nimgnu_create)
q = nsub.add_parser("add",
help="let package-user(s) install into a collector group's dirs")
q.add_argument("group")
q.add_argument("users", nargs="+", help="package-user(s) to add to the group")
q.set_defaults(func=cmd_nimgnu_add)
q = nsub.add_parser("remove", help="remove package-user(s) from a collector group")
q.add_argument("group")
q.add_argument("users", nargs="+")
q.set_defaults(func=cmd_nimgnu_remove)
q = nsub.add_parser("delete", help="delete a collector group + its user")
q.add_argument("name")
q.add_argument("--run", action="store_true",
help="actually delete (default is a dry-run)")
q.add_argument("--yes", action="store_true", help="skip the confirmation prompt")
q.set_defaults(func=cmd_nimgnu_delete)
p = sub.add_parser("list", help="list all package-users")
p.add_argument("packages", nargs="*", default=["all"])
p.set_defaults(func=cmd_list)
p = sub.add_parser("files_with_broken_id",
help="find files whose uid/gid don't resolve")
p.add_argument("packages", nargs="*", default=[])
p.add_argument("-o", "--outfile", help="where to write the list "
"(default /tmp/invalid_files.txt)")
p.set_defaults(func=cmd_files_with_broken_id)
args = ap.parse_args()
restore_wget_verification()
_hint_generate_ca_bundle()
ensure_wget_workaround()
_warn_init_mismatch()
warn_if_tools_are_stale()
_STATE["verbose"] = getattr(args, "verbose", False)
# first run: ask for the group-name prefixes (once), unless we're already
# running the config command or there's no terminal to prompt at.
if (not config_exists() and args.command != "config"
and sys.stdin.isatty() and sys.stdout.isatty()):
first_run_setup()
# THE EXIT CODE IS AN ANSWER. Every command's return value was thrown
# away here, so `packagemanager` exited 0 whatever happened -- including
# `service <name> status` on a service that is not running, which is the
# one command people put in an `if`. Only a non-zero INT becomes the exit
# status; a command that returns something else (or nothing) is unchanged.
rc = args.func(args)
if isinstance(rc, int) and not isinstance(rc, bool) and rc != 0:
sys.exit(rc)
if __name__ == "__main__":
try:
main()
except KeyboardInterrupt:
sys.exit(130)
except BrokenPipeError:
try:
devnull = os.open(os.devnull, os.O_WRONLY)
os.dup2(devnull, sys.stdout.fileno())
except OSError:
pass
sys.exit(0)