Packagemanager-LFS-PackageUser-System git · main
Package-user based package manager for LFS/BLFS
Shell 63.7% Markdown 35.8%git clone https://git.christianimmanuel.de/linux-from-scratch/Packagemanager-LFS-PackageUser-System.gitwget https://git.christianimmanuel.de/linux-from-scratch/Packagemanager-LFS-PackageUser-System/archive/Packagemanager-LFS-PackageUser-System.tar.gzAdd old version
OLD/blfs_parse_all | 97 +++++ OLD/blfs_parse_find_package | 99 +++++ OLD/blfs_parse_get_dependencies | 115 ++++++ OLD/blfs_parse_page | 542 +++++++++++++++++++++++++ OLD/packagemanager | 876 ++++++++++++++++++++++++++++++++++++++++ OLD/packagemanager_install | 418 +++++++++++++++++++ 6 files changed, 2147 insertions(+) diff --git a/OLD/blfs_parse_all b/OLD/blfs_parse_all new file mode 100755 index 0000000..f02e155 --- /dev/null +++ b/OLD/blfs_parse_all @@ -0,0 +1,97 @@ +#!/bin/python3 + +import os +import requests +from bs4 import BeautifulSoup +import tempfile + + +base_url = "https://www.linuxfromscratch.org/blfs/view/12.0/" +main_index_url = f"{base_url}index.html" +base_dir = tempfile.mkdtemp() +print(base_dir) + +class ChapterInfo: + def __init__(self): + self.chapter_name = "" + self.chapter_link = "" + self.chapter_pages = [] + +# Create the 'downloaded_pages' directory if it doesn't exist +if not os.path.exists(base_dir): + os.makedirs(base_dir) + +response = requests.get(main_index_url) +if response.status_code == 200: + # Save the HTML content to a file + with open(f"{base_dir}/index.html", "w", encoding="utf-8") as file: + file.write(response.text) + + # Create a BeautifulSoup object + soup = BeautifulSoup(response.text, 'html.parser') + + chapters = [] + for part in soup.select('div.book div.toc li.part'): + for chapter in part.select('li.chapter'): + chapter_info = ChapterInfo() + h4 = chapter.find('h4') + if h4: + chapter_info.chapter_name = ' '.join(h4.a.get_text(strip=True).split()) + chapter_info.chapter_link = h4.a['href'] + + ul = chapter.find('ul') + if ul: + chapter_info.chapter_pages = [(' '.join(a.get_text(strip=True).split()), a['href']) for a in ul.select('li a')] + + chapters.append(chapter_info) + + + + for chapter in chapters: + print("Chapter Name:", chapter.chapter_name) + print("Chapter Link:", chapter.chapter_link) + print("Chapter Pages:") + + chapter_url = base_url + chapter.chapter_link + chapter_response = requests.get(chapter_url) + + if chapter_response.status_code == 200: + # Create a directory for the chapter + chapter_file = os.path.join(base_dir, chapter.chapter_link) + chapter_dir = os.path.dirname(chapter_file) + print("1#################") + print(chapter_file) + print(chapter_dir) + print("2#################") + + if not os.path.exists(chapter_dir): + os.makedirs(chapter_dir) + + # Save the chapter page content + print(f"CH: {chapter_dir}") + with open(chapter_file, "wb") as f: + f.write(chapter_response.content) + + # Download and save chapter pages + for page_name, page_link in chapter.chapter_pages: + print(f"{page_name}\t{page_link}") + page_url = base_url + page_link + page_response = requests.get(page_url) + + if page_response.status_code == 200: + page_file = os.path.join(base_dir, page_link) + page_dir = os.path.dirname(page_file) + os.makedirs(page_dir, exist_ok=True) + + with open(page_file, "wb") as f: + f.write(page_response.content) + + +else: + print(f"Failed to fetch the page. Status code: {response.status_code}") + + +print() +print(base_dir) +#shutil.rmtree(base_dir) +#print("Temporary directory deleted.") diff --git a/OLD/blfs_parse_find_package b/OLD/blfs_parse_find_package new file mode 100755 index 0000000..07cb852 --- /dev/null +++ b/OLD/blfs_parse_find_package @@ -0,0 +1,99 @@ +#!/usr/bin/python3 + +import sys +import requests +import re +from bs4 import BeautifulSoup + +blfs_version = "svn" + +def extract_package_info(package_string): + version_pattern = r'-(\d+(\.\d+)*)$' + match = re.search(version_pattern, package_string) + + if match: + package_name_without_version = package_string[:match.start()] + package_version = match.group(1) + return package_name_without_version, package_version + else: + return package_string, None + +def get_package_name(url): + response = requests.get(url) + + if response.status_code != 200: + return "Error: Unable to fetch the webpage" + + soup_n = BeautifulSoup(response.content, 'html.parser') + name_link = soup_n.select_one("h1.sect1 > a") + #name_link = soup_n.select_one("div.sect1 > h1.sect1 > a") + return name_link['name'] + +def find_packages(package_name): + url = f"https://www.linuxfromscratch.org/blfs/view/{blfs_version}/" + response = requests.get(url) + + if response.status_code != 200: + return "Error: Unable to fetch the webpage" + + soup = BeautifulSoup(response.content, 'html.parser') + package_name = package_name.lower() + + results = [] + for package_link in soup.select("li.chapter > ul > li > a"): + link_text = package_link.text.strip().lower().replace(" ","-") + if package_name in link_text: + package_title = package_link.find_previous("h4").text + package_title = re.sub(r'\s+', ' ', package_title.strip()) + # package_found_name = package_link['name'] + package_url = f"https://www.linuxfromscratch.org/blfs/view/{blfs_version}/{package_link['href']}" + package_found_name = get_package_name(package_url) + results.append((package_title, package_found_name, package_url)) + + if results: + return results + else: + return "Package not found on the page" + +if __name__ == "__main__": + + # Initialize the quiet flag to False + quiet = False + + if len(sys.argv) == 1: + print("Find packages by name") + print(" blfs_parse_find_package {package_name}") + print("Find exact package name and return url") + print(" blfs_parse_find_package -q {package_name}") + sys.exit(1) + + # Check if -q was passed as a parameter + if "-q" in sys.argv: + quiet = True + # Remove the "-q" parameter from sys.argv + sys.argv.remove("-q") + + if len(sys.argv) != 2: + print("Usage: script.py [-q] <package_name>") + sys.exit(1) + + package_name = sys.argv[1].replace(" ","-") + found_packages = find_packages(package_name) + + if isinstance(found_packages, list): + if not quiet: + for package in found_packages: + print(f"Title: {package[0]}") + print(f"Name: {package[1]}") + print(f"Link: {package[2]}\n") + else: + for package in found_packages: + name, version = extract_package_info(package[1]) + if name.lower() == package_name.lower(): + print(package[2]) + sys.exit(1) + print(found_packages[0][2]) + sys.exit(0) + + else: + print(found_packages) diff --git a/OLD/blfs_parse_get_dependencies b/OLD/blfs_parse_get_dependencies new file mode 100755 index 0000000..92573d2 --- /dev/null +++ b/OLD/blfs_parse_get_dependencies @@ -0,0 +1,115 @@ +#!/bin/python3 + + +import re, sys, os +import argparse +import requests +import subprocess +from bs4 import BeautifulSoup, Tag +from urllib.parse import urlparse +from urllib.parse import urljoin + +file_path = '/tmp/packagemanager/install_links.txt' + +# Now you have a list of pairs from the file +#for pkg_name, url in stored_pairs: +# print(f'pkg_name: {pkg_name}, url: {url}') + +dependencies = {} +missing = [] +quiet = False + + +def extract_package_required_pks(url): + global dependencies + global missing + global quiet + if not quiet: + print() + print("#################") + print("FETCHING: ", os.path.basename(url.replace(".html",""))) + soup = None + pkg_name = None + pkg_name_short = None + deps = [] + # Send a GET request to the URL + response = requests.get(url) + + # Check if the request was successful (status code 200) + if response.status_code == 200: + # Parse the HTML content of the webpage + soup = BeautifulSoup(response.text, "html.parser") + else: + print(f"Failed to retrieve the webpage. Status code: {response.status_code}") + + # Extract name + div_element = soup.select_one('h1.sect1 > a') + pkg_name = div_element['name'] + + # Find the div element with class "package" + package_div = soup.find(class_="package") + + with open(file_path, 'a') as file: + file.write(f'{pkg_name}\t{url}\n') + + dependencies[pkg_name] = deps + if package_div: + # Find all <a class="xref"> elements within <p class="required"> under the package_div + + # Extract and join the titles of all found xref elements with a comma + def get_p(required_xref_elements): + for xref in required_xref_elements: + title = xref.get("title").split()[0] + if title: + href = xref.get("href") + # if not a local page + if href.startswith("https://"): + missing.append([title, href]) + else: + url_new = urljoin(url, href) + deps.append([title, url_new]) + for subarray in dependencies: + if title in dependencies: + return + extract_package_required_pks(url_new) + + get_p(package_div.select('p.required a.xref')) + get_p(package_div.select('p.recommended a.xref')) + + dependencies[pkg_name] = deps + + +if __name__ == "__main__": + + # Check if -q was passed as a parameter + if "-q" in sys.argv: + quiet = True + # Remove the "-q" parameter from sys.argv + sys.argv.remove("-q") + + parser = argparse.ArgumentParser(description="Fetch data from a URL") + parser.add_argument("url", help="URL of the webpage to fetch") + + args = parser.parse_args() + + url = args.url + + # Check if the URL is not a webpage (doesn't start with "https://" or "http://") + if not url.startswith(("https://", "http://")): + # Execute the blfs_parse_find_package script with -q and store its output in the 'url' variable + try: + blfs_output = subprocess.check_output(["blfs_parse_find_package", "-q", url], universal_newlines=True, stderr=subprocess.STDOUT).strip() + url = blfs_output + except subprocess.CalledProcessError as e: + if e.output: + url = e.output.strip() + else: + print(f"Error executing blfs_parse_find_package: {e}") + sys.exit(1) + + extract_package_required_pks(url) + if not quiet: + print("DEPENDENCIES: ", dependencies) + print("MISSING: ", missing) + else: + print(dependencies) diff --git a/OLD/blfs_parse_page b/OLD/blfs_parse_page new file mode 100755 index 0000000..2e7edb0 --- /dev/null +++ b/OLD/blfs_parse_page @@ -0,0 +1,542 @@ +#!/usr/bin/python3 + +##### just do wget if pkg is not there +##### remove pkg_dir if it exist before unpacking tar + + + +import argparse +import requests +from bs4 import BeautifulSoup, Tag +from urllib.parse import urlparse, urlsplit, unquote +import re +import os +import sys +import subprocess + +# Global variable to hold the BeautifulSoup object +soup = None + + +soup = None +extract_name = None +extract_name_version = None +is_module = False + +def fetch_webpage(url): + global soup, extract_name, extract_name_version, is_module + + # If there is a fragment in the URL, use it to find the specific div.sect2 + if is_module: + extract_name = is_module + + # Send a GET request to the URL without the fragment + url_without_fragment = url.split("#")[0] + response = requests.get(url_without_fragment) + + if response.status_code == 200: + soup = BeautifulSoup(response.text, "html.parser") + + # Search for the specific div.sect2 using the criteria you mentioned + soup = soup.select_one(f'div.sect2 h2.sect2 a[name="{is_module}"]').find_parent('div', class_='sect2') + + # Check if there's an h2.sect2 > a#asciidoc within the div.sect2 + else: + print(f"Failed to retrieve the webpage. Status code: {response.status_code}") + else: + # Handle the case where there's no fragment in the URL (your original logic) + response = requests.get(url) + + if response.status_code == 200: + soup = BeautifulSoup(response.text, "html.parser") + else: + print(f"Failed to retrieve the webpage. Status code: {response.status_code}") + + +def strip_package_name_full(name): + return re.sub("\(.*","",name).strip() + +def extractPackageNameFull(): + global soup + # Find the div element with class "sect1" followed by h1 with class "sect1" + div_element = None + if is_module: + div_element = soup.select_one('h2.sect2') + else: + div_element = soup.select_one('h1.sect1') + + if div_element: + # Extract and return the content within the div + content = div_element.get_text().strip() + return strip_package_name_full(content) + else: + print(f"extractPackageNameFull: nothing found") + return "" + + +def extractPackageName(): + global soup + name_link = None + if is_module: + name_link = soup.select_one("h2.sect2 > a") + else: + name_link = soup.select_one("h1.sect1 > a") + name = name_link['name'] + parts = name.split("-") + if len(parts) > 1 and parts[-1].replace(".","").isdigit(): + return "-".join(parts[:-1]) + else: + return name + +def extractPackageNameVersion(package_string): + version_pattern = r'-(\d+(\.\d+)+)$' + match = re.search(version_pattern, package_string) + + if match: + package_name_without_version = package_string[:match.start()] + package_version = match.group(1) + return package_name_without_version, package_version + else: + return package_string, None + +def extract_package_link(): + global soup + # Find the div element with class "package" + package_div = soup.find(class_="package") + + if package_div: + # Find all <a class="ulink"> elements within the div + ulink_elements = package_div.select('div.itemizedlist a.ulink') + + if ulink_elements: + for ulink_element in ulink_elements: + href = ulink_element.get("href") + if href and href.startswith("https"): + return href + + # If no HTTPS link is found, return the first one + first_href = ulink_elements[0].get("href") + return first_href + + print(f"extract_package_link: nothing found") + return "" + +def extract_package_md5(): + global soup + # Find the div element with class "package" + package_div = soup.find(class_="package") + + if package_div: + itemizedlists = package_div.find_all("div", class_="itemizedlist") + md5_sum = "" + + for itemizedlist in itemizedlists: + md5_p = itemizedlist.find("p", string=re.compile(r'\sDownload MD5 sum:\s*', re.IGNORECASE)) + if md5_p: + # Extract the text after "*Download MD5 sum:*" and remove any line breaks or leading/trailing spaces + md5_sum = md5_p.get_text() + md5_sum = re.sub(r'Download MD5 sum:', '', md5_sum, flags=re.IGNORECASE).strip().replace('\n', ' ') + + # Break after finding the first matching element + break + + return md5_sum + else: + print(f"extract_package_md5: nothing found") + return None + +def extract_additional_downloads(): + global soup + # Find the div element with class "package" + package_div = soup.find(class_="package") + + if package_div: + content = [] + h3_additional_downloads = package_div.find("h3", string=re.compile(r'\s*Additional Downloads\s*')) + h3_dependencies = package_div.find("h3", string=re.compile(r'\s* Dependencies\s*')) + + + #if h3_additional_downloads and h3_dependencies: + if h3_additional_downloads: + ulink_elements = [] + content = [] + + # Iterate through siblings of h3_additional_downloads until h3_dependencies is found + current_element = h3_additional_downloads.find_next_sibling() + while current_element and current_element != h3_dependencies: + # Find all <a> elements with class "ulink" within current_element + ulinks = current_element.find_all("a", class_="ulink") + for ulink in ulinks: + link = ulink.get("href") + if link: + content.append(link) + current_element = current_element.find_next_sibling() + + return content + else: + print(f"extract_additional_downloads: nothing found") + return [] + else: + print(f"extract_additional_downloads: nothing found") + return [] + + + +def extract_package_info(): + global soup + # Find the element with class "package" + package_element = soup.find(class_="package") + + if package_element: + # Find all p elements between "sect2" and the h3 element with content "Package Information" + sect2_element = None + if is_module: + sect2_element = package_element.find(class_="sect3") + else: + sect2_element = package_element.find(class_="sect2") + + if sect2_element: + package_info = [] + p_content = "" + break_div = "h3" + if is_module: + break_div = "h4" + + for element in sect2_element.next_siblings: + if element.name == break_div and "Package Information" in element.text: + break + if element.name == "p": + p_content = p_content + '\\n\\n' + ' '.join(line.strip() for line in element.text.strip().splitlines()) + + package_info.append(p_content.replace('"', '\\"')) + # Return the extracted package information as a single string + extracted_info = '\n'.join(package_info).lstrip("\\n") + return extracted_info + else: + print(f"extract_package_info: nothing found") + else: + print(f"extract_package_info: nothing found") + return "" + +def extract_package_required_pks(): + global soup + # Find the div element with class "package" + package_div = soup.find(class_="package") + + if package_div: + # Find all <a class="xref"> elements within <p class="required"> under the package_div + required_xref_elements = package_div.select('p.required a.xref') + + # Extract and join the titles of all found xref elements with a comma + titles = [xref.get("title").replace(" ","-") for xref in required_xref_elements if xref.get("title")] + + if titles: + return titles + else: + print(f"extract_package_required_pks: nothing found") + return [] + else: + print(f"extract_package_required_pks: nothing found") + return [] + +def extract_package_recommended_pks(): + global soup + # Find the div element with class "package" + package_div = soup.find(class_="package") + + if package_div: + # Find all <a class="xref"> elements within <p class="required"> under the package_div + required_xref_elements = package_div.select('p.recommended a.xref') + + # Extract and join the titles of all found xref elements with a comma + titles = [xref.get("title").replace(" ","-") for xref in required_xref_elements if xref.get("title")] + + if titles: + return titles + else: + return [] + print(f"extract_package_recommended_pks: nothing found") + else: + print(f"extract_package_recommended_pks: nothing found") + return [] + +def extract_package_optional_pks(): + global soup + # Find the div element with class "package" + package_div = soup.find(class_="package") + + if package_div: + # Find all <a class="xref"> elements within <p class="required"> under the package_div + required_xref_elements = package_div.select('p.optional a.xref') + + # Extract and join the titles of all found xref elements with a comma + titles = [xref.get("title").replace(" ","-") for xref in required_xref_elements if xref.get("title")] + + if titles: + return titles + else: + print(f"extract_package_optional_pks: nothing found") + return [] + else: + print(f"extract_package_optional_pks: nothing found") + return [] + +def extract_package_installed_content(): + global soup + # Find all div.seg within div.sect1 > div.content > div.seglistitem + seglistitems = None + if is_module: + seglistitems = soup.select('div.content div.seglistitem') + else: + seglistitems = soup.select('div.sect1 > div.content div.seglistitem') + + content_dict = {} + + for seglistitem in seglistitems: + segs = seglistitem.find_all("div", class_="seg") + + for seg in segs: + segtitle = seg.find("strong", class_="segtitle") + segbody = seg.find("span", class_="segbody") + + if segtitle and segbody: + segtitle_text = segtitle.get_text().strip().lower().replace(" ", "_").replace(":","") + segbody_text = ' '.join(segbody.get_text().split()) # Remove multiple spaces and newlines + + content_dict[segtitle_text] = segbody_text.replace(" and "," ") + + return content_dict + +def extract_package_commands(class_names): + global soup + # Find the div element with class "installation" + #command_div = soup.find(class_=class_names) + command_div = soup.select(class_names + " >*") + + if command_div: + contents = [] + + # Iterate through the elements within div.installation + for element in command_div: + if element.name == "h2": + d_content = "\n\n\n#### " + ''.join(line.strip() for line in element.get_text().splitlines()).replace('\n', '') + "\n" + contents.append(d_content) + elif element.name == "h3": + d_content = "\n\n### " + ''.join(line.strip() for line in element.get_text().splitlines()).replace('\n', '') + contents.append(d_content) + elif element.name == "h4": + d_content = "\n\n## " + ''.join(line.strip() for line in element.get_text().splitlines()).replace('\n', '') + contents.append(d_content) + elif element.name == "p": + # If it's a <p> element, add its text content with "# " prefix + p_content = ' '.join(line.strip() for line in element.get_text().splitlines()).replace('\n', ' ') + p_content = "\n\n# " + p_content.strip() + "\n" + contents.append(p_content) + elif element.name == "pre": + kbd_command = element.find("kbd", class_="command") + if kbd_command: + # If it's a <pre><kbd class="command"> element, add its text content + contents.append(kbd_command.get_text().strip()) + elif element.name == "div": + element_classes = element.get("class", []) + if isinstance(element, Tag) and "admon" in element.get("class", []): + element_classes.remove("admon") + contents.append("\n#### " + ''.join([element.upper() for element in element_classes]) + " ####") + for element_admon in element: + if element_admon.name == "p": + lines = [] + # Iterate through the contents of the p element + for item in element_admon.contents: + if isinstance(item, str): + # If it's a text node (not a child element), strip leading and trailing whitespace + line = re.sub(r'\s+', ' ', item.strip()) + if line: + lines.append(line) + elif item.name: # If it's a child element (has a name) + # Add the content of the child element without leading/trailing whitespace + child_text = item.get_text().strip() + child_text = re.sub(r'\s+', ' ', child_text) + if child_text: + lines.append(child_text) + + # Join the lines into a single string, replacing line breaks with spaces + content_str = ' '.join(lines) + contents.append("## " + content_str.replace(" , ", ", ")) + contents.append("") + + + + if contents: + joined_contents = '\n'.join(contents) + return joined_contents + else: + print(f"extract_package_commands({class_names}): nothing found") + return "" + else: + print(f"extract_package_commands({class_names}): nothing found") + return "" + +def extract_classnames_between_package_and_installation(): + global soup + # Find the div element with class "sect1" + sect1_div = soup.find("div", class_="sect1") + + if sect1_div: + installation_div = sect1_div.find("div", class_="package") + configuration_div = sect1_div.find("div", class_="installation") + + if installation_div and configuration_div: + # Find all siblings of div.installation that are on the same level + elements_between = [] + + for sibling in installation_div.next_siblings: + if sibling == configuration_div: + break + + if sibling.name: + elements_between.append(sibling["class"]) + + if elements_between: + classnames = [classname for element_classes in elements_between for classname in element_classes] + return classnames + +def format_array_to_bash_array(input_array): + formatted_elements = " ".join([f"'{element}'" for element in input_array]) + return f'({formatted_elements})' + +if __name__ == "__main__": + install_groups = [] + # get url from arguments + parser = argparse.ArgumentParser(description="Fetch data from a URL") + parser.add_argument("url", help="URL of the webpage to fetch") + + args = parser.parse_args() + url = args.url + + # Check if pkg is a module + parsed_url = urlsplit(url) + fragment = parsed_url.fragment + + # Remove URL encoding from the fragment + if fragment: + is_module = unquote(fragment) + install_groups.append(os.path.basename(parsed_url.path.strip('/')).replace(".html", "")) + + #url = "https://www.linuxfromscratch.org/blfs/view/12.0/x/qt5.html" + #url = "https://www.linuxfromscratch.org/blfs/view/12.0/xsoft/libreoffice.html" + #url = "https://www.linuxfromscratch.org/blfs/view/12.0/postlfs/mitkrb.html" + + # Fetch the webpage and parse it + fetch_webpage(url) + + print(f'\n\n–––––––––––––––––––––––––––––––––––––\n') + print(f'Getting content\n') + + if extract_name_version is None: + extract_name_version = extractPackageNameFull() + + # Get the extract_name + if extract_name is None: + extract_name, extracted_version = extractPackageNameVersion(extract_name_version) + extract_name = extractPackageName() + # Convert extract_name to lowercase + extract_name_lower = extract_name.lower() + + # Define the filename + filename = f"install_{extract_name_version}" + + pkg_link = extract_package_link() + pkg_name = basename = os.path.basename(pkg_link) + + installed_content = extract_package_installed_content() + # Extract the keys from the dictionary + installed_content_formatted_keys = [f'"{key.rstrip(":")}"' for key in installed_content.keys()] + + # Create the first string + installed_content_a = f"installed_content=({' '.join(installed_content_formatted_keys)})" + + # Create the second string by joining the values of the dictionary + installed_content_b = '\n'.join([f'{key}="{value}"' for key, value in installed_content.items()]) + + installation_command = extract_package_commands("div.installation") + + installation_configuration = extract_package_commands("div.configuration >div.sect3") + + classnames_between_package_and_installation = extract_classnames_between_package_and_installation() + + # Open the file for writing + with open(filename, "w") as script_file: + # Write the script content to the file + script_file.write(f'#!/bin/bash\n\n') + script_file.write(f'############################################\n### {extract_name_version}\n\n') + script_file.write(f'url="{url}"\n') + script_file.write(f'name="{extract_name}"\n') + script_file.write(f'name_version="{extract_name_version}"\n') + script_file.write(f'link="{pkg_link}"\n') + script_file.write(f'pkg="{pkg_name}"\n') + script_file.write(f'pkg_dir="" # this is getting set within the script\n') + script_file.write(f'md5_sum="{extract_package_md5()}"\n') + script_file.write(f'additional_links={format_array_to_bash_array(extract_additional_downloads())}\n') + script_file.write(f'info="{extract_package_info()}"\n') + script_file.write(f'required={format_array_to_bash_array(extract_package_required_pks())}\n') + script_file.write(f'recommended={format_array_to_bash_array(extract_package_recommended_pks())}\n') + script_file.write(f'optional={format_array_to_bash_array(extract_package_optional_pks())}\n') + script_file.write(f'{installed_content_a}\n') + script_file.write(f'{installed_content_b}\n') + if is_module: + script_file.write(f'install_groups={format_array_to_bash_array(install_groups)}\n') + + script_file.write(f'\n\n') + script_file.write(f'install_pkg() {{\n') + if classnames_between_package_and_installation: + script_file.write(f'#### BEFORE INSTALLATION ####\n\n') + for b_package in classnames_between_package_and_installation: + bc_package = "div." + b_package + script_file.write(f'{extract_package_commands(bc_package)}\n') + script_file.write(f'#### BEFORE INSTALLATION DONE ####\n\n') + script_file.write(f'#### INSTALLATION ####\n\n') + script_file.write(f'# Get tar/zip\n') + script_file.write(f'[ -f "$pkg" ] || wget -4 $link\n\n') + script_file.write(f'# Check md5sum\n') + script_file.write(f'if ! grep -wq "$md5_sum" <<< $(md5sum $pkg); then\n\techo "ERROR: md5sum check failed" && exit\nfi\n\n') + script_file.write(f'# Get additional links\n') + # LAST WORKED HERE: + script_file.write('for li in "${additional_links[@]}"; do wget -4 "$li"; done\n\n') + if not re.search(fr"^tar .* {re.escape(pkg_name)} .*", installation_command): + script_file.write(f'# Unpack tar/zip and get pkg_dir variable\n') + script_file.write(f'rm -rf tmp && mkdir tmp && cd tmp\n') + if pkg_name.endswith(".zip"): + script_file.write(f'unzip ../$pkg &&\nif [ "$(ls | wc -w)" = "1" ]; then\n\tpkg_dir="$(ls)"; [ -d ../$pkg_dir ] && rm -rf ../$pkg_dir; cd .. && mv tmp/* . && rm -r tmp\nelse\n\tcd ..\n\t[ -d $name_version ] && rm -rf $name_version\n\tmv tmp $name_version\n\tpkg_dir=$name_version\nfi\n') + elif pkg_name.endswith((".tar", ".tgz", ".tar.gz", ".tar.bz2", ".tar.xz")): + script_file.write(f'tar -xvf ../$pkg && if [ "$(ls | wc -w)" = "1" ]; then\n\tpkg_dir="$(ls)";\nelse\n\techo "ERROR: unpacked tar leaves unknown directory: \\"$(ls)\\"" && exit;\nfi &&\ncd ..\n[ -d $pkg_dir ] && rm -rf $pkg_dir\nmv tmp/* . && rm -r tmp\n') + script_file.write(f'cd $pkg_dir\n') + script_file.write(f'{installation_command}\n') + script_file.write(f'\n#### INSTALLATION DONE ####\n\n') + script_file.write(f'}}\n\n') + if installation_configuration: + script_file.write(f'\nconfigure_pkg() {{\n') + script_file.write(f'\n#### CONFIGURATION ####\n\n') + script_file.write('echo "## Configuration"\n\n') + script_file.write(f'{installation_configuration}\n') + script_file.write(f'\n#### CONFIGURATION DONE ####\n\n') + script_file.write(f'}}\n\n') + script_file.write(f'if [[ "$1" == "install" ]]; then install_pkg; else echo -e "you may want to run\\n$0 install"; fi\n') + if installation_configuration: + script_file.write(f'if [[ "$1" == "install" ]]; then configure_pkg; fi') + # script_file.write(f'cd ..; llist_package $(whoami) > pkg.lst 2>&1 & disown') + + + print(f'\n\n–––––––––––––––––––––––––––––––––––––') + print(f'\nSuccessfully created {filename}') + # Check the syntax of the script without executing it + try: + subprocess.run(["bash", "-n", filename], check=True) + print(f'\n–––––––––––––––––––––––––––––––––––––') + print("\nBash script syntax is valid.") + except subprocess.CalledProcessError as e: + print(f'\n–––––––––––––––––––––––––––––––––––––') + print("\n!!! Error in Bash script syntax !!!") + if e.stderr is not None: + print(e.stderr) + print(f'\n–––––––––––––––––––––––––––––––––––––') + print(f'\nFile path: {filename}') diff --git a/OLD/packagemanager b/OLD/packagemanager new file mode 100755 index 0000000..5c0ed6a --- /dev/null +++ b/OLD/packagemanager @@ -0,0 +1,876 @@ +#!/bin/python3 + +from urllib.parse import urlparse, urlsplit, unquote, urljoin +from bs4 import BeautifulSoup, Tag +from collections import defaultdict +import subprocess +import os, sys, re, csv + +import argparse +import requests + +import pwd +import grp + +users_info = {} + +install_package = None +install_url = None +install_dependencies = None +is_module = False + +# ANSI escape codes for text colors +class colors: + normal = '\033[0m' + black = '\033[30m' + red = '\033[31m' + green = '\033[32m' + yellow = '\033[33m' + blue = '\033[34m' + purple = '\033[35m' + cyan = '\033[36m' + white = '\033[37m' + +# ANSI escape codes for text styles +class styles: + bold = '\033[1m' + underline = '\033[4m' + +users = [] +users_broken = [] +class User: + def __init__(self): + self.search = None + self.name = None + self.uid = None + self.gid = None + self.home = None + self.pkg_list = None + + +get_dependencies = {} +get_not_installable = [] +base_dir = "/usr/src" +install_scripts_main = "/etc/pkgusr/install_scripts" +install_command = "" +package_reinstall = False + +ignore_list = None +ignore_deps = False +ignore_recommended = False + +soup_global = None + +file_install_links = '/tmp/packagemanager/install_links.txt' +try: open(file_install_links, 'w').close() +except Exception as e: print(f"Error: {e}") + +if not os.path.exists('/tmp/packagemanager/'): + os.makedirs('/tmp/packagemanager/') +if not os.path.exists(install_scripts_main): + os.makedirs(install_scripts_main) + +def printHeader(msg): + header_width = 40 + msg_length = len(msg) + num_hashes = max(header_width - msg_length - 6, 2) + print(f"\n{colors.blue}{'#' * header_width}{colors.normal}") + print(f"{colors.blue}### {msg} #{'#' * num_hashes} {colors.normal}") + +def printLine(): + print(f"{colors.blue}{'–' * 40}{colors.normal}") + return f"{colors.blue}{'–' * 40}{colors.normal}" + +def printSuccess(msg): + print(f"{colors.green}{msg}{colors.normal}") + +def printInfo(msg): + print(f"{colors.blue}{msg}{colors.normal}") + +def printWarning(msg): + print(f"{colors.yellow}{msg}{colors.normal}") + +def printError(msg): + print(f"{colors.red}{msg}{colors.normal}") + + +#################################################### +### oter functions + +# yes no request +def askYesNoQuestion(question): + while True: + response = input(question + " [y|n]: ").strip().lower() + if response == 'y': + return True + elif response == 'n': + return False + else: + print("Please enter 'y' for yes or 'n' for no.") + +def strip_package_name_full(name): + return re.sub("\(.*","",name).strip() + +def extract_package_name(soup): + global is_module + name_link = None + if is_module: + name_link = soup.select_one("h2.sect2 > a") + else: + name_link = soup.select_one("h1.sect1 > a") + name = name_link['name'] + parts = name.split("-") + if len(parts) > 1 and parts[-1].replace(".","").isdigit(): + return "-".join(parts[:-1]) + else: + return name + +# return name and version of name-verion + +def extract_package_name_full(soup): + global is_module + # Find the div element with class "sect1" followed by h1 with class "sect1" + div_element = None + if is_module: + div_element = soup.select_one('h2.sect2') + else: + div_element = soup.select_one('h1.sect1') + + if div_element: + # Extract and return the content within the div + content = div_element.get_text().strip() + return strip_package_name_full(content) + else: + print(f"extract_package_name_version: nothing found") + return "" + +def extractPackageNameVersion(package_string): + version_pattern = r'-(\d+(\.\d+)+)$' + match = re.search(version_pattern, package_string) + + if match: + package_name_without_version = package_string[:match.start()] + package_version = match.group(1) + return package_name_without_version, package_version + else: + return package_string, None + +def sortDependencies(dependencies): + global get_not_installable + ## SORT + # Create a defaultdict to store dependencies as sets + dependency_sets = defaultdict(set) + # Convert the dependencies dictionary to sets + for package, deps in dependencies.items(): + for dep, _ in deps: + dependency_sets[package].add(dep) + # Perform topological sorting + sorted_packages = [] + visited = set() + + def topologicalSort(package): + visited.add(package) + for dep in dependency_sets[package]: + if dep not in visited: + topologicalSort(dep) + sorted_packages.append(package) + + for package in dependencies.keys(): + if package not in visited: + topologicalSort(package) + + # Create a new dictionary with sorted packages + sorted_dependencies = {} + for pkg in sorted_packages: + try: + sorted_dependencies[pkg] = dependencies[pkg] + except KeyError: + # If dependencies[pkg] does not exist, add pkg to get_not_installable + get_not_installable.append(pkg) + return sorted_dependencies + + +def checkInstalledDependencies(dependencies, ignore_check): + global get_not_installable + installed = [] + missing = [] + version_mismatch = [] + + ### CHECK + for key in dependencies: + package_name = getNameFromDependencyFile(key) + name, version = extractPackageNameVersion(key) + check = checkPackage(package_name, version) + if ignore_check: + missing.append(key) + elif check == "True": + installed.append(key) + elif check == "False": + missing.append(key) + elif check == "Undefined": + get_not_installable.append(key) + else: + version_mismatch.append(f"{key} == {check}") + + return installed, missing, version_mismatch + +def getPkgSoup(url): + global soup_global + + # Send a GET request to the URL + response = requests.get(url) + # Check if the request was successful (status code 200) + if response.status_code == 200: + # Parse the HTML content of the webpage + soup_global = BeautifulSoup(response.text, "html.parser") + else: + print(f"Failed to retrieve the webpage. Status code: {response.status_code}") + +#################################################### +### packagemanager functions + +def findUserByName(username): + res = next((user for user in users if user.name == username), None) + if not res: + res = next((user for user in users if user.search == username), None) + return res + + + +def setUserInfo(packages): + if len(packages) == 1 and packages[0] == "all": + packages = listAllUsers() + + default_infos = ("name", "uid", "gid", "home", "pkg_list") + for username in packages: + default_info = {} + for f in default_infos: default_info[f] = None + user = findUserByName(username) + if not user: + user = User() + users.append(user) + user.search = username + + commands = { + "name": ["id", "-un", username], + "uid": ["id", "-u", username], + "gid": ["id", "-g", username], + "home": ["bash", "-c", f'[ -d "{base_dir}" ] && echo "{base_dir}/{username}"'], + "pkg_list": ["bash", "-c", f'[ -f "{base_dir}/{username}/pkg.lst" ] && echo "{base_dir}/{username}/pkg.lst"'] + } + + check_missing = False + for attribute, command in commands.items(): + try: + setattr(user, attribute, subprocess.check_output(command).decode().strip()) + except subprocess.CalledProcessError: + check_missing = True + setattr(user, attribute, None) + + if check_missing and not user in users_broken: + users_broken.append(user) + + +def formatUserTableCell(key, value): + if not value: + value = "None" + if key == "name": + return value.ljust(12) + if key == "uid": + return value.ljust(6) + if key == "gid": + return value.ljust(6) + if key == "home": + return value.ljust(25) + if key == "pkg_list": + return value.ljust(28) + + + +def printUserInfo(packages): + if len(packages) == 1 and packages[0] == "all": + packages = listAllUsers() + + # Print the values of user_info + printHeader(f"User infos") + print() + arg_to_attr = {"name": args.name, "uid": args.uid, "gid": args.gid, "home": args.home, "pkg_list": args.pkg_list} + + values_to_print = [] + values_all = [] + # Loop through the dictionary and append values to values_all or values_to_print accordingly + for arg_name, arg_value in arg_to_attr.items(): + if arg_value: + values_to_print.append(arg_name) + else: + values_all.append(arg_name) + + if len(values_to_print) == 0: + values_to_print = values_all + + print_header = " ".join(formatUserTableCell(value, value.capitalize()) for value in values_to_print) + print(print_header) + printLine() + + def printUsers(users): + for user_name in users: + user = findUserByName(user_name) + def printValues(values): + print_string = "" + for value in values: + #print_value_in = userinfo[value] + print_value_in = getattr(user, value) + print_value = formatUserTableCell(value, print_value_in) + if not print_value_in: + if value == "name": + print_value = formatUserTableCell("name", user.search) + print_value = colors.red + print_value + colors.normal + elif (value == "uid" or value == "gid" ) and not user.gid == user.uid: + print_value = colors.yellow + print_value + colors.normal + print_string = print_string + f"{print_value} " + print(print_string) + + + printValues(values_to_print) + #print(f"{userinfo['name']}\t{userinfo['uid']}\t{userinfo['gid']}\t{userinfo['home']}\t{userinfo['pkg_list']}") + + printUsers(packages) + print() + if len(users_broken) > 0: + printHeader("Broken") + print() + print(print_header) + printLine() + printUsers([u.search for u in users_broken]) + +def getUserInfo(packages): + setUserInfo(packages) + printUserInfo(packages) + +def checkPackageVersion(package_name_version): + package_name, package_version = extractPackageNameVersion(package_name_version) + file_path = f"/usr/src/{package_name}/install_{package_name}" + with open(file_path, 'r') as file: + for line in file: + line = line.strip() + if line.startswith("name_version="): + name_version = line.split("=")[1].replace('"',"") + name, version = extractPackageNameVersion(name_version) + return version + + +def checkPackage(package_name, version): + if not package_name: + return "Undefined" + + installed_version = None + printHeader(f"Checking {package_name} {version}") + + print(f"Checking '{package_name}'") + package_name = package_name.lower() + + check = "True" + try: + user_result = os.system(f"id {package_name} >/dev/null 2>&1") + + if user_result == 0: + printSuccess("User "+package_name+" exists.") + else: + printWarning("User "+package_name+" does not exist.") + check = False + except Exception as e: + print(f"Error checking user: {e}") + check = "False" + + # Check dir + directory = f"/usr/src/{package_name}" + if os.path.exists(directory): + printSuccess("Directory "+directory+" exists.") + else: + printWarning("Directory "+directory+" does not exist.") + check = "False" + + # Check pkg.list file -> installed... + file_path = f"{directory}/pkg.lst" + if os.path.exists(file_path) and os.path.getsize(file_path) > 0: + printSuccess("File "+file_path+" exists") + else: + printWarning("File "+file_path+" does not exist or is empty.") + check = "False" + + # Check version + file_install = f"{directory}/install_{package_name}" + if os.path.exists(file_install) and os.path.getsize(file_install) > 0: + printSuccess("File "+file_install+" exists") + version_is = checkPackageVersion(package_name) + if not version_is == version and version is not None: + printWarning(f"Version mismatch: {version}") + check = version + else: + printSuccess(f"Version match: {version}") + else: + printWarning("File "+file_install+" does not exist or is empty.") + + printLine() + if check == "True": + printSuccess("All checks passed.") + elif check == "False": + printError("Some checks failed.") + else: + printError("Version mismatch.") + + return check + +def getPackageUrl(pkg_name, quiet): + # Execute the blfs_parse_find_package script with -q and store its output in the 'url' variable + url = "" + try: + if quiet: + blfs_output = subprocess.check_output(["blfs_parse_find_package", "-q", pkg_name], universal_newlines=True, stderr=subprocess.STDOUT).strip() + else: + blfs_output = subprocess.check_output(["blfs_parse_find_package", pkg_name], universal_newlines=True, stderr=subprocess.STDOUT).strip() + url = blfs_output + except subprocess.CalledProcessError as e: + if e.output: + url = e.output.strip() + else: + print(f"Error executing blfs_parse_find_package: {e}") + sys.exit(1) + + if "not found" in url or not quiet: + printHeader(f"Search {pkg_name}") + print(url) + + if "not found" in url: + sys.exit(0) + return url + +def fetchDependencies(url, quiet, once, tab): + global is_module + if not url: + return + + tab_space = " " * tab + tab += 1 + global get_dependencies + global get_not_installable + global ignore_list + + basename = os.path.basename(url.replace(".html", "")) + + if basename.startswith("xorg7#"): + printError("\n#################") + printError(f"cannot fetch {basename}") + get_not_installable.append(basename) + return + + if not quiet: + print("\n#################") + print(f"{tab_space}FETCHING: ", basename) + print() + soup = None + deps = [] + # Send a GET request to the URL + response = requests.get(url) + + # Check if the request was successful (status code 200) + if response.status_code == 200: + is_module = False + + # Check if pkg is a module + parsed_url = urlsplit(url) + fragment = parsed_url.fragment + + # Remove URL encoding from the fragment + if fragment: + is_module = unquote(fragment) + + # Parse the HTML content of the webpage + soup = BeautifulSoup(response.text, "html.parser") + if is_module: + soup = soup.select_one(f'div.sect2 h2.sect2 a[name="{is_module}"]').find_parent('div', class_='sect2') + else: + print(f"Failed to retrieve the webpage. Status code: {response.status_code}") + + # Extract name + pkg_name_version = extract_package_name_full(soup).replace(" ", "-") + + # Find the div element with class "package" + + pg_name, version = extractPackageNameVersion(pkg_name_version) + pkg_name = extract_package_name(soup) + + package_div = soup.find(class_="package") + with open(file_install_links, 'a') as file: + file.write(f'{pkg_name}\t{pkg_name_version}\t{url}\n') + + #if once: + # sys.exit(0) + # return + + #get_dependencies[pkg_name_version] = deps + get_dependencies[pkg_name_version] = [] + loop_pkg = pkg_name_version + + if checkPackage(pkg_name, version) == "True": + printSuccess(f"{pkg_name} is installed") + return + + + if package_div: + # Find all <a class="xref"> elements within <p class="required"> under the package_div + + # Extract and join the titles of all found xref elements with a comma + def getP(required_xref_elements): + for xref in required_xref_elements: + title_full = strip_package_name_full(xref.get("title")) + title_full = title_full.replace(" ", "-") + if title_full: + href = xref.get("href") + # if not a local page + if href.startswith("https://"): + missing.append([title_full, href]) + else: + url_new = urljoin(url, href) + deps.append([title_full, url_new]) + check_title = False + check_ignore = False + for subarray in get_dependencies: + if title_full in get_dependencies and not loop_pkg == subarray: + check_title = True + for ignore in ignore_list: + if ignore.lower() in title_full.lower(): + printError(f"\n\n{printHeader('IGNORE')}\n\n{title_full}\n\n") + check_ignore = True + if not once and not check_title and not check_ignore and not ignore_deps: + fetchDependencies(url_new, quiet, False, tab) + + getP(package_div.select('p.required a.xref')) + if not ignore_recommended: + getP(package_div.select('p.recommended a.xref')) + + get_dependencies[pkg_name_version] = deps + +def getDependencies(url, quiet, once): + global get_dependencies + global get_not_installable + + + if not url.startswith(("https://", "http://")): + url = getPackageUrl(url, True) + + if not quiet: + printHeader("Dependencies") + print() + + fetchDependencies(url, quiet, False, 0) + + dependencies_sorted = sortDependencies(get_dependencies) + + if not quiet: + printHeader(f"{package_name} dependencies") + print("DEPENDENCIES: ", dependencies_sorted) + print("NOT INSTALLABLE: ", get_not_installable) + + return dependencies_sorted + +def getUrlFromDependencyFile(pkg_name): + with open(file_install_links, "r", encoding="utf8") as file: + reader = csv.reader(file, delimiter="\t") + for row in reader: + (name, name_version, url) = row + if name_version == pkg_name: + return url + +def getNameFromDependencyFile(pkg_name_version): + with open(file_install_links, "r", encoding="utf8") as file: + reader = csv.reader(file, delimiter="\t") + for row in reader: + (name, name_version, url) = row + if name_version == pkg_name_version: + return name + +def getInstallFile(pkg_name): + original_dir = os.getcwd() + file_dir = f'/tmp/packagemanager/install_files' + if not os.path.exists(file_dir): + os.makedirs(file_dir) + os.chdir(file_dir) + + url = getUrlFromDependencyFile(pkg_name) + + print("URL: ", url) + try: + subprocess.check_output(['blfs_parse_page', url], universal_newlines=True, stderr=subprocess.STDOUT).strip() + except subprocess.CalledProcessError as e: + print("Error getInstallFile:", e) + sys.exit(0) + + os.chdir(original_dir) + +def getInstallPkgCommands(pkg_name): + global install_command + global soup_global + + name, version = extractPackageNameVersion(pkg_name) + name = getNameFromDependencyFile(pkg_name) + install_file = f'/tmp/packagemanager/install_files/install_{pkg_name}' + printHeader(f"Get Install file {pkg_name}") + print() + if not os.path.isfile(install_file): + print(f"Cannot find install file {install_file}") + #sys.exit(0) + print(f"{install_file}") + #askYesNoQuestion("Do you wish to continue?") + try: + subprocess.run(["bash", "-n", install_file], check=True) + printSuccess(f'\n–––––––––––––––––––––––––––––––––––––') + printSuccess("\nBash script syntax is valid.") + except subprocess.CalledProcessError as e: + printError(f'\n–––––––––––––––––––––––––––––––––––––') + printError("\n!!! Error in Bash script syntax !!!") + if e.stderr is not None: + print(e.stderr) + print("Recheck the file!") + print(f"{install_file}") + install_command = install_command + f"packagemanager_install {name.lower()} {install_file};\n" + + +install_check_run = 0 +def installCheck(package_name_version, dependencies, ignore_check): + global install_check_run + install_check_run = install_check_run + 1 + + printHeader("Install") + # get dependencies + package_name = getNameFromDependencyFile(package_name_version) + name, version = extractPackageNameVersion(package_name_version) + if not package_name: + package_name = name + add_missing = None + if checkPackage(package_name, version) == "True" and not ignore_check: + printLine() + printError(f"{package_name} is already installed") + if install_check_run == 1: + if askYesNoQuestion(f"do you want to go on anyways?"): + add_missing = package_name_version + else: + exit(1) + printLine() + # check dependencies + installed, missing, version_mismatch = checkInstalledDependencies(dependencies, ignore_check) + if add_missing: + installed.append(add_missing) + printHeader("Dependencie summary:") + printSuccess(f"Installed: {installed}") + printError(f"Missing: {missing}") + printWarning(f"Version mismatch: {version_mismatch}") + + return installed, missing, version_mismatch + +def installMissing(pkgs): + printHeader("Install missing pkgs") + + print(pkgs) + for pkg in pkgs: + getInstallFile(pkg) + getInstallPkgCommands(pkg) + +def installCycle(package_name): + global package_reinstall + + url = "" + + # Check if the URL is not a webpage (doesn't start with "https://" or "http://") + if not package_name.startswith(("https://", "http://")): + url = getPackageUrl(package_name, True) + + # get global soup + getPkgSoup(url) + + if not package_reinstall: + dependencies = getDependencies(url, False, False) + installed, missing, version_mismatch = installCheck(package_name, dependencies, False) + else: + dependencies = getDependencies(url, False, True) + installed, missing, version_mismatch = installCheck(package_name, dependencies, True) + + installMissing(missing) + print(install_command) + +def listPackage(packages): + for package_name in packages: + if package_name == "all" and len(packages) == 1: + listAllUsers() + return + + print() + printLine() + printHeader(f"List {package_name}") + + +def listAllUsers(): + print() + printLine() + printHeader(f"List all found in /usr/src") + print() + directory_names = None + directory_names = sorted([d for d in os.listdir(base_dir) if os.path.isdir(os.path.join(base_dir, d))]) + directory_names = directory_names[::-1] + longest_name = len(max(directory_names, key=len)) + num_columns = 3 + c = 0 + row_str = "" + a_break = 0 + for d_name in directory_names: + c = c+1 + if d_name and d_name[0].lower() == "a": + a_break = 1 + + if a_break == 1 and d_name[0].lower() != "a": + a_break = 2 + row_str = row_str + f"\n{printLine()}\n" + c = 1 + row_str = row_str + "" + d_name.ljust(longest_name+1) + if c % num_columns == 0: + row_str = row_str + "\n" + + print(row_str) + printLine() + print(f"Total: {len(directory_names)}") + print(f"Strange: {c}") + return directory_names + +def reloadPkgList(package_name): + print() + printLine() + printHeader(f"Reload pkg list {package_name}") + print() + #setUserInfo([package_name]) + + package_dir = os.path.join(base_dir, package_name) + if os.path.exists(package_dir): + # Execute the bash command and redirect output to pkg.lst.new file + command = f"list_package {package_name} 2>&1 | tee /tmp/pkg.lst.new && cp {package_dir}/pkg.lst.new {package_dir}/pkg.lst" + subprocess.run(command, shell=True, check=True) + print(f"Command executed successfully for {package_name}.") + else: + print(f"Directory {package_dir} does not exist.") + + +def findFilesOrFoldersWithBrokenGidOrUid(): + def print_find(file): + print(file) + + def findFilesWithInvalidIds(root_dir, ignored_paths): + invalid_files = [] + + for foldername, subfolders, filenames in os.walk(root_dir): + ignore_flag = False + for ignored_path in ignored_paths: + if foldername.startswith(ignored_path): + ignore_flag = True + break + if ignore_flag: + continue + + for filename in filenames: + file_path = os.path.join(foldername, filename) + + try: + file_stat = os.lstat(file_path) # Use os.lstat() instead of os.stat() for symlinks + uid = file_stat.st_uid + gid = file_stat.st_gid + + try: + pwd.getpwuid(uid) + grp.getgrgid(gid) + except (KeyError, OverflowError): + invalid_files.append(file_path) + print_find(file_path) + + except (FileNotFoundError, PermissionError): + invalid_files.append(file_path) + print_find(file_path) + + return invalid_files + + root_directory = '/' + ignored_paths = [ '/mnt', '/proc'] + invalid_files = findFilesWithInvalidIds(root_directory, ignored_paths) + + print() + print(invalid_files) + + # Write invalid files to a file in /tmp directory + output_file_path = '/tmp/invalid_files.txt' + with open(output_file_path, 'w') as output_file: + for file_path in invalid_files: + output_file.write(file_path + '\n') + + # Inform user about the output file + print(f'\nInvalid files list written to: {output_file_path}\n') + return invalid_files + + +if __name__ == "__main__": + ignore_list + + parser = argparse.ArgumentParser(description="Package Manager") + parser.add_argument("command", choices=["info", "install", "reload-pkg-list", "remove", "check", "search", "dependencies", "list", "files_with_broken_id"], help="Usage: packagemanager check <package_name>") + parser.add_argument("packages", nargs='+', default=None, help="Packages to operate on") + parser.add_argument("-r", "--reinstall", action="store_true", help="reinstall") + parser.add_argument("-i", "--ignore", help="List of items to ignore, separated by commas") + parser.add_argument("-e", "--ignore-recommended", action="store_true", help="Ignore recommended packages") + parser.add_argument("-f", "--force", action="store_true", help="ignores dependencies") + parser.add_argument("-name", action="store_true", help="Specify name parameter for info") + parser.add_argument("-uid", action="store_true", help="Specify uid parameter for info") + parser.add_argument("-gid", action="store_true", help="Specify home parameter for info") + parser.add_argument("-pkg-list", action="store_true", help="Specify pkg-list parameter for info") + parser.add_argument("-home", action="store_true", help="Specify home parameter for info") + + # print(args.reinstall) + args = parser.parse_args() + packages = args.packages + + if args.ignore: + ignore_list = args.ignore.split(",") + if not ignore_list: + ignore_list = "" + + #package_name = args.package.replace(" ","-") + + if args.reinstall: + package_reinstall = True + + if args.force: + ignore_deps = True + if args.ignore_recommended: + ignore_recommended = True + + if args.command == "files_with_broken_id": + findFilesOrFoldersWithBrokenGidOrUid() + elif args.command == "reload-pkg-list": + for package_name in packages: + reloadPkgList(package_name) + elif args.command == "info": + getUserInfo(packages) + elif args.command == "list": + listPackage(packages) + elif args.command == "check": + checkPackage(package_name, None) + elif args.command == "install": + for package_name in packages: + installCycle(package_name) + if len(get_not_installable) > 0: + print("NOT INSTALLABLE: ", get_not_installable) + elif args.command == "dependencies": + for package_name in packages: + getDependencies(package_name, False, False) + elif args.command == "search": + for package_name in packages: + getPackageUrl(package_name, False) + else: + printError("Usage: packagemanager check <package_name>") + sys.exit(1) + diff --git a/OLD/packagemanager_install b/OLD/packagemanager_install new file mode 100755 index 0000000..c04e6a3 --- /dev/null +++ b/OLD/packagemanager_install @@ -0,0 +1,418 @@ +#!/bin/bash +# +# the package_short_name is the name for te created user +# packagemanager_install package_short_name path_to_install_file +# + + + +skel=/etc/pkgusr/skel-package +user_name="" +install_script="" +install_script_basename="" +install_user_dir="/usr/src/$user_name" +sourced=false +date_log="$(date +%Y-%m-%d_%H-%m-%S)" +home_user="n76310" +home_user_dir="/home/${home_user}" + +echoRed() { + echo -e "\033[0;31m$1\033[0m" +} + +echoGreen() { + echo -e "\033[0;32m$1\033[0m" +} + +echoOrange() { + echo -e "\033[0;33m$1\033[0m" +} + +echoBlue() { + echo -e "\033[0;34m$1\033[0m" +} + +addUser() { + if [ $# -eq 1 ]; then + set -- "$1" "$1" "$1" + fi + if [ $UID -ne 0 ]; then echo Please run this script as root. ; exit 1; fi + add_package_user "${1}" $2 10000 20000 $3 10000 20000 || exit 1 +} + +addCollectorGroup() { + group_name="$1" + if ! echo "$group_name" | grep -qw "audio"; then + if ! echo "$group_name" | grep -q "^u_"; then + echo "$group_name" | grep -q "^nimgnu_" || + group_name="nimgnu_$1" + fi + fi + + echoBlue "\n\nCreate Collector group $group_name" + if awk --field-separator=":" '{print $1}' /etc/group | grep $group_name && + awk --field-separator=":" '{print $1}' /etc/passwd | grep $group_name; then + echoGreen "\nGroup and user exist"; + return + fi + echo "$group_name" | grep -w "nimgnu_" && + echoRed "\ngroup_name missing" && + return + + echo "addUser "$group_name"" + echo -n "[y/N]: " + read check + [ "$check" != "y" ] && exit + addUser "$group_name" + echo "Created whyle installing $user_name" > info.txt + echo "Check groups by running group command" >> info.txt +} + +addUerToCollectorGroup() { + group_name="$1" + if ! echo "$group_name" | grep -qw "audio"; then + if ! echo "$group_name" | grep -q "^u_"; then + if ! echo "$group_name" | grep -q "^nimgnu_"; then + group_name="nimgnu_$1" + fi + fi + fi + + echoBlue "\n\nAdding user $username to group $group_name" + if ! echo "$group_name" | grep -q "^u_"; then + echo "$group_name" | grep -qw "nimgnu_" && + echoRed "group_name missing" && + return + fi + [ "$user_name" == "" ] && echoRed "user_name missing" && exit + checkIfUserInGroup $user_name $group_name && + echoGreen "$user_name is already a part of $group_name group." && + return + + usermod -a -G $group_name $user_name + + checkIfUserInGroup $user_name $group_name && + echoGreen "$user_name is now a part of $group_name group." && + return +} + +checkIfUserInGroup() { + username="$1" + groupname="$2" + cat /etc/group | grep "^$groupname:" | sed "s/.*://g;s/,/\n/g" | grep -w "^$username$" +} + +checkGroups() { + ! grep -q "^install_groups=(.*)" $install_script && + return + ! $sourced && eval "$(grep --color=never "^install_groups=(.*)" $install_script)" + echoOrange "\n\nThis user needs to be added to groups:" + echoOrange "A new user called \"$user_name\" may get created" + + for group in "${install_groups[@]}"; do + #if ! echo "$group" | grep -q "^u_"; then + # echo "$group" | grep -q "^nimgnu_" || + # group="nimgnu_$group" + #fi + if [ "$group" == "python-dependencies" ]; then + group="nimgnu_python-modules" + fi + echo "checkIfUserInGroup $user_name $group" + if ! checkIfUserInGroup "$user_name" "$group"; then + echoOrange "By adding to a group the user gets access to all files of the group" + echoOrange "\nDo you want to execute the following?" + grep -q -E "^$group:" /etc/group || + addCollectorGroup "$group" + echo -e " usermod -a -G $group $user_name" + echo -n "[y/N]: " + read check + [ "$check" == "y" ] && [ "$group $user_name" != " " ] && + addUerToCollectorGroup "$group" + fi + sleep 2 + done +} + +findAllGroupsOfDir() { + dir="$1" + for d in $(find $dir/* -maxdepth 0); do ls -ald "$d" | awk '{print $4}'; done | sort | uniq +} + +addUserToGroup() { + change_name="$1" + change_group="$2" + change_dir="$3" + if [ "$3" == "" ]; then + echo "ERROR" + echo "change_name=$1" + echo "change_group=$2" + echo "change_dir=$3" + exit + fi + checkIfUserInGroup $change_name $change_group && + return + echo "$change_group to $change_name" | tee /usr/src/$change_name/log/install_nimgnu_group-$date_log.log + + echoBlue "\n\n##########################\n" + find $change_dir -group $change_name + echoBlue "\n\n##########################\n" + echo "$dir" + echo -e " usermod -a -G $change_group $change_name" + echo -n "[y/N]: " + read check + + [ "$check" == "y" ] && ( + usermod -a -G $change_group $change_name + ) + +} + +installNimgnuGroup() { + echo -e "\n\nInstall groups for nimgnu_group" + echo "###${nimgnu_groups_dir[@]}##" + + for dir in "${nimgnu_groups_dir[@]}"; do + old_group="$(ls -ald $dir | awk '{print $4}')" + dir_groups=($(findAllGroupsOfDir $dir)) + echo "Found groups in $dir:" + echo -e "${dir_groups[@]}\n" + for g in ${dir_groups[@]}; do + if [ "$g" == "$old_group" ]; then + echo "nothing to change" + else + echo "addUserToGroup $g $user_name $dir" + addUserToGroup $g $user_name $dir + fi + done + for f in $dir/*; do + echo "############################" + echo "## Add user rights to group" + ls -ald $f + user_perms=$(stat -c "%A" $f | cut -c2-4) + chmod g=$user_perms $f + ls -ald $f + done + + + dirs_a=$(echo $dir) + echo "## $dir" + su -c "mkdir -p $dir" $user_name + if [ -d "$dir" ]; then + ls -ald $dir + parameter="" + if [ "$nimgnu_groups_dir_recrusive" == "true" ]; then + echo "" + echo "change permission recrusive for $user_name on;" + echo "$dir" + echo -n "[y/N]: " + read check + + [ "$check" == "y" ] && + parameter="-R" + fi + (chgrp -v $parameter "$user_name" "$dir" && + chmod -v 775 $parameter "$dir" && ls -ald "$dir") || + echoRed "ERROR: Failed to do\nchgrp -v $user_name $dir &&\nchmod -v 775 $dir" + else + echo "$dir: is not a valid directory (installNimgnuGroup)" + fi + + if ! [ "$old_group" == "" ] && ! [ "$user_name" == "install" ] && ! [ "$user_name" == "root" ] && + ! [ "$old_group" == "root" ] && ! [ "$user_name" == "$old_group" ]; then + addUserToGroup $user_name $old_group $dir + fi + done +} + +installNimbinUser() { + echo -e "\n\nInstall user for local package for the user (u_user)" + linenumber="$(cat /etc/pam.d/su_u_user | wc -l)" + if sed "${linenumber}q;d" /etc/pam.d/su_u_user | grep -q "success=1 default=ignore"; then + t="success=ignore default=1" + else + t="success=1 default=ignore" + fi + + echo "Make HOME 0750" + chmod -R 0750 "$install_user_dir" + + grep -q "$user_name).*USER.*\<$user_name\>.*\<$home_user\>" /etc/pam.d/su_u_user || + sed -i "$((1+$(grep -n 'case "$PAM_USER" in' /etc/pam.d/su_u_user | sed "s/\:.*$//g")))i\ \ $user_name) USERS=\"$user_name $home_user\"; ;;" /etc/pam.d/su_u_user + echo -e "\n\nModified pam.d/su" + cat /etc/pam.d/su_u_user + + if grep -q "^install_groups=(" $install_script; then + eval "$(grep "^install_groups=(" $install_script)" + checkGroups + if echo "${install_groups[@]}" | grep -wq u_xdg_runtime; then + rm -r $install_user_dir/.bash_profile + ln -s /etc/pkgusr/skel-u_xdg/.bash_profile $install_user_dir/.bash_profile + chmod 755 $install_user_dir/.bash_profile + chown $user_name: $install_user_dir/.bash_profile + fi + fi + if grep -q "^shared=true" $install_script; then + s_dir_nam="$(echo "$user_name" | sed "s/^u_//g")" + mkdir -p "$install_user_dir/${s_dir_nam^}_shared" + chmod g+sw "$install_user_dir/${s_dir_nam^}_shared" + chmod 770 "$install_user_dir/${s_dir_nam^}_shared" + chown -R $user_name: "$install_user_dir/${s_dir_nam^}_shared" + ln -s "$install_user_dir/${s_dir_nam^}_shared" "$home_user_dir/Shared" + echo "ln -s $install_user_dir/${s_dir_nam^}_shared $home_user_dir/Shared/" + chown -R $home_user: $home_user_dir/Shared/${s_dir_nam^}_shared + fi + + if grep -q "^app=" $install_script; then + eval "$(grep "^app=" $install_script)" + if which "$app"; then + ! [ -f "$home_user_dir/bin/$app" ] && + cat > $home_user_dir/bin/$app << EOF +#!/bin/bash + +su - u_$app -c "$app $(echo "${@}" | sed "s/\"/\\\"/g")" +EOF + cat > $home_user_dir/.local/share/applications/$app.desktop << EOF +[Desktop Entry] +Name=${app^} +Exec=script -c "$home_user_dir/bin/$app" /dev/null +Terminal=false +Type=Application +Icon=${app} +StartupWMClass=${app^} +EOF + chown ${home_user}: $home_user_dir/bin/$app + chmod +x $home_user_dir/bin/$app + chown ${home_user}: $home_user_dir/.local/share/applications/$app.desktop + echoGreen "Creating:" + else + echoRed "!!! Error: $app not found !!!" + echo "Not creating:" + fi + echo -e "\t$home_user_dir/bin/$app" + echo -e "\t$home_user_dir/.local/share/applications/$app.desktop" + fi + + usermod -a -G $user_name $home_user + echo -e "\n\n" +} + + +installPkg() { + user_name="$1" + install_user_dir="/usr/src/$user_name" + install_dir=/etc/pkgusr/install_scripts + install_script="$2" + install_script_basename="$(basename $2)" + install_log_dir=$install_user_dir/log + mkdir -p $install_log_dir + touch $install_log_dir/$install_$user_name-$date_log.diff + + [ -z "$install_script" ] && echo "Please provide an install script with \$2" + [ -z "$install_script" ] && exit + [ -f $install_script ] || echo "The install script \"$install_script\" doese not exist" + [ -f $install_script ] || exit + + echo -e "\033[0;34m" + echo "–––––––––––––––––––––––––––––––" + echo -e "Install \033[0;32m$user_name\033[0;34m" + echo "–––––––––––––––––––––––––––––––" + echo + echo -e "\033[0;31mFirst check the install file for:\n\033[0;33mlink, name, wrong commands like permission changes\ncheck if \"$user_name\" is the right username\n\033[0m" + echo -e "\033[0;33mPress enter to continue\033[0m" + read -n1 + echo + + + #################### + # source the install file + if grep -q "^install_pkg() {" "$install_script"; then + sourced=true + source $install_script + fi + + + #################### + # install user + id "$user_name" || + addUser $user_name + + id "$user_name" || + echo "The user $user_name could not be created." + + + #################### + # check for install_groups + checkGroups + + #################### + # copy install script + # special for old install scripts: + if grep -q "^source installBaseScript" $install_script; then + if ! [ -f $install_user_dir/installBaseScript ]; then + ln -s /etc/pkgusr/installBaseScript /$install_user_dir/installBaseScript && + chown $user_name: $install_user_dir/installBaseScript + fi + grep -q "sudo " $install_script && + cat $install_script | sed "s/\<sudo\> //g" > $install_user_dir/install_$user_name + vim $install_user_dir/install_$user_name + else + + #################### + # copy install script + touch $install_user_dir/install_$user_name + touch $install_user_dir/install_last + diff $install_user_dir/install_last $install_script >> $install_user_dir/log/install_$user_name-$date_log.diff + echo -e "##########################\n## !! DONT CHANGE ME !! ##\n## change instead: ##\n## $install_script ##\n## –––––––––––––––––––– ##\n## just install with: ##\n## packagemanager_install $user_name $install_script\n##########################" > $install_user_dir/install_last + cp $install_script $install_user_dir/$install_script_basename + cp $install_script $install_user_dir/install_last + cp $install_script $install_user_dir/install_$user_name + fi + + + + + #################### + # install package + # + # check if install_file installs a nimgnu_group + parameter="install" + if echo "$user_name" | grep "^nimgnu_"; then + installNimgnuGroup + parameter="" + fi + # check if install_file installs a u_user -> groups to run user applications like browser etc. + if echo "$user_name" | grep "^u_"; then + installNimbinUser + parameter="" + fi + + if grep -q "^install_pkg() {" "$install_script"; then + parameter="" + + grep -v '^[[:space:]]*$' "$install_script" | tail -n1 | grep -qw "install" && + parameter="install" + echo "$parameter" + fi + + # logs to $install_user_dir/packagemanager_install.log + chown $user_name: $install_user_dir/install_* + chown -R $user_name: $install_user_dir/log/ + su - -c "bash ~/install_$user_name $parameter 2>&1 | tee ~/log/packagemanager_install-$date_log.log" $user_name + + + if echo "$user_name" | grep "^u_"; then + gpasswd -d $user_name install # remove install group if nimgnu_group + fi + ################### + # write all installed files to /usr/src/$namepkg.lst + echo "" + echo -e "\033[0;31m!! Dont exit" + echo "Running mandb in backgronud" + mandb >/dev/null 2>&1 & + echo -e "!! Creating $install_user_dir/pkg.lst...\033[0m" + list_package $user_name 2>&1 > $install_user_dir/pkg.lst.new && + cp $install_user_dir/pkg.lst.new $install_user_dir/pkg.lst + +} + +installPkg $@