#include #include #include #include #include #include #include #include "download_unpack.h" // Function to create directories if they do not exist void make_directories(const char *path) { struct stat st = {0}; if (stat(path, &st) == -1) { printf("Creating directory: %s\n", path); if (mkdir(path, 0755) != 0) { perror("mkdir failed"); exit(EXIT_FAILURE); } } } // Function to download a single file void download_file(const char *url, const char *output_dir, int line) { char command[512]; // Move the cursor to the specific line printf("\033[%d;1H", line); // Move cursor to (line, 1) printf("Downloading: %s\n", url); snprintf(command, sizeof(command), "rm -f %s/%s", output_dir, strrchr(url, '/') + 1); system(command); // Remove the file if it already exists snprintf(command, sizeof(command), "wget -c -P %s %s", output_dir, url); // Added -c to continue download if (system(command) != 0) { fprintf(stderr, "Error downloading: %s\n", url); exit(EXIT_FAILURE); } } // Function to unpack a file void unpack_file(const char *file_path, const char *output_dir, int line) { char command[512]; // Move the cursor to the specific line printf("\033[%d;1H", line); // Move cursor to (line, 1) printf("Unpacking: %s\n", file_path); snprintf(command, sizeof(command), "test -f %s", file_path); if (system(command) != 0) { fprintf(stderr, "File %s does not exist, skipping unpack.\n", file_path); return; } snprintf(command, sizeof(command), "gunzip -c %s > %s/$(basename %s .gz)", file_path, output_dir, file_path); if (system(command) != 0) { fprintf(stderr, "Error unpacking: %s\n", file_path); exit(EXIT_FAILURE); } } // Function to download and unpack with status void process_file(const char *url, const char *download_dir, const char *unpack_dir, int line) { char file_path[512]; snprintf(file_path, sizeof(file_path), "%s/%s", download_dir, strrchr(url, '/') + 1); download_file(url, download_dir, line); // Pass the line number unpack_file(file_path, unpack_dir, line); // Pass the line number printf("Processed: %s\n", url); } // Function to process files in parallel void process_files_in_parallel(const char **urls, int url_count, const char *download_dir, const char *unpack_dir) { make_directories(download_dir); // Ensure download directory exists make_directories(unpack_dir); // Ensure unpack directory exists for (int i = 0; i < url_count; i++) { pid_t pid = fork(); if (pid == 0) { // Child process process_file(urls[i], download_dir, unpack_dir, i + 1); // Pass line number based on index exit(EXIT_SUCCESS); } else if (pid < 0) { perror("Fork failed"); exit(EXIT_FAILURE); } } // Parent process waits for all child processes for (int i = 0; i < url_count; i++) { wait(NULL); } } // Test function (used for main) int test_download_unpack() { printf("Fuuf"); const char *urls[] = { "https://datasets.imdbws.com/name.basics.tsv.gz", "https://datasets.imdbws.com/title.akas.tsv.gz", "https://datasets.imdbws.com/title.basics.tsv.gz", "https://datasets.imdbws.com/title.crew.tsv.gz", "https://datasets.imdbws.com/title.episode.tsv.gz", "https://datasets.imdbws.com/title.principals.tsv.gz", "https://datasets.imdbws.com/title.ratings.tsv.gz" }; int url_count = sizeof(urls) / sizeof(urls[0]); const char *download_dir = "imdb_gz"; const char *unpack_dir = "imdb"; process_files_in_parallel(urls, url_count, download_dir, unpack_dir); printf("All files processed successfully.\n"); return 0; }