From f523583ff3528f7fdaaae4747ed4da1c7c6a3131 Mon Sep 17 00:00:00 2001 From: Ethan Dye Date: Fri, 28 Jun 2024 20:41:11 -0600 Subject: [PATCH] Add `overlayfs-tools` back as submodule Now that the project is being actively maintained it makes more sense to have it included as a submodule, rather than maintaining a separate copy of the code for this project. Signed-off-by: Ethan Dye --- .github/workflows/test-action.yml | 2 + .gitmodules | 3 + README.md | 2 +- install.bash | 14 +- overlayfs-tools | 1 + overlayfs-tools/README.md | 42 --- overlayfs-tools/logic.c | 581 ------------------------------ overlayfs-tools/logic.h | 37 -- overlayfs-tools/main.c | 266 -------------- overlayfs-tools/makefile | 23 -- overlayfs-tools/sh.c | 98 ----- overlayfs-tools/sh.h | 20 - zram-config | 3 +- 13 files changed, 16 insertions(+), 1076 deletions(-) create mode 100644 .gitmodules create mode 160000 overlayfs-tools delete mode 100644 overlayfs-tools/README.md delete mode 100644 overlayfs-tools/logic.c delete mode 100644 overlayfs-tools/logic.h delete mode 100644 overlayfs-tools/main.c delete mode 100644 overlayfs-tools/makefile delete mode 100644 overlayfs-tools/sh.c delete mode 100644 overlayfs-tools/sh.h diff --git a/.github/workflows/test-action.yml b/.github/workflows/test-action.yml index c8b4581..69917cf 100644 --- a/.github/workflows/test-action.yml +++ b/.github/workflows/test-action.yml @@ -22,6 +22,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 + with: + submodules: recursive - name: Setup environment id: setup run: | diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..1cc197e --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "overlayfs-tools"] + path = overlayfs-tools + url = https://www.github.com/kmxz/overlayfs-tools diff --git a/README.md b/README.md index f796dab..0cf454d 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ Experimental Alpine support has also been added, other distributions may work bu ``` shell sudo apt-get install git -git clone https://github.com/ecdye/zram-config +git clone --recurse-submodules https://github.com/ecdye/zram-config sudo ./zram-config/install.bash ``` diff --git a/install.bash b/install.bash index 720047c..2aabcd6 100755 --- a/install.bash +++ b/install.bash @@ -29,12 +29,12 @@ if [[ -s /usr/local/sbin/zram-config ]] || [[ -s /usr/sbin/zram-config ]]; then exit 1 fi -if [[ $OS == "alpine" ]] && ! [[ "$(apk info 2> /dev/null | grep -E '^(gcc|make|fts-dev|linux-headers|util-linux-misc|musl-dev)' | tr '\n' ' ')" == "fts-dev gcc make util-linux-misc musl-dev linux-headers " ]]; then - echo "Installing needed packages (gcc, make, fts-dev, linux-headers, util-linux-misc, musl-dev)" - apk add gcc make fts-dev linux-headers util-linux-misc musl-dev || exit 1 -elif ! dpkg -s 'gcc' 'make' 'libc6-dev' &> /dev/null; then +if [[ $OS == "alpine" ]] && ! [[ "$(apk info 2> /dev/null | grep -E '^(gcc|meson|fts-dev|linux-headers|util-linux-misc|musl-dev)' | tr '\n' ' ')" == "fts-dev gcc meson util-linux-misc musl-dev linux-headers " ]]; then + echo "Installing needed packages (gcc, meson, fts-dev, linux-headers, util-linux-misc, musl-dev)" + apk add gcc meson fts-dev linux-headers util-linux-misc musl-dev || exit 1 +elif ! dpkg -s 'gcc' 'meson' 'libc6-dev' &> /dev/null; then echo "Installing needed packages (gcc, make, libc6-dev)" - apt-get install --yes gcc make libc6-dev || exit 1 + apt-get install --yes gcc meson libc6-dev || exit 1 fi UBUNTU_VERSION="$(grep -o '^VERSION_ID=.*$' /etc/os-release | cut -d'=' -f2 | tr -d '"')" @@ -45,7 +45,9 @@ if [[ $OS == "ubuntu" ]] && [[ $(bc -l <<< "$UBUNTU_VERSION >= 21.10") -eq 1 ]]; fi fi -make --always-make --directory="${BASEDIR}/overlayfs-tools" +meson setup --clearcache --reconfigure overlayfs-tools/builddir overlayfs-tools || exit 1 +meson compile -C overlayfs-tools/builddir || exit 1 +meson install -C overlayfs-tools/builddir || exit 1 echo "Installing zram-config files" if [[ "$OS" == "alpine" ]]; then diff --git a/overlayfs-tools b/overlayfs-tools new file mode 160000 index 0000000..162a5bf --- /dev/null +++ b/overlayfs-tools @@ -0,0 +1 @@ +Subproject commit 162a5bf6d90ed849bd1bbe999832d12f342f6092 diff --git a/overlayfs-tools/README.md b/overlayfs-tools/README.md deleted file mode 100644 index cc9922f..0000000 --- a/overlayfs-tools/README.md +++ /dev/null @@ -1,42 +0,0 @@ -overlayfs-tools -======== - -[OverlayFS](https://www.kernel.org/doc/Documentation/filesystems/overlayfs.txt) is the union filesystem provided by Linux kernel. - -This program comes provides three tools: -- **vacuum** - remove duplicated files in `upperdir` where `copy_up` is done but the file is not actually modified (see the sentence "the `copy_up` may turn out to be unnecessary" in the [Linux documentation](https://www.kernel.org/doc/Documentation/filesystems/overlayfs.txt)). This may reduce the size of `upperdir` without changing `lowerdir` or `overlay`. -- **diff** - show the list of actually changed files (the difference between `overlay` and `lowerdir`). A file with its type changed (i.e. from symbolic link to regular file) will shown as deleted then added, rather than modified. Similarly, for a opaque directory in `upperdir`, the corresponding directory in `lowerdir` (if exists) will be shown as entirely deleted, and a new directory with the same name added. File permission/owner changes will be simply shown as modified. -- **merge** - merge down the changes from `upperdir` to `lowerdir`. Unlike [aubrsync](http://aufs.sourceforge.net/aufs2/brsync/README.txt) for AuFS which bypasses the union filesystem mechanism, overlayfs-utils emulates the OverlayFS logic, which will be far more efficient. After this operation, `upperdir` will be empty and `lowerdir` will be the same as original `overlay`. -- **deref** - copy changes from `upperdir` to `uppernew` while unfolding redirect directories and metacopy regular files, so that new upperdir is compatible with legacy overlayfs driver. - -For safety reasons, vacuum and merge will not actually modify the filesystem, but generate a shell script to do the changes instead. - -Prerequisite and Building --------- - - $ make - -Example usage --------- - - # ./overlay diff -l /lower -u /upper - -See `./overlay --help` for more. - -Why sudo --------- - -As [Linux documentation](https://www.kernel.org/doc/Documentation/filesystems/overlayfs.txt) said, - -> A directory is made opaque by setting the xattr "trusted.overlay.opaque" to "y". - -However, only users with `CAP_SYS_ADMIN` can read `trusted.*` extended attributes. - -Warnings / limitations --------- - -- Only works for regular files and directories. Do not use it on OverlayFS with device files, socket files, etc.. -- Hard links may be broken (i.e. resulting in duplicated independent files). -- File owner, group and permission bits will be preserved. File timestamps, attributes and extended attributes might be lost. -- This program only works for OverlayFS with only one lower layer. -- It is recommended to have the OverlayFS unmounted before running this program. diff --git a/overlayfs-tools/logic.c b/overlayfs-tools/logic.c deleted file mode 100644 index 47ebfaa..0000000 --- a/overlayfs-tools/logic.c +++ /dev/null @@ -1,581 +0,0 @@ -#define _GNU_SOURCE -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "logic.h" -#include "sh.h" - -// exactly the same as in linux/fs.h -#define WHITEOUT_DEV 0 - -// exact the same as in fs/overlayfs/overlayfs.h -const char *ovl_opaque_xattr = "trusted.overlay.opaque"; -const char *ovl_redirect_xattr = "trusted.overlay.redirect"; -const char *ovl_metacopy_xattr = "trusted.overlay.metacopy"; - -#define MIN(X,Y) ((X) < (Y) ? (X) : (Y)) - -#define TRAILING_SLASH(ftype) (((ftype) == S_IFDIR) ? "/" : "") - -static inline mode_t file_type(const struct stat *status) { - return status->st_mode & S_IFMT; -} - -const char *ftype_name(mode_t type) { - switch (type) { - case S_IFDIR: - return "directory"; - case S_IFREG: - return "regular file"; - case S_IFLNK: - return "symbolic link"; - default: - return "special file"; - } -} - -const char *ftype_name_plural(mode_t type) { - switch (type) { - case S_IFDIR: - return "Directories"; - case S_IFREG: - return "Files"; - case S_IFLNK: - return "Symbolic links"; - default: - return "Special files"; - } -} - -static inline bool is_whiteout(const struct stat *status) { - return (file_type(status) == S_IFCHR) && (status->st_rdev == WHITEOUT_DEV); -} - -static inline mode_t permission_bits(const struct stat *status) { // not used yet. I haven't decided how to treat permission bit changes - return status->st_mode & (S_IRWXU | S_IRWXG | S_IRWXO | S_ISVTX); -} - -int is_opaque(const char *path, bool *output) { - char val; - ssize_t res = getxattr(path, ovl_opaque_xattr, &val, 1); - if ((res < 0) && (errno != ENODATA)) { - return -1; - } - *output = (res == 1 && val == 'y'); - return 0; -} - -int is_redirect(const char *path, bool *output) { - ssize_t res = getxattr(path, ovl_redirect_xattr, NULL, 0); - if ((res < 0) && (errno != ENODATA)) { - fprintf(stderr, "File %s redirect xattr can not be read.\n", path); - return -1; - } - *output = (res > 0); - return 0; -} - -int is_metacopy(const char *path, bool *output) { - ssize_t res = getxattr(path, ovl_metacopy_xattr, NULL, 0); - if ((res < 0) && (errno != ENODATA)) { - fprintf(stderr, "File %s metacopy xattr can not be read.\n", path); - return -1; - } - *output = (res >= 0); - return 0; -} - -// Treat redirect as opaque dir because it hides the tree in lower_path -// and we do not support following to redirected lower path -int is_opaquedir(const char *path, bool *output) { - bool opaque, redirect; - if (is_opaque(path, &opaque) < 0) { return -1; } - if (is_redirect(path, &redirect) < 0) { return -1; } - *output = opaque || redirect; - return 0; -} - -bool permission_identical(const struct stat *lower_status, const struct stat *upper_status) { - return (permission_bits(lower_status) == permission_bits(upper_status)) && (lower_status->st_uid == upper_status->st_uid) && (lower_status->st_gid == upper_status->st_gid); -} - -int read_chunk(int fd, char *buf, int len) { - ssize_t ret; - ssize_t remain = len; - while (remain > 0 && (ret = read(fd, buf, remain)) != 0) { - if (ret == -1) { - if (errno == EINTR) { - continue; - } - return -1; - } - remain -= ret; - buf += ret; - } - return len - remain; -} - -int regular_file_identical(const char *lower_path, const struct stat *lower_status, const char *upper_path, const struct stat *upper_status, bool *output) { - size_t blksize = (size_t) MIN(lower_status->st_blksize, upper_status->st_blksize); - if (lower_status->st_size != upper_status->st_size) { // different sizes - *output = false; - return 0; - } - bool metacopy, redirect; - if (is_metacopy(upper_path, &metacopy) < 0) { return -1; } - if (is_redirect(upper_path, &redirect) < 0) { return -1; } - if (metacopy) { - // metacopy means data is indentical, but redirect means it is not identical to lower_path - *output = !redirect; - return 0; - } - char lower_buffer[blksize]; - char upper_buffer[blksize]; - int lower_file = open(lower_path, O_RDONLY); - int upper_file = open(upper_path, O_RDONLY); - if (lower_file < 0) { - fprintf(stderr, "File %s can not be read for content.\n", lower_path); - return -1; - } - if (upper_file < 0) { - fprintf(stderr, "File %s can not be read for content.\n", upper_path); - return -1; - } - ssize_t read_lower; ssize_t read_upper; - do { // we can assume one will not reach EOF earlier than the other, as the file sizes are checked to be the same earlier - read_lower = read_chunk(lower_file, lower_buffer, blksize); - read_upper = read_chunk(upper_file, upper_buffer, blksize); - if (read_lower < 0) { - fprintf(stderr, "Error occured when reading file %s.\n", lower_path); - return -1; - } - if (read_upper < 0) { - fprintf(stderr, "Error occured when reading file %s.\n", upper_path); - return -1; - } - if (read_upper != read_lower) { // this should not happen as we've checked the sizes - fprintf(stderr, "Unexpected size difference: %s.\n", upper_path); - return -1; - } - if (memcmp(lower_buffer, upper_buffer, read_upper)) { *output = false; return 0; } // the output is by default false, but we still set it for ease of reading - } while (read_lower || read_upper); - *output = true; // now we can say they are identical - if (close(lower_file) || close(upper_file)) { return -1; } - return 0; -} - -int symbolic_link_identical(const char *lower_path, const char *upper_path, bool *output) { - char lower_buffer[PATH_MAX]; - char upper_buffer[PATH_MAX]; - ssize_t lower_len = readlink(lower_path, lower_buffer, PATH_MAX); - ssize_t upper_len = readlink(upper_path, upper_buffer, PATH_MAX); - if (lower_len < 0 || lower_len == PATH_MAX) { - fprintf(stderr, "Symbolic link %s cannot be resolved.\n", lower_path); - return -1; - } - if (upper_len < 0 || upper_len == PATH_MAX) { - fprintf(stderr, "Symbolic link %s cannot be resolved.\n", upper_path); - return -1; - } - lower_buffer[lower_len] = '\0'; - upper_buffer[upper_len] = '\0'; - *output = (strcmp(lower_buffer, upper_buffer) == 0); - return 0; -} - -static int vacuum_d(const char *lower_path, const char* upper_path, const size_t lower_root_len, const struct stat *lower_status, const struct stat *upper_status, FILE* script_stream, int *fts_instr) { - bool opaque; - if (is_opaquedir(upper_path, &opaque) < 0) { return -1; } - if (opaque) { // TODO: sometimes removing opaque directory (and combine with lower directory) might be better - *fts_instr = FTS_SKIP; - } - return 0; -} - -static int vacuum_dp(const char *lower_path, const char* upper_path, const size_t lower_root_len, const struct stat *lower_status, const struct stat *upper_status, FILE* script_stream, int *fts_instr) { - if (lower_status == NULL) { return 0; } // lower does not exist - if (file_type(lower_status) != S_IFDIR) { return 0; } - if (!permission_identical(lower_status, upper_status)) { return 0; } - bool opaque; - if (is_opaquedir(upper_path, &opaque) < 0) { - return -1; - } - if (opaque) { return 0; } - // this directory might be empty if all children are deleted in previous commands. but we simply don't test whether it's that case - return command(script_stream, "rmdir --ignore-fail-on-non-empty %U", upper_path); -} - -static int vacuum_f(const char *lower_path, const char* upper_path, const size_t lower_root_len, const struct stat *lower_status, const struct stat *upper_status, FILE* script_stream, int *fts_instr) { - if (lower_status == NULL) { return 0; } // lower does not exist - if (file_type(lower_status) != S_IFREG) { return 0; } - if (!permission_identical(lower_status, upper_status)) { return 0; } - bool identical; - if (regular_file_identical(lower_path, lower_status, upper_path, upper_status, &identical) < 0) { - return -1; - } - if (!identical) { return 0; } - return command(script_stream, "rm %U", upper_path); -} - -static int vacuum_sl(const char *lower_path, const char* upper_path, const size_t lower_root_len, const struct stat *lower_status, const struct stat *upper_status, FILE* script_stream, int *fts_instr) { - if (lower_status == NULL) { return 0; } // lower does not exist - if (file_type(lower_status) != S_IFLNK) { return 0; } - if (!permission_identical(lower_status, upper_status)) { return 0; } - bool identical; - if (symbolic_link_identical(lower_path, upper_path, &identical) < 0) { - return -1; - } - if (!identical) { return 0; } - return command(script_stream, "rm %U", upper_path); -} - -void print_only_in(const char *path) { - char *dirc = strdup(path); - char *basec = strdup(path); - char *dname = dirname(dirc); - char *bname = basename(basec); - printf("Only in %s: %s\n", dname, bname); - free(dirc); - free(basec); -} - -void print_removed(const char *lower_path, const size_t lower_root_len, mode_t lower_type) { - if (brief) { - print_only_in(lower_path); - } else { - printf("Removed: %s%s\n", &lower_path[lower_root_len], TRAILING_SLASH(lower_type)); - } -} - -void print_added(const char *lower_path, const size_t lower_root_len, const char *upper_path, mode_t upper_type) { - if (brief) { - print_only_in(upper_path); - } else { - printf("Added: %s%s\n", &lower_path[lower_root_len], TRAILING_SLASH(upper_type)); - } -} - -void print_replaced(const char *lower_path, const size_t lower_root_len, mode_t lower_type, const char *upper_path, mode_t upper_type) { - if (brief) { - printf("File %s is a %s while file %s is a %s\n", lower_path, ftype_name(lower_type), upper_path, ftype_name(upper_type)); - } else { - if (lower_type != S_IFDIR) { // dir removed already printed by list_deleted_files() - print_removed(lower_path, lower_root_len, lower_type); - } - print_added(lower_path, lower_root_len, upper_path, upper_type); - } -} - -void print_modified(const char *lower_path, const size_t lower_root_len, mode_t lower_type, const char *upper_path, bool identical) { - if (brief) { - if (!identical) { // brief format does not print permission difference - printf("%s %s and %s differ\n", ftype_name_plural(lower_type), lower_path, upper_path); - } - } else { - printf("Modified: %s%s\n", &lower_path[lower_root_len], TRAILING_SLASH(lower_type)); - } -} - -int list_deleted_files(const char *lower_path, size_t lower_root_len, mode_t upper_type) { // This WORKS with files and itself is listed. However, prefixs are WRONG! - // brief format needs to print only first level deleted children under opaque dir - bool children = (brief && (upper_type == S_IFDIR)); - if (!verbose && !children) { - if (!brief || upper_type == S_IFCHR) { // dir replaced already printed by print_replaced() - print_removed(lower_path, lower_root_len, S_IFDIR); - } - return 0; - } - FTSENT *cur; - char *paths[2] = {(char *) lower_path, NULL }; - FTS *ftsp = fts_open(paths, FTS_NOCHDIR | FTS_PHYSICAL, NULL); - if (ftsp == NULL) { return -1; } - int return_val = 0; - while (((cur = fts_read(ftsp)) != NULL) && (return_val == 0)) { - switch (cur->fts_info) { - case FTS_D: - // brief format does not need to print deleted grand children under opaque dir - if (children && cur->fts_level > 0) { - print_removed(cur->fts_path, lower_root_len, S_IFDIR); - fts_set(ftsp, cur, FTS_SKIP); - } - break; // do nothing - case FTS_DP: - // brief format does not need to print deleted dir under opaque dir itself - if (!children) { - print_removed(cur->fts_path, lower_root_len, S_IFDIR); - } - break; - case FTS_F: - print_removed(cur->fts_path, lower_root_len, S_IFREG); - break; - case FTS_SL: - print_removed(cur->fts_path, lower_root_len, S_IFLNK); - break; - case FTS_DEFAULT: - fprintf(stderr, "File %s is a special file (device or pipe). We cannot handle that.\n", cur->fts_path); - return_val = -1; - break; - default: - fprintf(stderr, "Error occured when opening %s.\n", cur->fts_path); - return_val = -1; - } - } - if (errno) { return_val = -1; } // if no error happened, fts_read will "sets the external variable errno to 0" according to the documentation - return fts_close(ftsp) || return_val; -} - -static int diff_d(const char *lower_path, const char* upper_path, const size_t lower_root_len, const struct stat *lower_status, const struct stat *upper_status, FILE* script_stream, int *fts_instr) { - bool opaque = false; - bool lower_exist = (lower_status != NULL); - if (lower_exist) { - if (file_type(lower_status) == S_IFDIR) { - if (is_opaquedir(upper_path, &opaque) < 0) { return -1; } - if (opaque) { - if (list_deleted_files(lower_path, lower_root_len, S_IFDIR) < 0) { return -1; } - } else { - if (!permission_identical(lower_status, upper_status)) { - print_modified(lower_path, lower_root_len, S_IFDIR, upper_path, true); - } - return 0; // children must be recursed, and directory itself does not need to be printed - } - } else { // other types of files - print_replaced(lower_path, lower_root_len, file_type(lower_status), upper_path, S_IFDIR); - } - } - if (!(verbose || (brief && opaque))) { // brief format needs to print children of opaque dir - *fts_instr = FTS_SKIP; - } - if (!lower_exist || (!brief && opaque)) { // brief format does not need to print opaque dir itself - print_added(lower_path, lower_root_len, upper_path, S_IFDIR); - } - return 0; -} - -static int diff_f(const char *lower_path, const char* upper_path, const size_t lower_root_len, const struct stat *lower_status, const struct stat *upper_status, FILE* script_stream, int *fts_instr) { - bool identical; - if (lower_status != NULL) { - switch (file_type(lower_status)) { - case S_IFREG: - if (regular_file_identical(lower_path, lower_status, upper_path, upper_status, &identical) < 0) { - return -1; - } - if (!(identical && permission_identical(lower_status, upper_status))) { - print_modified(lower_path, lower_root_len, S_IFREG, upper_path, identical); - } - return 0; - case S_IFDIR: - if (list_deleted_files(lower_path, lower_root_len, S_IFREG) < 0) { return -1; } - /* fallthrough */ - case S_IFLNK: - print_replaced(lower_path, lower_root_len, file_type(lower_status), upper_path, S_IFREG); - return 0; - default: - fprintf(stderr, "File %s is a special file (device or pipe). We cannot handle that.\n", lower_path); - return -1; - } - } - print_added(lower_path, lower_root_len, upper_path, S_IFREG); - return 0; -} - -static int diff_sl(const char *lower_path, const char* upper_path, const size_t lower_root_len, const struct stat *lower_status, const struct stat *upper_status, FILE* script_stream, int *fts_instr) { - bool identical; - if (lower_status != NULL) { - switch (file_type(lower_status)) { - case S_IFDIR: - if (list_deleted_files(lower_path, lower_root_len, S_IFLNK) < 0) { return -1; } - /* fallthrough */ - case S_IFREG: - print_replaced(lower_path, lower_root_len, file_type(lower_status), upper_path, S_IFLNK); - return 0; - case S_IFLNK: - if (symbolic_link_identical(lower_path, upper_path, &identical) < 0) { - return -1; - } - if (!(identical && permission_identical(lower_status, upper_status))) { - print_modified(lower_path, lower_root_len, S_IFLNK, upper_path, identical); - } - return 0; - default: - fprintf(stderr, "File %s is a special file (device or pipe). We cannot handle that.\n", lower_path); - return -1; - } - } - print_added(lower_path, lower_root_len, upper_path, S_IFLNK); - return 0; -} - -static int diff_whiteout(const char *lower_path, const char* upper_path, const size_t lower_root_len, const struct stat *lower_status, const struct stat *upper_status, FILE* script_stream, int *fts_instr) { - if (lower_status != NULL) { - if (file_type(lower_status) == S_IFDIR) { - if (list_deleted_files(lower_path, lower_root_len, S_IFCHR) < 0) { return -1; } - } else { - print_removed(lower_path, lower_root_len, file_type(lower_status)); - } - } // else: whiteouting a nonexistent file? must be an error. but we ignore that :) - return 0; -} - -static int merge_d(const char *lower_path, const char* upper_path, const size_t lower_root_len, const struct stat *lower_status, const struct stat *upper_status, FILE* script_stream, int *fts_instr) { - bool redirect; - if (is_redirect(upper_path, &redirect) < 0) { return -1; } - // merging redirects is not supported, we must abort merge so redirected lower (under whiteout) won't be deleted - // upper_path may be hiding the directory in lower_path, but there may be another redirect upper pointing at it - if (redirect) { - fprintf(stderr, "Found redirect on %s. Merging redirect is not supported - Abort.\n", upper_path); - return -1; - } - if (lower_status != NULL) { - if (file_type(lower_status) == S_IFDIR) { - bool opaque = false; - if (is_opaquedir(upper_path, &opaque) < 0) { return -1; } - if (opaque) { - if (command(script_stream, "rm -r %L", lower_path) < 0) { return -1; }; - } else { - if (!permission_identical(lower_status, upper_status)) { - command(script_stream, "chmod --reference %U %L", upper_path, lower_path); - } - return 0; // children must be recursed, and directory itself does not need to be printed - } - } else { - command(script_stream, "rm %L", lower_path); - } - } - *fts_instr = FTS_SKIP; - return command(script_stream, "mv -T %U %L", upper_path, lower_path); -} - -static int merge_dp(const char *lower_path, const char* upper_path, const size_t lower_root_len, const struct stat *lower_status, const struct stat *upper_status, FILE* script_stream, int *fts_instr) { - if (lower_status != NULL) { - if (file_type(lower_status) == S_IFDIR) { - bool opaque = false; - if (is_opaquedir(upper_path, &opaque) < 0) { return -1; } - if (!opaque) { // delete the directory: it should be empty already - return command(script_stream, "rmdir %U", upper_path); - } - } - } - return 0; -} - -static int merge_f(const char *lower_path, const char* upper_path, const size_t lower_root_len, const struct stat *lower_status, const struct stat *upper_status, FILE* script_stream, int *fts_instr) { - bool metacopy, redirect; - if (is_metacopy(upper_path, &metacopy) < 0) { return -1; } - if (is_redirect(upper_path, &redirect) < 0) { return -1; } - // merging metacopy is not supported, we must abort merge so lower data won't be deleted - if (metacopy || redirect) { - fprintf(stderr, "Found metacopy/redirect on %s. Merging metacopy/redirect is not supported - Abort.\n", upper_path); - return -1; - } - return command(script_stream, "rm -rf %L", lower_path) || command(script_stream, "mv -T %U %L", upper_path, lower_path); -} - -static int merge_sl(const char *lower_path, const char* upper_path, const size_t lower_root_len, const struct stat *lower_status, const struct stat *upper_status, FILE* script_stream, int *fts_instr) { - return command(script_stream, "rm -rf %L", lower_path) || command(script_stream, "mv -T %U %L", upper_path, lower_path); -} - -static int merge_whiteout(const char *lower_path, const char* upper_path, const size_t lower_root_len, const struct stat *lower_status, const struct stat *upper_status, FILE* script_stream, int *fts_instr) { - return command(script_stream, "rm -r %L", lower_path) || command(script_stream, "rm %U", upper_path); -} - -typedef int (*TRAVERSE_CALLBACK)(const char *lower_path, const char* upper_path, const size_t lower_root_len, const struct stat *lower_status, const struct stat *upper_status, FILE* script_stream, int *fts_instr); - -int traverse(const char *lower_root, const char *upper_root, FILE* script_stream, TRAVERSE_CALLBACK callback_d, TRAVERSE_CALLBACK callback_dp, TRAVERSE_CALLBACK callback_f, TRAVERSE_CALLBACK callback_sl, TRAVERSE_CALLBACK callback_whiteout) { // returns 0 on success - FTSENT *cur; - char *paths[2] = {(char *) upper_root, NULL }; - char lower_path[PATH_MAX]; - strcpy(lower_path, lower_root); - size_t upper_root_len = strlen(upper_root); - size_t lower_root_len = strlen(lower_root); - FTS *ftsp = fts_open(paths, FTS_NOCHDIR | FTS_PHYSICAL, NULL); - if (ftsp == NULL) { return -1; } - int return_val = 0; - while ((return_val == 0) && ((cur = fts_read(ftsp)) != NULL)) { - TRAVERSE_CALLBACK callback = NULL; - switch (cur->fts_info) { - case FTS_D: - callback = callback_d; - break; - case FTS_DP: - callback = callback_dp; - break; - case FTS_F: - callback = callback_f; - break; - case FTS_SL: - callback = callback_sl; - break; - case FTS_DEFAULT: - if (is_whiteout(cur->fts_statp)) { - callback = callback_whiteout; - } else { - return_val = -1; - fprintf(stderr, "File %s is a special file (device or pipe). We cannot handle that.\n", cur->fts_path); - } - break; - default: - return_val = -1; - fprintf(stderr, "Error occured when opening %s.\n", cur->fts_path); - } - if (callback != NULL) { - int fts_instr = 0; - struct stat lower_status; - bool lower_exist = true; - strcpy(&lower_path[lower_root_len], &(cur->fts_path[upper_root_len])); - if (lstat(lower_path, &lower_status) != 0) { - if (errno == ENOENT || errno == ENOTDIR) { // the corresponding lower file (or its ancestor) does not exist at all - lower_exist = false; - } else { // stat failed for some unknown reason - fprintf(stderr, "Failed to stat %s.\n", lower_path); - return_val = -1; - break; // do not call callback in this case - } - } - return_val = callback(lower_path, cur->fts_path, lower_root_len, lower_exist ? &lower_status : NULL, cur->fts_statp, script_stream, &fts_instr); // return_val must previously be 0 - if (fts_instr) { - fts_set(ftsp, cur, fts_instr); - } - } - } - if (errno) { return_val = -1; } // if no error happened, fts_read will "sets the external variable errno to 0" according to the documentation - return fts_close(ftsp) || return_val; -} - -static int deref_d(const char *mnt_path, const char* upper_path, const size_t mnt_root_len, const struct stat *mnt_status, const struct stat *upper_status, FILE* script_stream, int *fts_instr) { - bool redirect; - if (is_redirect(upper_path, &redirect) < 0) { return -1; } - if (!redirect) { return 0; } - *fts_instr = FTS_SKIP; - return command(script_stream, "rm -rf %U", upper_path) || command(script_stream, "cp -a %M %U", mnt_path, upper_path); -} - -static int deref_f(const char *mnt_path, const char* upper_path, const size_t mnt_root_len, const struct stat *mnt_status, const struct stat *upper_status, FILE* script_stream, int *fts_instr) { - bool metacopy; - if (is_metacopy(upper_path, &metacopy) < 0) { return -1; } - if (!metacopy) { return 0; } - return command(script_stream, "rm -r %U", upper_path) || command(script_stream, "cp -a %M %U", mnt_path, upper_path); -} - -int vacuum(const char* lowerdir, const char* upperdir, FILE* script_stream) { - return traverse(lowerdir, upperdir, script_stream, vacuum_d, vacuum_dp, vacuum_f, vacuum_sl, NULL); -} - -int diff(const char* lowerdir, const char* upperdir) { - return traverse(lowerdir, upperdir, NULL, diff_d, NULL, diff_f, diff_sl, diff_whiteout); -} - -int merge(const char* lowerdir, const char* upperdir, FILE* script_stream) { - return traverse(lowerdir, upperdir, script_stream, merge_d, merge_dp, merge_f, merge_sl, merge_whiteout); -} - -int deref(const char* mountdir, const char* upperdir, FILE* script_stream) { - return traverse(mountdir, upperdir, script_stream, deref_d, NULL, deref_f, NULL, NULL); -} diff --git a/overlayfs-tools/logic.h b/overlayfs-tools/logic.h deleted file mode 100644 index 374e517..0000000 --- a/overlayfs-tools/logic.h +++ /dev/null @@ -1,37 +0,0 @@ -/* - * logic.h / logic.c - * - * the logic for the three feature functions - */ - -#ifndef OVERLAYFS_TOOLS_LOGIC_H -#define OVERLAYFS_TOOLS_LOGIC_H - -#include - -extern bool verbose; -extern bool brief; - -/* - * feature function. will take very long time to complete. returns 0 on success - */ -int vacuum(const char* lowerdir, const char* upperdir, FILE* script_stream); - -/* - * feature function. will take very long time to complete. returns 0 on success - */ -int diff(const char* lowerdir, const char* upperdir); - -/* - * feature function. will take very long time to complete. returns 0 on success - */ -int merge(const char* lowerdir, const char* upperdir, FILE* script_stream); - -/* - * Unfold metacopy and redirect upper. - * - * mountdir is required and lowerdir is irrelevant. - */ -int deref(const char* mountdir, const char* upperdir, FILE* script_stream); - -#endif //OVERLAYFS_TOOLS_LOGIC_H diff --git a/overlayfs-tools/main.c b/overlayfs-tools/main.c deleted file mode 100644 index bea469f..0000000 --- a/overlayfs-tools/main.c +++ /dev/null @@ -1,266 +0,0 @@ -/* - * main.c - * - * the command line user interface - */ -#define _GNU_SOURCE -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#ifndef _SYS_STAT_H - #include -#endif -#include "logic.h" -#include "sh.h" - -#define STRING_BUFFER_SIZE PATH_MAX * 2 - -// currently, brief and verbose are mutually exclusive -bool verbose; -bool brief; -bool yes; - -void print_help(const char *program) { - printf("Usage: %s command options\n", program); - puts(""); - puts("Commands:"); - puts(" vacuum - remove duplicated files in upperdir where copy_up is done but the file is not actually modified"); - puts(" diff - show the list of actually changed files"); - puts(" merge - merge all changes from upperdir to lowerdir, and clear upperdir"); - puts(" deref - copy changes from upperdir to a new upperdir unfolding redirect and metacopy"); - puts(""); - puts("Options:"); - puts(" -l, --lowerdir=LOWERDIR the lowerdir of OverlayFS (required)"); - puts(" -u, --upperdir=UPPERDIR the upperdir of OverlayFS (required)"); - puts(" -m, --mountdir=MOUNTDIR the mountdir of OverlayFS (optional)"); - puts(" -L, --lowernew=LOWERNEW the lowerdir of new OverlayFS (optional)"); - puts(" -U, --uppernew=UPPERNEW the upperdir of new OverlayFS (optional)"); - puts(" -y --yes don't prompt if OverlayFS is still mounted (optional)"); - puts(" -v, --verbose with diff action only: when a directory only exists in one version, still list every file of the directory"); - puts(" -b, --brief with diff action only: conform to output of diff --brief --recursive --no-dereference"); - puts(" -h, --help show this help text"); - puts(""); - puts("See https://github.com/kmxz/overlayfs-tools/ for warnings and more information."); -} - -bool starts_with(const char *haystack, const char* needle) { - return strncmp(needle, haystack, strlen(needle)) == 0; -} - -bool is_mounted(const char *lower, const char *upper) { - FILE *f = fopen("/proc/mounts", "r"); - if (!f) { - fprintf(stderr, "Cannot read /proc/mounts to test whether OverlayFS is mounted.\n"); - return true; - } - char buf[STRING_BUFFER_SIZE]; - while (fgets(buf, STRING_BUFFER_SIZE, f)) { - if (!starts_with(buf, "overlay")) { - continue; - } - if (strlen(buf) == STRING_BUFFER_SIZE) { - fprintf(stderr, "OverlayFS line in /proc/mounts is too long.\n"); - return true; - } - char *m_lower = strstr(buf, "lowerdir="); - char *m_upper = strstr(buf, "upperdir="); - if (m_lower == NULL || m_upper == NULL) { - fprintf(stderr, "Cannot extract information from OverlayFS line in /proc/mounts.\n"); - return true; - } - m_lower = &(m_lower[strlen("lowerdir=")]); - m_upper = &(m_upper[strlen("upperdir=")]); - if (!(strncmp(lower, m_lower, strlen(lower)) && strncmp(upper, m_upper, strlen(upper)))) { - printf("The OverlayFS involved is still mounted.\n"); - return true; - } - } - return false; -} - -bool check_mounted(const char *lower, const char *upper) { - if (is_mounted(lower, upper) && !yes) { - printf("It is strongly recommended to unmount OverlayFS first. Still continue (not recommended)?: \n"); - int r = getchar(); - if (r != 'Y' && r != 'y') { - return true; - } - } - return false; -} - -bool directory_exists(const char *path) { - struct stat sb; - if (lstat(path, &sb) != 0) { return false; } - return (sb.st_mode & S_IFMT) == S_IFDIR; -} - -bool directory_create(const char *name, const char *path) { - if (mkdir(path, 0755) == 0 || errno == EEXIST) { return true; } - fprintf(stderr, "%s directory '%s' does not exist and cannot be created.\n", name, path); - exit(EXIT_FAILURE); -} - -bool real_check_xattr_trusted(const char *tmp_path, int tmp_file) { - int ret = fsetxattr(tmp_file, "trusted.overlay.test", "naive", 5, 0); - close(tmp_file); - if (ret) { return false; } - char verify_buffer[10]; - if (getxattr(tmp_path, "trusted.overlay.test", verify_buffer, 10) != 5) { return false; } - return !strncmp(verify_buffer, "naive", 5); -} - -bool check_xattr_trusted(const char *upper) { - char tmp_path[PATH_MAX]; - strcpy(tmp_path, upper); - strcat(tmp_path, "/.xattr_test_XXXXXX.tmp"); - int tmp_file = mkstemps(tmp_path, 4); - if (tmp_file < 0) { return false; } - bool ret = real_check_xattr_trusted(tmp_path, tmp_file); - unlink(tmp_path); - return ret; -} - -int main(int argc, char *argv[]) { - - char *lower = NULL; - char *upper = NULL; - char *dir, *mnt = NULL; - - static struct option long_options[] = { - { "lowerdir", required_argument, 0, 'l' }, - { "upperdir", required_argument, 0, 'u' }, - { "mountdir", required_argument, 0, 'm' }, - { "lowernew", required_argument, 0, 'L' }, - { "uppernew", required_argument, 0, 'U' }, - { "yes", no_argument , 0, 'y' }, - { "help", no_argument , 0, 'h' }, - { "verbose", no_argument , 0, 'v' }, - { "brief", no_argument , 0, 'b' }, - { 0, 0, 0, 0 } - }; - - int opt = 0; - int long_index = 0; - while ((opt = getopt_long_only(argc, argv, "l:u:m:L:U:yhvb", long_options, &long_index)) != -1) { - switch (opt) { - case 'l': - lower = realpath(optarg, NULL); - if (lower) { vars[LOWERDIR] = lower; } - break; - case 'u': - upper = realpath(optarg, NULL); - if (upper) { vars[UPPERDIR] = upper; } - break; - case 'm': - mnt = realpath(optarg, NULL); - if (mnt) { vars[MOUNTDIR] = mnt; } - break; - case 'L': - directory_create("New lowerdir", optarg); - dir = realpath(optarg, NULL); - if (dir) { vars[LOWERNEW] = dir; } - break; - case 'U': - directory_create("New upperdir", optarg); - dir = realpath(optarg, NULL); - if (dir) { vars[UPPERNEW] = dir; } - break; - case 'y': - yes = true; - break; - case 'h': - print_help(argv[0]); - return EXIT_SUCCESS; - case 'v': - verbose = true; - brief = false; - break; - case 'b': - verbose = false; - brief = true; - break; - default: - fprintf(stderr, "Option %c is not supported.\n", opt); - goto see_help; - } - } - - if (!lower) { - fprintf(stderr, "Lower directory not specified.\n"); - goto see_help; - } - if (!directory_exists(lower)) { - fprintf(stderr, "Lower directory cannot be opened.\n"); - goto see_help; - } - if (!upper) { - fprintf(stderr, "Upper directory not specified.\n"); - goto see_help; - } - if (!directory_exists(upper)) { - fprintf(stderr, "Upper directory cannot be opened.\n"); - goto see_help; - } - if (!check_xattr_trusted(upper)) { - fprintf(stderr, "The program cannot write trusted.* xattr. Try run again as root.\n"); - return EXIT_FAILURE; - } - // Relax check for mounted overlay if we are not going to modify lowerdir/upperdir - if ((!vars[LOWERNEW] || !vars[UPPERNEW]) && check_mounted(lower, upper)) { - return EXIT_FAILURE; - } - - if (optind == argc - 1) { - int out; - char filename_template[] = "overlay-tools-XXXXXX.sh"; - FILE *script = NULL; - if (strcmp(argv[optind], "diff") == 0) { - out = diff(lower, upper); - } else if (strcmp(argv[optind], "vacuum") == 0) { - script = create_shell_script(filename_template); - if (script == NULL) { fprintf(stderr, "Script file cannot be created.\n"); return EXIT_FAILURE; } - out = vacuum(lower, upper, script); - } else if (strcmp(argv[optind], "merge") == 0) { - script = create_shell_script(filename_template); - if (script == NULL) { fprintf(stderr, "Script file cannot be created.\n"); return EXIT_FAILURE; } - out = merge(lower, upper, script); - } else if (strcmp(argv[optind], "deref") == 0) { - if (!mnt || !vars[UPPERNEW]) { fprintf(stderr, "'deref' command requires --uppernew and --mountdir.\n"); return EXIT_FAILURE; } - if (!directory_exists(mnt)) { - fprintf(stderr, "OverlayFS mount directory cannot be opened.\n"); - goto see_help; - } - script = create_shell_script(filename_template); - if (script == NULL) { fprintf(stderr, "Script file cannot be created.\n"); return EXIT_FAILURE; } - out = deref(mnt, upper, script); - } else { - fprintf(stderr, "Action not supported.\n"); - goto see_help; - } - if (script != NULL) { - printf("The script %s is created. Run the script to do the actual work please. Remember to run it when the OverlayFS is not mounted.\n", filename_template); - fclose(script); - } - if (out) { - fprintf(stderr, "Action aborted due to fatal error.\n"); - return EXIT_FAILURE; - } - return EXIT_SUCCESS; - } - - fprintf(stderr, "Please specify one action.\n"); - -see_help: - fprintf(stderr, "Try '%s --help' for more information.\n", argv[0]); - return EXIT_FAILURE; - -} diff --git a/overlayfs-tools/makefile b/overlayfs-tools/makefile deleted file mode 100644 index 6de467a..0000000 --- a/overlayfs-tools/makefile +++ /dev/null @@ -1,23 +0,0 @@ -CC = gcc -CFLAGS = -Wall -std=c99 -LDFLAGS = -lm -ifneq (,$(wildcard /etc/alpine-release)) - LDFLAGS += -lfts -endif - -all: overlay - -overlay: main.o logic.o sh.o - $(CC) main.o logic.o sh.o -o overlay $(LDFLAGS) - -main.o: main.c logic.h - $(CC) $(CFLAGS) -c main.c - -logic.o: logic.c logic.h sh.h - $(CC) $(CFLAGS) -c logic.c - -sh.o: sh.c sh.h - $(CC) $(CFLAGS) -c sh.c - -clean: - $(RM) main.o logic.o sh.o overlay diff --git a/overlayfs-tools/sh.c b/overlayfs-tools/sh.c deleted file mode 100644 index 98565ac..0000000 --- a/overlayfs-tools/sh.c +++ /dev/null @@ -1,98 +0,0 @@ -#define _GNU_SOURCE -#include -#include -#include -#include -#include -#include -#include "sh.h" - -char * vars[NUM_VARS]; -const char * var_names[NUM_VARS] = { - "LOWERDIR", - "UPPERDIR", - "MOUNTDIR", - "LOWERNEW", - "UPPERNEW", -}; - -int quote(const char *filename, FILE *output); - -FILE* create_shell_script(char *tmp_path_buffer) { - int tmp_file = mkstemps(tmp_path_buffer, 3); // the 3 is for suffix length (".sh") - if (tmp_file < 0) { return NULL; } - fchmod(tmp_file, S_IRWXU); // chmod to 0700 - FILE* f = fdopen(tmp_file, "w"); - if (f == NULL) { return NULL; } - fprintf(f, "#!/usr/bin/env bash\n"); - fprintf(f, "set -x\n"); - time_t rawtime; - time (&rawtime); - fprintf(f, "# This shell script is generated by overlayfs-tools on %s\n", ctime (&rawtime)); - for (int i=0; i < NUM_VARS; i++) { - if (vars[i]) { - fprintf(f, "%s=", var_names[i]); - if (quote(vars[i], f) < 0) { return NULL; } - if (fputc('\n', f) == EOF) { return NULL; } - } - } - // Non-empty *NEW vars make a backup copy and override *DIR vars - if (vars[LOWERNEW]) { - fprintf(f, "rm -rf \"$LOWERNEW\"\n"); - fprintf(f, "cp -a \"$LOWERDIR\" \"$LOWERNEW\"\n"); - fprintf(f, "LOWERDIR="); - if (quote(vars[LOWERNEW], f) < 0) { return NULL; } - if (fputc('\n', f) == EOF) { return NULL; } - } - if (vars[UPPERNEW]) { - fprintf(f, "rm -rf \"$UPPERNEW\"\n"); - fprintf(f, "cp -a \"$UPPERDIR\" \"$UPPERNEW\"\n"); - fprintf(f, "UPPERDIR="); - if (quote(vars[UPPERNEW], f) < 0) { return NULL; } - if (fputc('\n', f) == EOF) { return NULL; } - } - return f; -} - -int quote(const char *filename, FILE *output) { - if (fputc('\'', output) == EOF) { return -1; } - for (const char *s = filename; *s != '\0'; s++) { - if (*s == '\'') { - if (fprintf(output, "'\"'\"'") < 0) { return -1; } - } else { - if (fputc(*s, output) == EOF) { return -1; } - } - } - if (fputc('\'', output) == EOF) { return -1; } - return 0; -} - -int substitue(char what, const char *filename, FILE *output) { - int i; - for (i=0; i < NUM_VARS; i++) - if (vars[i] && var_names[i][0] == what) - break; - if (i == NUM_VARS) { return -1; } - // filename prefix must match the var value - int prefix = strlen(vars[i]); - if (strncmp(filename, vars[i], prefix)) { return -1; } - filename += prefix; - fprintf(output, "\"$%s\"", var_names[i]); - return quote(filename, output); -} - -int command(FILE *output, const char *command_format, ...) { - va_list arg; - va_start(arg, command_format); - for (size_t i = 0; command_format[i] != '\0'; i++) { - if (command_format[i] == '%') { - const char *s = va_arg(arg, char *); - if (substitue(command_format[++i], s, output) < 0) { return -1; } - } else { - if (fputc(command_format[i], output) == EOF) { return -1; } - } - } - va_end(arg); - if (fputc('\n', output) == EOF) { return -1; } - return 0; -} diff --git a/overlayfs-tools/sh.h b/overlayfs-tools/sh.h deleted file mode 100644 index 19d9bdb..0000000 --- a/overlayfs-tools/sh.h +++ /dev/null @@ -1,20 +0,0 @@ -#ifndef OVERLAYFS_TOOLS_SH_H -#define OVERLAYFS_TOOLS_SH_H - -enum { - LOWERDIR, - UPPERDIR, - MOUNTDIR, - LOWERNEW, - UPPERNEW, - NUM_VARS -}; - -extern const char *var_names[NUM_VARS]; -extern char *vars[NUM_VARS]; - -FILE* create_shell_script(char *tmp_path_buffer); - -int command(FILE *output, const char *command_format, ...); - -#endif //OVERLAYFS_TOOLS_SH_H diff --git a/zram-config b/zram-config index 3fb258a..558c1d4 100755 --- a/zram-config +++ b/zram-config @@ -126,8 +126,7 @@ createZdir() { mergeOverlay() { echo "mergeOverlay: Beginning merge of ${ZDIR}${BIND_DIR}." >> "$ZLOG" ls -la "$ZDIR" "${ZDIR}${BIND_DIR}" "${ZDIR}${ZRAM_DEV}/upper" >> "$ZLOG" - cd /usr/local/lib/zram-config/ || return 1 - (./overlay merge --yes --lowerdir="${ZDIR}${BIND_DIR}" --upperdir="${ZDIR}${ZRAM_DEV}/upper") &>> "$ZLOG" || return 1 + overlay merge --yes --lowerdir="${ZDIR}${BIND_DIR}" --upperdir="${ZDIR}${ZRAM_DEV}/upper" &>> "$ZLOG" || return 1 bash -x -- *.sh &>> "$ZLOG" rm -fv -- *.sh &>> "$ZLOG" || return 1 echo "mergeOverlay: Merge of ${ZDIR}${BIND_DIR} complete." >> "$ZLOG"