From 67267307019324ffd0cefcb5f23e377856980b85 Mon Sep 17 00:00:00 2001 From: Erik Flodin Date: Mon, 10 Feb 2025 20:58:01 +0100 Subject: [PATCH] parse_encrypt: Don't let e.g. "*.ext" match files in subdirs This matches the behavior before 3.4.0. Silent errors from ls-files to avoid warnings about e.g. directories that aren't readable and also list files that would have been encrypted had they not been tracked in git (#521). Fix the patterns written to info/exclude so that they match the same files as are encrypted (e.g. *.key should only match .key files in the topdir, not in subdirs). --- test/conftest.py | 8 +-- test/test_encryption.py | 5 +- test/test_unit_exclude_encrypted.py | 6 +- test/test_unit_parse_encrypt.py | 6 +- yadm | 98 +++++++++++++++++++++-------- 5 files changed, 87 insertions(+), 36 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 455e5c6..0523355 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -609,14 +609,14 @@ disable-scdaemon env["GNUPGHOME"] = home # this pre-populates std files in the GNUPGHOME - runner(["gpg", "-k"], env=env) + runner(["gpg", "-k"], env=env, report=False) def register_gpg_password(password): """Publish a new GPG mock password and flush cached passwords""" home.join("mock-password").write(password) - runner(["gpgconf", "--reload", "gpg-agent"], env=env) + runner(["gpgconf", "--reload", "gpg-agent"], env=env, report=False) yield data(home, register_gpg_password) - runner(["gpgconf", "--kill", "gpg-agent"], env=env) - runner(["gpgconf", "--remove-socketdir", "gpg-agent"], env=env) + runner(["gpgconf", "--kill", "gpg-agent"], env=env, report=False) + runner(["gpgconf", "--remove-socketdir", "gpg-agent"], env=env, report=False) diff --git a/test/test_encryption.py b/test/test_encryption.py index 23d8e37..33b7394 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -92,6 +92,7 @@ def encrypt_targets(yadm_cmd, paths): paths.work.join("globs dir/globs file2").write("globs file2") expected.append("globs dir/globs file2") paths.encrypt.write("globs*\n", mode="a") + paths.encrypt.write("globs d*/globs*\n", mode="a") # blank lines paths.encrypt.write("\n \n\t\n", mode="a") @@ -404,8 +405,8 @@ def test_encrypt_added_to_exclude(runner, yadm_cmd, paths, gnupg): run = runner(yadm_cmd("encrypt"), env=env) - assert "test-encrypt-data" in paths.repo.join("info/exclude").read() - assert "original-data" in paths.repo.join("info/exclude").read() + assert "test-encrypt-data" in exclude_file.read() + assert "original-data" in exclude_file.read() assert run.success assert run.err == "" diff --git a/test/test_unit_exclude_encrypted.py b/test/test_unit_exclude_encrypted.py index 99db336..bbe29fd 100644 --- a/test/test_unit_exclude_encrypted.py +++ b/test/test_unit_exclude_encrypted.py @@ -24,7 +24,7 @@ def test_exclude_encrypted(runner, tmpdir, yadm, encrypt_exists, auto_exclude, e if exclude == "outdated": exclude_file.write(f"original-exclude\n{header}outdated\n", ensure=True) elif exclude == "up-to-date": - exclude_file.write(f"original-exclude\n{header}test-encrypt-data\n", ensure=True) + exclude_file.write(f"original-exclude\n{header}/test-encrypt-data\n", ensure=True) script = f""" YADM_TEST=1 source {yadm} @@ -42,9 +42,9 @@ def test_exclude_encrypted(runner, tmpdir, yadm, encrypt_exists, auto_exclude, e if encrypt_exists: assert exclude_file.exists() if exclude == "missing": - assert exclude_file.read() == f"{header}test-encrypt-data\n" + assert exclude_file.read() == f"{header}/test-encrypt-data\n" else: - assert exclude_file.read() == ("original-exclude\n" f"{header}test-encrypt-data\n") + assert exclude_file.read() == ("original-exclude\n" f"{header}/test-encrypt-data\n") if exclude != "up-to-date": assert f"Updating {exclude_file}" in run.out else: diff --git a/test/test_unit_parse_encrypt.py b/test/test_unit_parse_encrypt.py index 2acac15..935bea4 100644 --- a/test/test_unit_parse_encrypt.py +++ b/test/test_unit_parse_encrypt.py @@ -100,10 +100,11 @@ def create_test_encrypt_data(paths): edata += "*card1\n" # matches same file as the one above paths.work.join("wildcard1").write("", ensure=True) paths.work.join("wildcard2").write("", ensure=True) + paths.work.join("subdir/wildcard1").write("", ensure=True) expected.add("wildcard1") expected.add("wildcard2") - edata += "dirwild*\n" + edata += "dirwild*/file*\n" paths.work.join("dirwildcard/file1").write("", ensure=True) paths.work.join("dirwildcard/file2").write("", ensure=True) expected.add("dirwildcard/file1") @@ -125,6 +126,9 @@ def create_test_encrypt_data(paths): expected.add("ex ex/file4") expected.add("ex ex/file6.text") + paths.work.join("dirwildcard/file7.ex").write("", ensure=True) + expected.add("dirwildcard/file7.ex") + # double star edata += "doublestar/**/file*\n" edata += "!**/file3\n" diff --git a/yadm b/yadm index f0c1403..2dbb540 100755 --- a/yadm +++ b/yadm @@ -61,6 +61,7 @@ PROC_VERSION="/proc/version" OPERATING_SYSTEM="Unknown" ENCRYPT_INCLUDE_FILES="unparsed" +NO_ENCRYPT_TRACKED_FILES=() LEGACY_WARNING_ISSUED=0 INVALID_ALT=() @@ -1042,6 +1043,12 @@ function encrypt() { printf '%s\n' "${ENCRYPT_INCLUDE_FILES[@]}" echo + if [ ${#NO_ENCRYPT_TRACKED_FILES[@]} -gt 0 ]; then + echo "Warning: The following files are tracked and will NOT be encrypted:" + printf '%s\n' "${NO_ENCRYPT_TRACKED_FILES[@]}" + echo + fi + # encrypt all files which match the globs if tar -f - -c "${ENCRYPT_INCLUDE_FILES[@]}" | _encrypt_to "$YADM_ARCHIVE"; then echo "Wrote new file: $YADM_ARCHIVE" @@ -1468,38 +1475,59 @@ function version() { function exclude_encrypted() { + local auto_exclude auto_exclude=$(config --bool yadm.auto-exclude) [ "$auto_exclude" == "false" ] && return 0 - exclude_path="${YADM_REPO}/info/exclude" - newline=$'\n' - exclude_flag="# yadm-auto-excludes" - exclude_header="${exclude_flag}${newline}" + # do nothing if there is no YADM_ENCRYPT + [ -e "$YADM_ENCRYPT" ] || return 0 + + readonly exclude_path="${YADM_REPO}/info/exclude" + readonly newline=$'\n' + readonly exclude_flag="# yadm-auto-excludes" + + local exclude_header="${exclude_flag}${newline}" exclude_header="${exclude_header}# This section is managed by yadm." exclude_header="${exclude_header}${newline}" exclude_header="${exclude_header}# Any edits below will be lost." exclude_header="${exclude_header}${newline}" - # do nothing if there is no YADM_ENCRYPT - [ -e "$YADM_ENCRYPT" ] || return 0 - # read encrypt - encrypt_data="" - while IFS='' read -r line || [ -n "$line" ]; do - encrypt_data="${encrypt_data}${line}${newline}" + local encrypt_data="" + local pattern + while IFS='' read -r pattern || [ -n "$pattern" ]; do + case ${pattern:0:1} in + \#) + pattern="" + ;; + !) + # Prepend / to the pattern so that it matches the same files as in + # parse_encrypt (i.e. only from the root) + pattern="!/${pattern:1}$newline" + ;; + *) + if ! [[ $pattern =~ ^[[:blank:]]*(#|$) ]]; then + pattern="/$pattern$newline" + else + pattern="" + fi + ;; + esac + encrypt_data="${encrypt_data}${pattern}" done <"$YADM_ENCRYPT" # read info/exclude - unmanaged="" - managed="" + local unmanaged="" + local managed="" if [ -e "$exclude_path" ]; then - flag_seen=0 + local -i flag_seen=0 + local line while IFS='' read -r line || [ -n "$line" ]; do [ "$line" = "$exclude_flag" ] && flag_seen=1 - if [ "$flag_seen" -eq 0 ]; then - unmanaged="${unmanaged}${line}${newline}" - else + if ((flag_seen)); then managed="${managed}${line}${newline}" + else + unmanaged="${unmanaged}${line}${newline}" fi done <"$exclude_path" fi @@ -1926,26 +1954,44 @@ function parse_encrypt() { local -a exclude local -a include - while IFS= read -r pattern; do - case $pattern in - \#*) + local pattern + while IFS='' read -r pattern || [ -n "$pattern" ]; do + case ${pattern:0:1} in + \#) # Ignore comments ;; - !*) - exclude+=("--exclude=${pattern:1}") + !) + exclude+=("--exclude=/${pattern:1}") ;; *) - if ! [[ $pattern =~ ^[[:blank:]]*$ ]]; then + if ! [[ $pattern =~ ^[[:blank:]]*(#|$) ]]; then include+=("$pattern") fi ;; esac done <"$YADM_ENCRYPT" - if [[ ${#include} -gt 0 ]]; then - while IFS= read -r filename; do - ENCRYPT_INCLUDE_FILES+=("${filename%/}") - done <<<"$("$GIT_PROGRAM" ls-files --others "${exclude[@]}" -- "${include[@]}")" + if [ ${#include[@]} -gt 0 ]; then + while IFS='' read -r filename; do + if [ -n "$filename" ]; then + ENCRYPT_INCLUDE_FILES+=("${filename%/}") + fi + done <<<"$( + "$GIT_PROGRAM" --glob-pathspecs ls-files --others \ + "${exclude[@]}" -- "${include[@]}" 2>/dev/null + )" + + [ "$YADM_COMMAND" = "encrypt" ] || return + + # List files that matches encryption pattern but is tracked + while IFS='' read -r filename; do + if [ -n "$filename" ]; then + NO_ENCRYPT_TRACKED_FILES+=("${filename%/}") + fi + done <<<"$( + "$GIT_PROGRAM" --glob-pathspecs ls-files \ + "${exclude[@]}" -- "${include[@]}" + )" fi }