From 67267307019324ffd0cefcb5f23e377856980b85 Mon Sep 17 00:00:00 2001 From: Erik Flodin Date: Mon, 10 Feb 2025 20:58:01 +0100 Subject: [PATCH 1/7] parse_encrypt: Don't let e.g. "*.ext" match files in subdirs This matches the behavior before 3.4.0. Silent errors from ls-files to avoid warnings about e.g. directories that aren't readable and also list files that would have been encrypted had they not been tracked in git (#521). Fix the patterns written to info/exclude so that they match the same files as are encrypted (e.g. *.key should only match .key files in the topdir, not in subdirs). --- test/conftest.py | 8 +-- test/test_encryption.py | 5 +- test/test_unit_exclude_encrypted.py | 6 +- test/test_unit_parse_encrypt.py | 6 +- yadm | 98 +++++++++++++++++++++-------- 5 files changed, 87 insertions(+), 36 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 455e5c6..0523355 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -609,14 +609,14 @@ disable-scdaemon env["GNUPGHOME"] = home # this pre-populates std files in the GNUPGHOME - runner(["gpg", "-k"], env=env) + runner(["gpg", "-k"], env=env, report=False) def register_gpg_password(password): """Publish a new GPG mock password and flush cached passwords""" home.join("mock-password").write(password) - runner(["gpgconf", "--reload", "gpg-agent"], env=env) + runner(["gpgconf", "--reload", "gpg-agent"], env=env, report=False) yield data(home, register_gpg_password) - runner(["gpgconf", "--kill", "gpg-agent"], env=env) - runner(["gpgconf", "--remove-socketdir", "gpg-agent"], env=env) + runner(["gpgconf", "--kill", "gpg-agent"], env=env, report=False) + runner(["gpgconf", "--remove-socketdir", "gpg-agent"], env=env, report=False) diff --git a/test/test_encryption.py b/test/test_encryption.py index 23d8e37..33b7394 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -92,6 +92,7 @@ def encrypt_targets(yadm_cmd, paths): paths.work.join("globs dir/globs file2").write("globs file2") expected.append("globs dir/globs file2") paths.encrypt.write("globs*\n", mode="a") + paths.encrypt.write("globs d*/globs*\n", mode="a") # blank lines paths.encrypt.write("\n \n\t\n", mode="a") @@ -404,8 +405,8 @@ def test_encrypt_added_to_exclude(runner, yadm_cmd, paths, gnupg): run = runner(yadm_cmd("encrypt"), env=env) - assert "test-encrypt-data" in paths.repo.join("info/exclude").read() - assert "original-data" in paths.repo.join("info/exclude").read() + assert "test-encrypt-data" in exclude_file.read() + assert "original-data" in exclude_file.read() assert run.success assert run.err == "" diff --git a/test/test_unit_exclude_encrypted.py b/test/test_unit_exclude_encrypted.py index 99db336..bbe29fd 100644 --- a/test/test_unit_exclude_encrypted.py +++ b/test/test_unit_exclude_encrypted.py @@ -24,7 +24,7 @@ def test_exclude_encrypted(runner, tmpdir, yadm, encrypt_exists, auto_exclude, e if exclude == "outdated": exclude_file.write(f"original-exclude\n{header}outdated\n", ensure=True) elif exclude == "up-to-date": - exclude_file.write(f"original-exclude\n{header}test-encrypt-data\n", ensure=True) + exclude_file.write(f"original-exclude\n{header}/test-encrypt-data\n", ensure=True) script = f""" YADM_TEST=1 source {yadm} @@ -42,9 +42,9 @@ def test_exclude_encrypted(runner, tmpdir, yadm, encrypt_exists, auto_exclude, e if encrypt_exists: assert exclude_file.exists() if exclude == "missing": - assert exclude_file.read() == f"{header}test-encrypt-data\n" + assert exclude_file.read() == f"{header}/test-encrypt-data\n" else: - assert exclude_file.read() == ("original-exclude\n" f"{header}test-encrypt-data\n") + assert exclude_file.read() == ("original-exclude\n" f"{header}/test-encrypt-data\n") if exclude != "up-to-date": assert f"Updating {exclude_file}" in run.out else: diff --git a/test/test_unit_parse_encrypt.py b/test/test_unit_parse_encrypt.py index 2acac15..935bea4 100644 --- a/test/test_unit_parse_encrypt.py +++ b/test/test_unit_parse_encrypt.py @@ -100,10 +100,11 @@ def create_test_encrypt_data(paths): edata += "*card1\n" # matches same file as the one above paths.work.join("wildcard1").write("", ensure=True) paths.work.join("wildcard2").write("", ensure=True) + paths.work.join("subdir/wildcard1").write("", ensure=True) expected.add("wildcard1") expected.add("wildcard2") - edata += "dirwild*\n" + edata += "dirwild*/file*\n" paths.work.join("dirwildcard/file1").write("", ensure=True) paths.work.join("dirwildcard/file2").write("", ensure=True) expected.add("dirwildcard/file1") @@ -125,6 +126,9 @@ def create_test_encrypt_data(paths): expected.add("ex ex/file4") expected.add("ex ex/file6.text") + paths.work.join("dirwildcard/file7.ex").write("", ensure=True) + expected.add("dirwildcard/file7.ex") + # double star edata += "doublestar/**/file*\n" edata += "!**/file3\n" diff --git a/yadm b/yadm index f0c1403..2dbb540 100755 --- a/yadm +++ b/yadm @@ -61,6 +61,7 @@ PROC_VERSION="/proc/version" OPERATING_SYSTEM="Unknown" ENCRYPT_INCLUDE_FILES="unparsed" +NO_ENCRYPT_TRACKED_FILES=() LEGACY_WARNING_ISSUED=0 INVALID_ALT=() @@ -1042,6 +1043,12 @@ function encrypt() { printf '%s\n' "${ENCRYPT_INCLUDE_FILES[@]}" echo + if [ ${#NO_ENCRYPT_TRACKED_FILES[@]} -gt 0 ]; then + echo "Warning: The following files are tracked and will NOT be encrypted:" + printf '%s\n' "${NO_ENCRYPT_TRACKED_FILES[@]}" + echo + fi + # encrypt all files which match the globs if tar -f - -c "${ENCRYPT_INCLUDE_FILES[@]}" | _encrypt_to "$YADM_ARCHIVE"; then echo "Wrote new file: $YADM_ARCHIVE" @@ -1468,38 +1475,59 @@ function version() { function exclude_encrypted() { + local auto_exclude auto_exclude=$(config --bool yadm.auto-exclude) [ "$auto_exclude" == "false" ] && return 0 - exclude_path="${YADM_REPO}/info/exclude" - newline=$'\n' - exclude_flag="# yadm-auto-excludes" - exclude_header="${exclude_flag}${newline}" + # do nothing if there is no YADM_ENCRYPT + [ -e "$YADM_ENCRYPT" ] || return 0 + + readonly exclude_path="${YADM_REPO}/info/exclude" + readonly newline=$'\n' + readonly exclude_flag="# yadm-auto-excludes" + + local exclude_header="${exclude_flag}${newline}" exclude_header="${exclude_header}# This section is managed by yadm." exclude_header="${exclude_header}${newline}" exclude_header="${exclude_header}# Any edits below will be lost." exclude_header="${exclude_header}${newline}" - # do nothing if there is no YADM_ENCRYPT - [ -e "$YADM_ENCRYPT" ] || return 0 - # read encrypt - encrypt_data="" - while IFS='' read -r line || [ -n "$line" ]; do - encrypt_data="${encrypt_data}${line}${newline}" + local encrypt_data="" + local pattern + while IFS='' read -r pattern || [ -n "$pattern" ]; do + case ${pattern:0:1} in + \#) + pattern="" + ;; + !) + # Prepend / to the pattern so that it matches the same files as in + # parse_encrypt (i.e. only from the root) + pattern="!/${pattern:1}$newline" + ;; + *) + if ! [[ $pattern =~ ^[[:blank:]]*(#|$) ]]; then + pattern="/$pattern$newline" + else + pattern="" + fi + ;; + esac + encrypt_data="${encrypt_data}${pattern}" done <"$YADM_ENCRYPT" # read info/exclude - unmanaged="" - managed="" + local unmanaged="" + local managed="" if [ -e "$exclude_path" ]; then - flag_seen=0 + local -i flag_seen=0 + local line while IFS='' read -r line || [ -n "$line" ]; do [ "$line" = "$exclude_flag" ] && flag_seen=1 - if [ "$flag_seen" -eq 0 ]; then - unmanaged="${unmanaged}${line}${newline}" - else + if ((flag_seen)); then managed="${managed}${line}${newline}" + else + unmanaged="${unmanaged}${line}${newline}" fi done <"$exclude_path" fi @@ -1926,26 +1954,44 @@ function parse_encrypt() { local -a exclude local -a include - while IFS= read -r pattern; do - case $pattern in - \#*) + local pattern + while IFS='' read -r pattern || [ -n "$pattern" ]; do + case ${pattern:0:1} in + \#) # Ignore comments ;; - !*) - exclude+=("--exclude=${pattern:1}") + !) + exclude+=("--exclude=/${pattern:1}") ;; *) - if ! [[ $pattern =~ ^[[:blank:]]*$ ]]; then + if ! [[ $pattern =~ ^[[:blank:]]*(#|$) ]]; then include+=("$pattern") fi ;; esac done <"$YADM_ENCRYPT" - if [[ ${#include} -gt 0 ]]; then - while IFS= read -r filename; do - ENCRYPT_INCLUDE_FILES+=("${filename%/}") - done <<<"$("$GIT_PROGRAM" ls-files --others "${exclude[@]}" -- "${include[@]}")" + if [ ${#include[@]} -gt 0 ]; then + while IFS='' read -r filename; do + if [ -n "$filename" ]; then + ENCRYPT_INCLUDE_FILES+=("${filename%/}") + fi + done <<<"$( + "$GIT_PROGRAM" --glob-pathspecs ls-files --others \ + "${exclude[@]}" -- "${include[@]}" 2>/dev/null + )" + + [ "$YADM_COMMAND" = "encrypt" ] || return + + # List files that matches encryption pattern but is tracked + while IFS='' read -r filename; do + if [ -n "$filename" ]; then + NO_ENCRYPT_TRACKED_FILES+=("${filename%/}") + fi + done <<<"$( + "$GIT_PROGRAM" --glob-pathspecs ls-files \ + "${exclude[@]}" -- "${include[@]}" + )" fi } From d4796108f404cf68a351f465c1010213f3d281b4 Mon Sep 17 00:00:00 2001 From: Erik Flodin Date: Thu, 27 Feb 2025 21:18:07 +0100 Subject: [PATCH 2/7] Automatically exclude alt links and template files unless yadm.auto-exclude is set to false (#234, #465). Alt files exclude pattern will be written to $GIT_DIR/info/exclude.yadm-alt and encrypt files exclude patthern to ...yadm-encrypt. Then these two files will be merged together and added to $GIT_DIR/info/exclude whenever one of them has changed. --- test/test_alt.py | 23 ++++++ test/test_unit_exclude_encrypted.py | 7 +- yadm | 119 ++++++++++++++++------------ yadm.1 | 15 +++- 4 files changed, 110 insertions(+), 54 deletions(-) diff --git a/test/test_alt.py b/test/test_alt.py index ef421f7..138d609 100644 --- a/test/test_alt.py +++ b/test/test_alt.py @@ -217,6 +217,29 @@ def test_auto_alt(runner, yadm_cmd, paths, autoalt): assert str(paths.work.join(source_file)) not in linked +@pytest.mark.usefixtures("ds1_copy") +@pytest.mark.parametrize("autoexclude", [None, "true", "false"]) +def test_alt_exclude(runner, yadm_cmd, paths, autoexclude): + """Test alt exclude""" + + # set the value of auto-exclude + if autoexclude: + os.system(" ".join(yadm_cmd("config", "yadm.auto-exclude", autoexclude))) + + utils.create_alt_files(paths, "##default") + run = runner(yadm_cmd("alt", "-d")) + assert run.success + + run = runner(yadm_cmd("status", "-z", "-uall", "--ignored")) + assert run.success + assert run.err == "" + status = run.out.split("\0") + + for link_path in TEST_PATHS: + flags = "??" if autoexclude == "false" else "!!" + assert f"{flags} {link_path}" in status + + @pytest.mark.usefixtures("ds1_copy") def test_stale_link_removal(runner, yadm_cmd, paths): """Stale links to alternative files are removed diff --git a/test/test_unit_exclude_encrypted.py b/test/test_unit_exclude_encrypted.py index bbe29fd..1daa891 100644 --- a/test/test_unit_exclude_encrypted.py +++ b/test/test_unit_exclude_encrypted.py @@ -9,7 +9,12 @@ import pytest def test_exclude_encrypted(runner, tmpdir, yadm, encrypt_exists, auto_exclude, exclude): """Test exclude_encrypted()""" - header = "# yadm-auto-excludes\n# This section is managed by yadm.\n# Any edits below will be lost.\n" + header = """\ +# yadm-auto-excludes +# This section is managed by yadm. +# Any edits below will be lost. +# yadm encrypt +""" config_function = 'function config() { echo "false";}' if auto_exclude: diff --git a/yadm b/yadm index 2dbb540..2e8bea2 100755 --- a/yadm +++ b/yadm @@ -691,6 +691,11 @@ function set_local_alt_values() { } function alt_linking() { + local -a exclude=() + + local log="debug" + [ -n "$loud" ] && log="echo" + local -i index for ((index = 0; index < ${#alt_targets[@]}; ++index)); do local target="${alt_targets[$index]}" @@ -709,17 +714,17 @@ function alt_linking() { if [[ -n "$template_processor" ]]; then template "$template_processor" "$source" "$target" elif [[ "$do_copy" -eq 1 ]]; then - debug "Copying $source to $target" - [[ -n "$loud" ]] && echo "Copying $source to $target" - + $log "Copying $source to $target" cp -f "$source" "$target" else - debug "Linking $source to $target" - [[ -n "$loud" ]] && echo "Linking $source to $target" - + $log "Linking $source to $target" ln_relative "$source" "$target" fi + + exclude+=("${target#"$YADM_WORK"}") done + + update_exclude alt "${exclude[@]}" } function ln_relative() { @@ -1473,18 +1478,35 @@ function version() { # ****** Utility Functions ****** -function exclude_encrypted() { +function update_exclude() { local auto_exclude auto_exclude=$(config --bool yadm.auto-exclude) [ "$auto_exclude" == "false" ] && return 0 - # do nothing if there is no YADM_ENCRYPT - [ -e "$YADM_ENCRYPT" ] || return 0 + local exclude_path="${YADM_REPO}/info/exclude" + local newline=$'\n' - readonly exclude_path="${YADM_REPO}/info/exclude" - readonly newline=$'\n' - readonly exclude_flag="# yadm-auto-excludes" + local part_path="$exclude_path.yadm-$1" + local part_str + part_str=$(join_string "$newline" "${@:2}") + + if [ -e "$part_path" ]; then + if [ "$part_str" = "$(<"$part_path")" ]; then + return + fi + + rm -f "$part_path" + elif [ -z "$part_str" ]; then + return + fi + + if [ -n "$part_str" ]; then + assert_parent "$part_path" + cat >"$part_path" <<<"$part_str" + fi + + local exclude_flag="# yadm-auto-excludes" local exclude_header="${exclude_flag}${newline}" exclude_header="${exclude_header}# This section is managed by yadm." @@ -1492,30 +1514,6 @@ function exclude_encrypted() { exclude_header="${exclude_header}# Any edits below will be lost." exclude_header="${exclude_header}${newline}" - # read encrypt - local encrypt_data="" - local pattern - while IFS='' read -r pattern || [ -n "$pattern" ]; do - case ${pattern:0:1} in - \#) - pattern="" - ;; - !) - # Prepend / to the pattern so that it matches the same files as in - # parse_encrypt (i.e. only from the root) - pattern="!/${pattern:1}$newline" - ;; - *) - if ! [[ $pattern =~ ^[[:blank:]]*(#|$) ]]; then - pattern="/$pattern$newline" - else - pattern="" - fi - ;; - esac - encrypt_data="${encrypt_data}${pattern}" - done <"$YADM_ENCRYPT" - # read info/exclude local unmanaged="" local managed="" @@ -1532,14 +1530,39 @@ function exclude_encrypted() { done <"$exclude_path" fi - if [ "${exclude_header}${encrypt_data}" != "$managed" ]; then + local exclude_str="" + for suffix in alt encrypt; do + if [ -e "${exclude_path}.yadm-$suffix" ]; then + local header="# yadm $suffix$newline" + exclude_str="$exclude_str$header$(<"$exclude_path".yadm-"$suffix")" + fi + done + + if [ "${exclude_header}${exclude_str}${newline}" != "$managed" ]; then debug "Updating ${exclude_path}" - assert_parent "$exclude_path" - printf "%s" "${unmanaged}${exclude_header}${encrypt_data}" >"$exclude_path" + cat >"$exclude_path" <<<"${unmanaged}${exclude_header}${exclude_str}" fi return 0 +} +function exclude_encrypted() { + local -a exclude=() + + if [ -r "$YADM_ENCRYPT" ]; then + local pattern + while IFS='' read -r pattern || [ -n "$pattern" ]; do + # Prepend / to the pattern so that it matches the same files as in + # parse_encrypt (i.e. only from the root) + if [ "${pattern:0:1}" = "!" ]; then + exclude+=("!/${pattern:1}") + elif ! [[ $pattern =~ ^[[:blank:]]*(#|$) ]]; then + exclude+=("/$pattern") + fi + done <"$YADM_ENCRYPT" + fi + + update_exclude encrypt "${exclude[@]}" } function query_distro() { @@ -1956,19 +1979,11 @@ function parse_encrypt() { local pattern while IFS='' read -r pattern || [ -n "$pattern" ]; do - case ${pattern:0:1} in - \#) - # Ignore comments - ;; - !) - exclude+=("--exclude=/${pattern:1}") - ;; - *) - if ! [[ $pattern =~ ^[[:blank:]]*(#|$) ]]; then - include+=("$pattern") - fi - ;; - esac + if [ "${pattern:0:1}" = "!" ]; then + exclude+=("--exclude=/${pattern:1}") + elif ! [[ $pattern =~ ^[[:blank:]]*(#|$) ]]; then + include+=("$pattern") + fi done <"$YADM_ENCRYPT" if [ ${#include[@]} -gt 0 ]; then diff --git a/yadm.1 b/yadm.1 index caa37de..52ef994 100644 --- a/yadm.1 +++ b/yadm.1 @@ -363,7 +363,8 @@ you may still run "yadm alt" manually to create the alternate links. This feature is enabled by default. .TP .B yadm.auto-exclude -Disable the automatic exclusion of patterns defined in +Disable the automatic exclusion of created alternate links, template files and +patterns defined in .IR $HOME/.config/yadm/encrypt . This feature is enabled by default. .TP @@ -614,6 +615,12 @@ configuration. Even if disabled, links can be manually created by running .BR "yadm alt" . +Created links are automatically added to the repository's +.I info/exclude +file. This can be disabled using the +.I yadm.auto-exclude +configuration. + Class is a special value which is stored locally on each host (inside the local repository). To use alternate symlinks using class, you must set the value of class using the configuration @@ -748,6 +755,12 @@ would look like: <%+ whatever.extra %> <% fi -%> +Created files are automatically added to the repository's +.I info/exclude +file. This can be disabled using the +.I yadm.auto-exclude +configuration. + .SH ENCRYPTION It can be useful to manage confidential files, like SSH or GPG keys, across From 9ff5e0965077756cd9d8bdde82231456a60d2ad3 Mon Sep 17 00:00:00 2001 From: AaronYoung5 Date: Sun, 2 Mar 2025 22:05:15 +0100 Subject: [PATCH 3/7] Add support for negative alt conditions (#522) --- test/test_unit_score_file.py | 73 ++++++++++++++++++++++++++++++++++++ yadm | 27 ++++++++----- yadm.1 | 24 +++++++++--- 3 files changed, 109 insertions(+), 15 deletions(-) diff --git a/test/test_unit_score_file.py b/test/test_unit_score_file.py index 9952c0c..d5412c1 100644 --- a/test/test_unit_score_file.py +++ b/test/test_unit_score_file.py @@ -321,3 +321,76 @@ def test_underscores_and_upper_case_in_distro_and_family(runner, yadm): assert run.success assert run.err == "" assert run.out == expected + + +def test_negative_class_condition(runner, yadm): + """Test negative class condition: returns 0 when matching and proper score when not matching.""" + script = f""" + YADM_TEST=1 source {yadm} + local_class="testclass" + local_classes=("testclass") + + # 0 + score=0 + score_file "filename##~class.testclass" "dest" + echo "score: $score" + + # 16 + score=0 + score_file "filename##~class.badclass" "dest" + echo "score2: $score" + + # 16 + score=0 + score_file "filename##~c.badclass" "dest" + echo "score3: $score" + """ + run = runner(command=["bash"], inp=script) + assert run.success + output = run.out.strip().splitlines() + assert output[0] == "score: 0" + assert output[1] == "score2: 16" + assert output[2] == "score3: 16" + + +def test_negative_combined_conditions(runner, yadm): + """Test negative conditions for multiple alt types: returns 0 when matching and proper score when not matching.""" + script = f""" + YADM_TEST=1 source {yadm} + local_class="testclass" + local_classes=("testclass") + local_distro="testdistro" + + # (0) + (0) = 0 + score=0 + score_file "filename##~class.testclass,~distro.testdistro" "dest" + echo "score: $score" + + # (1000 + 16) + (1000 + 4) = 2020 + score=0 + score_file "filename##class.testclass,distro.testdistro" "dest" + echo "score2: $score" + + # 0 (negated class condition) + score=0 + score_file "filename##~class.badclass,~distro.testdistro" "dest" + echo "score3: $score" + + # (1000 + 16) + (4) = 1020 + score=0 + score_file "filename##class.testclass,~distro.baddistro" "dest" + echo "score4: $score" + + # (1000 + 16) + (16) = 1032 + score=0 + score_file "filename##class.testclass,~class.badclass" "dest" + echo "score5: $score" + """ + run = runner(command=["bash"], inp=script) + assert run.success + output = run.out.strip().splitlines() + assert output[0] == "score: 0" + assert output[1] == "score2: 2020" + assert output[2] == "score3: 0" + assert output[3] == "score4: 1020" + assert output[4] == "score5: 1032" diff --git a/yadm b/yadm index 2e8bea2..dccb7c9 100755 --- a/yadm +++ b/yadm @@ -180,32 +180,39 @@ function score_file() { local value=${field#*.} [ "$field" = "$label" ] && value="" # when .value is omitted + # Check for negative condition prefix (e.g., "~