diff --git a/test/test_alt.py b/test/test_alt.py index ef421f7..d9e6431 100644 --- a/test/test_alt.py +++ b/test/test_alt.py @@ -40,15 +40,15 @@ def test_alt_source(runner, paths, tracked, encrypt, exclude, yadm_alt): source_file_content = link_path + "##default" source_file = basepath.join(source_file_content) link_file = paths.work.join(link_path) + if link_path == utils.ALT_DIR: + source_file = source_file.join(utils.CONTAINED) + link_file = link_file.join(utils.CONTAINED) if tracked or (encrypt and not exclude): assert link_file.islink() target = py.path.local(os.path.realpath(link_file)) - if target.isfile(): - assert link_file.read() == source_file_content - assert str(source_file) in linked - else: - assert link_file.join(utils.CONTAINED).read() == source_file_content - assert str(source_file) in linked + assert target.isfile() + assert link_file.read() == source_file_content + assert str(source_file) in linked else: assert not link_file.exists() assert str(source_file) not in linked @@ -73,6 +73,9 @@ def test_relative_link(runner, paths, yadm_alt): source_file_content = link_path + "##default" source_file = basepath.join(source_file_content) link_file = paths.work.join(link_path) + if link_path == utils.ALT_DIR: + source_file = source_file.join(utils.CONTAINED) + link_file = link_file.join(utils.CONTAINED) link = link_file.readlink() relpath = os.path.relpath(source_file, start=os.path.dirname(link_file)) assert link == relpath @@ -128,15 +131,17 @@ def test_alt_conditions(runner, paths, tst_arch, tst_sys, tst_distro, tst_distro linked = utils.parse_alt_output(run.out) for link_path in TEST_PATHS: - source_file = link_path + suffix - assert paths.work.join(link_path).islink() - target = py.path.local(os.path.realpath(paths.work.join(link_path))) - if target.isfile(): - assert paths.work.join(link_path).read() == source_file - assert str(paths.work.join(source_file)) in linked - else: - assert paths.work.join(link_path).join(utils.CONTAINED).read() == source_file - assert str(paths.work.join(source_file)) in linked + source_file_content = link_path + suffix + source_file = paths.work.join(source_file_content) + link_file = paths.work.join(link_path) + if link_path == utils.ALT_DIR: + source_file = source_file.join(utils.CONTAINED) + link_file = link_file.join(utils.CONTAINED) + assert link_file.islink() + target = py.path.local(os.path.realpath(link_file)) + assert target.isfile() + assert link_file.read() == source_file_content + assert str(source_file) in linked @pytest.mark.usefixtures("ds1_copy") @@ -156,6 +161,7 @@ def test_alt_templates(runner, paths, kind, label): suffix = f"##{label}.{kind}" if kind is None: suffix = f"##{label}" + utils.create_alt_files(paths, suffix) run = runner([paths.pgm, "-Y", yadm_dir, "--yadm-data", yadm_data, "alt"]) assert run.success @@ -163,11 +169,15 @@ def test_alt_templates(runner, paths, kind, label): created = utils.parse_alt_output(run.out, linked=False) for created_path in TEST_PATHS: - if created_path != utils.ALT_DIR: - source_file = created_path + suffix - assert paths.work.join(created_path).isfile() - assert paths.work.join(created_path).read().strip() == source_file - assert str(paths.work.join(source_file)) in created + source_file_content = created_path + suffix + source_file = paths.work.join(source_file_content) + created_file = paths.work.join(created_path) + if created_path == utils.ALT_DIR: + source_file = source_file.join(utils.CONTAINED) + created_file = created_file.join(utils.CONTAINED) + assert created_file.isfile() + assert created_file.read().strip() == source_file_content + assert str(source_file) in created @pytest.mark.usefixtures("ds1_copy") @@ -201,20 +211,22 @@ def test_auto_alt(runner, yadm_cmd, paths, autoalt): linked = utils.parse_alt_output(run.out) for link_path in TEST_PATHS: - source_file = link_path + "##default" + source_file_content = link_path + "##default" + source_file = paths.work.join(source_file_content) + link_file = paths.work.join(link_path) + if link_path == utils.ALT_DIR: + source_file = source_file.join(utils.CONTAINED) + link_file = link_file.join(utils.CONTAINED) + if autoalt == "false": - assert not paths.work.join(link_path).exists() + assert not link_file.exists() else: - assert paths.work.join(link_path).islink() - target = py.path.local(os.path.realpath(paths.work.join(link_path))) - if target.isfile(): - assert paths.work.join(link_path).read() == source_file - # no linking output when run via auto-alt - assert str(paths.work.join(source_file)) not in linked - else: - assert paths.work.join(link_path).join(utils.CONTAINED).read() == source_file - # no linking output when run via auto-alt - assert str(paths.work.join(source_file)) not in linked + assert link_file.islink() + target = py.path.local(os.path.realpath(link_file)) + assert target.isfile() + assert link_file.read() == source_file_content + # no linking output when run via auto-alt + assert str(source_file) not in linked @pytest.mark.usefixtures("ds1_copy") @@ -239,16 +251,18 @@ def test_stale_link_removal(runner, yadm_cmd, paths): linked = utils.parse_alt_output(run.out) # assert the proper linking has occurred - for stale_path in TEST_PATHS: - source_file = stale_path + "##class." + tst_class - assert paths.work.join(stale_path).islink() - target = py.path.local(os.path.realpath(paths.work.join(stale_path))) - if target.isfile(): - assert paths.work.join(stale_path).read() == source_file - assert str(paths.work.join(source_file)) in linked - else: - assert paths.work.join(stale_path).join(utils.CONTAINED).read() == source_file - assert str(paths.work.join(source_file)) in linked + for link_path in TEST_PATHS: + source_file_content = link_path + f"##class.{tst_class}" + source_file = paths.work.join(source_file_content) + link_file = paths.work.join(link_path) + if link_path == utils.ALT_DIR: + source_file = source_file.join(utils.CONTAINED) + link_file = link_file.join(utils.CONTAINED) + assert link_file.islink() + target = py.path.local(os.path.realpath(link_file)) + assert target.isfile() + assert link_file.read() == source_file_content + assert str(source_file) in linked # change the class so there are no valid alternates utils.set_local(paths, "class", "changedclass") @@ -261,9 +275,53 @@ def test_stale_link_removal(runner, yadm_cmd, paths): # assert the linking is removed for stale_path in TEST_PATHS: - source_file = stale_path + "##class." + tst_class - assert not paths.work.join(stale_path).exists() - assert str(paths.work.join(source_file)) not in linked + source_file_content = stale_path + f"##class.{tst_class}" + source_file = paths.work.join(source_file_content) + stale_file = paths.work.join(stale_path) + if stale_path == utils.ALT_DIR: + source_file = source_file.join(utils.CONTAINED) + stale_file = stale_file.join(utils.CONTAINED) + assert not stale_file.exists() + assert str(source_file) not in linked + + +@pytest.mark.usefixtures("ds1_copy") +def test_legacy_dir_link_removal(runner, yadm_cmd, paths): + """Legacy link to alternative dir is removed + + This test ensures that a legacy dir alternative (i.e. symlink to the dir + itself) is converted to indiividual links. + """ + + utils.create_alt_files(paths, "##default") + + # Create legacy link + link_dir = paths.work.join(utils.ALT_DIR) + link_dir.mksymlinkto(link_dir.basename + "##default") + assert link_dir.islink() + + # run alt to trigger linking + run = runner(yadm_cmd("alt")) + assert run.success + assert run.err == "" + linked = utils.parse_alt_output(run.out) + + # assert legacy link is removed + assert not link_dir.islink() + + # assert the proper linking has occurred + for link_path in TEST_PATHS: + source_file_content = link_path + "##default" + source_file = paths.work.join(source_file_content) + link_file = paths.work.join(link_path) + if link_path == utils.ALT_DIR: + source_file = source_file.join(utils.CONTAINED) + link_file = link_file.join(utils.CONTAINED) + assert link_file.islink() + target = py.path.local(os.path.realpath(link_file)) + assert target.isfile() + assert link_file.read() == source_file_content + assert str(source_file) in linked @pytest.mark.usefixtures("ds1_repo_copy") diff --git a/test/test_unit_remove_stale_links.py b/test/test_unit_remove_stale_links.py deleted file mode 100644 index 275832d..0000000 --- a/test/test_unit_remove_stale_links.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Unit tests: remove_stale_links""" - -import os - -import pytest - - -@pytest.mark.parametrize("linked", [True, False]) -@pytest.mark.parametrize("kind", ["file", "symlink"]) -def test_remove_stale_links(runner, yadm, tmpdir, kind, linked): - """Test remove_stale_links()""" - - source_file = tmpdir.join("source_file") - source_file.write("source file", ensure=True) - link = tmpdir.join("link") - - if kind == "file": - link.write("link file", ensure=True) - else: - os.system(f"ln -s {source_file} {link}") - - alt_linked = "" - if linked: - alt_linked = source_file - - script = f""" - YADM_TEST=1 source {yadm} - possible_alt_targets=({link}) - alt_linked=({alt_linked}) - function rm() {{ echo rm "$@"; }} - remove_stale_links - """ - - run = runner(command=["bash"], inp=script) - assert run.err == "" - if kind == "symlink" and not linked: - assert f"rm -f {link}" in run.out - else: - assert run.out == "" diff --git a/test/test_unit_score_file.py b/test/test_unit_score_file.py index 9952c0c..293010e 100644 --- a/test/test_unit_score_file.py +++ b/test/test_unit_score_file.py @@ -39,13 +39,11 @@ TEMPLATE_LABELS = ["t", "template", "yadm"] -def calculate_score(filename): +def calculate_score(conditions): """Calculate the expected score""" # pylint: disable=too-many-branches score = 0 - _, conditions = filename.split("##", 1) - for condition in conditions.split(","): label = condition value = None @@ -111,70 +109,70 @@ def test_score_values(runner, yadm, default, arch, system, distro, cla, host, us local_distro = "testDISTro" local_host = "testHost" local_user = "testUser" - filenames = {"filename##": 0} + conditions = {"": 0} if default: - for filename in list(filenames): + for condition in list(conditions): for label in CONDITION[default]["labels"]: - newfile = filename - if not newfile.endswith("##"): - newfile += "," - newfile += label - filenames[newfile] = calculate_score(newfile) + newcond = condition + if newcond: + newcond += "," + newcond += label + conditions[newcond] = calculate_score(newcond) if arch: - for filename in list(filenames): + for condition in list(conditions): for match in [True, False]: for label in CONDITION[arch]["labels"]: - newfile = filename - if not newfile.endswith("##"): - newfile += "," - newfile += ".".join([label, local_arch if match else "badarch"]) - filenames[newfile] = calculate_score(newfile) + newcond = condition + if newcond: + newcond += "," + newcond += ".".join([label, local_arch if match else "badarch"]) + conditions[newcond] = calculate_score(newcond) if system: - for filename in list(filenames): + for condition in list(conditions): for match in [True, False]: for label in CONDITION[system]["labels"]: - newfile = filename - if not newfile.endswith("##"): - newfile += "," - newfile += ".".join([label, local_system if match else "badsys"]) - filenames[newfile] = calculate_score(newfile) + newcond = condition + if newcond: + newcond += "," + newcond += ".".join([label, local_system if match else "badsys"]) + conditions[newcond] = calculate_score(newcond) if distro: - for filename in list(filenames): + for condition in list(conditions): for match in [True, False]: for label in CONDITION[distro]["labels"]: - newfile = filename - if not newfile.endswith("##"): - newfile += "," - newfile += ".".join([label, local_distro if match else "baddistro"]) - filenames[newfile] = calculate_score(newfile) + newcond = condition + if newcond: + newcond += "," + newcond += ".".join([label, local_distro if match else "baddistro"]) + conditions[newcond] = calculate_score(newcond) if cla: - for filename in list(filenames): + for condition in list(conditions): for match in [True, False]: for label in CONDITION[cla]["labels"]: - newfile = filename - if not newfile.endswith("##"): - newfile += "," - newfile += ".".join([label, local_class if match else "badclass"]) - filenames[newfile] = calculate_score(newfile) + newcond = condition + if newcond: + newcond += "," + newcond += ".".join([label, local_class if match else "badclass"]) + conditions[newcond] = calculate_score(newcond) if host: - for filename in list(filenames): + for condition in list(conditions): for match in [True, False]: for label in CONDITION[host]["labels"]: - newfile = filename - if not newfile.endswith("##"): - newfile += "," - newfile += ".".join([label, local_host if match else "badhost"]) - filenames[newfile] = calculate_score(newfile) + newcond = condition + if newcond: + newcond += "," + newcond += ".".join([label, local_host if match else "badhost"]) + conditions[newcond] = calculate_score(newcond) if user: - for filename in list(filenames): + for condition in list(conditions): for match in [True, False]: for label in CONDITION[user]["labels"]: - newfile = filename - if not newfile.endswith("##"): - newfile += "," - newfile += ".".join([label, local_user if match else "baduser"]) - filenames[newfile] = calculate_score(newfile) + newcond = condition + if newcond: + newcond += "," + newcond += ".".join([label, local_user if match else "baduser"]) + conditions[newcond] = calculate_score(newcond) script = f""" YADM_TEST=1 source {yadm} @@ -187,15 +185,15 @@ def test_score_values(runner, yadm, default, arch, system, distro, cla, host, us local_host={local_host} local_user={local_user} """ - expected = "" - for filename, score in filenames.items(): + expected = [] + for condition, score in conditions.items(): script += f""" - score_file "{filename}" "dest" - echo "{filename}" - echo "$score" + score_file "source" "target" "{condition}" + echo "{condition}=$score" """ - expected += filename + "\n" - expected += str(score) + "\n" + expected.append(f"{condition}={score}") + expected.append("") + expected = "\n".join(expected) run = runner(command=["bash"], inp=script) assert run.success assert run.err == "" @@ -206,15 +204,15 @@ def test_score_values(runner, yadm, default, arch, system, distro, cla, host, us def test_extensions(runner, yadm, ext): """Verify extensions do not effect scores""" local_user = "testuser" - filename = f"filename##u.{local_user}" + condition = f"u.{local_user}" if ext: - filename += f",{ext}.xyz" + condition += f",{ext}.xyz" expected = "" script = f""" YADM_TEST=1 source {yadm} score=0 local_user={local_user} - score_file "{filename}" + score_file "source" "target" "{condition}" echo "$score" """ expected = f'{1000 + CONDITION["user"]["modifier"]}\n' @@ -232,15 +230,15 @@ def test_score_values_templates(runner, yadm): local_distro = "testdistro" local_host = "testhost" local_user = "testuser" - filenames = {"filename##": 0} + conditions = {"": 0} - for filename in list(filenames): + for condition in list(conditions): for label in TEMPLATE_LABELS: - newfile = filename - if not newfile.endswith("##"): - newfile += "," - newfile += ".".join([label, "testtemplate"]) - filenames[newfile] = calculate_score(newfile) + newcond = condition + if newcond: + newcond += "," + newcond += ".".join([label, "testtemplate"]) + conditions[newcond] = calculate_score(newcond) script = f""" YADM_TEST=1 source {yadm} @@ -252,15 +250,15 @@ def test_score_values_templates(runner, yadm): local_host={local_host} local_user={local_user} """ - expected = "" - for filename, score in filenames.items(): + expected = [] + for condition, score in conditions.items(): script += f""" - score_file "{filename}" "dest" - echo "{filename}" - echo "$score" + score_file "source" "target" "{condition}" + echo "{condition}=$score" """ - expected += filename + "\n" - expected += str(score) + "\n" + expected.append(f"{condition}={score}") + expected.append("") + expected = "\n".join(expected) run = runner(command=["bash"], inp=script) assert run.success assert run.err == "" @@ -281,7 +279,7 @@ def test_template_recording(runner, yadm, processor_generated): YADM_TEST=1 source {yadm} function record_score() {{ [ -n "$4" ] && echo "template recorded"; }} {mock} - score_file "testfile##template.kind" + score_file "source" "target" "template.kind" """ run = runner(command=["bash"], inp=script) assert run.success @@ -293,13 +291,13 @@ def test_underscores_and_upper_case_in_distro_and_family(runner, yadm): """Test replacing spaces with underscores and lowering case in distro / distro_family""" local_distro = "test distro" local_distro_family = "test family" - filenames = { - "filename##distro.Test Distro": 1004, - "filename##distro.test-distro": 0, - "filename##distro.test_distro": 1004, - "filename##distro_family.test FAMILY": 1008, - "filename##distro_family.test-family": 0, - "filename##distro_family.test_family": 1008, + conditions = { + "distro.Test Distro": 1004, + "distro.test-distro": 0, + "distro.test_distro": 1004, + "distro_family.test FAMILY": 1008, + "distro_family.test-family": 0, + "distro_family.test_family": 1008, } script = f""" @@ -308,15 +306,15 @@ def test_underscores_and_upper_case_in_distro_and_family(runner, yadm): local_distro="{local_distro}" local_distro_family="{local_distro_family}" """ - expected = "" - for filename, score in filenames.items(): + expected = [] + for condition, score in conditions.items(): script += f""" - score_file "{filename}" - echo "{filename}" - echo "$score" + score_file "source" "target" "{condition}" + echo "{condition}=$score" """ - expected += filename + "\n" - expected += str(score) + "\n" + expected.append(f"{condition}={score}") + expected.append("") + expected = "\n".join(expected) run = runner(command=["bash"], inp=script) assert run.success assert run.err == "" diff --git a/test/utils.py b/test/utils.py index 7e6a36d..b600bdc 100644 --- a/test/utils.py +++ b/test/utils.py @@ -61,12 +61,8 @@ def create_alt_files( new_dir = basepath.join(ALT_DIR + suffix).join(CONTAINED) new_dir.write(ALT_DIR + suffix, ensure=True) - # Do not test directory support for jinja alternates - test_paths = [new_file1, new_file2] - test_names = [ALT_FILE1, ALT_FILE2] - if not re.match(r"##(t$|t\.|template|yadm)", suffix): - test_paths += [new_dir] - test_names += [ALT_DIR] + test_paths = [new_file1, new_file2, new_dir] + test_names = [ALT_FILE1, ALT_FILE2, ALT_DIR] for test_path in test_paths: if content: diff --git a/yadm b/yadm index 2a3ff87..a5995d4 100755 --- a/yadm +++ b/yadm @@ -167,7 +167,7 @@ function main() { function score_file() { local source="$1" local target="$2" - local conditions="${source#*##}" + local conditions="$3" score=0 local template_processor="" @@ -210,17 +210,13 @@ function score_file() { continue ;; t | template | yadm) - if [ -d "$source" ]; then - INVALID_ALT+=("$source") + template_processor=$(choose_template_processor "$value") + if [ -n "$template_processor" ]; then + delta=0 + elif [ -n "$loud" ]; then + echo "No supported template processor for template $source" else - template_processor=$(choose_template_processor "$value") - if [ -n "$template_processor" ]; then - delta=0 - elif [ -n "$loud" ]; then - echo "No supported template processor for template $source" - else - debug "No supported template processor for template $source" - fi + debug "No supported template processor for template $source" fi ;; *) @@ -562,42 +558,52 @@ function alt() { local alt_scores=() local alt_template_processors=() - # For removing stale links - local possible_alt_targets=() - - local alt_source - for alt_source in "${tracked_files[@]}" "${ENCRYPT_INCLUDE_FILES[@]}"; do - local conditions="${alt_source#*##}" - if [ "$alt_source" = "$conditions" ]; then + local filename + for filename in "${tracked_files[@]}" "${ENCRYPT_INCLUDE_FILES[@]}"; do + local suffix="${filename#*##}" + if [ "$filename" = "$suffix" ]; then continue fi + local conditions="${suffix%%/*}" + suffix="${suffix:${#conditions}}" - local target_base="${alt_source%%##*}" - alt_source="${YADM_BASE}/${target_base}##${conditions%%/*}" - local alt_target="${YADM_BASE}/${target_base}" - if [ "${alt_target#"$YADM_ALT/"}" != "$alt_target" ]; then - target_base="${alt_target#"$YADM_ALT/"}" + local target="${YADM_BASE}/${filename%%##*}" + if [ "${target#"$YADM_ALT/"}" != "$target" ]; then + target="${YADM_BASE}/${target#"$YADM_ALT/"}" + fi + local source="${YADM_BASE}/${filename}" + + # If conditions are given on a directory we check if this alt, without the + # filename part, has a target that's a symlink pointing at this source + # (which was the legacy behavior for yadm) and if so remove this target. + if [ -n "$suffix" ]; then + if [ -L "$target" ]; then + if [ "$target" -ef "${YADM_BASE}/${filename%"$suffix"}" ]; then + rm -f "$target" + fi + fi + target="$target$suffix" fi - alt_target="${YADM_BASE}/${target_base}" - if ! in_list "$alt_target" "${possible_alt_targets[@]}"; then - possible_alt_targets+=("$alt_target") + # Remove target if it's a symlink pointing at source + if [ -L "$target" ] && [ "$target" -ef "$source" ]; then + rm -f "$target" fi - score_file "$alt_source" "$alt_target" + score_file "${source}" "$target" "$conditions" done local alt_linked=() alt_linking - remove_stale_links report_invalid_alts } function report_invalid_alts() { [ "$LEGACY_WARNING_ISSUED" = "1" ] && return [ "${#INVALID_ALT[@]}" = "0" ] && return - local path_list + local path_list="" + local invalid for invalid in "${INVALID_ALT[@]}"; do path_list="$path_list * $invalid"$'\n' done @@ -627,25 +633,6 @@ EOF printf '%s\n' "$msg" >&2 } -function remove_stale_links() { - # review alternate candidates for stale links - # if a possible alt IS linked, but it's source is not part of alt_linked, - # remove it. - if readlink_available; then - for stale_candidate in "${possible_alt_targets[@]}"; do - if [ -L "$stale_candidate" ]; then - src=$(readlink "$stale_candidate" 2>/dev/null) - if [ -n "$src" ]; then - for review_link in "${alt_linked[@]}"; do - [ "$src" = "$review_link" ] && continue 2 - done - rm -f "$stale_candidate" - fi - fi - done - fi -} - function set_local_alt_values() { local -a all_classes @@ -2187,10 +2174,6 @@ function esh_available() { command -v "$ESH_PROGRAM" &>/dev/null && return return 1 } -function readlink_available() { - command -v "readlink" &>/dev/null && return - return 1 -} # ****** Directory translations ****** diff --git a/yadm.1 b/yadm.1 index f16a34e..5a75905 100644 --- a/yadm.1 +++ b/yadm.1 @@ -604,8 +604,16 @@ If running on a system, with class set to "Work", the link will be: If no "##default" version exists and no files have valid conditions, then no link will be created. -Links are also created for directories named this way, as long as they have at -least one yadm managed file within them. +Conditions can also be used on directories and will then apply on all files +within the directory. The following files: + + - $HOME/path/example.txt##os.Linux + - $HOME/path/subdir/other.txt##os.Linux + +Would give the same result as: + + - $HOME/path##os.Linux/example.txt + - $HOME/path##os.Linux/subdir/other.txt yadm will automatically create these links by default. This can be disabled using the