"""Patch ria-, ora-, ria_utils-, and clone-tests to work with modified ria_utils
The ria-utils-patches use an abstract path representation for RIA-store elements.
This patch adapts the tests that use `ria_utils.create_store` and
`ria_utils.create_ds_in_store` to these modifications.
"""
from __future__ import annotations
import logging
import shutil
import stat
from pathlib import (
Path,
PurePosixPath,
)
from urllib.request import pathname2url
from datalad.api import (
Dataset,
clone,
create_sibling_ria,
)
from datalad.cmd import (
WitlessRunner as Runner,
NoCapture,
)
from datalad.customremotes.ria_utils import (
UnknownLayoutVersion,
create_ds_in_store,
create_store,
get_layout_locations,
)
from datalad.distributed.ora_remote import (
LocalIO,
SSHRemoteIO,
)
from datalad.distributed.tests.ria_utils import (
common_init_opts,
get_all_files,
populate_dataset,
)
from datalad.support.exceptions import (
CommandError,
IncompleteResultsError,
)
from datalad.support.network import get_local_file_url
from datalad.tests.utils_pytest import (
SkipTest,
assert_equal,
assert_false,
assert_in,
assert_not_in,
assert_raises,
assert_repo_status,
assert_result_count,
assert_status,
assert_true,
create_tree,
has_symlink_capability,
known_failure_githubci_win,
known_failure_windows,
rmtree,
serve_path_via_http,
skip_if_adjusted_branch,
swallow_logs,
with_tempfile,
)
from . import apply_patch
[docs]
def local_path2pure_posix_path(path: Path | str):
return PurePosixPath(pathname2url(str(path)))
# taken from datalad-core@864dc4ae24c8aac0ec4003604543b86de4735732
[docs]
@with_tempfile
def patched__postclonetest_prepare(lcl, storepath, storepath2, link):
from datalad.customremotes.ria_utils import (
create_ds_in_store,
create_store,
get_layout_locations,
)
from datalad.distributed.ora_remote import LocalIO
create_tree(lcl,
tree={
'ds': {
'test.txt': 'some',
'subdir': {
'subds': {'testsub.txt': 'somemore'},
'subgit': {'testgit.txt': 'even more'}
},
},
})
lcl = Path(lcl)
storepath = Path(storepath)
storepath2 = Path(storepath2)
# PATCH: introduce `ppp_storepath` and `ppp_storepath2` and use them instead
# of `storepath` and `storepath2`.
ppp_storepath = local_path2pure_posix_path(storepath)
ppp_storepath2 = local_path2pure_posix_path(storepath2)
link = Path(link)
link.symlink_to(storepath)
# create a local dataset with a subdataset
subds = Dataset(lcl / 'ds' / 'subdir' / 'subds').create(force=True)
subds.save()
# add a plain git dataset as well
subgit = Dataset(lcl / 'ds' / 'subdir' / 'subgit').create(force=True,
annex=False)
subgit.save()
ds = Dataset(lcl / 'ds').create(force=True)
ds.save(version_tag='original')
assert_repo_status(ds.path)
io = LocalIO()
# Have a second store with valid ORA remote. This should not interfere with
# reconfiguration of the first one, when that second store is not the one we
# clone from. However, don't push data into it for easier get-based testing
# later on.
# Doing this first, so datasets in "first"/primary store know about this.
create_store(io, ppp_storepath2, '1')
url2 = "ria+{}".format(get_local_file_url(str(storepath2)))
for d in (ds, subds, subgit):
create_ds_in_store(io, ppp_storepath2, d.id, '2', '1')
d.create_sibling_ria(url2, "anotherstore", new_store_ok=True)
d.push('.', to='anotherstore', data='nothing')
store2_loc, _, _ = get_layout_locations(1, ppp_storepath2, d.id)
Runner(cwd=str(store2_loc)).run(['git', 'update-server-info'])
# Now the store to clone from:
create_store(io, ppp_storepath, '1')
# URL to use for upload. Point is, that this should be invalid for the clone
# so that autoenable would fail. Therefore let it be based on a to be
# deleted symlink
upl_url = "ria+{}".format(get_local_file_url(str(link)))
for d in (ds, subds, subgit):
# TODO: create-sibling-ria required for config! => adapt to RF'd
# creation (missed on rebase?)
create_ds_in_store(io, ppp_storepath, d.id, '2', '1')
d.create_sibling_ria(upl_url, "store", new_store_ok=True)
if d is not subgit:
# Now, simulate the problem by reconfiguring the special remote to
# not be autoenabled.
# Note, however, that the actual intention is a URL, that isn't
# valid from the point of view of the clone (doesn't resolve, no
# credentials, etc.) and therefore autoenabling on git-annex-init
# when datalad-cloning would fail to succeed.
Runner(cwd=d.path).run(['git', 'annex', 'enableremote',
'store-storage',
'autoenable=false'])
d.push('.', to='store')
store_loc, _, _ = get_layout_locations(1, ppp_storepath, d.id)
Runner(cwd=str(store_loc)).run(['git', 'update-server-info'])
link.unlink()
# We should now have a store with datasets that have an autoenabled ORA
# remote relying on an inaccessible URL.
# datalad-clone is supposed to reconfigure based on the URL we cloned from.
# Test this feature for cloning via HTTP, SSH and FILE URLs.
return ds.id
# taken from datalad-core@864dc4ae24c8aac0ec4003604543b86de4735732
[docs]
@known_failure_githubci_win # in datalad/git-annex as e.g. of 20201218
@with_tempfile(mkdir=True)
@with_tempfile
@with_tempfile
def patched_test_ria_postclone_noannex(dspath=None, storepath=None, clonepath=None):
# Test for gh-5186: Cloning from local FS, shouldn't lead to annex
# initializing origin.
dspath = Path(dspath)
storepath = Path(storepath)
clonepath = Path(clonepath)
# PATCH: introduce `ppp_storepath` and use it instead of `storepath`.
ppp_storepath = local_path2pure_posix_path(storepath)
from datalad.customremotes.ria_utils import (
create_ds_in_store,
create_store,
get_layout_locations,
)
from datalad.distributed.ora_remote import LocalIO
# First create a dataset in a RIA store the standard way
somefile = dspath / 'a_file.txt'
somefile.write_text('irrelevant')
ds = Dataset(dspath).create(force=True)
io = LocalIO()
create_store(io, ppp_storepath, '1')
lcl_url = "ria+{}".format(get_local_file_url(str(storepath)))
create_ds_in_store(io, ppp_storepath, ds.id, '2', '1')
ds.create_sibling_ria(lcl_url, "store", new_store_ok=True)
ds.push('.', to='store')
# now, remove annex/ tree from store in order to see, that clone
# doesn't cause annex to recreate it.
store_loc, _, _ = get_layout_locations(1, storepath, ds.id)
annex = store_loc / 'annex'
rmtree(str(annex))
assert_false(annex.exists())
clone_url = get_local_file_url(str(storepath), compatibility='git') + \
'#{}'.format(ds.id)
clone("ria+{}".format(clone_url), clonepath)
# no need to test the cloning itself - we do that over and over in here
# bare repo in store still has no local annex:
assert_false(annex.exists())
# taken from datalad-core@864dc4ae24c8aac0ec4003604543b86de4735732
[docs]
@with_tempfile
def patched_test_setup_store(io_cls, io_args, store=None):
io = io_cls(*io_args)
store = Path(store)
# PATCH: introduce `ppp_store` and use it instead of `store`
ppp_store = local_path2pure_posix_path(store)
version_file = store / 'ria-layout-version'
error_logs = store / 'error_logs'
# invalid version raises:
assert_raises(UnknownLayoutVersion, create_store, io, ppp_store, '2')
# non-existing path should work:
create_store(io, ppp_store, '1')
assert_true(version_file.exists())
assert_true(error_logs.exists())
assert_true(error_logs.is_dir())
assert_equal([f for f in error_logs.iterdir()], [])
# empty target directory should work as well:
rmtree(str(store))
store.mkdir(exist_ok=False)
create_store(io, ppp_store, '1')
assert_true(version_file.exists())
assert_true(error_logs.exists())
assert_true(error_logs.is_dir())
assert_equal([f for f in error_logs.iterdir()], [])
# re-execution also fine:
create_store(io, ppp_store, '1')
# but version conflict with existing target isn't:
version_file.write_text("2|unknownflags\n")
assert_raises(ValueError, create_store, io, ppp_store, '1')
# TODO: check output reporting conflicting version "2"
# taken from datalad-core@864dc4ae24c8aac0ec4003604543b86de4735732
[docs]
@with_tempfile
def patched_test_setup_ds_in_store(io_cls, io_args, store=None):
io = io_cls(*io_args)
store = Path(store)
# PATCH: introduce `ppp_store` and use it instead of `store`
ppp_store = local_path2pure_posix_path(store)
# ATM create_ds_in_store doesn't care what kind of ID is provided
dsid = "abc123456"
ds_path = store / dsid[:3] / dsid[3:] # store layout version 1
version_file = ds_path / 'ria-layout-version'
archives = ds_path / 'archives'
objects = ds_path / 'annex' / 'objects'
git_config = ds_path / 'config'
# invalid store version:
assert_raises(UnknownLayoutVersion,
create_ds_in_store, io, ppp_store, dsid, '1', 'abc')
# invalid obj version:
assert_raises(UnknownLayoutVersion,
create_ds_in_store, io, ppp_store, dsid, 'abc', '1')
# version 1
create_store(io, ppp_store, '1')
create_ds_in_store(io, ppp_store, dsid, '1', '1')
for p in [ds_path, archives, objects]:
assert_true(p.is_dir(), msg="Not a directory: %s" % str(p))
for p in [version_file]:
assert_true(p.is_file(), msg="Not a file: %s" % str(p))
assert_equal(version_file.read_text(), "1\n")
# conflicting version exists at target:
assert_raises(ValueError, create_ds_in_store, io, ppp_store, dsid, '2', '1')
# version 2
# Note: The only difference between version 1 and 2 are supposed to be the
# key paths (dirhashlower vs mixed), which has nothing to do with
# setup routine.
rmtree(str(store))
create_store(io, ppp_store, '1')
create_ds_in_store(io, ppp_store, dsid, '2', '1')
for p in [ds_path, archives, objects]:
assert_true(p.is_dir(), msg="Not a directory: %s" % str(p))
for p in [version_file]:
assert_true(p.is_file(), msg="Not a file: %s" % str(p))
assert_equal(version_file.read_text(), "2\n")
# taken from datalad-core@864dc4ae24c8aac0ec4003604543b86de4735732
[docs]
@with_tempfile(mkdir=True)
@serve_path_via_http
@with_tempfile
def patched_test_initremote(store_path=None, store_url=None, ds_path=None):
ds = Dataset(ds_path).create()
store_path = Path(store_path)
# PATCH: introduce `ppp_store_path` and use it instead of `store_path`
ppp_store_path = local_path2pure_posix_path(store_path)
url = "ria+" + store_url
init_opts = common_init_opts + ['url={}'.format(url)]
# fail when there's no RIA store at the destination
assert_raises(CommandError, ds.repo.init_remote, 'ora-remote',
options=init_opts)
# Doesn't actually create a remote if it fails
assert_not_in('ora-remote',
[cfg['name']
for uuid, cfg in ds.repo.get_special_remotes().items()]
)
# now make it a store
io = LocalIO()
create_store(io, ppp_store_path, '1')
create_ds_in_store(io, ppp_store_path, ds.id, '2', '1')
# fails on non-RIA URL
assert_raises(CommandError, ds.repo.init_remote, 'ora-remote',
options=common_init_opts + ['url={}'
''.format(store_path.as_uri())]
)
# Doesn't actually create a remote if it fails
assert_not_in('ora-remote',
[cfg['name']
for uuid, cfg in ds.repo.get_special_remotes().items()]
)
ds.repo.init_remote('ora-remote', options=init_opts)
assert_in('ora-remote',
[cfg['name']
for uuid, cfg in ds.repo.get_special_remotes().items()]
)
assert_repo_status(ds.path)
# git-annex:remote.log should have:
# - url
# - common_init_opts
# - archive_id (which equals ds id)
remote_log = ds.repo.call_git(['cat-file', 'blob', 'git-annex:remote.log'],
read_only=True)
assert_in("url={}".format(url), remote_log)
[assert_in(c, remote_log) for c in common_init_opts]
assert_in("archive-id={}".format(ds.id), remote_log)
# taken from datalad-core@864dc4ae24c8aac0ec4003604543b86de4735732
# TODO: on crippled FS copytree to populate store doesn't seem to work.
# Or may be it's just the serving via HTTP that doesn't work.
# Either way, after copytree and fsck, whereis doesn't report
# the store as an available source.
[docs]
@skip_if_adjusted_branch
@known_failure_windows # see gh-4469
@with_tempfile(mkdir=True)
@serve_path_via_http
@with_tempfile
def patched_test_read_access(store_path=None, store_url=None, ds_path=None):
ds = Dataset(ds_path).create()
populate_dataset(ds)
files = [Path('one.txt'), Path('subdir') / 'two']
store_path = Path(store_path)
# PATCH: introduce `ppp_store_path` and use it instead of `store_path`
ppp_store_path = local_path2pure_posix_path(store_path)
url = "ria+" + store_url
init_opts = common_init_opts + ['url={}'.format(url)]
io = LocalIO()
create_store(io, ppp_store_path, '1')
create_ds_in_store(io, ppp_store_path, ds.id, '2', '1')
ds.repo.init_remote('ora-remote', options=init_opts)
fsck_results = ds.repo.fsck(remote='ora-remote', fast=True)
# Note: Failures in the special remote will show up as a success=False
# result for fsck -> the call itself would not fail.
for r in fsck_results:
if "note" in r:
# we could simply assert "note" to not be in r, but we want proper
# error reporting - content of note, not just its unexpected
# existence.
assert_equal(r["success"], "true",
msg="git-annex-fsck failed with ORA over HTTP: %s" % r)
assert_equal(r["error-messages"], [])
store_uuid = ds.siblings(name='ora-remote',
return_type='item-or-list',
result_renderer='disabled')['annex-uuid']
here_uuid = ds.siblings(name='here',
return_type='item-or-list',
result_renderer='disabled')['annex-uuid']
# nothing in store yet:
for f in files:
known_sources = ds.repo.whereis(str(f))
assert_in(here_uuid, known_sources)
assert_not_in(store_uuid, known_sources)
annex_obj_target = str(store_path / ds.id[:3] / ds.id[3:]
/ 'annex' / 'objects')
shutil.rmtree(annex_obj_target)
shutil.copytree(src=str(ds.repo.dot_git / 'annex' / 'objects'),
dst=annex_obj_target)
ds.repo.fsck(remote='ora-remote', fast=True)
# all in store now:
for f in files:
known_sources = ds.repo.whereis(str(f))
assert_in(here_uuid, known_sources)
assert_in(store_uuid, known_sources)
ds.drop('.')
res = ds.get('.')
assert_equal(len(res), 4)
assert_result_count(res, 4, status='ok', type='file', action='get',
message="from ora-remote...")
# try whether the reported access URL is correct
one_url = ds.repo.whereis('one.txt', output='full'
)[store_uuid]['urls'].pop()
assert_status('ok', ds.download_url(urls=[one_url], path=str(ds.pathobj / 'dummy')))
# taken from datalad-core@864dc4ae24c8aac0ec4003604543b86de4735732
[docs]
@with_tempfile
@with_tempfile
def patched_test_initremote_basic(url, io, store, ds_path, link):
ds_path = Path(ds_path)
store = Path(store)
# PATCH: introduce `ppp_store` and use it instead of `store`
ppp_store = local_path2pure_posix_path(store)
link = Path(link)
ds = Dataset(ds_path).create()
populate_dataset(ds)
init_opts = common_init_opts + ['url={}'.format(url)]
# fails on non-existing storage location
assert_raises(CommandError,
ds.repo.init_remote, 'ria-remote', options=init_opts)
# Doesn't actually create a remote if it fails
assert_not_in('ria-remote',
[cfg['name']
for uuid, cfg in ds.repo.get_special_remotes().items()]
)
# fails on non-RIA URL
assert_raises(CommandError, ds.repo.init_remote, 'ria-remote',
options=common_init_opts + ['url={}'.format(store.as_uri())]
)
# Doesn't actually create a remote if it fails
assert_not_in('ria-remote',
[cfg['name']
for uuid, cfg in ds.repo.get_special_remotes().items()]
)
# set up store:
create_store(io, ppp_store, '1')
# still fails, since ds isn't setup in the store
assert_raises(CommandError,
ds.repo.init_remote, 'ria-remote', options=init_opts)
# Doesn't actually create a remote if it fails
assert_not_in('ria-remote',
[cfg['name']
for uuid, cfg in ds.repo.get_special_remotes().items()]
)
# set up the dataset as well
create_ds_in_store(io, ppp_store, ds.id, '2', '1')
# now should work
ds.repo.init_remote('ria-remote', options=init_opts)
assert_in('ria-remote',
[cfg['name']
for uuid, cfg in ds.repo.get_special_remotes().items()]
)
assert_repo_status(ds.path)
# git-annex:remote.log should have:
# - url
# - common_init_opts
# - archive_id (which equals ds id)
remote_log = ds.repo.call_git(['cat-file', 'blob', 'git-annex:remote.log'],
read_only=True)
assert_in("url={}".format(url), remote_log)
[assert_in(c, remote_log) for c in common_init_opts]
assert_in("archive-id={}".format(ds.id), remote_log)
# re-configure with invalid URL should fail:
assert_raises(
CommandError,
ds.repo.call_annex,
['enableremote', 'ria-remote'] + common_init_opts + [
'url=ria+file:///non-existing'])
# but re-configure with valid URL should work
if has_symlink_capability():
link.symlink_to(store)
new_url = 'ria+{}'.format(link.as_uri())
ds.repo.call_annex(
['enableremote', 'ria-remote'] + common_init_opts + [
'url={}'.format(new_url)])
# git-annex:remote.log should have:
# - url
# - common_init_opts
# - archive_id (which equals ds id)
remote_log = ds.repo.call_git(['cat-file', 'blob',
'git-annex:remote.log'],
read_only=True)
assert_in("url={}".format(new_url), remote_log)
[assert_in(c, remote_log) for c in common_init_opts]
assert_in("archive-id={}".format(ds.id), remote_log)
# we can deal with --sameas, which leads to a special remote not having a
# 'name' property, but only a 'sameas-name'. See gh-4259
try:
ds.repo.init_remote('ora2',
options=init_opts + ['--sameas', 'ria-remote'])
except CommandError as e:
if 'Invalid option `--sameas' in e.stderr:
# annex too old - doesn't know --sameas
pass
else:
raise
# TODO: - check output of failures to verify it's failing the right way
# - might require to run initremote directly to get the output
# taken from datalad-core@864dc4ae24c8aac0ec4003604543b86de4735732
[docs]
@known_failure_windows # see gh-4469
@with_tempfile
@with_tempfile
@with_tempfile
def patched_test_remote_layout(host, dspath, store, archiv_store):
dspath = Path(dspath)
store = Path(store)
archiv_store = Path(archiv_store)
# PATCH: introduce `ppp_store` and use it instead of `store`
ppp_store = local_path2pure_posix_path(store)
ppp_archiv_store = local_path2pure_posix_path(archiv_store)
ds = Dataset(dspath).create()
populate_dataset(ds)
assert_repo_status(ds.path)
# set up store:
io = SSHRemoteIO(host) if host else LocalIO()
if host:
store_url = "ria+ssh://{host}{path}".format(host=host,
path=store)
arch_url = "ria+ssh://{host}{path}".format(host=host,
path=archiv_store)
else:
store_url = "ria+{}".format(store.as_uri())
arch_url = "ria+{}".format(archiv_store.as_uri())
create_store(io, ppp_store, '1')
# TODO: Re-establish test for version 1
# version 2: dirhash
create_ds_in_store(io, ppp_store, ds.id, '2', '1')
# add special remote
init_opts = common_init_opts + ['url={}'.format(store_url)]
ds.repo.init_remote('store', options=init_opts)
# copy files into the RIA store
ds.push('.', to='store')
# we should see the exact same annex object tree
dsgit_dir, archive_dir, dsobj_dir = \
get_layout_locations(1, store, ds.id)
store_objects = get_all_files(dsobj_dir)
local_objects = get_all_files(ds.pathobj / '.git' / 'annex' / 'objects')
assert_equal(len(store_objects), 4)
if not ds.repo.is_managed_branch():
# with managed branches the local repo uses hashdirlower instead
# TODO: However, with dataset layout version 1 this should therefore
# work on adjusted branch the same way
# TODO: Wonder whether export-archive-ora should account for that and
# rehash according to target layout.
assert_equal(sorted([p for p in store_objects]),
sorted([p for p in local_objects])
)
if not io.get_7z():
raise SkipTest("No 7z available in RIA store")
# we can simply pack up the content of the remote into a
# 7z archive and place it in the right location to get a functional
# archive remote
create_store(io, ppp_archiv_store, '1')
create_ds_in_store(io, ppp_archiv_store, ds.id, '2', '1')
whereis = ds.repo.whereis('one.txt')
dsgit_dir, archive_dir, dsobj_dir = \
get_layout_locations(1, archiv_store, ds.id)
ds.export_archive_ora(archive_dir / 'archive.7z')
init_opts = common_init_opts + ['url={}'.format(arch_url)]
ds.repo.init_remote('archive', options=init_opts)
# now fsck the new remote to get the new special remote indexed
ds.repo.fsck(remote='archive', fast=True)
assert_equal(len(ds.repo.whereis('one.txt')), len(whereis) + 1)
# test creating an archive with filters on files
ds.export_archive_ora(archive_dir / 'archive2.7z', annex_wanted='(include=*.txt)')
# test with wanted expression of a specific remote
ds.repo.set_preferred_content("wanted", "include=subdir/*", remote="store")
ds.export_archive_ora(archive_dir / 'archive3.7z', remote="store")
# test with the current sha
ds.export_archive_ora(
archive_dir / 'archive4.7z',
froms=ds.repo.get_revisions()[1],
)
# taken from datalad-core@864dc4ae24c8aac0ec4003604543b86de4735732
[docs]
@known_failure_windows # see gh-4469
@with_tempfile
@with_tempfile
def patched_test_version_check(host, dspath, store):
dspath = Path(dspath)
store = Path(store)
# PATCH: introduce `ppp_store` and use it instead of `store`
ppp_store = local_path2pure_posix_path(store)
ds = Dataset(dspath).create()
populate_dataset(ds)
assert_repo_status(ds.path)
# set up store:
io = SSHRemoteIO(host) if host else LocalIO()
if host:
store_url = "ria+ssh://{host}{path}".format(host=host,
path=store)
else:
store_url = "ria+{}".format(store.as_uri())
create_store(io, ppp_store, '1')
# TODO: Re-establish test for version 1
# version 2: dirhash
create_ds_in_store(io, ppp_store, ds.id, '2', '1')
# add special remote
init_opts = common_init_opts + ['url={}'.format(store_url)]
ds.repo.init_remote('store', options=init_opts)
ds.push('.', to='store')
# check version files
remote_ds_tree_version_file = store / 'ria-layout-version'
dsgit_dir, archive_dir, dsobj_dir = \
get_layout_locations(1, store, ds.id)
remote_obj_tree_version_file = dsgit_dir / 'ria-layout-version'
assert_true(remote_ds_tree_version_file.exists())
assert_true(remote_obj_tree_version_file.exists())
with open(str(remote_ds_tree_version_file), 'r') as f:
assert_equal(f.read().strip(), '1')
with open(str(remote_obj_tree_version_file), 'r') as f:
assert_equal(f.read().strip(), '2')
# Accessing the remote should not yield any output regarding versioning,
# since it's the "correct" version. Note that "fsck" is an arbitrary choice.
# We need just something to talk to the special remote.
with swallow_logs(new_level=logging.INFO) as cml:
ds.repo.fsck(remote='store', fast=True)
# TODO: For some reason didn't get cml.assert_logged to assert
# "nothing was logged"
assert not cml.out
# Now fake-change the version
with open(str(remote_obj_tree_version_file), 'w') as f:
f.write('X\n')
# Now we should see a message about it
with swallow_logs(new_level=logging.INFO) as cml:
ds.repo.fsck(remote='store', fast=True)
cml.assert_logged(level="INFO",
msg="Remote object tree reports version X",
regex=False)
# reading still works:
ds.drop('.')
assert_status('ok', ds.get('.'))
# but writing doesn't:
with open(str(Path(ds.path) / 'new_file'), 'w') as f:
f.write("arbitrary addition")
ds.save(message="Add a new_file")
with assert_raises((CommandError, IncompleteResultsError)):
ds.push('new_file', to='store')
# However, we can force it by configuration
ds.config.add("annex.ora-remote.store.force-write", "true", scope='local')
ds.push('new_file', to='store')
# taken from datalad-core@864dc4ae24c8aac0ec4003604543b86de4735732
# git-annex-testremote is way too slow on crippled FS.
# Use is_managed_branch() as a proxy and skip only here
# instead of in a decorator
[docs]
@skip_if_adjusted_branch
@known_failure_windows # see gh-4469
@with_tempfile
@with_tempfile
def patched_test_gitannex(host, store, dspath):
dspath = Path(dspath)
store = Path(store)
# PATCH: introduce `ppp_store` and use it instead of `store`
ppp_store = local_path2pure_posix_path(store)
ds = Dataset(dspath).create()
populate_dataset(ds)
assert_repo_status(ds.path)
# set up store:
io = SSHRemoteIO(host) if host else LocalIO()
if host:
store_url = "ria+ssh://{host}{path}".format(host=host,
path=store)
else:
store_url = "ria+{}".format(store.as_uri())
create_store(io, ppp_store, '1')
# TODO: Re-establish test for version 1
# version 2: dirhash
create_ds_in_store(io, ppp_store, ds.id, '2', '1')
# add special remote
init_opts = common_init_opts + ['url={}'.format(store_url)]
ds.repo.init_remote('store', options=init_opts)
from datalad.support.external_versions import external_versions
if '8.20200330' < external_versions['cmd:annex'] < '8.20200624':
# https://git-annex.branchable.com/bugs/testremote_breeds_way_too_many_instances_of_the_externals_remote/?updated
raise SkipTest(
"git-annex might lead to overwhelming number of external "
"special remote instances")
# run git-annex-testremote
# note, that we don't want to capture output. If something goes wrong we
# want to see it in test build's output log.
ds.repo._call_annex(['testremote', 'store'], protocol=NoCapture)
# taken from datalad-core@864dc4ae24c8aac0ec4003604543b86de4735732
[docs]
@known_failure_windows
@with_tempfile
@with_tempfile
@with_tempfile
def patched_test_push_url(storepath=None, dspath=None, blockfile=None):
dspath = Path(dspath)
store = Path(storepath)
# PATCH: introduce `ppp_store` and use it instead of `store`
ppp_store = local_path2pure_posix_path(store)
blockfile = Path(blockfile)
blockfile.touch()
ds = Dataset(dspath).create()
populate_dataset(ds)
assert_repo_status(ds.path)
repo = ds.repo
# set up store:
io = LocalIO()
store_url = "ria+{}".format(store.as_uri())
create_store(io, ppp_store, '1')
create_ds_in_store(io, ppp_store, ds.id, '2', '1')
# initremote fails with invalid url (not a ria+ URL):
invalid_url = (store.parent / "non-existent").as_uri()
init_opts = common_init_opts + ['url={}'.format(store_url),
'push-url={}'.format(invalid_url)]
assert_raises(CommandError, ds.repo.init_remote, 'store', options=init_opts)
# initremote succeeds with valid but inaccessible URL (pointing to a file
# instead of a store):
block_url = "ria+" + blockfile.as_uri()
init_opts = common_init_opts + ['url={}'.format(store_url),
'push-url={}'.format(block_url)]
repo.init_remote('store', options=init_opts)
store_uuid = ds.siblings(name='store',
return_type='item-or-list')['annex-uuid']
here_uuid = ds.siblings(name='here',
return_type='item-or-list')['annex-uuid']
# but a push will fail:
assert_raises(CommandError, ds.repo.call_annex,
['copy', 'one.txt', '--to', 'store'])
# reconfigure w/ local overwrite:
repo.config.add("remote.store.ora-push-url", store_url, scope='local')
# push works now:
repo.call_annex(['copy', 'one.txt', '--to', 'store'])
# remove again (config and file from store)
repo.call_annex(['move', 'one.txt', '--from', 'store'])
repo.config.unset("remote.store.ora-push-url", scope='local')
repo.call_annex(['fsck', '-f', 'store'])
known_sources = repo.whereis('one.txt')
assert_in(here_uuid, known_sources)
assert_not_in(store_uuid, known_sources)
# reconfigure (this time committed)
init_opts = common_init_opts + ['url={}'.format(store_url),
'push-url={}'.format(store_url)]
repo.enable_remote('store', options=init_opts)
# push works now:
repo.call_annex(['copy', 'one.txt', '--to', 'store'])
known_sources = repo.whereis('one.txt')
assert_in(here_uuid, known_sources)
assert_in(store_uuid, known_sources)
# taken from datalad-core@864dc4ae24c8aac0ec4003604543b86de4735732
# Skipping on adjusted branch as a proxy for crippledFS. Write permissions of
# the owner on a directory can't be revoked on VFAT. "adjusted branch" is a
# bit broad but covers the CI cases. And everything RIA/ORA doesn't currently
# properly run on crippled/windows anyway. Needs to be more precise when
# RF'ing will hopefully lead to support on windows in principle.
[docs]
@skip_if_adjusted_branch
@known_failure_windows
@with_tempfile
@with_tempfile
def patched_test_permission(host, storepath, dspath):
# Test whether ORA correctly revokes and obtains write permissions within
# the annex object tree. That is: Revoke after ORA pushed a key to store
# in order to allow the object tree to safely be used with an ephemeral
# clone. And on removal obtain write permissions, like annex would
# internally on a drop (but be sure to restore if something went wrong).
dspath = Path(dspath)
storepath = Path(storepath)
# PATCH: introduce `ppp_storepath` and use it instead of `storepath`
ppp_storepath = local_path2pure_posix_path(storepath)
ds = Dataset(dspath).create()
populate_dataset(ds)
ds.save()
assert_repo_status(ds.path)
testfile = 'one.txt'
# set up store:
io = SSHRemoteIO(host) if host else LocalIO()
if host:
store_url = "ria+ssh://{host}{path}".format(host=host,
path=storepath)
else:
store_url = "ria+{}".format(storepath.as_uri())
create_store(io, ppp_storepath, '1')
create_ds_in_store(io, ppp_storepath, ds.id, '2', '1')
_, _, obj_tree = get_layout_locations(1, storepath, ds.id)
assert_true(obj_tree.is_dir())
file_key_in_store = obj_tree / 'X9' / '6J' / 'MD5E-s8--7e55db001d319a94b0b713529a756623.txt' / 'MD5E-s8--7e55db001d319a94b0b713529a756623.txt'
init_opts = common_init_opts + ['url={}'.format(store_url)]
ds.repo.init_remote('store', options=init_opts)
store_uuid = ds.siblings(name='store',
return_type='item-or-list')['annex-uuid']
here_uuid = ds.siblings(name='here',
return_type='item-or-list')['annex-uuid']
known_sources = ds.repo.whereis(testfile)
assert_in(here_uuid, known_sources)
assert_not_in(store_uuid, known_sources)
assert_false(file_key_in_store.exists())
ds.repo.call_annex(['copy', testfile, '--to', 'store'])
known_sources = ds.repo.whereis(testfile)
assert_in(here_uuid, known_sources)
assert_in(store_uuid, known_sources)
assert_true(file_key_in_store.exists())
# Revoke write permissions from parent dir in-store to test whether we
# still can drop (if we can obtain the permissions). Note, that this has
# no effect on VFAT.
file_key_in_store.parent.chmod(file_key_in_store.parent.stat().st_mode &
~stat.S_IWUSR)
# we can't directly delete; key in store should be protected
assert_raises(PermissionError, file_key_in_store.unlink)
# ORA can still drop, since it obtains permission to:
ds.repo.call_annex(['drop', testfile, '--from', 'store'])
known_sources = ds.repo.whereis(testfile)
assert_in(here_uuid, known_sources)
assert_not_in(store_uuid, known_sources)
assert_false(file_key_in_store.exists())
# Overwrite `_postclonetest_prepare` to handle paths properly
apply_patch(
'datalad.core.distributed.tests.test_clone',
None,
'_postclonetest_prepare',
patched__postclonetest_prepare,
'modify _postclonetest_prepare to use PurePosixPath-arguments '
'in RIA-methodes'
)
apply_patch(
'datalad.core.distributed.tests.test_clone',
None,
'test_ria_postclone_noannex',
patched_test_ria_postclone_noannex,
'modify test_ria_postclone_noannex to use PurePosixPath-arguments '
'in RIA-methods'
)
apply_patch(
'datalad.customremotes.tests.test_ria_utils',
None,
'_test_setup_store',
patched_test_setup_store,
'modify _test_setup_store to use PurePosixPath-arguments in RIA-methods'
)
apply_patch(
'datalad.customremotes.tests.test_ria_utils',
None,
'_test_setup_ds_in_store',
patched_test_setup_ds_in_store,
'modify _test_setup_ds_in_store to use PurePosixPath-arguments '
'in RIA-methods'
)
apply_patch(
'datalad.distributed.tests.test_ora_http',
None,
'test_initremote',
patched_test_initremote,
'modify test_initremote to use PurePosixPath-arguments in RIA-methods'
)
apply_patch(
'datalad.distributed.tests.test_ora_http',
None,
'test_read_access',
patched_test_read_access,
'modify test_read_access to use PurePosixPath-arguments in RIA-methods'
)
apply_patch(
'datalad.distributed.tests.test_ria_basics',
None,
'_test_initremote_basic',
patched_test_initremote_basic,
'modify _test_initremote_basic to use PurePosixPath-arguments '
'in RIA-methods'
)
apply_patch(
'datalad.distributed.tests.test_ria_basics',
None,
'_test_remote_layout',
patched_test_remote_layout,
'modify _test_remote_layout to use PurePosixPath-arguments in RIA-methods'
)
apply_patch(
'datalad.distributed.tests.test_ria_basics',
None,
'_test_version_check',
patched_test_version_check,
'modify _test_version_check to use PurePosixPath-arguments in RIA-methods'
)
apply_patch(
'datalad.distributed.tests.test_ria_basics',
None,
'_test_gitannex',
patched_test_gitannex,
'modify _test_gitannex to use PurePosixPath-arguments in RIA-methods'
)
apply_patch(
'datalad.distributed.tests.test_ria_basics',
None,
'test_push_url',
patched_test_push_url,
'modify test_push_url to use PurePosixPath-arguments in RIA-methods'
)
apply_patch(
'datalad.distributed.tests.test_ria_basics',
None,
'_test_permission',
patched_test_permission,
'modify _test_permission to use PurePosixPath-arguments in RIA-methods'
)