Skip to content

Commit

Permalink
cmd/gitannex: Add layout modes for compatibility
Browse files Browse the repository at this point in the history
  • Loading branch information
@dmcardle
dmcardle committed Apr 10, 2024
1 parent 04128f9 commit 306ccee
Show file tree
Hide file tree
Showing 4 changed files with 398 additions and 123 deletions.
286 changes: 208 additions & 78 deletions cmd/gitannex/e2e_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,35 @@ import (
"github.com/rclone/rclone/lib/buildinfo"
)

func skipE2eTestIfNecessary(t *testing.T) {
if testing.Short() {
t.Skip("Skipping due to short mode.")
}

// TODO: Support e2e tests on Windows. Need to evaluate the semantics of the
// HOME and PATH environment variables.
switch runtime.GOOS {
case "darwin",
"freebsd",
"linux",
"netbsd",
"openbsd",
"plan9",
"solaris":
default:
t.Skipf("GOOS %q is not supported.", runtime.GOOS)
}

if err := checkRcloneBinaryVersion(); err != nil {
t.Skipf("Skipping due to rclone version: %s", err)
}

if _, err := exec.LookPath("git-annex"); err != nil {
t.Skipf("Skipping because git-annex was not found: %s", err)
}

}

// checkRcloneBinaryVersion runs whichever rclone is on the PATH and checks
// whether it reports a version that matches the test's expectations. Returns
// nil when the version is the expected version, otherwise returns an error.
Expand Down Expand Up @@ -50,6 +79,108 @@ func checkRcloneBinaryVersion() error {
return nil
}

// countFilesRecursively returns the number of files nested underneath `dir`. It
// counts files only and excludes directories.
func countFilesRecursively(t *testing.T, dir string) int {
remoteFiles, err := os.ReadDir(dir)
require.NoError(t, err)

var count int
for _, f := range remoteFiles {
if f.IsDir() {
count += countFilesRecursively(t, filepath.Join(dir, f.Name()))
} else {
count++
}
}

return count
}

type e2eTestingContext struct {
tempDir string
binDir string
homeDir string
configDir string
rcloneConfigDir string
ephemeralRepoDir string
}

// makeE2eTestingContext returns a new testing context rooted. Before it
// returns, it creates following directory structure rooted under `t.TempDir()`:
//
// .
// |-- bin
// | `-- git-annex-remote-rclone-builtin -> ${PATH_TO_RCLONE_BINARY}
// |-- ephemeralRepo
// `-- user
// `-- .config
// `-- rclone
// `-- rclone.conf
func makeE2eTestingContext(t *testing.T) e2eTestingContext {
tempDir := t.TempDir()

binDir := filepath.Join(tempDir, "bin")
homeDir := filepath.Join(tempDir, "user")
configDir := filepath.Join(homeDir, ".config")
rcloneConfigDir := filepath.Join(configDir, "rclone")
ephemeralRepoDir := filepath.Join(tempDir, "ephemeralRepo")

for _, dir := range []string{binDir, homeDir, configDir, rcloneConfigDir, ephemeralRepoDir} {
require.NoError(t, os.Mkdir(dir, 0700))
}

return e2eTestingContext{
tempDir: tempDir,
binDir: binDir,
homeDir: homeDir,
configDir: configDir,
rcloneConfigDir: rcloneConfigDir,
ephemeralRepoDir: ephemeralRepoDir,
}
}

// Install the symlink that enables git-annex to invoke "rclone gitannex"
// without explicitly specifying the subcommand.
func (e *e2eTestingContext) installRcloneGitannexSymlink(t *testing.T) {
rcloneBinaryPath, err := exec.LookPath("rclone")
require.NoError(t, err)
require.NoError(t, os.Symlink(
rcloneBinaryPath,
filepath.Join(e.binDir, "git-annex-remote-rclone-builtin")))
}

func (e *e2eTestingContext) installRcloneConfig(t *testing.T) {
// Install the rclone.conf file that defines the remote.
rcloneConfigPath := filepath.Join(e.rcloneConfigDir, "rclone.conf")
rcloneConfigContents := "[MyRcloneRemote]\ntype = local"
require.NoError(t, os.WriteFile(rcloneConfigPath, []byte(rcloneConfigContents), 0600))
}

// Runs a command with variadic number of args with HOME pointing to the
// ephemeral home directory the bin directory on the PATH.
func (e *e2eTestingContext) runInRepo(t *testing.T, command string, args ...string) {
fmt.Printf("+ %s %v\n", command, args)
cmd := exec.Command(command, args...)
cmd.Dir = e.ephemeralRepoDir
cmd.Env = []string{
"HOME=" + e.homeDir,
"PATH=" + os.Getenv("PATH") + ":" + e.binDir,
}
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
require.NoError(t, cmd.Run())
}

func (e *e2eTestingContext) createGitRepo(t *testing.T) {
e.runInRepo(t, "git", "annex", "version")
e.runInRepo(t, "git", "config", "--global", "user.name", "User Name")
e.runInRepo(t, "git", "config", "--global", "user.email", "user@example.com")
e.runInRepo(t, "git", "config", "--global", "init.defaultBranch", "main")
e.runInRepo(t, "git", "init")
e.runInRepo(t, "git", "annex", "init")
}

// This end-to-end test runs `git annex testremote` in a temporary git repo.
// This test will be skipped unless the `rclone` binary on PATH reports the
// expected version.
Expand All @@ -64,31 +195,7 @@ func checkRcloneBinaryVersion() error {
// parameters like repo layouts, and runtime may suffer from a combinatorial
// explosion.
func TestEndToEnd(t *testing.T) {
if testing.Short() {
t.Skip("Skipping due to short mode.")
}

// TODO: Support this test on Windows. Need to evaluate the semantics of the
// HOME and PATH environment variables.
switch runtime.GOOS {
case "darwin",
"freebsd",
"linux",
"netbsd",
"openbsd",
"plan9",
"solaris":
default:
t.Skipf("GOOS %q is not supported.", runtime.GOOS)
}

if err := checkRcloneBinaryVersion(); err != nil {
t.Skipf("Skipping due to rclone version: %s", err)
}

if _, err := exec.LookPath("git-annex"); err != nil {
t.Skipf("Skipping because git-annex was not found: %s", err)
}
skipE2eTestIfNecessary(t)

// Create a temp directory and chdir there, just in case.
originalWd, err := os.Getwd()
Expand All @@ -97,63 +204,86 @@ func TestEndToEnd(t *testing.T) {
require.NoError(t, os.Chdir(tempDir))
defer func() { require.NoError(t, os.Chdir(originalWd)) }()

// Flesh out subdirectories of the temp directory:
//
// .
// |-- bin
// | `-- git-annex-remote-rclone-builtin -> ${PATH_TO_RCLONE_BINARY}
// |-- ephemeralRepo
// `-- user
// `-- .config
// `-- rclone
// `-- rclone.conf
testingContext := makeE2eTestingContext(t)
testingContext.installRcloneGitannexSymlink(t)
testingContext.installRcloneConfig(t)
testingContext.createGitRepo(t)

binDir := filepath.Join(tempDir, "bin")
homeDir := filepath.Join(tempDir, "user")
configDir := filepath.Join(homeDir, ".config")
rcloneConfigDir := filepath.Join(configDir, "rclone")
ephemeralRepoDir := filepath.Join(tempDir, "ephemeralRepo")
for _, dir := range []string{binDir, homeDir, configDir, rcloneConfigDir, ephemeralRepoDir} {
require.NoError(t, os.Mkdir(dir, 0700))
testingContext.runInRepo(t, "git", "annex", "initremote", "MyTestRemote",
"type=external", "externaltype=rclone-builtin", "encryption=none",
"rcloneremotename=MyRcloneRemote", "rcloneprefix="+testingContext.ephemeralRepoDir)

testingContext.runInRepo(t, "git", "annex", "testremote", "MyTestRemote")
}

// For each layout mode, ensure that we're compatible with data written by
// git-annex-remote-rclone.
func TestEndToEndRepoLayoutCompat(t *testing.T) {
skipE2eTestIfNecessary(t)

if _, err := exec.LookPath("git-annex-remote-rclone"); err != nil {
t.Skipf("Skipping because git-annex-remote-rclone was not found: %s", err)
}

// Install the symlink that enables git-annex to invoke "rclone gitannex"
// without explicitly specifying the subcommand.
rcloneBinaryPath, err := exec.LookPath("rclone")
require.NoError(t, err)
require.NoError(t, os.Symlink(
rcloneBinaryPath,
filepath.Join(binDir, "git-annex-remote-rclone-builtin")))
for _, mode := range allLayoutModes() {
t.Run(string(mode), func(t *testing.T) {
// Create a temp directory and chdir there, just in case.
originalWd, err := os.Getwd()
require.NoError(t, err)
tempDir := t.TempDir()
require.NoError(t, os.Chdir(tempDir))
defer func() { require.NoError(t, os.Chdir(originalWd)) }()

// Install the rclone.conf file that defines the remote.
rcloneConfigPath := filepath.Join(rcloneConfigDir, "rclone.conf")
rcloneConfigContents := "[MyRcloneRemote]\ntype = local"
require.NoError(t, os.WriteFile(rcloneConfigPath, []byte(rcloneConfigContents), 0600))
tc := makeE2eTestingContext(t)
tc.installRcloneGitannexSymlink(t)
tc.installRcloneConfig(t)
tc.createGitRepo(t)

// NOTE: These commands must be run with HOME pointing at an ephemeral
// directory, rather than the real home directory.
cmds := [][]string{
{"git", "annex", "version"},
{"git", "config", "--global", "user.name", "User Name"},
{"git", "config", "--global", "user.email", "user@example.com"},
{"git", "init"},
{"git", "annex", "init"},
{"git", "annex", "initremote", "MyTestRemote",
"type=external", "externaltype=rclone-builtin", "encryption=none",
"rcloneremotename=MyRcloneRemote", "rcloneprefix=" + ephemeralRepoDir},
{"git", "annex", "testremote", "MyTestRemote"},
}

for _, args := range cmds {
fmt.Printf("+ %v\n", args)
cmd := exec.Command(args[0], args[1:]...)
cmd.Dir = ephemeralRepoDir
cmd.Env = []string{
"HOME=" + homeDir,
"PATH=" + os.Getenv("PATH") + ":" + binDir,
}
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
require.NoError(t, cmd.Run())
remoteStorage := filepath.Join(tc.tempDir, "remotePrefix")
require.NoError(t, os.Mkdir(remoteStorage, 0777))

tc.runInRepo(t,
"git", "annex", "initremote", "Control",
"type=external", "externaltype=rclone", "encryption=none",
"target=MyRcloneRemote",
"rclone_layout="+string(mode),
"prefix="+remoteStorage)

tc.runInRepo(t,
"git", "annex", "initremote", "Experiment",
"type=external", "externaltype=rclone-builtin", "encryption=none",
"rcloneremotename=MyRcloneRemote",
"rclonelayout="+string(mode),
"rcloneprefix="+remoteStorage)

fooFilePath := filepath.Join(tc.ephemeralRepoDir, "foo")
require.NoError(t, os.WriteFile(fooFilePath, []byte{1, 2, 3, 4}, 0700))
tc.runInRepo(t, "git", "annex", "add", "foo")
tc.runInRepo(t, "git", "commit", "-m", "Add foo file")
// Git-annex objects are not writable, which prevents `testing` from
// cleaning up the temp directory. We can work around this by
// explicitly dropping any files we add to the annex.
t.Cleanup(func() { tc.runInRepo(t, "git", "annex", "drop", "--force", "foo") })

require.Equal(t, 0, countFilesRecursively(t, remoteStorage))

tc.runInRepo(t, "git", "annex", "copy", "--to=Control", "foo")
require.Equal(t, 1, countFilesRecursively(t, remoteStorage))

tc.runInRepo(t, "git", "annex", "fsck", "--from=Experiment", "foo")
require.Equal(t, 1, countFilesRecursively(t, remoteStorage))

tc.runInRepo(t, "git", "annex", "drop", "--from=Experiment", "--force", "foo")
require.Equal(t, 0, countFilesRecursively(t, remoteStorage))

tc.runInRepo(t, "git", "annex", "copy", "--to=Experiment", "foo")
require.Equal(t, 1, countFilesRecursively(t, remoteStorage))

tc.runInRepo(t, "git", "annex", "fsck", "--from=Control", "foo")
require.Equal(t, 1, countFilesRecursively(t, remoteStorage))

tc.runInRepo(t, "git", "annex", "drop", "--from=Control", "--force", "foo")
require.Equal(t, 0, countFilesRecursively(t, remoteStorage))
})
}
}

0 comments on commit 306ccee

Please sign in to comment.