Skip to content

Commit

Permalink
Remove dir fuzz error
Browse files Browse the repository at this point in the history
  • Loading branch information
@angelini
angelini committed Oct 28, 2022
1 parent bddb60b commit 6c31500
Show file tree
Hide file tree
Showing 3 changed files with 168 additions and 47 deletions.
146 changes: 99 additions & 47 deletions cmd/fuzz-test/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package main

import (
"context"
"errors"
"fmt"
"io/fs"
stdlog "log"
Expand Down Expand Up @@ -180,7 +179,7 @@ func (o AddFileOp) Apply() error {
}

func (o AddFileOp) String() string {
return fmt.Sprintf("AddFileOp(%s, %d)", Join(o.dir, o.name), len(o.content))
return fmt.Sprintf("AddFile(%s, %d)", Join(o.dir, o.name), len(o.content))
}

type UpdateFileOp struct {
Expand All @@ -198,7 +197,7 @@ func newUpdateFileOp(base string, project int64) Operation {
return SkipOp{}
}

return AddFileOp{
return UpdateFileOp{
base: base,
dir: dir,
name: files[rand.Intn(len(files))],
Expand All @@ -211,7 +210,7 @@ func (o UpdateFileOp) Apply() error {
}

func (o UpdateFileOp) String() string {
return fmt.Sprintf("UpdateFileOp(%s, %d)", Join(o.dir, o.name), len(o.content))
return fmt.Sprintf("UpdateFile(%s, %d)", Join(o.dir, o.name), len(o.content))
}

type AddDirOp struct {
Expand Down Expand Up @@ -277,7 +276,36 @@ func (o RemoveFileOp) Apply() error {
}

func (o RemoveFileOp) String() string {
return fmt.Sprintf("RemoveFileOp(%s)", Join(o.dir, o.name))
return fmt.Sprintf("RemoveFile(%s)", Join(o.dir, o.name))
}

type RemoveDirOp struct {
base string
dir string
name string
}

func newRemoveDirOp(base string, project int64) Operation {
dir := fmt.Sprint(project)
dirs := objectFilter(walkDir(Join(base, dir)), typeDirectory)

if len(dirs) == 0 {
return SkipOp{}
}

return RemoveDirOp{
base: base,
dir: dir,
name: dirs[rand.Intn(len(dirs))],
}
}

func (o RemoveDirOp) Apply() error {
return os.RemoveAll(Join(o.base, o.dir, o.name))
}

func (o RemoveDirOp) String() string {
return fmt.Sprintf("RemoveDir(%s)", Join(o.dir, o.name))
}

type AddSymlinkOp struct {
Expand Down Expand Up @@ -320,12 +348,12 @@ func (o AddSymlinkOp) Apply() error {
}

func (o AddSymlinkOp) String() string {
return fmt.Sprintf("AddSymlinkOp(%s, %s)", Join(o.dir, o.target), Join(o.dir, o.name))
return fmt.Sprintf("AddSymlink(%s, %s)", Join(o.dir, o.target), Join(o.dir, o.name))
}

type OpConstructor func(dir string, project int64) Operation

var opConstructors = []OpConstructor{newAddFileOp, newUpdateFileOp, newAddDirOp, newRemoveFileOp, newAddSymlinkOp}
var opConstructors = []OpConstructor{newAddFileOp, newUpdateFileOp, newAddDirOp, newRemoveFileOp, newRemoveDirOp, newAddSymlinkOp}

func randomOperation(baseDir string, project int64) Operation {
var operation Operation = SkipOp{}
Expand All @@ -340,71 +368,97 @@ func randomOperation(baseDir string, project int64) Operation {
return operation
}

func createDirs(projects int) (string, string, string, string, error) {
type Directories struct {
base string
reset string
previous string
step string
}

func createDirectories(projects int) (*Directories, error) {
var dirs []string

for _, name := range []string{"base", "continue", "reset", "step"} {
for _, name := range []string{"base", "reset", "previous", "step"} {
dir, err := os.MkdirTemp("", fmt.Sprintf("dl-ft-%s-", name))
if err != nil {
return "", "", "", "", fmt.Errorf("cannot create tmp dir: %w", err)
return nil, fmt.Errorf("cannot create tmp dir: %w", err)
}

for projectIdx := 1; projectIdx <= projects; projectIdx++ {
err = os.MkdirAll(filepath.Join(dir, fmt.Sprint(projectIdx)), 0755)
if err != nil {
return "", "", "", "", fmt.Errorf("cannot create project dir: %w", err)
return nil, fmt.Errorf("cannot create project dir: %w", err)
}
}
dirs = append(dirs, dir)
}

return dirs[0], dirs[1], dirs[2], dirs[3], nil
return &Directories{
base: dirs[0],
reset: dirs[1],
previous: dirs[2],
step: dirs[3],
}, nil
}

func (d *Directories) Log(ctx context.Context) {
logger.Info(ctx, "base", zap.String("path", d.base))
logger.Info(ctx, "reset", zap.String("path", d.reset))
logger.Info(ctx, "prev", zap.String("path", d.previous))
logger.Info(ctx, "step", zap.String("path", d.step))
}

func runIteration(ctx context.Context, client *dlc.Client, project int64, operation Operation, baseDir, continueDir, resetDir, stepDir string) error {
func (d *Directories) RemoveAll() {
os.RemoveAll(d.base)
os.RemoveAll(d.reset)
os.RemoveAll(d.previous)
os.RemoveAll(d.step)
}

func runIteration(ctx context.Context, client *dlc.Client, project int64, operation Operation, dirs *Directories) (int64, error) {
err := operation.Apply()
if err != nil {
return fmt.Errorf("failed to apply operation %s: %w", operation.String(), err)
return -1, fmt.Errorf("failed to apply operation %s: %w", operation.String(), err)
}

version, _, err := client.Update(ctx, project, projectDir(baseDir, project))
version, _, err := client.Update(ctx, project, projectDir(dirs.base, project))
if err != nil {
return fmt.Errorf("failed to update project %d: %w", project, err)
return -1, fmt.Errorf("failed to update project %d: %w", project, err)
}

_, _, err = client.Rebuild(ctx, project, "", nil, projectDir(continueDir, project), "")
os.RemoveAll(projectDir(dirs.reset, project))
err = os.MkdirAll(projectDir(dirs.reset, project), 0755)
if err != nil {
return fmt.Errorf("failed to rebuild continue project %d: %w", project, err)
return -1, fmt.Errorf("failed to create reset dir %s: %w", projectDir(dirs.reset, project), err)
}

os.RemoveAll(projectDir(resetDir, project))
err = os.MkdirAll(projectDir(resetDir, project), 0755)
_, _, err = client.Rebuild(ctx, project, "", nil, projectDir(dirs.reset, project), "")
if err != nil {
return fmt.Errorf("failed to create reset dir %s: %w", projectDir(resetDir, project), err)
return -1, fmt.Errorf("failed to rebuild from reset, project %d: %w", project, err)
}

_, _, err = client.Rebuild(ctx, project, "", nil, projectDir(resetDir, project), "")
_, _, err = client.Rebuild(ctx, project, "", nil, projectDir(dirs.previous, project), "")
if err != nil {
return fmt.Errorf("failed to rebuild reset project %d: %w", project, err)
return -1, fmt.Errorf("failed to rebuild from previous version, project %d: %w", project, err)
}

os.RemoveAll(projectDir(stepDir, project))
err = os.MkdirAll(projectDir(stepDir, project), 0755)
os.RemoveAll(projectDir(dirs.step, project))
err = os.MkdirAll(projectDir(dirs.step, project), 0755)
if err != nil {
return fmt.Errorf("failed to create step dir %s: %w", projectDir(stepDir, project), err)
return -1, fmt.Errorf("failed to create step dir %s: %w", projectDir(dirs.step, project), err)
}

stepVersion := int64(rand.Intn(int(version)))
_, _, err = client.Rebuild(ctx, project, "", &stepVersion, projectDir(stepDir, project), "")
_, _, err = client.Rebuild(ctx, project, "", &stepVersion, projectDir(dirs.step, project), "")
if err != nil {
return fmt.Errorf("failed to rebuild step project %d: %w", project, err)
return -1, fmt.Errorf("failed to rebuild step to version %v, project %d: %w", stepVersion, project, err)
}
_, _, err = client.Rebuild(ctx, project, "", &version, projectDir(stepDir, project), "")
_, _, err = client.Rebuild(ctx, project, "", &version, projectDir(dirs.step, project), "")
if err != nil {
return fmt.Errorf("failed to rebuild step project %d: %w", project, err)
return -1, fmt.Errorf("failed to rebuild step from version %v, project %d: %w", stepVersion, project, err)
}

return nil
return stepVersion, nil
}

type MatchError struct {
Expand Down Expand Up @@ -507,35 +561,35 @@ func logOpLog(ctx context.Context, opLog []Operation) {
}
}

func verifyDirs(ctx context.Context, projects int, baseDir, continueDir, resetDir, stepDir string) error {
func verifyDirs(ctx context.Context, projects int, dirs *Directories, stepVersion int64) error {
for projectIdx := 1; projectIdx <= projects; projectIdx++ {
project := int64(projectIdx)

matchErrors, err := compareDirs(project, projectDir(baseDir, project), projectDir(resetDir, project))
matchErrors, err := compareDirs(project, projectDir(dirs.base, project), projectDir(dirs.reset, project))
if err != nil {
return fmt.Errorf("failed to compare base & reset dirs: %w", err)
}
if len(matchErrors) > 0 {
logMatchErrors(ctx, matchErrors)
return errors.New("reset directory match error")
return fmt.Errorf("reset directory match error, project %d", project)
}

matchErrors, err = compareDirs(project, projectDir(baseDir, project), projectDir(continueDir, project))
matchErrors, err = compareDirs(project, projectDir(dirs.base, project), projectDir(dirs.previous, project))
if err != nil {
return fmt.Errorf("failed to compare base & continue dirs: %w", err)
return fmt.Errorf("failed to compare base & previous dirs: %w", err)
}
if len(matchErrors) > 0 {
logMatchErrors(ctx, matchErrors)
return errors.New("continue directory match error")
return fmt.Errorf("previous version directory match error, project %d", project)
}

matchErrors, err = compareDirs(project, projectDir(baseDir, project), projectDir(stepDir, project))
matchErrors, err = compareDirs(project, projectDir(dirs.base, project), projectDir(dirs.step, project))
if err != nil {
return fmt.Errorf("failed to compare base & step dirs: %w", err)
}
if len(matchErrors) > 0 {
logMatchErrors(ctx, matchErrors)
return errors.New("step directory match error")
return fmt.Errorf("step from verion %d directory match error, project %d", stepVersion, project)
}
}
return nil
Expand All @@ -552,29 +606,27 @@ func fuzzTest(ctx context.Context, client *dlc.Client, projects, iterations int)
}
}

baseDir, continueDir, resetDir, stepDir, err := createDirs(projects)
dirs, err := createDirectories(projects)
if err != nil {
return err
}
defer os.RemoveAll(baseDir)
defer os.RemoveAll(continueDir)
defer os.RemoveAll(resetDir)
defer os.RemoveAll(stepDir)
// defer dirs.RemoveAll()
dirs.Log(ctx)

var opLog []Operation

for iterIdx := 0; iterIdx < iterations; iterIdx++ {
project := int64(rand.Intn(projects) + 1)

operation := randomOperation(baseDir, project)
operation := randomOperation(dirs.base, project)
opLog = append(opLog, operation)

err := runIteration(ctx, client, project, operation, baseDir, continueDir, resetDir, stepDir)
stepVersion, err := runIteration(ctx, client, project, operation, dirs)
if err != nil {
return fmt.Errorf("failed to run iteration %d: %w", iterIdx, err)
}

err = verifyDirs(ctx, projects, baseDir, continueDir, resetDir, stepDir)
err = verifyDirs(ctx, projects, dirs, stepVersion)
if err != nil {
logOpLog(ctx, opLog)
return err
Expand Down
64 changes: 64 additions & 0 deletions test/client_combined_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -621,3 +621,67 @@ func TestCombinedWithPrefixDirectoryBug(t *testing.T) {
"abc": {content: "abc v2"},
})
}

func TestCombinedAddAndRemoveInDirBug(t *testing.T) {
tc := util.NewTestCtx(t, auth.Project, 1)
defer tc.Close()

writeProject(tc, 1, 1)
writeEmptyDir(tc, 1, 1, nil, "a/")

c, fs, close := createTestClient(tc)
defer close()

tmpDir := emptyTmpDir(t)
defer os.RemoveAll(tmpDir)

rebuild(tc, c, 1, nil, tmpDir, nil, expectedResponse{
version: 1,
count: 1,
})

verifyDir(t, tmpDir, 1, map[string]expectedFile{
"a/": {content: "", fileType: typeDirectory},
})

writeFile(t, tmpDir, "a/b", "a/b v1")

update(tc, c, 1, tmpDir, expectedResponse{
version: 2,
count: 1,
})

removeFile(t, tmpDir, "a/b")

update(tc, c, 1, tmpDir, expectedResponse{
version: 3,
count: 2,
})

updateStream := newMockUpdateServer(tc.Context(), 1, map[string]expectedObject{
"b": {content: "b v4"},
"a/": {deleted: true},
})
err := fs.Update(updateStream)
require.NoError(t, err, "fs.Update")

os.RemoveAll(tmpDir)
err = os.MkdirAll(tmpDir, 0755)
require.NoError(t, err, "recreate tmp dir %v", tmpDir)

rebuild(tc, c, 1, i(2), tmpDir, nil, expectedResponse{
version: 2,
count: 1,
})

debugObjects(tc)

rebuild(tc, c, 1, nil, tmpDir, nil, expectedResponse{
version: 4,
count: 2,
})

verifyDir(t, tmpDir, 4, map[string]expectedFile{
"b": {content: "b v4"},
})
}
5 changes: 5 additions & 0 deletions test/shared_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,11 @@ func writeFile(t *testing.T, dir string, path string, content string) {
require.NoError(t, err, "write file %v", path)
}

func removeFile(t *testing.T, dir string, path string) {
err := os.Remove(filepath.Join(dir, path))
require.NoError(t, err, "remove file %v", path)
}

func emptyTmpDir(t *testing.T) string {
dir, err := os.MkdirTemp("", "dateilager_tests_")
require.NoError(t, err, "create temp dir")
Expand Down

0 comments on commit 6c31500

Please sign in to comment.