Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Remove dir fuzz error #20

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

Remove dir fuzz error #20

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
146 changes: 99 additions & 47 deletions cmd/fuzz-test/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package main

import (
"context"
"errors"
"fmt"
"io/fs"
stdlog "log"
Expand Down Expand Up @@ -180,7 +179,7 @@ func (o AddFileOp) Apply() error {
}

func (o AddFileOp) String() string {
return fmt.Sprintf("AddFileOp(%s, %d)", Join(o.dir, o.name), len(o.content))
return fmt.Sprintf("AddFile(%s, %d)", Join(o.dir, o.name), len(o.content))
}

type UpdateFileOp struct {
Expand All @@ -198,7 +197,7 @@ func newUpdateFileOp(base string, project int64) Operation {
return SkipOp{}
}

return AddFileOp{
return UpdateFileOp{
base: base,
dir: dir,
name: files[rand.Intn(len(files))],
Expand All @@ -211,7 +210,7 @@ func (o UpdateFileOp) Apply() error {
}

func (o UpdateFileOp) String() string {
return fmt.Sprintf("UpdateFileOp(%s, %d)", Join(o.dir, o.name), len(o.content))
return fmt.Sprintf("UpdateFile(%s, %d)", Join(o.dir, o.name), len(o.content))
}

type AddDirOp struct {
Expand Down Expand Up @@ -277,7 +276,36 @@ func (o RemoveFileOp) Apply() error {
}

func (o RemoveFileOp) String() string {
return fmt.Sprintf("RemoveFileOp(%s)", Join(o.dir, o.name))
return fmt.Sprintf("RemoveFile(%s)", Join(o.dir, o.name))
}

type RemoveDirOp struct {
base string
dir string
name string
}

func newRemoveDirOp(base string, project int64) Operation {
dir := fmt.Sprint(project)
dirs := objectFilter(walkDir(Join(base, dir)), typeDirectory)

if len(dirs) == 0 {
return SkipOp{}
}

return RemoveDirOp{
base: base,
dir: dir,
name: dirs[rand.Intn(len(dirs))],
}
}

func (o RemoveDirOp) Apply() error {
return os.RemoveAll(Join(o.base, o.dir, o.name))
}

func (o RemoveDirOp) String() string {
return fmt.Sprintf("RemoveDir(%s)", Join(o.dir, o.name))
}

type AddSymlinkOp struct {
Expand Down Expand Up @@ -320,12 +348,12 @@ func (o AddSymlinkOp) Apply() error {
}

func (o AddSymlinkOp) String() string {
return fmt.Sprintf("AddSymlinkOp(%s, %s)", Join(o.dir, o.target), Join(o.dir, o.name))
return fmt.Sprintf("AddSymlink(%s, %s)", Join(o.dir, o.target), Join(o.dir, o.name))
}

type OpConstructor func(dir string, project int64) Operation

var opConstructors = []OpConstructor{newAddFileOp, newUpdateFileOp, newAddDirOp, newRemoveFileOp, newAddSymlinkOp}
var opConstructors = []OpConstructor{newAddFileOp, newUpdateFileOp, newAddDirOp, newRemoveFileOp, newRemoveDirOp, newAddSymlinkOp}

func randomOperation(baseDir string, project int64) Operation {
var operation Operation = SkipOp{}
Expand All @@ -340,71 +368,97 @@ func randomOperation(baseDir string, project int64) Operation {
return operation
}

func createDirs(projects int) (string, string, string, string, error) {
type Directories struct {
base string
reset string
previous string
step string
}

func createDirectories(projects int) (*Directories, error) {
var dirs []string

for _, name := range []string{"base", "continue", "reset", "step"} {
for _, name := range []string{"base", "reset", "previous", "step"} {
dir, err := os.MkdirTemp("", fmt.Sprintf("dl-ft-%s-", name))
if err != nil {
return "", "", "", "", fmt.Errorf("cannot create tmp dir: %w", err)
return nil, fmt.Errorf("cannot create tmp dir: %w", err)
}

for projectIdx := 1; projectIdx <= projects; projectIdx++ {
err = os.MkdirAll(filepath.Join(dir, fmt.Sprint(projectIdx)), 0755)
if err != nil {
return "", "", "", "", fmt.Errorf("cannot create project dir: %w", err)
return nil, fmt.Errorf("cannot create project dir: %w", err)
}
}
dirs = append(dirs, dir)
}

return dirs[0], dirs[1], dirs[2], dirs[3], nil
return &Directories{
base: dirs[0],
reset: dirs[1],
previous: dirs[2],
step: dirs[3],
}, nil
}

func (d *Directories) Log(ctx context.Context) {
logger.Info(ctx, "base", zap.String("path", d.base))
logger.Info(ctx, "reset", zap.String("path", d.reset))
logger.Info(ctx, "prev", zap.String("path", d.previous))
logger.Info(ctx, "step", zap.String("path", d.step))
}

func runIteration(ctx context.Context, client *dlc.Client, project int64, operation Operation, baseDir, continueDir, resetDir, stepDir string) error {
func (d *Directories) RemoveAll() {
os.RemoveAll(d.base)
os.RemoveAll(d.reset)
os.RemoveAll(d.previous)
os.RemoveAll(d.step)
}

func runIteration(ctx context.Context, client *dlc.Client, project int64, operation Operation, dirs *Directories) (int64, error) {
err := operation.Apply()
if err != nil {
return fmt.Errorf("failed to apply operation %s: %w", operation.String(), err)
return -1, fmt.Errorf("failed to apply operation %s: %w", operation.String(), err)
}

version, _, err := client.Update(ctx, project, projectDir(baseDir, project))
version, _, err := client.Update(ctx, project, projectDir(dirs.base, project))
if err != nil {
return fmt.Errorf("failed to update project %d: %w", project, err)
return -1, fmt.Errorf("failed to update project %d: %w", project, err)
}

_, _, err = client.Rebuild(ctx, project, "", nil, projectDir(continueDir, project), "")
os.RemoveAll(projectDir(dirs.reset, project))
err = os.MkdirAll(projectDir(dirs.reset, project), 0755)
if err != nil {
return fmt.Errorf("failed to rebuild continue project %d: %w", project, err)
return -1, fmt.Errorf("failed to create reset dir %s: %w", projectDir(dirs.reset, project), err)
}

os.RemoveAll(projectDir(resetDir, project))
err = os.MkdirAll(projectDir(resetDir, project), 0755)
_, _, err = client.Rebuild(ctx, project, "", nil, projectDir(dirs.reset, project), "")
if err != nil {
return fmt.Errorf("failed to create reset dir %s: %w", projectDir(resetDir, project), err)
return -1, fmt.Errorf("failed to rebuild from reset, project %d: %w", project, err)
}

_, _, err = client.Rebuild(ctx, project, "", nil, projectDir(resetDir, project), "")
_, _, err = client.Rebuild(ctx, project, "", nil, projectDir(dirs.previous, project), "")
if err != nil {
return fmt.Errorf("failed to rebuild reset project %d: %w", project, err)
return -1, fmt.Errorf("failed to rebuild from previous version, project %d: %w", project, err)
}

os.RemoveAll(projectDir(stepDir, project))
err = os.MkdirAll(projectDir(stepDir, project), 0755)
os.RemoveAll(projectDir(dirs.step, project))
err = os.MkdirAll(projectDir(dirs.step, project), 0755)
if err != nil {
return fmt.Errorf("failed to create step dir %s: %w", projectDir(stepDir, project), err)
return -1, fmt.Errorf("failed to create step dir %s: %w", projectDir(dirs.step, project), err)
}

stepVersion := int64(rand.Intn(int(version)))
_, _, err = client.Rebuild(ctx, project, "", &stepVersion, projectDir(stepDir, project), "")
_, _, err = client.Rebuild(ctx, project, "", &stepVersion, projectDir(dirs.step, project), "")
if err != nil {
return fmt.Errorf("failed to rebuild step project %d: %w", project, err)
return -1, fmt.Errorf("failed to rebuild step to version %v, project %d: %w", stepVersion, project, err)
}
_, _, err = client.Rebuild(ctx, project, "", &version, projectDir(stepDir, project), "")
_, _, err = client.Rebuild(ctx, project, "", &version, projectDir(dirs.step, project), "")
if err != nil {
return fmt.Errorf("failed to rebuild step project %d: %w", project, err)
return -1, fmt.Errorf("failed to rebuild step from version %v, project %d: %w", stepVersion, project, err)
}

return nil
return stepVersion, nil
}

type MatchError struct {
Expand Down Expand Up @@ -507,35 +561,35 @@ func logOpLog(ctx context.Context, opLog []Operation) {
}
}

func verifyDirs(ctx context.Context, projects int, baseDir, continueDir, resetDir, stepDir string) error {
func verifyDirs(ctx context.Context, projects int, dirs *Directories, stepVersion int64) error {
for projectIdx := 1; projectIdx <= projects; projectIdx++ {
project := int64(projectIdx)

matchErrors, err := compareDirs(project, projectDir(baseDir, project), projectDir(resetDir, project))
matchErrors, err := compareDirs(project, projectDir(dirs.base, project), projectDir(dirs.reset, project))
if err != nil {
return fmt.Errorf("failed to compare base & reset dirs: %w", err)
}
if len(matchErrors) > 0 {
logMatchErrors(ctx, matchErrors)
return errors.New("reset directory match error")
return fmt.Errorf("reset directory match error, project %d", project)
}

matchErrors, err = compareDirs(project, projectDir(baseDir, project), projectDir(continueDir, project))
matchErrors, err = compareDirs(project, projectDir(dirs.base, project), projectDir(dirs.previous, project))
if err != nil {
return fmt.Errorf("failed to compare base & continue dirs: %w", err)
return fmt.Errorf("failed to compare base & previous dirs: %w", err)
}
if len(matchErrors) > 0 {
logMatchErrors(ctx, matchErrors)
return errors.New("continue directory match error")
return fmt.Errorf("previous version directory match error, project %d", project)
}

matchErrors, err = compareDirs(project, projectDir(baseDir, project), projectDir(stepDir, project))
matchErrors, err = compareDirs(project, projectDir(dirs.base, project), projectDir(dirs.step, project))
if err != nil {
return fmt.Errorf("failed to compare base & step dirs: %w", err)
}
if len(matchErrors) > 0 {
logMatchErrors(ctx, matchErrors)
return errors.New("step directory match error")
return fmt.Errorf("step from verion %d directory match error, project %d", stepVersion, project)
}
}
return nil
Expand All @@ -552,29 +606,27 @@ func fuzzTest(ctx context.Context, client *dlc.Client, projects, iterations int)
}
}

baseDir, continueDir, resetDir, stepDir, err := createDirs(projects)
dirs, err := createDirectories(projects)
if err != nil {
return err
}
defer os.RemoveAll(baseDir)
defer os.RemoveAll(continueDir)
defer os.RemoveAll(resetDir)
defer os.RemoveAll(stepDir)
// defer dirs.RemoveAll()
dirs.Log(ctx)

var opLog []Operation

for iterIdx := 0; iterIdx < iterations; iterIdx++ {
project := int64(rand.Intn(projects) + 1)

operation := randomOperation(baseDir, project)
operation := randomOperation(dirs.base, project)
opLog = append(opLog, operation)

err := runIteration(ctx, client, project, operation, baseDir, continueDir, resetDir, stepDir)
stepVersion, err := runIteration(ctx, client, project, operation, dirs)
if err != nil {
return fmt.Errorf("failed to run iteration %d: %w", iterIdx, err)
}

err = verifyDirs(ctx, projects, baseDir, continueDir, resetDir, stepDir)
err = verifyDirs(ctx, projects, dirs, stepVersion)
if err != nil {
logOpLog(ctx, opLog)
return err
Expand Down
64 changes: 64 additions & 0 deletions test/client_combined_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -621,3 +621,67 @@ func TestCombinedWithPrefixDirectoryBug(t *testing.T) {
"abc": {content: "abc v2"},
})
}

func TestCombinedAddAndRemoveInDirBug(t *testing.T) {
tc := util.NewTestCtx(t, auth.Project, 1)
defer tc.Close()

writeProject(tc, 1, 1)
writeEmptyDir(tc, 1, 1, nil, "a/")

c, fs, close := createTestClient(tc)
defer close()

tmpDir := emptyTmpDir(t)
defer os.RemoveAll(tmpDir)

rebuild(tc, c, 1, nil, tmpDir, nil, expectedResponse{
version: 1,
count: 1,
})

verifyDir(t, tmpDir, 1, map[string]expectedFile{
"a/": {content: "", fileType: typeDirectory},
})

writeFile(t, tmpDir, "a/b", "a/b v1")

update(tc, c, 1, tmpDir, expectedResponse{
version: 2,
count: 1,
})

removeFile(t, tmpDir, "a/b")

update(tc, c, 1, tmpDir, expectedResponse{
version: 3,
count: 2,
})

updateStream := newMockUpdateServer(tc.Context(), 1, map[string]expectedObject{
"b": {content: "b v4"},
"a/": {deleted: true},
})
err := fs.Update(updateStream)
require.NoError(t, err, "fs.Update")

os.RemoveAll(tmpDir)
err = os.MkdirAll(tmpDir, 0755)
require.NoError(t, err, "recreate tmp dir %v", tmpDir)

rebuild(tc, c, 1, i(2), tmpDir, nil, expectedResponse{
version: 2,
count: 1,
})

debugObjects(tc)

rebuild(tc, c, 1, nil, tmpDir, nil, expectedResponse{
version: 4,
count: 2,
})

verifyDir(t, tmpDir, 4, map[string]expectedFile{
"b": {content: "b v4"},
})
}
5 changes: 5 additions & 0 deletions test/shared_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,11 @@ func writeFile(t *testing.T, dir string, path string, content string) {
require.NoError(t, err, "write file %v", path)
}

func removeFile(t *testing.T, dir string, path string) {
err := os.Remove(filepath.Join(dir, path))
require.NoError(t, err, "remove file %v", path)
}

func emptyTmpDir(t *testing.T) string {
dir, err := os.MkdirTemp("", "dateilager_tests_")
require.NoError(t, err, "create temp dir")
Expand Down