diff --git a/storage/filesystem/dotgit/dotgit.go b/storage/filesystem/dotgit/dotgit.go
index 6c386f799..2be2bae3e 100644
--- a/storage/filesystem/dotgit/dotgit.go
+++ b/storage/filesystem/dotgit/dotgit.go
@@ -943,6 +943,7 @@ func (d *DotGit) walkReferencesTree(refs *[]*plumbing.Reference, relPath []strin
 	files, err := d.fs.ReadDir(d.fs.Join(relPath...))
 	if err != nil {
 		if os.IsNotExist(err) {
+			// a race happened, and our directory is gone now
 			return nil
 		}
 
@@ -960,6 +961,10 @@ func (d *DotGit) walkReferencesTree(refs *[]*plumbing.Reference, relPath []strin
 		}
 
 		ref, err := d.readReferenceFile(".", strings.Join(newRelPath, "/"))
+		if os.IsNotExist(err) {
+			// a race happened, and our file is gone now
+			continue
+		}
 		if err != nil {
 			return err
 		}
diff --git a/storage/filesystem/dotgit/dotgit_test.go b/storage/filesystem/dotgit/dotgit_test.go
index a8f0eb754..63c9eb015 100644
--- a/storage/filesystem/dotgit/dotgit_test.go
+++ b/storage/filesystem/dotgit/dotgit_test.go
@@ -864,3 +864,71 @@ func (s *SuiteDotGit) TestIncBytes(c *C) {
 		c.Assert(overflow, Equals, test.overflow)
 	}
 }
+
+// this filesystem wrapper returns os.ErrNotExist if the file matches
+// the provided paths list
+type notExistsFS struct {
+	billy.Filesystem
+
+	paths []string
+}
+
+func (f *notExistsFS) matches(path string) bool {
+	p := filepath.ToSlash(path)
+	for _, n := range f.paths {
+		if p == n {
+			return true
+		}
+	}
+	return false
+}
+
+func (f *notExistsFS) Open(filename string) (billy.File, error) {
+	if f.matches(filename) {
+		return nil, os.ErrNotExist
+	}
+
+	return f.Filesystem.Open(filename)
+}
+
+func (f *notExistsFS) ReadDir(path string) ([]os.FileInfo, error) {
+	if f.matches(path) {
+		return nil, os.ErrNotExist
+	}
+
+	return f.Filesystem.ReadDir(path)
+}
+
+func (s *SuiteDotGit) TestDeletedRefs(c *C) {
+	fs, clean := s.TemporalFilesystem()
+	defer clean()
+
+	dir := New(&notExistsFS{
+		Filesystem: fs,
+		paths: []string{
+			"refs/heads/bar",
+			"refs/heads/baz",
+		},
+	})
+
+	err := dir.SetRef(plumbing.NewReferenceFromStrings(
+		"refs/heads/foo",
+		"e8d3ffab552895c19b9fcf7aa264d277cde33881",
+	), nil)
+	c.Assert(err, IsNil)
+	err = dir.SetRef(plumbing.NewReferenceFromStrings(
+		"refs/heads/bar",
+		"a8d3ffab552895c19b9fcf7aa264d277cde33881",
+	), nil)
+	c.Assert(err, IsNil)
+	err = dir.SetRef(plumbing.NewReferenceFromStrings(
+		"refs/heads/baz/baz",
+		"a8d3ffab552895c19b9fcf7aa264d277cde33881",
+	), nil)
+	c.Assert(err, IsNil)
+
+	refs, err := dir.Refs()
+	c.Assert(err, IsNil)
+	c.Assert(refs, HasLen, 1)
+	c.Assert(refs[0].Name(), Equals, plumbing.ReferenceName("refs/heads/foo"))
+}