WIP digest_key

server/devpi_server/filestore.py

@@ -374,6 +374,9 @@ def store(self, user, index, basename, content_or_file, *, dir_hash_spec=None, h
             hashdir_a=hashdir_a, hashdir_b=hashdir_b, filename=basename)
         entry = MutableFileEntry(key)
         entry.file_set_content(content_or_file, hashes=hashes)
+        digest_key = entry.get_digest_key()
+        with digest_key.update() as digest_paths:
+            digest_paths.add(entry.relpath)
         return entry
 
 
@@ -507,8 +510,8 @@ def meta(self):
     def file_exists(self):
         return self.tx.io_file.exists(self.file_path_info)
 
-    def file_delete(self):
-        return self.tx.io_file.delete(self.file_path_info)
+    def file_delete(self, *, is_last_of_hash):
+        return self.tx.io_file.delete(self.file_path_info, is_last_of_hash=is_last_of_hash)
 
     def file_size(self):
         return self.tx.io_file.size(self.file_path_info)
@@ -581,11 +584,21 @@ def __hash__(self):
         return hash(self.relpath)
 
     def delete(self, *, file_only=False):
-        self.file_delete()
+        digest_key = self.get_digest_key()
+        with digest_key.update() as digest_paths:
+            digest_paths.discard(self.relpath)
+        is_last_of_hash = False
+        if not digest_paths:
+            digest_key.delete()
+            is_last_of_hash = True
+        self.file_delete(is_last_of_hash=is_last_of_hash)
         if not file_only:
             self.key.delete()
             self._meta = {}
 
+    def get_digest_key(self):
+        return self.tx.keyfs.DIGESTPATHS(digest=self.hashes[DEFAULT_HASH_TYPE])
+
     def has_existing_metadata(self):
         return bool(self.hashes and self.last_modified)
 

server/devpi_server/filestore_fs.py

@@ -81,7 +81,7 @@ def __exit__(self, cls, val, tb):
                 LazyChangesFormatter({}, files_commit, files_del))
         return True
 
-    def delete(self, path):
+    def delete(self, path, *, is_last_of_hash):  # noqa: ARG002
         assert isinstance(path, FilePathInfo)
         path = str(self.basedir / "+files" / path.relpath)
         old = self._dirty_files.get(path)

server/devpi_server/interfaces.py

@@ -61,7 +61,7 @@ class IIOFile(Interface):
     def commit() -> None:
         """ Commit changed files to storage. """
 
-    def delete(path: FilePathInfo) -> None:
+    def delete(path: FilePathInfo, *, is_last_of_hash: bool) -> None:
         """ Deletes the file at path. """
 
     def exists(path: FilePathInfo) -> bool:
@@ -266,7 +266,7 @@ def __init__(self, conn: Any) -> None:
         self.dirty_files = conn.dirty_files
         self.storage = conn.storage
 
-    def io_file_delete(self, path: FilePathInfo) -> None:
+    def io_file_delete(self, path: FilePathInfo, *, is_last_of_hash: bool) -> None:  # noqa: ARG002
         return self.conn.io_file_delete(path.relpath)
 
     def io_file_exists(self, path: FilePathInfo) -> bool:

server/devpi_server/model.py

@@ -1939,6 +1939,9 @@ def add_keys(xom, keyfs):
     keyfs.add_key("STAGEFILE",
                   "{user}/{index}/+f/{hashdir_a}/{hashdir_b}/{filename}", dict)
 
+    # files related
+    keyfs.add_key("DIGESTPATHS", "{digest}", set)
+
     sub = EventSubscribers(xom)
     keyfs.PROJVERSION.on_key_change(sub.on_changed_version_config)
     keyfs.STAGEFILE.on_key_change(sub.on_changed_file_entry)

server/devpi_server/replica.py

@@ -956,15 +956,23 @@ def importer(self, serial, key, val, back_serial, session):
         keyfs = self.xom.keyfs
         relpath = key.relpath
         entry = self.xom.filestore.get_file_entry_from_key(key, meta=val)
-        if val is None:
-            if back_serial >= 0:
+        if val is None and back_serial >= 0:
+            # check for existence with metadata from old serial
+            with keyfs.read_transaction(at_serial=back_serial):
+                entry = self.xom.filestore.get_file_entry(relpath)
+                digest_key = entry.get_digest_key()
+                file_exists = entry.file_exists()
+            if file_exists:
+                # check if there is no remaining reference at current serial
+                with keyfs.read_transaction():
+                    is_last_of_hash = not digest_key.get()
                 with keyfs.filestore_transaction():
-                    # file was deleted, still might never have been replicated
-                    if entry.file_exists():
-                        threadlog.info("mark for deletion: %s", relpath)
-                        entry.file_delete()
-                self.shared_data.errors.remove(entry)
-                return
+                    threadlog.info(
+                        "mark for deletion: %s, is_last_of_hash %s",
+                        relpath, is_last_of_hash)
+                    entry.file_delete(is_last_of_hash=is_last_of_hash)
+            self.shared_data.errors.remove(entry)
+            return
         if entry.last_modified is None:
             # there is no remote file
             self.shared_data.errors.remove(entry)

server/devpi_server/views.py

@@ -1740,6 +1740,9 @@ def iter_cache_remote_file(stage, entry, url):
                     f,
                     last_modified=r.headers.get("last-modified", None),
                     hashes=file_streamer.hashes)
+                digest_key = entry.get_digest_key()
+                with digest_key.update() as digest_paths:
+                    digest_paths.add(entry.relpath)
                 if entry.project:
                     stage = xom.model.getstage(entry.user, entry.index)
                     # for mirror indexes this makes sure the project is in the database

server/test_devpi_server/test_filestore.py

@@ -147,7 +147,7 @@ def test_file_delete(self, filestore, gen):
         content = b""
         entry1.file_set_content(content, hashes=get_hashes(content))
         assert entry1.file_exists()
-        entry1.file_delete()
+        entry1.file_delete(is_last_of_hash=True)
         assert not entry1.file_exists()
 
     def test_relpathentry(self, filestore, gen):
@@ -299,7 +299,7 @@ def test_file_tx_commit(filestore, gen):
     filestore.keyfs.commit_transaction_in_thread()
     filestore.keyfs.begin_transaction_in_thread(write=True)
     assert filepath.exists()
-    entry.file_delete()
+    entry.file_delete(is_last_of_hash=True)
     assert filepath.exists()
     assert not entry.file_exists()
     filestore.keyfs.commit_transaction_in_thread()

server/test_devpi_server/test_mirror.py

@@ -1202,7 +1202,7 @@ def test_redownload_locally_removed_release(file_digest, mapp, simpypi):
         assert tx.io_file.exists(file_path_info)
     # now remove the local copy
     with mapp.xom.keyfs.write_transaction() as tx:
-        tx.io_file.delete(file_path_info)
+        tx.io_file.delete(file_path_info, is_last_of_hash=True)
     with mapp.xom.keyfs.read_transaction() as tx:
         assert not tx.io_file.exists(file_path_info)
     serial = mapp.xom.keyfs.get_current_serial()

server/test_devpi_server/test_replica.py

@@ -761,7 +761,7 @@ def test_fetch_later_deleted(self, gen, xom, replica_xom):
 
         # then we delete
         with xom.keyfs.write_transaction():
-            entry.file_delete()
+            entry.file_delete(is_last_of_hash=True)
             entry.delete()
         with xom.keyfs.read_transaction():
             assert not entry.file_exists()

server/test_devpi_server/test_views.py

@@ -1355,7 +1355,7 @@ def test_push_from_pypi_mirror_switch_to_use_external_urls(httpget, mapp, pypist
         # and remove the file to simulate a cleanup
         linkstore = pypistage.get_linkstore_perstage("hello", "1.0")
         (link,) = linkstore.get_links()
-        link.entry.file_delete()
+        link.entry.file_delete(is_last_of_hash=True)
     assert pypistage.use_external_url
     # we should now get a redirect when trying to get the file
     r = testapp.xget(302, URL(pkg_url).joinpath(tag['href']).url)
@@ -1918,7 +1918,7 @@ def test_mirror_use_external_urls(mapp, simpypi, testapp, xom):
     r = testapp.xget(200, link, follow=False)
     # now remove the file, but keep metadata in place
     with testapp.xom.keyfs.write_transaction():
-        getentry(testapp, path).file_delete()
+        getentry(testapp, path).file_delete(is_last_of_hash=True)
     # we should get a redirect to the original URL again
     r = testapp.xget(302, link, follow=False)
 
@@ -2064,7 +2064,7 @@ def test_delete_package_where_file_was_deleted(mapp, testapp):
     path = link.href.replace(api.index, '/' + api.stagename)
     with testapp.xom.keyfs.write_transaction():
         # simulate removal of the file from file system outside of devpi
-        getentry(testapp, path).file_delete()
+        getentry(testapp, path).file_delete(is_last_of_hash=True)
     # delete the whole release
     testapp.xdel(200, api.index + "/pkg5/2.6")
     # check that there are no more releases listed
@@ -2187,7 +2187,7 @@ def test_delete_removed_toxresult(mapp, testapp, tox_result_data):
         entry = getentry(testapp, URL(toxlink1.href).path)
         assert entry.file_exists()
         # remove the file from filesystem
-        entry.file_delete()
+        entry.file_delete(is_last_of_hash=True)
         assert not entry.file_exists()
     # the link entry should still exist
     vv = get_view_version_links(testapp, api.index, "pkg6", "2.6")

web/tests/test_views_toxresults.py

@@ -93,7 +93,7 @@ def test_testdata_missing(mapp, testapp, tox_result_data):
         linkstore = stage.get_linkstore_perstage(link.project, link.version)
         toxresult_link, = linkstore.get_links(rel="toxresult", for_entrypath=link)
         # delete the tox result file
-        toxresult_link.entry.file_delete()
+        toxresult_link.entry.file_delete(is_last_of_hash=True)
     r = testapp.xget(200, api.index, headers=dict(accept="text/html"))
     assert '.toxresult' not in r.unicode_body