Use threadlog + devpi_srcpath better

server/devpi_server/keyfs_sqlite_fs.py

@@ -42,11 +42,7 @@ def __repr__(self):
     @classmethod
     def from_content(cls, path, content_or_file):
         self = DirtyFile(path)
-        if hasattr(content_or_file, "filestore"):
-            # ELink
-            devpi_srcpath = content_or_file.filestore.keyfs.basedir / content_or_file.entry._storepath
-        else:
-            devpi_srcpath = getattr(content_or_file, "devpi_srcpath", None)
+        devpi_srcpath = getattr(content_or_file, "devpi_srcpath", None)
         if devpi_srcpath is not None:
             dirname = os.path.dirname(self.tmppath)
             if not os.path.exists(dirname):

server/devpi_server/model.py

@@ -1419,15 +1419,16 @@ def devpiserver_get_stage_customizer_classes():
 
 class ELink(object):
     """ model Link using entrypathes for referencing. """
-    __slots__ = ('_entry', 'basename', 'filestore', 'linkdict', 'project', 'version')
+    __slots__ = ('_entry', 'basename', 'filestore', 'linkdict', 'project', 'version', 'devpi_srcpath')
 
-    def __init__(self, filestore, linkdict, project, version):
+    def __init__(self, filestore, linkdict, project, version, *, devpi_srcpath=None):
         self._entry = notset
         self.filestore = filestore
         self.linkdict = linkdict
         self.basename = posixpath.basename(self.entrypath)
         self.project = project
         self.version = version
+        self.devpi_srcpath = devpi_srcpath
 
     @property
     def relpath(self):

server/devpi_server/views.py

@@ -40,12 +40,11 @@
 from .filestore import BadGateway
 from .filestore import get_checksum_error
 from .fileutil import buffered_iterator
-from .model import InvalidIndex, InvalidIndexconfig, InvalidUser, InvalidUserconfig
+from .model import InvalidIndex, InvalidIndexconfig, InvalidUser, InvalidUserconfig, ELink
 from .model import ReadonlyIndex
 from .model import RemoveValue
 from .readonly import get_mutable_deepcopy
 from .log import thread_push_log, thread_pop_log, threadlog
-
 from .auth import Auth
 
 devpiweb_hookimpl = HookimplMarker("devpiweb")
@@ -1465,66 +1464,55 @@ def snapshot(self):
         target_index = json["target_index"]
         target_stage = None
         source_stage, projects = context.stage.list_projects()[0]
+        front = source_stage.name + "/+f/"  # used to set the devpi_srcpath
         failures = []
         notices = []
         tot_wheels = 0
         tot_project_versions = 0
         if context.user.getstage(target_index):
             apireturn(409, "index %r already exists" % target_index)
+        threadlog.info("snapshot %s", source_stage.name)
         try:
             target_stage = context.user.create_stage(target_index)
             for project_name in projects:
-                print(f"doing {project_name} ..")
+                threadlog.info("snapshot doing %s", project_name)
                 project_versions = source_stage.list_versions(project_name)
                 tot_project_versions += len(project_versions)
                 for version in project_versions:
-                    print(f" {project_name}-{version} ..")
+                    threadlog.debug("snapshot doing %s-%s", project_name, version)
                     try:
                         linkstore = source_stage.get_linkstore_perstage(project_name, version)
                         release_links = linkstore.get_links("releasefile", None)
-                        verdata = source_stage.get_versiondata_perstage(project_name, version)
                     except Exception as err:
                         failure = f"failed during read of {project_name}-{version}: {err}\n{traceback.format_exc()}"
-                        print(failure)
+                        threadlog.error("%s", failure)
                         failures.append(failure)
                         continue
-                    # fixup verdata coming from source_stage for target_stage:
-                    new_elinks = []
-                    elinks = verdata.get('+elinks', None)
-                    if not elinks:
-                        msg = f"{project_name}-{version} has no +elinks: {elinks}"
-                        notices.append(msg)
-                        print(msg)
-                        continue
-                    for elink in elinks:
-                        try:
-                            front = source_stage.name + "/+f/"
-                            entrypath = elink['entrypath']
-                            assert entrypath.startswith(front), elink
-                            new_elink = dict(elink)
-                            new_elink['entrypath'] = entrypath.replace(front, f"{context.user.name}/{target_index}/+f/")
-                            new_elinks.append(new_elink)
-                        except Exception as err:
-                            msg = f"error with elink {elink}: {err}"
-                            failures.append(msg)
-                            print(msg)
-                    # have to set on _data cause `verdata` itself is ReadOnly:
-                    verdata._data['+elinks'] = new_elinks
-                    # fixup done.
-                    # we now have to set that version data into the target stage:
-                    target_stage.set_versiondata(verdata)
                     # then we can create the (hard-)links for each release:
                     for link in release_links:
-                        print(f"  {link.entrypath}")
-                        new_link = target_stage.store_releasefile(
-                            project_name, version,
-                            link.basename, link)
+                        threadlog.debug("snapshot doing %s", link.entrypath)
+                        d = dict(link.linkdict)
+                        entrypath = d.get('entrypath', None)
+                        if entrypath is None or not entrypath.startswith(front):
+                            # unhandled case TODO ?
+                            failures.append(f"{project_name}-{version}: bad entrypath for {d}")
+                            continue
+                        d['entrypath'] = entrypath.replace(front, f"{context.user.name}/{target_index}/+f/")
+                        target_stage.set_versiondata({'name': project_name, 'version': version})
+                        linkstore = target_stage.get_linkstore_perstage(project_name, version, readonly=False)
+                        devpi_srcpath = linkstore.filestore.keyfs.basedir / "+files" / link.entrypath
+                        new_release_link = ELink(target_stage.filestore, d, project_name, version,
+                                                 devpi_srcpath=devpi_srcpath)
+                        new_link = linkstore.create_linked_entry(
+                            rel="releasefile",
+                            basename=link.basename,
+                            content_or_file=new_release_link,
+                            last_modified=None)
                         new_link.add_log(
                             'snapshot', request.authenticated_userid, src=context.stage.name)
                     tot_wheels += len(release_links)
         except Exception as err:
-            print(f"Got error during snapshot {source_stage}: {err}")
-            traceback.print_exc()
+            threadlog.error("Got error during snapshot %s: %s\n%s", source_stage, err, traceback.format_exc())
             if target_stage is not None:
                 target_stage.delete()
             apireturn(500,