diff mercurial/commandserver.py @ 40999:dcac24ec935b

commandserver: preload repository in master server and reuse its file cache

This greatly speeds up repository operation with lots of obsolete markers:

  $ ls -lh .hg/store/obsstore
  -rw-r--r-- 1 yuya yuya 21M Dec  2 17:55 .hg/store/obsstore
  $ time hg log -G -l10 --pager no
  (hg)           1.79s user 0.13s system 99% cpu 1.919 total
  (chg uncached) 0.00s user 0.01s system 0% cpu 1.328 total
  (chg cached)   0.00s user 0.00s system 3% cpu 0.180 total

As you can see, the implementation of the preloader function is highly
experimental. It works, but I'm yet to be sure how things can be organized.
So I don't want to formalize the API at this point.

--- a/mercurial/commandserver.py	Wed Oct 31 22:19:03 2018 +0900
+++ b/mercurial/commandserver.py	Wed Oct 31 22:43:08 2018 +0900
@@ -28,6 +28,7 @@
     error,
     loggingutil,
     pycompat,
+    repocache,
     util,
     vfs as vfsmod,
 )
@@ -511,6 +512,11 @@
         self._oldsigchldhandler = None
         self._workerpids = set()  # updated by signal handler; do not iterate
         self._socketunlinked = None
+        # experimental config: cmdserver.max-repo-cache
+        maxlen = ui.configint(b'cmdserver', b'max-repo-cache')
+        if maxlen < 0:
+            raise error.Abort(_('negative max-repo-cache size not allowed'))
+        self._repoloader = repocache.repoloader(ui, maxlen)
 
     def init(self):
         self._sock = socket.socket(socket.AF_UNIX)
@@ -525,6 +531,7 @@
         o = signal.signal(signal.SIGCHLD, self._sigchldhandler)
         self._oldsigchldhandler = o
         self._socketunlinked = False
+        self._repoloader.start()
 
     def _unlinksocket(self):
         if not self._socketunlinked:
@@ -537,6 +544,7 @@
         self._mainipc.close()
         self._workeripc.close()
         self._unlinksocket()
+        self._repoloader.stop()
         # don't kill child processes as they have active clients, just wait
         self._reapworkers(0)
 
@@ -590,6 +598,10 @@
                 return
             raise
 
+        # Future improvement: On Python 3.7, maybe gc.freeze() can be used
+        # to prevent COW memory from being touched by GC.
+        # https://instagram-engineering.com/
+        #   copy-on-write-friendly-python-garbage-collection-ad6ed5233ddf
         pid = os.fork()
         if pid:
             try:
@@ -622,8 +634,7 @@
             if inst.args[0] == errno.EINTR:
                 return
             raise
-
-        self.ui.log(b'cmdserver', b'repository: %s\n', path)
+        self._repoloader.load(path)
 
     def _sigchldhandler(self, signal, frame):
         self._reapworkers(os.WNOHANG)
@@ -671,3 +682,9 @@
 
         repo.__class__ = unixcmdserverrepo
         repo._cmdserveripc = self._workeripc
+
+        cachedrepo = self._repoloader.get(repo.root)
+        if cachedrepo is None:
+            return
+        repo.ui.log(b'repocache', b'repo from cache: %s\n', repo.root)
+        repocache.copycache(cachedrepo, repo)