changeset 20882:5dffd06f1e50

transaction: add support for non-append files

This adds support for normal, non-append-only files in transactions.  For
example, .hg/store/fncache and .hg/store/phaseroots should be written as part of
the transaction, but are not append only files.

This adds a journal.backupfiles along side the normal journal. This tracks which
files have been backed up as part of the transaction.  transaction.addbackup()
creates a backup of the file (using a hardlink), which is later used to recover
in the event of the transaction failing.

Using a seperate journal allows the repository to still be used by older
versions of Mercurial. A future patch will use this functionality and add tests
for it.

--- a/mercurial/transaction.py	Mon Mar 24 15:57:47 2014 -0700
+++ b/mercurial/transaction.py	Mon Mar 24 15:21:51 2014 -0700
@@ -12,8 +12,8 @@
 # GNU General Public License version 2 or any later version.
 
 from i18n import _
-import errno
-import error
+import errno, os
+import error, util
 
 def active(func):
     def _active(self, *args, **kwds):
@@ -23,7 +23,7 @@
         return func(self, *args, **kwds)
     return _active
 
-def _playback(journal, report, opener, entries, unlink=True):
+def _playback(journal, report, opener, entries, backupentries, unlink=True):
     for f, o, ignore in entries:
         if o or not unlink:
             try:
@@ -39,7 +39,24 @@
             except (IOError, OSError), inst:
                 if inst.errno != errno.ENOENT:
                     raise
+
+    backupfiles = []
+    for f, b, ignore in backupentries:
+        filepath = opener.join(f)
+        backuppath = opener.join(b)
+        try:
+            util.copyfile(backuppath, filepath)
+            backupfiles.append(b)
+        except IOError:
+            report(_("failed to recover %s\n") % f)
+            raise
+
     opener.unlink(journal)
+    backuppath = "%s.backupfiles" % journal
+    if opener.exists(backuppath):
+        opener.unlink(backuppath)
+    for f in backupfiles:
+        opener.unlink(f)
 
 class transaction(object):
     def __init__(self, report, opener, journal, after=None, createmode=None,
@@ -64,13 +81,18 @@
         self.onclose = onclose
         self.onabort = onabort
         self.entries = []
+        self.backupentries = []
         self.map = {}
+        self.backupmap = {}
         self.journal = journal
         self._queue = []
 
+        self.backupjournal = "%s.backupfiles" % journal
         self.file = opener.open(self.journal, "w")
+        self.backupsfile = opener.open(self.backupjournal, 'w')
         if createmode is not None:
             opener.chmod(self.journal, createmode & 0666)
+            opener.chmod(self.backupjournal, createmode & 0666)
 
     def __del__(self):
         if self.journal:
@@ -78,22 +100,36 @@
 
     @active
     def startgroup(self):
-        self._queue.append([])
+        self._queue.append(([], []))
 
     @active
     def endgroup(self):
         q = self._queue.pop()
-        d = ''.join(['%s\0%d\n' % (x[0], x[1]) for x in q])
-        self.entries.extend(q)
+        self.entries.extend(q[0])
+        self.backupentries.extend(q[1])
+
+        offsets = []
+        backups = []
+        for f, o, _ in q[0]:
+            offsets.append((f, o))
+
+        for f, b, _ in q[1]:
+            backups.append((f, b))
+
+        d = ''.join(['%s\0%d\n' % (f, o) for f, o in offsets])
         self.file.write(d)
         self.file.flush()
 
+        d = ''.join(['%s\0%s\0' % (f, b) for f, b in backups])
+        self.backupsfile.write(d)
+        self.backupsfile.flush()
+
     @active
     def add(self, file, offset, data=None):
-        if file in self.map:
+        if file in self.map or file in self.backupmap:
             return
         if self._queue:
-            self._queue[-1].append((file, offset, data))
+            self._queue[-1][0].append((file, offset, data))
             return
 
         self.entries.append((file, offset, data))
@@ -103,9 +139,43 @@
         self.file.flush()
 
     @active
+    def addbackup(self, file, hardlink=True):
+        """Adds a backup of the file to the transaction
+
+        Calling addbackup() creates a hardlink backup of the specified file
+        that is used to recover the file in the event of the transaction
+        aborting.
+
+        * `file`: the file path, relative to .hg/store
+        * `hardlink`: use a hardlink to quickly create the backup
+        """
+
+        if file in self.map or file in self.backupmap:
+            return
+        backupfile = "journal.%s" % file
+        if self.opener.exists(file):
+            filepath = self.opener.join(file)
+            backuppath = self.opener.join(backupfile)
+            util.copyfiles(filepath, backuppath, hardlink=hardlink)
+        else:
+            self.add(file, 0)
+            return
+
+        if self._queue:
+            self._queue[-1][1].append((file, backupfile))
+            return
+
+        self.backupentries.append((file, backupfile, None))
+        self.backupmap[file] = len(self.backupentries) - 1
+        self.backupsfile.write("%s\0%s\0" % (file, backupfile))
+        self.backupsfile.flush()
+
+    @active
     def find(self, file):
         if file in self.map:
             return self.entries[self.map[file]]
+        if file in self.backupmap:
+            return self.backupentries[self.backupmap[file]]
         return None
 
     @active
@@ -153,6 +223,11 @@
             self.after()
         if self.opener.isfile(self.journal):
             self.opener.unlink(self.journal)
+        if self.opener.isfile(self.backupjournal):
+            self.opener.unlink(self.backupjournal)
+            for f, b, _ in self.backupentries:
+                self.opener.unlink(b)
+        self.backupentries = []
         self.journal = None
 
     @active
@@ -171,16 +246,18 @@
             self.onabort()
 
         try:
-            if not self.entries:
+            if not self.entries and not self.backupentries:
                 if self.journal:
                     self.opener.unlink(self.journal)
+                if self.backupjournal:
+                    self.opener.unlink(self.backupjournal)
                 return
 
             self.report(_("transaction abort!\n"))
 
             try:
                 _playback(self.journal, self.report, self.opener,
-                          self.entries, False)
+                          self.entries, self.backupentries, False)
                 self.report(_("rollback completed\n"))
             except Exception:
                 self.report(_("rollback failed - please run hg recover\n"))
@@ -189,7 +266,19 @@
 
 
 def rollback(opener, file, report):
+    """Rolls back the transaction contained in the given file
+
+    Reads the entries in the specified file, and the corresponding
+    '*.backupfiles' file, to recover from an incomplete transaction.
+
+    * `file`: a file containing a list of entries, specifying where
+    to truncate each file.  The file should contain a list of
+    file\0offset pairs, delimited by newlines. The corresponding
+    '*.backupfiles' file should contain a list of file\0backupfile
+    pairs, delimited by \0.
+    """
     entries = []
+    backupentries = []
 
     fp = opener.open(file)
     lines = fp.readlines()
@@ -201,4 +290,14 @@
         except ValueError:
             report(_("couldn't read journal entry %r!\n") % l)
 
-    _playback(file, report, opener, entries)
+    backupjournal = "%s.backupfiles" % file
+    if opener.exists(backupjournal):
+        fp = opener.open(backupjournal)
+        data = fp.read()
+        if len(data) > 0:
+            parts = data.split('\0')
+            for i in xrange(0, len(parts), 2):
+                f, b = parts[i:i + 1]
+                backupentries.append((f, b, None))
+
+    _playback(file, report, opener, entries, backupentries)