Mercurial > hg
comparison mercurial/transaction.py @ 20882:5dffd06f1e50
transaction: add support for non-append files
This adds support for normal, non-append-only files in transactions. For
example, .hg/store/fncache and .hg/store/phaseroots should be written as part of
the transaction, but are not append only files.
This adds a journal.backupfiles along side the normal journal. This tracks which
files have been backed up as part of the transaction. transaction.addbackup()
creates a backup of the file (using a hardlink), which is later used to recover
in the event of the transaction failing.
Using a seperate journal allows the repository to still be used by older
versions of Mercurial. A future patch will use this functionality and add tests
for it.
author | Durham Goode <durham@fb.com> |
---|---|
date | Mon, 24 Mar 2014 15:21:51 -0700 |
parents | 3c47677a8d04 |
children | 203908968644 |
comparison
equal
deleted
inserted
replaced
20881:3c47677a8d04 | 20882:5dffd06f1e50 |
---|---|
10 # | 10 # |
11 # This software may be used and distributed according to the terms of the | 11 # This software may be used and distributed according to the terms of the |
12 # GNU General Public License version 2 or any later version. | 12 # GNU General Public License version 2 or any later version. |
13 | 13 |
14 from i18n import _ | 14 from i18n import _ |
15 import errno | 15 import errno, os |
16 import error | 16 import error, util |
17 | 17 |
18 def active(func): | 18 def active(func): |
19 def _active(self, *args, **kwds): | 19 def _active(self, *args, **kwds): |
20 if self.count == 0: | 20 if self.count == 0: |
21 raise error.Abort(_( | 21 raise error.Abort(_( |
22 'cannot use transaction when it is already committed/aborted')) | 22 'cannot use transaction when it is already committed/aborted')) |
23 return func(self, *args, **kwds) | 23 return func(self, *args, **kwds) |
24 return _active | 24 return _active |
25 | 25 |
26 def _playback(journal, report, opener, entries, unlink=True): | 26 def _playback(journal, report, opener, entries, backupentries, unlink=True): |
27 for f, o, ignore in entries: | 27 for f, o, ignore in entries: |
28 if o or not unlink: | 28 if o or not unlink: |
29 try: | 29 try: |
30 fp = opener(f, 'a') | 30 fp = opener(f, 'a') |
31 fp.truncate(o) | 31 fp.truncate(o) |
37 try: | 37 try: |
38 opener.unlink(f) | 38 opener.unlink(f) |
39 except (IOError, OSError), inst: | 39 except (IOError, OSError), inst: |
40 if inst.errno != errno.ENOENT: | 40 if inst.errno != errno.ENOENT: |
41 raise | 41 raise |
42 | |
43 backupfiles = [] | |
44 for f, b, ignore in backupentries: | |
45 filepath = opener.join(f) | |
46 backuppath = opener.join(b) | |
47 try: | |
48 util.copyfile(backuppath, filepath) | |
49 backupfiles.append(b) | |
50 except IOError: | |
51 report(_("failed to recover %s\n") % f) | |
52 raise | |
53 | |
42 opener.unlink(journal) | 54 opener.unlink(journal) |
55 backuppath = "%s.backupfiles" % journal | |
56 if opener.exists(backuppath): | |
57 opener.unlink(backuppath) | |
58 for f in backupfiles: | |
59 opener.unlink(f) | |
43 | 60 |
44 class transaction(object): | 61 class transaction(object): |
45 def __init__(self, report, opener, journal, after=None, createmode=None, | 62 def __init__(self, report, opener, journal, after=None, createmode=None, |
46 onclose=None, onabort=None): | 63 onclose=None, onabort=None): |
47 """Begin a new transaction | 64 """Begin a new transaction |
62 self.opener = opener | 79 self.opener = opener |
63 self.after = after | 80 self.after = after |
64 self.onclose = onclose | 81 self.onclose = onclose |
65 self.onabort = onabort | 82 self.onabort = onabort |
66 self.entries = [] | 83 self.entries = [] |
84 self.backupentries = [] | |
67 self.map = {} | 85 self.map = {} |
86 self.backupmap = {} | |
68 self.journal = journal | 87 self.journal = journal |
69 self._queue = [] | 88 self._queue = [] |
70 | 89 |
90 self.backupjournal = "%s.backupfiles" % journal | |
71 self.file = opener.open(self.journal, "w") | 91 self.file = opener.open(self.journal, "w") |
92 self.backupsfile = opener.open(self.backupjournal, 'w') | |
72 if createmode is not None: | 93 if createmode is not None: |
73 opener.chmod(self.journal, createmode & 0666) | 94 opener.chmod(self.journal, createmode & 0666) |
95 opener.chmod(self.backupjournal, createmode & 0666) | |
74 | 96 |
75 def __del__(self): | 97 def __del__(self): |
76 if self.journal: | 98 if self.journal: |
77 self._abort() | 99 self._abort() |
78 | 100 |
79 @active | 101 @active |
80 def startgroup(self): | 102 def startgroup(self): |
81 self._queue.append([]) | 103 self._queue.append(([], [])) |
82 | 104 |
83 @active | 105 @active |
84 def endgroup(self): | 106 def endgroup(self): |
85 q = self._queue.pop() | 107 q = self._queue.pop() |
86 d = ''.join(['%s\0%d\n' % (x[0], x[1]) for x in q]) | 108 self.entries.extend(q[0]) |
87 self.entries.extend(q) | 109 self.backupentries.extend(q[1]) |
110 | |
111 offsets = [] | |
112 backups = [] | |
113 for f, o, _ in q[0]: | |
114 offsets.append((f, o)) | |
115 | |
116 for f, b, _ in q[1]: | |
117 backups.append((f, b)) | |
118 | |
119 d = ''.join(['%s\0%d\n' % (f, o) for f, o in offsets]) | |
88 self.file.write(d) | 120 self.file.write(d) |
89 self.file.flush() | 121 self.file.flush() |
90 | 122 |
123 d = ''.join(['%s\0%s\0' % (f, b) for f, b in backups]) | |
124 self.backupsfile.write(d) | |
125 self.backupsfile.flush() | |
126 | |
91 @active | 127 @active |
92 def add(self, file, offset, data=None): | 128 def add(self, file, offset, data=None): |
93 if file in self.map: | 129 if file in self.map or file in self.backupmap: |
94 return | 130 return |
95 if self._queue: | 131 if self._queue: |
96 self._queue[-1].append((file, offset, data)) | 132 self._queue[-1][0].append((file, offset, data)) |
97 return | 133 return |
98 | 134 |
99 self.entries.append((file, offset, data)) | 135 self.entries.append((file, offset, data)) |
100 self.map[file] = len(self.entries) - 1 | 136 self.map[file] = len(self.entries) - 1 |
101 # add enough data to the journal to do the truncate | 137 # add enough data to the journal to do the truncate |
102 self.file.write("%s\0%d\n" % (file, offset)) | 138 self.file.write("%s\0%d\n" % (file, offset)) |
103 self.file.flush() | 139 self.file.flush() |
104 | 140 |
105 @active | 141 @active |
142 def addbackup(self, file, hardlink=True): | |
143 """Adds a backup of the file to the transaction | |
144 | |
145 Calling addbackup() creates a hardlink backup of the specified file | |
146 that is used to recover the file in the event of the transaction | |
147 aborting. | |
148 | |
149 * `file`: the file path, relative to .hg/store | |
150 * `hardlink`: use a hardlink to quickly create the backup | |
151 """ | |
152 | |
153 if file in self.map or file in self.backupmap: | |
154 return | |
155 backupfile = "journal.%s" % file | |
156 if self.opener.exists(file): | |
157 filepath = self.opener.join(file) | |
158 backuppath = self.opener.join(backupfile) | |
159 util.copyfiles(filepath, backuppath, hardlink=hardlink) | |
160 else: | |
161 self.add(file, 0) | |
162 return | |
163 | |
164 if self._queue: | |
165 self._queue[-1][1].append((file, backupfile)) | |
166 return | |
167 | |
168 self.backupentries.append((file, backupfile, None)) | |
169 self.backupmap[file] = len(self.backupentries) - 1 | |
170 self.backupsfile.write("%s\0%s\0" % (file, backupfile)) | |
171 self.backupsfile.flush() | |
172 | |
173 @active | |
106 def find(self, file): | 174 def find(self, file): |
107 if file in self.map: | 175 if file in self.map: |
108 return self.entries[self.map[file]] | 176 return self.entries[self.map[file]] |
177 if file in self.backupmap: | |
178 return self.backupentries[self.backupmap[file]] | |
109 return None | 179 return None |
110 | 180 |
111 @active | 181 @active |
112 def replace(self, file, offset, data=None): | 182 def replace(self, file, offset, data=None): |
113 ''' | 183 ''' |
151 self.entries = [] | 221 self.entries = [] |
152 if self.after: | 222 if self.after: |
153 self.after() | 223 self.after() |
154 if self.opener.isfile(self.journal): | 224 if self.opener.isfile(self.journal): |
155 self.opener.unlink(self.journal) | 225 self.opener.unlink(self.journal) |
226 if self.opener.isfile(self.backupjournal): | |
227 self.opener.unlink(self.backupjournal) | |
228 for f, b, _ in self.backupentries: | |
229 self.opener.unlink(b) | |
230 self.backupentries = [] | |
156 self.journal = None | 231 self.journal = None |
157 | 232 |
158 @active | 233 @active |
159 def abort(self): | 234 def abort(self): |
160 '''abort the transaction (generally called on error, or when the | 235 '''abort the transaction (generally called on error, or when the |
169 | 244 |
170 if self.onabort is not None: | 245 if self.onabort is not None: |
171 self.onabort() | 246 self.onabort() |
172 | 247 |
173 try: | 248 try: |
174 if not self.entries: | 249 if not self.entries and not self.backupentries: |
175 if self.journal: | 250 if self.journal: |
176 self.opener.unlink(self.journal) | 251 self.opener.unlink(self.journal) |
252 if self.backupjournal: | |
253 self.opener.unlink(self.backupjournal) | |
177 return | 254 return |
178 | 255 |
179 self.report(_("transaction abort!\n")) | 256 self.report(_("transaction abort!\n")) |
180 | 257 |
181 try: | 258 try: |
182 _playback(self.journal, self.report, self.opener, | 259 _playback(self.journal, self.report, self.opener, |
183 self.entries, False) | 260 self.entries, self.backupentries, False) |
184 self.report(_("rollback completed\n")) | 261 self.report(_("rollback completed\n")) |
185 except Exception: | 262 except Exception: |
186 self.report(_("rollback failed - please run hg recover\n")) | 263 self.report(_("rollback failed - please run hg recover\n")) |
187 finally: | 264 finally: |
188 self.journal = None | 265 self.journal = None |
189 | 266 |
190 | 267 |
191 def rollback(opener, file, report): | 268 def rollback(opener, file, report): |
269 """Rolls back the transaction contained in the given file | |
270 | |
271 Reads the entries in the specified file, and the corresponding | |
272 '*.backupfiles' file, to recover from an incomplete transaction. | |
273 | |
274 * `file`: a file containing a list of entries, specifying where | |
275 to truncate each file. The file should contain a list of | |
276 file\0offset pairs, delimited by newlines. The corresponding | |
277 '*.backupfiles' file should contain a list of file\0backupfile | |
278 pairs, delimited by \0. | |
279 """ | |
192 entries = [] | 280 entries = [] |
281 backupentries = [] | |
193 | 282 |
194 fp = opener.open(file) | 283 fp = opener.open(file) |
195 lines = fp.readlines() | 284 lines = fp.readlines() |
196 fp.close() | 285 fp.close() |
197 for l in lines: | 286 for l in lines: |
199 f, o = l.split('\0') | 288 f, o = l.split('\0') |
200 entries.append((f, int(o), None)) | 289 entries.append((f, int(o), None)) |
201 except ValueError: | 290 except ValueError: |
202 report(_("couldn't read journal entry %r!\n") % l) | 291 report(_("couldn't read journal entry %r!\n") % l) |
203 | 292 |
204 _playback(file, report, opener, entries) | 293 backupjournal = "%s.backupfiles" % file |
294 if opener.exists(backupjournal): | |
295 fp = opener.open(backupjournal) | |
296 data = fp.read() | |
297 if len(data) > 0: | |
298 parts = data.split('\0') | |
299 for i in xrange(0, len(parts), 2): | |
300 f, b = parts[i:i + 1] | |
301 backupentries.append((f, b, None)) | |
302 | |
303 _playback(file, report, opener, entries, backupentries) |