Mercurial > hg
comparison mercurial/revlog.py @ 5445:64cf1c853674
revlog: reduce memory usage in addgroup
- use a buffer to extract the delta from a chunk
- avoid concatenating to a compressed delta
- use a buffer to directly extra full text from a trivial delta
- delete chunk and delta objects after use
author | Matt Mackall <mpm@selenic.com> |
---|---|
date | Thu, 11 Oct 2007 00:46:47 -0500 |
parents | 61462e7d62ed |
children | e038738714fd |
comparison
equal
deleted
inserted
replaced
5444:a0952e4e52eb | 5445:64cf1c853674 |
---|---|
1138 # this can happen if two branches make the same change | 1138 # this can happen if two branches make the same change |
1139 # if unique: | 1139 # if unique: |
1140 # raise RevlogError(_("already have %s") % hex(node[:4])) | 1140 # raise RevlogError(_("already have %s") % hex(node[:4])) |
1141 chain = node | 1141 chain = node |
1142 continue | 1142 continue |
1143 delta = chunk[80:] | 1143 delta = buffer(chunk, 80) |
1144 del chunk | |
1144 | 1145 |
1145 for p in (p1, p2): | 1146 for p in (p1, p2): |
1146 if not p in self.nodemap: | 1147 if not p in self.nodemap: |
1147 raise LookupError(_("unknown parent %s") % short(p)) | 1148 raise LookupError(_("unknown parent %s") % short(p)) |
1148 | 1149 |
1157 # version is not the one we have a delta against. We use | 1158 # version is not the one we have a delta against. We use |
1158 # the size of the previous full rev as a proxy for the | 1159 # the size of the previous full rev as a proxy for the |
1159 # current size. | 1160 # current size. |
1160 | 1161 |
1161 if chain == prev: | 1162 if chain == prev: |
1162 tempd = compress(delta) | 1163 cdelta = compress(delta) |
1163 cdelta = tempd[0] + tempd[1] | 1164 cdeltalen = len(cdelta[0]) + len(cdelta[1]) |
1164 textlen = mdiff.patchedsize(textlen, delta) | 1165 textlen = mdiff.patchedsize(textlen, delta) |
1165 | 1166 |
1166 if chain != prev or (end - start + len(cdelta)) > textlen * 2: | 1167 if chain != prev or (end - start + cdeltalen) > textlen * 2: |
1167 # flush our writes here so we can read it in revision | 1168 # flush our writes here so we can read it in revision |
1168 if dfh: | 1169 if dfh: |
1169 dfh.flush() | 1170 dfh.flush() |
1170 ifh.flush() | 1171 ifh.flush() |
1171 text = self.revision(chain) | 1172 text = self.revision(chain) |
1172 text = mdiff.patches(text, [delta]) | 1173 if len(text) == 0: |
1174 # skip over trivial delta header | |
1175 text = buffer(delta, 12) | |
1176 else: | |
1177 text = mdiff.patches(text, [delta]) | |
1178 del delta | |
1173 chk = self._addrevision(text, transaction, link, p1, p2, None, | 1179 chk = self._addrevision(text, transaction, link, p1, p2, None, |
1174 ifh, dfh) | 1180 ifh, dfh) |
1175 if not dfh and not self._inline: | 1181 if not dfh and not self._inline: |
1176 # addrevision switched from inline to conventional | 1182 # addrevision switched from inline to conventional |
1177 # reopen the index | 1183 # reopen the index |
1179 ifh = self.opener(self.indexfile, "a") | 1185 ifh = self.opener(self.indexfile, "a") |
1180 if chk != node: | 1186 if chk != node: |
1181 raise RevlogError(_("consistency error adding group")) | 1187 raise RevlogError(_("consistency error adding group")) |
1182 textlen = len(text) | 1188 textlen = len(text) |
1183 else: | 1189 else: |
1184 e = (offset_type(end, 0), len(cdelta), textlen, base, | 1190 e = (offset_type(end, 0), cdeltalen, textlen, base, |
1185 link, self.rev(p1), self.rev(p2), node) | 1191 link, self.rev(p1), self.rev(p2), node) |
1186 self.index.insert(-1, e) | 1192 self.index.insert(-1, e) |
1187 self.nodemap[node] = r | 1193 self.nodemap[node] = r |
1188 entry = self._io.packentry(e, self.node, self.version, r) | 1194 entry = self._io.packentry(e, self.node, self.version, r) |
1189 if self._inline: | 1195 if self._inline: |
1190 ifh.write(entry) | 1196 ifh.write(entry) |
1191 ifh.write(cdelta) | 1197 ifh.write(cdelta[0]) |
1198 ifh.write(cdelta[1]) | |
1192 self.checkinlinesize(transaction, ifh) | 1199 self.checkinlinesize(transaction, ifh) |
1193 if not self._inline: | 1200 if not self._inline: |
1194 dfh = self.opener(self.datafile, "a") | 1201 dfh = self.opener(self.datafile, "a") |
1195 ifh = self.opener(self.indexfile, "a") | 1202 ifh = self.opener(self.indexfile, "a") |
1196 else: | 1203 else: |
1197 dfh.write(cdelta) | 1204 dfh.write(cdelta[0]) |
1205 dfh.write(cdelta[1]) | |
1198 ifh.write(entry) | 1206 ifh.write(entry) |
1199 | 1207 |
1200 t, r, chain, prev = r, r + 1, node, node | 1208 t, r, chain, prev = r, r + 1, node, node |
1201 base = self.base(t) | 1209 base = self.base(t) |
1202 start = self.start(base) | 1210 start = self.start(base) |