Mercurial > hg
comparison hgext/convert/p4.py @ 25882:97a9f7602014 stable
convert: when getting file from Perforce concatenate data at the end
As it turned out, even when getting relatively small files, concatenating
string data every time when new chunk is received is very inefficient.
Maintaining a string list of data chunks and concatenating everything in one go
at the end seems much more efficient - in my testing it made getting 40 MB file
7 times faster, whilst converting of a particularly big changelist with some big
files went down from 20 hours to 3 hours.
author | Eugene Baranov <eug.baranov@gmail.com> |
---|---|
date | Thu, 30 Jul 2015 00:58:05 +0100 |
parents | 584044e5ad57 |
children | b810b59eca62 |
comparison
equal
deleted
inserted
replaced
25881:9de443515f1d | 25882:97a9f7602014 |
---|---|
214 lasterror = None | 214 lasterror = None |
215 while True: | 215 while True: |
216 stdout = util.popen(cmd, mode='rb') | 216 stdout = util.popen(cmd, mode='rb') |
217 | 217 |
218 mode = None | 218 mode = None |
219 contents = "" | 219 contents = [] |
220 keywords = None | 220 keywords = None |
221 | 221 |
222 for d in loaditer(stdout): | 222 for d in loaditer(stdout): |
223 code = d["code"] | 223 code = d["code"] |
224 data = d.get("data") | 224 data = d.get("data") |
250 keywords = self.re_keywords_old | 250 keywords = self.re_keywords_old |
251 elif "k" in flags: | 251 elif "k" in flags: |
252 keywords = self.re_keywords | 252 keywords = self.re_keywords |
253 | 253 |
254 elif code == "text" or code == "binary": | 254 elif code == "text" or code == "binary": |
255 contents += data | 255 contents.append(data) |
256 | 256 |
257 lasterror = None | 257 lasterror = None |
258 | 258 |
259 if not lasterror: | 259 if not lasterror: |
260 break | 260 break |
261 | 261 |
262 if mode is None: | 262 if mode is None: |
263 return None, None | 263 return None, None |
264 | |
265 contents = ''.join(contents) | |
264 | 266 |
265 if keywords: | 267 if keywords: |
266 contents = keywords.sub("$\\1$", contents) | 268 contents = keywords.sub("$\\1$", contents) |
267 if mode == "l" and contents.endswith("\n"): | 269 if mode == "l" and contents.endswith("\n"): |
268 contents = contents[:-1] | 270 contents = contents[:-1] |