streamclone: use backgroundfilecloser (issue4889)
Closing files that have been appended to is slow on Windows/NTFS.
CloseHandle() calls on this platform often take 1-10ms - and that's
on my i7-6700K Skylake processor with a modern and fast SSD. Contrast
with other I/O operations, such as writing data, which take <100us.
This means that creating/appending thousands of files can add
significant overhead. For example, cloning mozilla-central creates
~232,000 revlog files. Assuming 1ms per CloseHandle(), that yields
232s (3:52) of wall time waiting for file closes!
The impact of this overhead can be measured most directly when applying
stream clone bundles. Applying these files is effectively uncompressing
a tar archive (read: it's very fast).
Using a RAM disk (read: no I/O wait), the difference in wall time for a
`hg debugapplystreamclonebundle` for a ~1731 MB mozilla-central bundle
between Windows and Linux from the same machine is drastic:
Linux: ~12.8s (128MB/s)
Windows: ~352.0s (4.7MB/s)
Windows is ~27.5x slower. Yikes!
After this patch:
Linux: ~12.8s (128MB/s)
Windows: ~102.1s (16.1MB/s)
Windows is now ~3.4x faster. Unfortunately, it is still ~8x slower than
Linux. Profiling reveals a few hot code paths that could likely be
improved. But those are for other patches.
This patch introduces test-clone-uncompressed.t because existing tests
of `clone --uncompressed` are scattered about and adding a variation for
background thread closing to e.g. test-http.t doesn't feel correct.
$ hg init t
$ cd t
$ mkdir -p beans
$ for b in kidney navy turtle borlotti black pinto; do
> echo $b > beans/$b
> done
$ mkdir -p mammals/Procyonidae
$ for m in cacomistle coatimundi raccoon; do
> echo $m > mammals/Procyonidae/$m
> done
$ echo skunk > mammals/skunk
$ echo fennel > fennel
$ echo fenugreek > fenugreek
$ echo fiddlehead > fiddlehead
$ hg addremove
adding beans/black
adding beans/borlotti
adding beans/kidney
adding beans/navy
adding beans/pinto
adding beans/turtle
adding fennel
adding fenugreek
adding fiddlehead
adding mammals/Procyonidae/cacomistle
adding mammals/Procyonidae/coatimundi
adding mammals/Procyonidae/raccoon
adding mammals/skunk
$ hg commit -m "commit #0"
$ hg debugwalk
f beans/black beans/black
f beans/borlotti beans/borlotti
f beans/kidney beans/kidney
f beans/navy beans/navy
f beans/pinto beans/pinto
f beans/turtle beans/turtle
f fennel fennel
f fenugreek fenugreek
f fiddlehead fiddlehead
f mammals/Procyonidae/cacomistle mammals/Procyonidae/cacomistle
f mammals/Procyonidae/coatimundi mammals/Procyonidae/coatimundi
f mammals/Procyonidae/raccoon mammals/Procyonidae/raccoon
f mammals/skunk mammals/skunk
$ hg debugwalk -I.
f beans/black beans/black
f beans/borlotti beans/borlotti
f beans/kidney beans/kidney
f beans/navy beans/navy
f beans/pinto beans/pinto
f beans/turtle beans/turtle
f fennel fennel
f fenugreek fenugreek
f fiddlehead fiddlehead
f mammals/Procyonidae/cacomistle mammals/Procyonidae/cacomistle
f mammals/Procyonidae/coatimundi mammals/Procyonidae/coatimundi
f mammals/Procyonidae/raccoon mammals/Procyonidae/raccoon
f mammals/skunk mammals/skunk
$ cd mammals
$ hg debugwalk
f beans/black ../beans/black
f beans/borlotti ../beans/borlotti
f beans/kidney ../beans/kidney
f beans/navy ../beans/navy
f beans/pinto ../beans/pinto
f beans/turtle ../beans/turtle
f fennel ../fennel
f fenugreek ../fenugreek
f fiddlehead ../fiddlehead
f mammals/Procyonidae/cacomistle Procyonidae/cacomistle
f mammals/Procyonidae/coatimundi Procyonidae/coatimundi
f mammals/Procyonidae/raccoon Procyonidae/raccoon
f mammals/skunk skunk
$ hg debugwalk -X ../beans
f fennel ../fennel
f fenugreek ../fenugreek
f fiddlehead ../fiddlehead
f mammals/Procyonidae/cacomistle Procyonidae/cacomistle
f mammals/Procyonidae/coatimundi Procyonidae/coatimundi
f mammals/Procyonidae/raccoon Procyonidae/raccoon
f mammals/skunk skunk
$ hg debugwalk -I '*k'
f mammals/skunk skunk
$ hg debugwalk -I 'glob:*k'
f mammals/skunk skunk
$ hg debugwalk -I 'relglob:*k'
f beans/black ../beans/black
f fenugreek ../fenugreek
f mammals/skunk skunk
$ hg debugwalk -I 'relglob:*k' .
f mammals/skunk skunk
$ hg debugwalk -I 're:.*k$'
f beans/black ../beans/black
f fenugreek ../fenugreek
f mammals/skunk skunk
$ hg debugwalk -I 'relre:.*k$'
f beans/black ../beans/black
f fenugreek ../fenugreek
f mammals/skunk skunk
$ hg debugwalk -I 'path:beans'
f beans/black ../beans/black
f beans/borlotti ../beans/borlotti
f beans/kidney ../beans/kidney
f beans/navy ../beans/navy
f beans/pinto ../beans/pinto
f beans/turtle ../beans/turtle
$ hg debugwalk -I 'relpath:detour/../../beans'
f beans/black ../beans/black
f beans/borlotti ../beans/borlotti
f beans/kidney ../beans/kidney
f beans/navy ../beans/navy
f beans/pinto ../beans/pinto
f beans/turtle ../beans/turtle
$ hg debugwalk .
f mammals/Procyonidae/cacomistle Procyonidae/cacomistle
f mammals/Procyonidae/coatimundi Procyonidae/coatimundi
f mammals/Procyonidae/raccoon Procyonidae/raccoon
f mammals/skunk skunk
$ hg debugwalk -I.
f mammals/Procyonidae/cacomistle Procyonidae/cacomistle
f mammals/Procyonidae/coatimundi Procyonidae/coatimundi
f mammals/Procyonidae/raccoon Procyonidae/raccoon
f mammals/skunk skunk
$ hg debugwalk Procyonidae
f mammals/Procyonidae/cacomistle Procyonidae/cacomistle
f mammals/Procyonidae/coatimundi Procyonidae/coatimundi
f mammals/Procyonidae/raccoon Procyonidae/raccoon
$ cd Procyonidae
$ hg debugwalk .
f mammals/Procyonidae/cacomistle cacomistle
f mammals/Procyonidae/coatimundi coatimundi
f mammals/Procyonidae/raccoon raccoon
$ hg debugwalk ..
f mammals/Procyonidae/cacomistle cacomistle
f mammals/Procyonidae/coatimundi coatimundi
f mammals/Procyonidae/raccoon raccoon
f mammals/skunk ../skunk
$ cd ..
$ hg debugwalk ../beans
f beans/black ../beans/black
f beans/borlotti ../beans/borlotti
f beans/kidney ../beans/kidney
f beans/navy ../beans/navy
f beans/pinto ../beans/pinto
f beans/turtle ../beans/turtle
$ hg debugwalk .
f mammals/Procyonidae/cacomistle Procyonidae/cacomistle
f mammals/Procyonidae/coatimundi Procyonidae/coatimundi
f mammals/Procyonidae/raccoon Procyonidae/raccoon
f mammals/skunk skunk
$ hg debugwalk .hg
abort: path 'mammals/.hg' is inside nested repo 'mammals' (glob)
[255]
$ hg debugwalk ../.hg
abort: path contains illegal component: .hg
[255]
$ cd ..
$ hg debugwalk -Ibeans
f beans/black beans/black
f beans/borlotti beans/borlotti
f beans/kidney beans/kidney
f beans/navy beans/navy
f beans/pinto beans/pinto
f beans/turtle beans/turtle
$ hg debugwalk -I '{*,{b,m}*/*}k'
f beans/black beans/black
f fenugreek fenugreek
f mammals/skunk mammals/skunk
$ hg debugwalk -Ibeans mammals
$ hg debugwalk -Inon-existent
$ hg debugwalk -Inon-existent -Ibeans/black
f beans/black beans/black
$ hg debugwalk -Ibeans beans/black
f beans/black beans/black exact
$ hg debugwalk -Ibeans/black beans
f beans/black beans/black
$ hg debugwalk -Xbeans/black beans
f beans/borlotti beans/borlotti
f beans/kidney beans/kidney
f beans/navy beans/navy
f beans/pinto beans/pinto
f beans/turtle beans/turtle
$ hg debugwalk -Xbeans/black -Ibeans
f beans/borlotti beans/borlotti
f beans/kidney beans/kidney
f beans/navy beans/navy
f beans/pinto beans/pinto
f beans/turtle beans/turtle
$ hg debugwalk -Xbeans/black beans/black
f beans/black beans/black exact
$ hg debugwalk -Xbeans/black -Ibeans/black
$ hg debugwalk -Xbeans beans/black
f beans/black beans/black exact
$ hg debugwalk -Xbeans -Ibeans/black
$ hg debugwalk 'glob:mammals/../beans/b*'
f beans/black beans/black
f beans/borlotti beans/borlotti
$ hg debugwalk '-X*/Procyonidae' mammals
f mammals/skunk mammals/skunk
$ hg debugwalk path:mammals
f mammals/Procyonidae/cacomistle mammals/Procyonidae/cacomistle
f mammals/Procyonidae/coatimundi mammals/Procyonidae/coatimundi
f mammals/Procyonidae/raccoon mammals/Procyonidae/raccoon
f mammals/skunk mammals/skunk
$ hg debugwalk ..
abort: .. not under root '$TESTTMP/t' (glob)
[255]
$ hg debugwalk beans/../..
abort: beans/../.. not under root '$TESTTMP/t' (glob)
[255]
$ hg debugwalk .hg
abort: path contains illegal component: .hg
[255]
$ hg debugwalk beans/../.hg
abort: path contains illegal component: .hg
[255]
$ hg debugwalk beans/../.hg/data
abort: path contains illegal component: .hg/data (glob)
[255]
$ hg debugwalk beans/.hg
abort: path 'beans/.hg' is inside nested repo 'beans' (glob)
[255]
Test absolute paths:
$ hg debugwalk `pwd`/beans
f beans/black beans/black
f beans/borlotti beans/borlotti
f beans/kidney beans/kidney
f beans/navy beans/navy
f beans/pinto beans/pinto
f beans/turtle beans/turtle
$ hg debugwalk `pwd`/..
abort: $TESTTMP/t/.. not under root '$TESTTMP/t' (glob)
[255]
Test patterns:
$ hg debugwalk glob:\*
f fennel fennel
f fenugreek fenugreek
f fiddlehead fiddlehead
#if eol-in-paths
$ echo glob:glob > glob:glob
$ hg addremove
adding glob:glob
warning: filename contains ':', which is reserved on Windows: 'glob:glob'
$ hg debugwalk glob:\*
f fennel fennel
f fenugreek fenugreek
f fiddlehead fiddlehead
f glob:glob glob:glob
$ hg debugwalk glob:glob
glob: No such file or directory
$ hg debugwalk glob:glob:glob
f glob:glob glob:glob exact
$ hg debugwalk path:glob:glob
f glob:glob glob:glob exact
$ rm glob:glob
$ hg addremove
removing glob:glob
#endif
$ hg debugwalk 'glob:**e'
f beans/turtle beans/turtle
f mammals/Procyonidae/cacomistle mammals/Procyonidae/cacomistle
$ hg debugwalk 're:.*[kb]$'
f beans/black beans/black
f fenugreek fenugreek
f mammals/skunk mammals/skunk
$ hg debugwalk path:beans/black
f beans/black beans/black exact
$ hg debugwalk path:beans//black
f beans/black beans/black exact
$ hg debugwalk relglob:Procyonidae
$ hg debugwalk 'relglob:Procyonidae/**'
f mammals/Procyonidae/cacomistle mammals/Procyonidae/cacomistle
f mammals/Procyonidae/coatimundi mammals/Procyonidae/coatimundi
f mammals/Procyonidae/raccoon mammals/Procyonidae/raccoon
$ hg debugwalk 'relglob:Procyonidae/**' fennel
f fennel fennel exact
f mammals/Procyonidae/cacomistle mammals/Procyonidae/cacomistle
f mammals/Procyonidae/coatimundi mammals/Procyonidae/coatimundi
f mammals/Procyonidae/raccoon mammals/Procyonidae/raccoon
$ hg debugwalk beans 'glob:beans/*'
f beans/black beans/black
f beans/borlotti beans/borlotti
f beans/kidney beans/kidney
f beans/navy beans/navy
f beans/pinto beans/pinto
f beans/turtle beans/turtle
$ hg debugwalk 'glob:mamm**'
f mammals/Procyonidae/cacomistle mammals/Procyonidae/cacomistle
f mammals/Procyonidae/coatimundi mammals/Procyonidae/coatimundi
f mammals/Procyonidae/raccoon mammals/Procyonidae/raccoon
f mammals/skunk mammals/skunk
$ hg debugwalk 'glob:mamm**' fennel
f fennel fennel exact
f mammals/Procyonidae/cacomistle mammals/Procyonidae/cacomistle
f mammals/Procyonidae/coatimundi mammals/Procyonidae/coatimundi
f mammals/Procyonidae/raccoon mammals/Procyonidae/raccoon
f mammals/skunk mammals/skunk
$ hg debugwalk 'glob:j*'
$ hg debugwalk NOEXIST
NOEXIST: * (glob)
#if fifo
$ mkfifo fifo
$ hg debugwalk fifo
fifo: unsupported file type (type is fifo)
#endif
$ rm fenugreek
$ hg debugwalk fenugreek
f fenugreek fenugreek exact
$ hg rm fenugreek
$ hg debugwalk fenugreek
f fenugreek fenugreek exact
$ touch new
$ hg debugwalk new
f new new exact
$ mkdir ignored
$ touch ignored/file
$ echo '^ignored$' > .hgignore
$ hg debugwalk ignored
$ hg debugwalk ignored/file
f ignored/file ignored/file exact
Test listfile and listfile0
$ $PYTHON -c "file('listfile0', 'wb').write('fenugreek\0new\0')"
$ hg debugwalk -I 'listfile0:listfile0'
f fenugreek fenugreek
f new new
$ $PYTHON -c "file('listfile', 'wb').write('fenugreek\nnew\r\nmammals/skunk\n')"
$ hg debugwalk -I 'listfile:listfile'
f fenugreek fenugreek
f mammals/skunk mammals/skunk
f new new
$ cd ..
$ hg debugwalk -R t t/mammals/skunk
f mammals/skunk t/mammals/skunk exact
$ mkdir t2
$ cd t2
$ hg debugwalk -R ../t ../t/mammals/skunk
f mammals/skunk ../t/mammals/skunk exact
$ hg debugwalk --cwd ../t mammals/skunk
f mammals/skunk mammals/skunk exact
$ cd ..
Test split patterns on overflow
$ cd t
$ echo fennel > overflow.list
$ $PYTHON -c "for i in xrange(20000 / 100): print 'x' * 100" >> overflow.list
$ echo fenugreek >> overflow.list
$ hg debugwalk 'listfile:overflow.list' 2>&1 | grep -v '^xxx'
f fennel fennel exact
f fenugreek fenugreek exact
$ cd ..