Mercurial > hg
annotate mercurial/worker.py @ 51413:8fc92193a2cf
phases: use a more generic way to trigger a phases computation for perf
Querying the tip most revision will require the cache to warm the same as
calling the dedicated method. This avoid using a method that is mostly meant for
internal use and will be renamed in a coming changesets.
author | Pierre-Yves David <pierre-yves.david@octobus.net> |
---|---|
date | Thu, 22 Feb 2024 10:56:05 +0100 |
parents | d0b8bbf603d7 |
children |
rev | line source |
---|---|
18635
fed06dd07665
worker: count the number of CPUs
Bryan O'Sullivan <bryano@fb.com>
parents:
diff
changeset
|
1 # worker.py - master-slave parallelism support |
fed06dd07665
worker: count the number of CPUs
Bryan O'Sullivan <bryano@fb.com>
parents:
diff
changeset
|
2 # |
fed06dd07665
worker: count the number of CPUs
Bryan O'Sullivan <bryano@fb.com>
parents:
diff
changeset
|
3 # Copyright 2013 Facebook, Inc. |
fed06dd07665
worker: count the number of CPUs
Bryan O'Sullivan <bryano@fb.com>
parents:
diff
changeset
|
4 # |
fed06dd07665
worker: count the number of CPUs
Bryan O'Sullivan <bryano@fb.com>
parents:
diff
changeset
|
5 # This software may be used and distributed according to the terms of the |
fed06dd07665
worker: count the number of CPUs
Bryan O'Sullivan <bryano@fb.com>
parents:
diff
changeset
|
6 # GNU General Public License version 2 or any later version. |
fed06dd07665
worker: count the number of CPUs
Bryan O'Sullivan <bryano@fb.com>
parents:
diff
changeset
|
7 |
25992
2d76f8a2d831
worker: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
25660
diff
changeset
|
8 |
2d76f8a2d831
worker: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
25660
diff
changeset
|
9 import os |
48870
df56e6bd37f6
py3: use pickle directly
Gregory Szorc <gregory.szorc@gmail.com>
parents:
46819
diff
changeset
|
10 import pickle |
49280
311fcc5a65f6
thirdparty: remove Python 2-specific selectors2 copy
Manuel Jacob <me@manueljacob.de>
parents:
49269
diff
changeset
|
11 import selectors |
25992
2d76f8a2d831
worker: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
25660
diff
changeset
|
12 import signal |
2d76f8a2d831
worker: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
25660
diff
changeset
|
13 import sys |
35427
02b36e860e0b
workers: implemented worker on windows
Wojciech Lis <wlis@fb.com>
parents:
34646
diff
changeset
|
14 import threading |
35432
86b8cc1f244e
worker: make windows workers daemons
Wojciech Lis <wlis@fb.com>
parents:
35431
diff
changeset
|
15 import time |
25992
2d76f8a2d831
worker: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
25660
diff
changeset
|
16 |
2d76f8a2d831
worker: use absolute_import
Gregory Szorc <gregory.szorc@gmail.com>
parents:
25660
diff
changeset
|
17 from .i18n import _ |
30396 | 18 from . import ( |
30635
a150173da1c1
py3: replace os.environ with encoding.environ (part 2 of 5)
Pulkit Goyal <7895pulkit@gmail.com>
parents:
30521
diff
changeset
|
19 encoding, |
30396 | 20 error, |
30639
d524c88511a7
py3: replace os.name with pycompat.osname (part 1 of 2)
Pulkit Goyal <7895pulkit@gmail.com>
parents:
30635
diff
changeset
|
21 pycompat, |
30521
86cd09bc13ba
worker: use os._exit for posix worker in all cases
Jun Wu <quark@fb.com>
parents:
30425
diff
changeset
|
22 scmutil, |
30396 | 23 ) |
18635
fed06dd07665
worker: count the number of CPUs
Bryan O'Sullivan <bryano@fb.com>
parents:
diff
changeset
|
24 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42522
diff
changeset
|
25 |
18635
fed06dd07665
worker: count the number of CPUs
Bryan O'Sullivan <bryano@fb.com>
parents:
diff
changeset
|
26 def countcpus(): |
fed06dd07665
worker: count the number of CPUs
Bryan O'Sullivan <bryano@fb.com>
parents:
diff
changeset
|
27 '''try to count the number of CPUs on the system''' |
26568
c0501c26b05c
worker: restore old countcpus code (issue4869)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26063
diff
changeset
|
28 |
c0501c26b05c
worker: restore old countcpus code (issue4869)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26063
diff
changeset
|
29 # posix |
18635
fed06dd07665
worker: count the number of CPUs
Bryan O'Sullivan <bryano@fb.com>
parents:
diff
changeset
|
30 try: |
43506
9f70512ae2cf
cleanup: remove pointless r-prefixes on single-quoted strings
Augie Fackler <augie@google.com>
parents:
43077
diff
changeset
|
31 n = int(os.sysconf('SC_NPROCESSORS_ONLN')) |
26568
c0501c26b05c
worker: restore old countcpus code (issue4869)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26063
diff
changeset
|
32 if n > 0: |
c0501c26b05c
worker: restore old countcpus code (issue4869)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26063
diff
changeset
|
33 return n |
c0501c26b05c
worker: restore old countcpus code (issue4869)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26063
diff
changeset
|
34 except (AttributeError, ValueError): |
c0501c26b05c
worker: restore old countcpus code (issue4869)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26063
diff
changeset
|
35 pass |
c0501c26b05c
worker: restore old countcpus code (issue4869)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26063
diff
changeset
|
36 |
c0501c26b05c
worker: restore old countcpus code (issue4869)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26063
diff
changeset
|
37 # windows |
c0501c26b05c
worker: restore old countcpus code (issue4869)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26063
diff
changeset
|
38 try: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
39 n = int(encoding.environ[b'NUMBER_OF_PROCESSORS']) |
26568
c0501c26b05c
worker: restore old countcpus code (issue4869)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26063
diff
changeset
|
40 if n > 0: |
c0501c26b05c
worker: restore old countcpus code (issue4869)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26063
diff
changeset
|
41 return n |
c0501c26b05c
worker: restore old countcpus code (issue4869)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26063
diff
changeset
|
42 except (KeyError, ValueError): |
c0501c26b05c
worker: restore old countcpus code (issue4869)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26063
diff
changeset
|
43 pass |
c0501c26b05c
worker: restore old countcpus code (issue4869)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26063
diff
changeset
|
44 |
c0501c26b05c
worker: restore old countcpus code (issue4869)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26063
diff
changeset
|
45 return 1 |
18636
dcb27c153a40
worker: estimate whether it's worth running a task in parallel
Bryan O'Sullivan <bryano@fb.com>
parents:
18635
diff
changeset
|
46 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42522
diff
changeset
|
47 |
18636
dcb27c153a40
worker: estimate whether it's worth running a task in parallel
Bryan O'Sullivan <bryano@fb.com>
parents:
18635
diff
changeset
|
48 def _numworkers(ui): |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
49 s = ui.config(b'worker', b'numcpus') |
18636
dcb27c153a40
worker: estimate whether it's worth running a task in parallel
Bryan O'Sullivan <bryano@fb.com>
parents:
18635
diff
changeset
|
50 if s: |
dcb27c153a40
worker: estimate whether it's worth running a task in parallel
Bryan O'Sullivan <bryano@fb.com>
parents:
18635
diff
changeset
|
51 try: |
dcb27c153a40
worker: estimate whether it's worth running a task in parallel
Bryan O'Sullivan <bryano@fb.com>
parents:
18635
diff
changeset
|
52 n = int(s) |
dcb27c153a40
worker: estimate whether it's worth running a task in parallel
Bryan O'Sullivan <bryano@fb.com>
parents:
18635
diff
changeset
|
53 if n >= 1: |
dcb27c153a40
worker: estimate whether it's worth running a task in parallel
Bryan O'Sullivan <bryano@fb.com>
parents:
18635
diff
changeset
|
54 return n |
dcb27c153a40
worker: estimate whether it's worth running a task in parallel
Bryan O'Sullivan <bryano@fb.com>
parents:
18635
diff
changeset
|
55 except ValueError: |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
56 raise error.Abort(_(b'number of cpus must be an integer')) |
18636
dcb27c153a40
worker: estimate whether it's worth running a task in parallel
Bryan O'Sullivan <bryano@fb.com>
parents:
18635
diff
changeset
|
57 return min(max(countcpus(), 4), 32) |
dcb27c153a40
worker: estimate whether it's worth running a task in parallel
Bryan O'Sullivan <bryano@fb.com>
parents:
18635
diff
changeset
|
58 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42522
diff
changeset
|
59 |
48901
cc0e059d2af8
worker: remove Python 2 support code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
48875
diff
changeset
|
60 def ismainthread(): |
cc0e059d2af8
worker: remove Python 2 support code
Gregory Szorc <gregory.szorc@gmail.com>
parents:
48875
diff
changeset
|
61 return threading.current_thread() == threading.main_thread() |
44248
12491abf93bd
worker: manually buffer reads from pickle stream
Jan Alexander Steffens (heftig) <jan.steffens@gmail.com>
parents:
44226
diff
changeset
|
62 |
12491abf93bd
worker: manually buffer reads from pickle stream
Jan Alexander Steffens (heftig) <jan.steffens@gmail.com>
parents:
44226
diff
changeset
|
63 |
50999
d0b8bbf603d7
openvms: consider worker costly on OpenVMS
Jean-Francois Pieronne <jf.pieronne@laposte.net>
parents:
49899
diff
changeset
|
64 if ( |
d0b8bbf603d7
openvms: consider worker costly on OpenVMS
Jean-Francois Pieronne <jf.pieronne@laposte.net>
parents:
49899
diff
changeset
|
65 pycompat.isposix and pycompat.sysplatform != b'OpenVMS' |
d0b8bbf603d7
openvms: consider worker costly on OpenVMS
Jean-Francois Pieronne <jf.pieronne@laposte.net>
parents:
49899
diff
changeset
|
66 ) or pycompat.iswindows: |
38730
69ed2cff4277
worker: rename variable to reflect constant
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38729
diff
changeset
|
67 _STARTUP_COST = 0.01 |
38731
ef3838a47503
worker: ability to disable thread unsafe tasks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38730
diff
changeset
|
68 # The Windows worker is thread based. If tasks are CPU bound, threads |
ef3838a47503
worker: ability to disable thread unsafe tasks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38730
diff
changeset
|
69 # in the presence of the GIL result in excessive context switching and |
ef3838a47503
worker: ability to disable thread unsafe tasks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38730
diff
changeset
|
70 # this overhead can slow down execution. |
ef3838a47503
worker: ability to disable thread unsafe tasks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38730
diff
changeset
|
71 _DISALLOW_THREAD_UNSAFE = pycompat.iswindows |
18636
dcb27c153a40
worker: estimate whether it's worth running a task in parallel
Bryan O'Sullivan <bryano@fb.com>
parents:
18635
diff
changeset
|
72 else: |
38730
69ed2cff4277
worker: rename variable to reflect constant
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38729
diff
changeset
|
73 _STARTUP_COST = 1e30 |
38731
ef3838a47503
worker: ability to disable thread unsafe tasks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38730
diff
changeset
|
74 _DISALLOW_THREAD_UNSAFE = False |
18636
dcb27c153a40
worker: estimate whether it's worth running a task in parallel
Bryan O'Sullivan <bryano@fb.com>
parents:
18635
diff
changeset
|
75 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42522
diff
changeset
|
76 |
38731
ef3838a47503
worker: ability to disable thread unsafe tasks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38730
diff
changeset
|
77 def worthwhile(ui, costperop, nops, threadsafe=True): |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
45825
diff
changeset
|
78 """try to determine whether the benefit of multiple processes can |
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
45825
diff
changeset
|
79 outweigh the cost of starting them""" |
38731
ef3838a47503
worker: ability to disable thread unsafe tasks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38730
diff
changeset
|
80 |
ef3838a47503
worker: ability to disable thread unsafe tasks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38730
diff
changeset
|
81 if not threadsafe and _DISALLOW_THREAD_UNSAFE: |
ef3838a47503
worker: ability to disable thread unsafe tasks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38730
diff
changeset
|
82 return False |
ef3838a47503
worker: ability to disable thread unsafe tasks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38730
diff
changeset
|
83 |
18636
dcb27c153a40
worker: estimate whether it's worth running a task in parallel
Bryan O'Sullivan <bryano@fb.com>
parents:
18635
diff
changeset
|
84 linear = costperop * nops |
dcb27c153a40
worker: estimate whether it's worth running a task in parallel
Bryan O'Sullivan <bryano@fb.com>
parents:
18635
diff
changeset
|
85 workers = _numworkers(ui) |
38730
69ed2cff4277
worker: rename variable to reflect constant
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38729
diff
changeset
|
86 benefit = linear - (_STARTUP_COST * workers + linear / workers) |
18636
dcb27c153a40
worker: estimate whether it's worth running a task in parallel
Bryan O'Sullivan <bryano@fb.com>
parents:
18635
diff
changeset
|
87 return benefit >= 0.15 |
18637
ac4dbceeb14a
worker: partition a list (of tasks) into equal-sized chunks
Bryan O'Sullivan <bryano@fb.com>
parents:
18636
diff
changeset
|
88 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42522
diff
changeset
|
89 |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42522
diff
changeset
|
90 def worker( |
49545
3556f0392808
lfs: avoid closing connections when the worker doesn't fork
Matt Harbison <matt_harbison@yahoo.com>
parents:
49309
diff
changeset
|
91 ui, |
3556f0392808
lfs: avoid closing connections when the worker doesn't fork
Matt Harbison <matt_harbison@yahoo.com>
parents:
49309
diff
changeset
|
92 costperarg, |
3556f0392808
lfs: avoid closing connections when the worker doesn't fork
Matt Harbison <matt_harbison@yahoo.com>
parents:
49309
diff
changeset
|
93 func, |
3556f0392808
lfs: avoid closing connections when the worker doesn't fork
Matt Harbison <matt_harbison@yahoo.com>
parents:
49309
diff
changeset
|
94 staticargs, |
3556f0392808
lfs: avoid closing connections when the worker doesn't fork
Matt Harbison <matt_harbison@yahoo.com>
parents:
49309
diff
changeset
|
95 args, |
3556f0392808
lfs: avoid closing connections when the worker doesn't fork
Matt Harbison <matt_harbison@yahoo.com>
parents:
49309
diff
changeset
|
96 hasretval=False, |
3556f0392808
lfs: avoid closing connections when the worker doesn't fork
Matt Harbison <matt_harbison@yahoo.com>
parents:
49309
diff
changeset
|
97 threadsafe=True, |
3556f0392808
lfs: avoid closing connections when the worker doesn't fork
Matt Harbison <matt_harbison@yahoo.com>
parents:
49309
diff
changeset
|
98 prefork=None, |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42522
diff
changeset
|
99 ): |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
45825
diff
changeset
|
100 """run a function, possibly in parallel in multiple worker |
18638
047110c0e2a8
worker: allow a function to be run in multiple worker processes
Bryan O'Sullivan <bryano@fb.com>
parents:
18637
diff
changeset
|
101 processes. |
047110c0e2a8
worker: allow a function to be run in multiple worker processes
Bryan O'Sullivan <bryano@fb.com>
parents:
18637
diff
changeset
|
102 |
047110c0e2a8
worker: allow a function to be run in multiple worker processes
Bryan O'Sullivan <bryano@fb.com>
parents:
18637
diff
changeset
|
103 returns a progress iterator |
047110c0e2a8
worker: allow a function to be run in multiple worker processes
Bryan O'Sullivan <bryano@fb.com>
parents:
18637
diff
changeset
|
104 |
047110c0e2a8
worker: allow a function to be run in multiple worker processes
Bryan O'Sullivan <bryano@fb.com>
parents:
18637
diff
changeset
|
105 costperarg - cost of a single task |
047110c0e2a8
worker: allow a function to be run in multiple worker processes
Bryan O'Sullivan <bryano@fb.com>
parents:
18637
diff
changeset
|
106 |
42455
5ca136bbd3f6
worker: support parallelization of functions with return values
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
40989
diff
changeset
|
107 func - function to run. It is expected to return a progress iterator. |
18638
047110c0e2a8
worker: allow a function to be run in multiple worker processes
Bryan O'Sullivan <bryano@fb.com>
parents:
18637
diff
changeset
|
108 |
047110c0e2a8
worker: allow a function to be run in multiple worker processes
Bryan O'Sullivan <bryano@fb.com>
parents:
18637
diff
changeset
|
109 staticargs - arguments to pass to every invocation of the function |
047110c0e2a8
worker: allow a function to be run in multiple worker processes
Bryan O'Sullivan <bryano@fb.com>
parents:
18637
diff
changeset
|
110 |
047110c0e2a8
worker: allow a function to be run in multiple worker processes
Bryan O'Sullivan <bryano@fb.com>
parents:
18637
diff
changeset
|
111 args - arguments to split into chunks, to pass to individual |
047110c0e2a8
worker: allow a function to be run in multiple worker processes
Bryan O'Sullivan <bryano@fb.com>
parents:
18637
diff
changeset
|
112 workers |
38731
ef3838a47503
worker: ability to disable thread unsafe tasks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38730
diff
changeset
|
113 |
42455
5ca136bbd3f6
worker: support parallelization of functions with return values
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
40989
diff
changeset
|
114 hasretval - when True, func and the current function return an progress |
42522
d29db0a0c4eb
update: fix spurious unclean status bug shown by previous commit
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
42455
diff
changeset
|
115 iterator then a dict (encoded as an iterator that yield many (False, ..) |
d29db0a0c4eb
update: fix spurious unclean status bug shown by previous commit
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
42455
diff
changeset
|
116 then a (True, dict)). The dicts are joined in some arbitrary order, so |
d29db0a0c4eb
update: fix spurious unclean status bug shown by previous commit
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
42455
diff
changeset
|
117 overlapping keys are a bad idea. |
42455
5ca136bbd3f6
worker: support parallelization of functions with return values
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
40989
diff
changeset
|
118 |
38731
ef3838a47503
worker: ability to disable thread unsafe tasks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38730
diff
changeset
|
119 threadsafe - whether work items are thread safe and can be executed using |
ef3838a47503
worker: ability to disable thread unsafe tasks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38730
diff
changeset
|
120 a thread-based worker. Should be disabled for CPU heavy tasks that don't |
ef3838a47503
worker: ability to disable thread unsafe tasks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38730
diff
changeset
|
121 release the GIL. |
49545
3556f0392808
lfs: avoid closing connections when the worker doesn't fork
Matt Harbison <matt_harbison@yahoo.com>
parents:
49309
diff
changeset
|
122 |
3556f0392808
lfs: avoid closing connections when the worker doesn't fork
Matt Harbison <matt_harbison@yahoo.com>
parents:
49309
diff
changeset
|
123 prefork - a parameterless Callable that is invoked prior to forking the |
3556f0392808
lfs: avoid closing connections when the worker doesn't fork
Matt Harbison <matt_harbison@yahoo.com>
parents:
49309
diff
changeset
|
124 process. fork() is only used on non-Windows platforms, but is also not |
3556f0392808
lfs: avoid closing connections when the worker doesn't fork
Matt Harbison <matt_harbison@yahoo.com>
parents:
49309
diff
changeset
|
125 called on POSIX platforms if the work amount doesn't warrant a worker. |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
45825
diff
changeset
|
126 """ |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
127 enabled = ui.configbool(b'worker', b'enabled') |
46240
a42502e9ae6d
worker: POSIX only supports workers from main thread (issue6460)
Joerg Sonnenberger <joerg@bec.de>
parents:
45942
diff
changeset
|
128 if enabled and _platformworker is _posixworker and not ismainthread(): |
a42502e9ae6d
worker: POSIX only supports workers from main thread (issue6460)
Joerg Sonnenberger <joerg@bec.de>
parents:
45942
diff
changeset
|
129 # The POSIX worker has to install a handler for SIGCHLD. |
a42502e9ae6d
worker: POSIX only supports workers from main thread (issue6460)
Joerg Sonnenberger <joerg@bec.de>
parents:
45942
diff
changeset
|
130 # Python up to 3.9 only allows this in the main thread. |
a42502e9ae6d
worker: POSIX only supports workers from main thread (issue6460)
Joerg Sonnenberger <joerg@bec.de>
parents:
45942
diff
changeset
|
131 enabled = False |
a42502e9ae6d
worker: POSIX only supports workers from main thread (issue6460)
Joerg Sonnenberger <joerg@bec.de>
parents:
45942
diff
changeset
|
132 |
38731
ef3838a47503
worker: ability to disable thread unsafe tasks
Gregory Szorc <gregory.szorc@gmail.com>
parents:
38730
diff
changeset
|
133 if enabled and worthwhile(ui, costperarg, len(args), threadsafe=threadsafe): |
49545
3556f0392808
lfs: avoid closing connections when the worker doesn't fork
Matt Harbison <matt_harbison@yahoo.com>
parents:
49309
diff
changeset
|
134 return _platformworker( |
3556f0392808
lfs: avoid closing connections when the worker doesn't fork
Matt Harbison <matt_harbison@yahoo.com>
parents:
49309
diff
changeset
|
135 ui, func, staticargs, args, hasretval, prefork=prefork |
3556f0392808
lfs: avoid closing connections when the worker doesn't fork
Matt Harbison <matt_harbison@yahoo.com>
parents:
49309
diff
changeset
|
136 ) |
18638
047110c0e2a8
worker: allow a function to be run in multiple worker processes
Bryan O'Sullivan <bryano@fb.com>
parents:
18637
diff
changeset
|
137 return func(*staticargs + (args,)) |
047110c0e2a8
worker: allow a function to be run in multiple worker processes
Bryan O'Sullivan <bryano@fb.com>
parents:
18637
diff
changeset
|
138 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42522
diff
changeset
|
139 |
49545
3556f0392808
lfs: avoid closing connections when the worker doesn't fork
Matt Harbison <matt_harbison@yahoo.com>
parents:
49309
diff
changeset
|
140 def _posixworker(ui, func, staticargs, args, hasretval, prefork=None): |
18638
047110c0e2a8
worker: allow a function to be run in multiple worker processes
Bryan O'Sullivan <bryano@fb.com>
parents:
18637
diff
changeset
|
141 workers = _numworkers(ui) |
18708
86524a70c0f6
worker: fix a race in SIGINT handling
Bryan O'Sullivan <bryano@fb.com>
parents:
18707
diff
changeset
|
142 oldhandler = signal.getsignal(signal.SIGINT) |
86524a70c0f6
worker: fix a race in SIGINT handling
Bryan O'Sullivan <bryano@fb.com>
parents:
18707
diff
changeset
|
143 signal.signal(signal.SIGINT, signal.SIG_IGN) |
30413 | 144 pids, problem = set(), [0] |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42522
diff
changeset
|
145 |
30410
7a5d6e2fd2d5
worker: move killworkers and waitforworkers up
Jun Wu <quark@fb.com>
parents:
30396
diff
changeset
|
146 def killworkers(): |
30423
237b2883cbd8
worker: make sure killworkers() never be interrupted by another SIGCHLD
Yuya Nishihara <yuya@tcha.org>
parents:
30422
diff
changeset
|
147 # unregister SIGCHLD handler as all children will be killed. This |
237b2883cbd8
worker: make sure killworkers() never be interrupted by another SIGCHLD
Yuya Nishihara <yuya@tcha.org>
parents:
30422
diff
changeset
|
148 # function shouldn't be interrupted by another SIGCHLD; otherwise pids |
237b2883cbd8
worker: make sure killworkers() never be interrupted by another SIGCHLD
Yuya Nishihara <yuya@tcha.org>
parents:
30422
diff
changeset
|
149 # could be updated while iterating, which would cause inconsistency. |
237b2883cbd8
worker: make sure killworkers() never be interrupted by another SIGCHLD
Yuya Nishihara <yuya@tcha.org>
parents:
30422
diff
changeset
|
150 signal.signal(signal.SIGCHLD, oldchldhandler) |
30410
7a5d6e2fd2d5
worker: move killworkers and waitforworkers up
Jun Wu <quark@fb.com>
parents:
30396
diff
changeset
|
151 # if one worker bails, there's no good reason to wait for the rest |
7a5d6e2fd2d5
worker: move killworkers and waitforworkers up
Jun Wu <quark@fb.com>
parents:
30396
diff
changeset
|
152 for p in pids: |
7a5d6e2fd2d5
worker: move killworkers and waitforworkers up
Jun Wu <quark@fb.com>
parents:
30396
diff
changeset
|
153 try: |
7a5d6e2fd2d5
worker: move killworkers and waitforworkers up
Jun Wu <quark@fb.com>
parents:
30396
diff
changeset
|
154 os.kill(p, signal.SIGTERM) |
49309
d54b213c4380
py3: catch ProcessLookupError instead of checking errno == ESRCH
Manuel Jacob <me@manueljacob.de>
parents:
49303
diff
changeset
|
155 except ProcessLookupError: |
d54b213c4380
py3: catch ProcessLookupError instead of checking errno == ESRCH
Manuel Jacob <me@manueljacob.de>
parents:
49303
diff
changeset
|
156 pass |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42522
diff
changeset
|
157 |
30412
7bc25549e084
worker: allow waitforworkers to be non-blocking
Jun Wu <quark@fb.com>
parents:
30411
diff
changeset
|
158 def waitforworkers(blocking=True): |
30414
5069a8a40b1b
worker: make waitforworkers reentrant
Jun Wu <quark@fb.com>
parents:
30413
diff
changeset
|
159 for pid in pids.copy(): |
5069a8a40b1b
worker: make waitforworkers reentrant
Jun Wu <quark@fb.com>
parents:
30413
diff
changeset
|
160 p = st = 0 |
49302
ee4537e365c8
py3: remove retry on EINTR errno
Manuel Jacob <me@manueljacob.de>
parents:
49280
diff
changeset
|
161 try: |
ee4537e365c8
py3: remove retry on EINTR errno
Manuel Jacob <me@manueljacob.de>
parents:
49280
diff
changeset
|
162 p, st = os.waitpid(pid, (0 if blocking else os.WNOHANG)) |
49303
dfdf85f37215
py3: catch ChildProcessError instead of checking errno == ECHILD
Manuel Jacob <me@manueljacob.de>
parents:
49302
diff
changeset
|
163 except ChildProcessError: |
dfdf85f37215
py3: catch ChildProcessError instead of checking errno == ECHILD
Manuel Jacob <me@manueljacob.de>
parents:
49302
diff
changeset
|
164 # child would already be reaped, but pids yet been |
dfdf85f37215
py3: catch ChildProcessError instead of checking errno == ECHILD
Manuel Jacob <me@manueljacob.de>
parents:
49302
diff
changeset
|
165 # updated (maybe interrupted just after waitpid) |
dfdf85f37215
py3: catch ChildProcessError instead of checking errno == ECHILD
Manuel Jacob <me@manueljacob.de>
parents:
49302
diff
changeset
|
166 pids.discard(pid) |
31063
18fb3cf572b4
worker: ignore meaningless exit status indication returned by os.waitpid()
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
30639
diff
changeset
|
167 if not p: |
18fb3cf572b4
worker: ignore meaningless exit status indication returned by os.waitpid()
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
30639
diff
changeset
|
168 # skip subsequent steps, because child process should |
18fb3cf572b4
worker: ignore meaningless exit status indication returned by os.waitpid()
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
30639
diff
changeset
|
169 # be still running in this case |
18fb3cf572b4
worker: ignore meaningless exit status indication returned by os.waitpid()
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
30639
diff
changeset
|
170 continue |
18fb3cf572b4
worker: ignore meaningless exit status indication returned by os.waitpid()
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
30639
diff
changeset
|
171 pids.discard(p) |
18fb3cf572b4
worker: ignore meaningless exit status indication returned by os.waitpid()
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
30639
diff
changeset
|
172 st = _exitstatus(st) |
30410
7a5d6e2fd2d5
worker: move killworkers and waitforworkers up
Jun Wu <quark@fb.com>
parents:
30396
diff
changeset
|
173 if st and not problem[0]: |
7a5d6e2fd2d5
worker: move killworkers and waitforworkers up
Jun Wu <quark@fb.com>
parents:
30396
diff
changeset
|
174 problem[0] = st |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42522
diff
changeset
|
175 |
30415
e8fb03cfbbde
worker: add a SIGCHLD handler to collect worker immediately
Jun Wu <quark@fb.com>
parents:
30414
diff
changeset
|
176 def sigchldhandler(signum, frame): |
e8fb03cfbbde
worker: add a SIGCHLD handler to collect worker immediately
Jun Wu <quark@fb.com>
parents:
30414
diff
changeset
|
177 waitforworkers(blocking=False) |
30424
f2d13eb85198
worker: kill workers after all zombie processes are reaped
Yuya Nishihara <yuya@tcha.org>
parents:
30423
diff
changeset
|
178 if problem[0]: |
f2d13eb85198
worker: kill workers after all zombie processes are reaped
Yuya Nishihara <yuya@tcha.org>
parents:
30423
diff
changeset
|
179 killworkers() |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42522
diff
changeset
|
180 |
30415
e8fb03cfbbde
worker: add a SIGCHLD handler to collect worker immediately
Jun Wu <quark@fb.com>
parents:
30414
diff
changeset
|
181 oldchldhandler = signal.signal(signal.SIGCHLD, sigchldhandler) |
31696
9d3d56aa1a9f
worker: flush ui buffers before running the worker
David Soria Parra <davidsp@fb.com>
parents:
31119
diff
changeset
|
182 ui.flush() |
32112
31763785094b
worker: rewrite error handling so os._exit covers all cases
Jun Wu <quark@fb.com>
parents:
32043
diff
changeset
|
183 parentpid = os.getpid() |
38729
9e6afe7fca31
worker: use one pipe per posix worker and select() in parent process
Danny Hooper <hooper@google.com>
parents:
38535
diff
changeset
|
184 pipes = [] |
42522
d29db0a0c4eb
update: fix spurious unclean status bug shown by previous commit
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
42455
diff
changeset
|
185 retval = {} |
49545
3556f0392808
lfs: avoid closing connections when the worker doesn't fork
Matt Harbison <matt_harbison@yahoo.com>
parents:
49309
diff
changeset
|
186 |
3556f0392808
lfs: avoid closing connections when the worker doesn't fork
Matt Harbison <matt_harbison@yahoo.com>
parents:
49309
diff
changeset
|
187 if prefork: |
3556f0392808
lfs: avoid closing connections when the worker doesn't fork
Matt Harbison <matt_harbison@yahoo.com>
parents:
49309
diff
changeset
|
188 prefork() |
3556f0392808
lfs: avoid closing connections when the worker doesn't fork
Matt Harbison <matt_harbison@yahoo.com>
parents:
49309
diff
changeset
|
189 |
45376
26eb62bd0550
posixworker: avoid creating workers that end up getting no work
Martin von Zweigbergk <martinvonz@google.com>
parents:
44248
diff
changeset
|
190 for pargs in partition(args, min(workers, len(args))): |
38729
9e6afe7fca31
worker: use one pipe per posix worker and select() in parent process
Danny Hooper <hooper@google.com>
parents:
38535
diff
changeset
|
191 # Every worker gets its own pipe to send results on, so we don't have to |
9e6afe7fca31
worker: use one pipe per posix worker and select() in parent process
Danny Hooper <hooper@google.com>
parents:
38535
diff
changeset
|
192 # implement atomic writes larger than PIPE_BUF. Each forked process has |
9e6afe7fca31
worker: use one pipe per posix worker and select() in parent process
Danny Hooper <hooper@google.com>
parents:
38535
diff
changeset
|
193 # its own pipe's descriptors in the local variables, and the parent |
9e6afe7fca31
worker: use one pipe per posix worker and select() in parent process
Danny Hooper <hooper@google.com>
parents:
38535
diff
changeset
|
194 # process has the full list of pipe descriptors (and it doesn't really |
9e6afe7fca31
worker: use one pipe per posix worker and select() in parent process
Danny Hooper <hooper@google.com>
parents:
38535
diff
changeset
|
195 # care what order they're in). |
9e6afe7fca31
worker: use one pipe per posix worker and select() in parent process
Danny Hooper <hooper@google.com>
parents:
38535
diff
changeset
|
196 rfd, wfd = os.pipe() |
9e6afe7fca31
worker: use one pipe per posix worker and select() in parent process
Danny Hooper <hooper@google.com>
parents:
38535
diff
changeset
|
197 pipes.append((rfd, wfd)) |
32112
31763785094b
worker: rewrite error handling so os._exit covers all cases
Jun Wu <quark@fb.com>
parents:
32043
diff
changeset
|
198 # make sure we use os._exit in all worker code paths. otherwise the |
31763785094b
worker: rewrite error handling so os._exit covers all cases
Jun Wu <quark@fb.com>
parents:
32043
diff
changeset
|
199 # worker may do some clean-ups which could cause surprises like |
31763785094b
worker: rewrite error handling so os._exit covers all cases
Jun Wu <quark@fb.com>
parents:
32043
diff
changeset
|
200 # deadlock. see sshpeer.cleanup for example. |
31763785094b
worker: rewrite error handling so os._exit covers all cases
Jun Wu <quark@fb.com>
parents:
32043
diff
changeset
|
201 # override error handling *before* fork. this is necessary because |
31763785094b
worker: rewrite error handling so os._exit covers all cases
Jun Wu <quark@fb.com>
parents:
32043
diff
changeset
|
202 # exception (signal) may arrive after fork, before "pid =" assignment |
31763785094b
worker: rewrite error handling so os._exit covers all cases
Jun Wu <quark@fb.com>
parents:
32043
diff
changeset
|
203 # completes, and other exception handler (dispatch.py) can lead to |
31763785094b
worker: rewrite error handling so os._exit covers all cases
Jun Wu <quark@fb.com>
parents:
32043
diff
changeset
|
204 # unexpected code path without os._exit. |
31763785094b
worker: rewrite error handling so os._exit covers all cases
Jun Wu <quark@fb.com>
parents:
32043
diff
changeset
|
205 ret = -1 |
31763785094b
worker: rewrite error handling so os._exit covers all cases
Jun Wu <quark@fb.com>
parents:
32043
diff
changeset
|
206 try: |
31763785094b
worker: rewrite error handling so os._exit covers all cases
Jun Wu <quark@fb.com>
parents:
32043
diff
changeset
|
207 pid = os.fork() |
31763785094b
worker: rewrite error handling so os._exit covers all cases
Jun Wu <quark@fb.com>
parents:
32043
diff
changeset
|
208 if pid == 0: |
31763785094b
worker: rewrite error handling so os._exit covers all cases
Jun Wu <quark@fb.com>
parents:
32043
diff
changeset
|
209 signal.signal(signal.SIGINT, oldhandler) |
31763785094b
worker: rewrite error handling so os._exit covers all cases
Jun Wu <quark@fb.com>
parents:
32043
diff
changeset
|
210 signal.signal(signal.SIGCHLD, oldchldhandler) |
30521
86cd09bc13ba
worker: use os._exit for posix worker in all cases
Jun Wu <quark@fb.com>
parents:
30425
diff
changeset
|
211 |
32112
31763785094b
worker: rewrite error handling so os._exit covers all cases
Jun Wu <quark@fb.com>
parents:
32043
diff
changeset
|
212 def workerfunc(): |
38729
9e6afe7fca31
worker: use one pipe per posix worker and select() in parent process
Danny Hooper <hooper@google.com>
parents:
38535
diff
changeset
|
213 for r, w in pipes[:-1]: |
9e6afe7fca31
worker: use one pipe per posix worker and select() in parent process
Danny Hooper <hooper@google.com>
parents:
38535
diff
changeset
|
214 os.close(r) |
9e6afe7fca31
worker: use one pipe per posix worker and select() in parent process
Danny Hooper <hooper@google.com>
parents:
38535
diff
changeset
|
215 os.close(w) |
32112
31763785094b
worker: rewrite error handling so os._exit covers all cases
Jun Wu <quark@fb.com>
parents:
32043
diff
changeset
|
216 os.close(rfd) |
49269
395f28064826
worker: avoid potential partial write of pickled data
Manuel Jacob <me@manueljacob.de>
parents:
49233
diff
changeset
|
217 with os.fdopen(wfd, 'wb') as wf: |
395f28064826
worker: avoid potential partial write of pickled data
Manuel Jacob <me@manueljacob.de>
parents:
49233
diff
changeset
|
218 for result in func(*(staticargs + (pargs,))): |
395f28064826
worker: avoid potential partial write of pickled data
Manuel Jacob <me@manueljacob.de>
parents:
49233
diff
changeset
|
219 pickle.dump(result, wf) |
395f28064826
worker: avoid potential partial write of pickled data
Manuel Jacob <me@manueljacob.de>
parents:
49233
diff
changeset
|
220 wf.flush() |
32112
31763785094b
worker: rewrite error handling so os._exit covers all cases
Jun Wu <quark@fb.com>
parents:
32043
diff
changeset
|
221 return 0 |
31763785094b
worker: rewrite error handling so os._exit covers all cases
Jun Wu <quark@fb.com>
parents:
32043
diff
changeset
|
222 |
31763785094b
worker: rewrite error handling so os._exit covers all cases
Jun Wu <quark@fb.com>
parents:
32043
diff
changeset
|
223 ret = scmutil.callcatch(ui, workerfunc) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42522
diff
changeset
|
224 except: # parent re-raises, child never returns |
32112
31763785094b
worker: rewrite error handling so os._exit covers all cases
Jun Wu <quark@fb.com>
parents:
32043
diff
changeset
|
225 if os.getpid() == parentpid: |
31763785094b
worker: rewrite error handling so os._exit covers all cases
Jun Wu <quark@fb.com>
parents:
32043
diff
changeset
|
226 raise |
31763785094b
worker: rewrite error handling so os._exit covers all cases
Jun Wu <quark@fb.com>
parents:
32043
diff
changeset
|
227 exctype = sys.exc_info()[0] |
31763785094b
worker: rewrite error handling so os._exit covers all cases
Jun Wu <quark@fb.com>
parents:
32043
diff
changeset
|
228 force = not issubclass(exctype, KeyboardInterrupt) |
31763785094b
worker: rewrite error handling so os._exit covers all cases
Jun Wu <quark@fb.com>
parents:
32043
diff
changeset
|
229 ui.traceback(force=force) |
31763785094b
worker: rewrite error handling so os._exit covers all cases
Jun Wu <quark@fb.com>
parents:
32043
diff
changeset
|
230 finally: |
31763785094b
worker: rewrite error handling so os._exit covers all cases
Jun Wu <quark@fb.com>
parents:
32043
diff
changeset
|
231 if os.getpid() != parentpid: |
31118
a91c62752d08
worker: flush messages written by child processes before exit
Yuya Nishihara <yuya@tcha.org>
parents:
31063
diff
changeset
|
232 try: |
a91c62752d08
worker: flush messages written by child processes before exit
Yuya Nishihara <yuya@tcha.org>
parents:
31063
diff
changeset
|
233 ui.flush() |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42522
diff
changeset
|
234 except: # never returns, no re-raises |
32112
31763785094b
worker: rewrite error handling so os._exit covers all cases
Jun Wu <quark@fb.com>
parents:
32043
diff
changeset
|
235 pass |
30521
86cd09bc13ba
worker: use os._exit for posix worker in all cases
Jun Wu <quark@fb.com>
parents:
30425
diff
changeset
|
236 finally: |
32112
31763785094b
worker: rewrite error handling so os._exit covers all cases
Jun Wu <quark@fb.com>
parents:
32043
diff
changeset
|
237 os._exit(ret & 255) |
30413 | 238 pids.add(pid) |
38729
9e6afe7fca31
worker: use one pipe per posix worker and select() in parent process
Danny Hooper <hooper@google.com>
parents:
38535
diff
changeset
|
239 selector = selectors.DefaultSelector() |
9e6afe7fca31
worker: use one pipe per posix worker and select() in parent process
Danny Hooper <hooper@google.com>
parents:
38535
diff
changeset
|
240 for rfd, wfd in pipes: |
9e6afe7fca31
worker: use one pipe per posix worker and select() in parent process
Danny Hooper <hooper@google.com>
parents:
38535
diff
changeset
|
241 os.close(wfd) |
49899
3eef8baf6b92
worker: avoid reading 1 byte at a time from the OS pipe
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
49545
diff
changeset
|
242 # Buffering is needed for performance, but it also presents a problem: |
3eef8baf6b92
worker: avoid reading 1 byte at a time from the OS pipe
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
49545
diff
changeset
|
243 # selector doesn't take the buffered data into account, |
3eef8baf6b92
worker: avoid reading 1 byte at a time from the OS pipe
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
49545
diff
changeset
|
244 # so we have to arrange it so that the buffers are empty when select is called |
3eef8baf6b92
worker: avoid reading 1 byte at a time from the OS pipe
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
49545
diff
changeset
|
245 # (see [peek_nonblock]) |
3eef8baf6b92
worker: avoid reading 1 byte at a time from the OS pipe
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
49545
diff
changeset
|
246 selector.register(os.fdopen(rfd, 'rb', 4096), selectors.EVENT_READ) |
3eef8baf6b92
worker: avoid reading 1 byte at a time from the OS pipe
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
49545
diff
changeset
|
247 |
3eef8baf6b92
worker: avoid reading 1 byte at a time from the OS pipe
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
49545
diff
changeset
|
248 def peek_nonblock(f): |
3eef8baf6b92
worker: avoid reading 1 byte at a time from the OS pipe
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
49545
diff
changeset
|
249 os.set_blocking(f.fileno(), False) |
3eef8baf6b92
worker: avoid reading 1 byte at a time from the OS pipe
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
49545
diff
changeset
|
250 res = f.peek() |
3eef8baf6b92
worker: avoid reading 1 byte at a time from the OS pipe
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
49545
diff
changeset
|
251 os.set_blocking(f.fileno(), True) |
3eef8baf6b92
worker: avoid reading 1 byte at a time from the OS pipe
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
49545
diff
changeset
|
252 return res |
3eef8baf6b92
worker: avoid reading 1 byte at a time from the OS pipe
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
49545
diff
changeset
|
253 |
3eef8baf6b92
worker: avoid reading 1 byte at a time from the OS pipe
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
49545
diff
changeset
|
254 def load_all(f): |
3eef8baf6b92
worker: avoid reading 1 byte at a time from the OS pipe
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
49545
diff
changeset
|
255 # The pytype error likely goes away on a modern version of |
3eef8baf6b92
worker: avoid reading 1 byte at a time from the OS pipe
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
49545
diff
changeset
|
256 # pytype having a modern typeshed snapshot. |
3eef8baf6b92
worker: avoid reading 1 byte at a time from the OS pipe
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
49545
diff
changeset
|
257 # pytype: disable=wrong-arg-types |
3eef8baf6b92
worker: avoid reading 1 byte at a time from the OS pipe
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
49545
diff
changeset
|
258 yield pickle.load(f) |
3eef8baf6b92
worker: avoid reading 1 byte at a time from the OS pipe
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
49545
diff
changeset
|
259 while len(peek_nonblock(f)) > 0: |
3eef8baf6b92
worker: avoid reading 1 byte at a time from the OS pipe
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
49545
diff
changeset
|
260 yield pickle.load(f) |
3eef8baf6b92
worker: avoid reading 1 byte at a time from the OS pipe
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
49545
diff
changeset
|
261 # pytype: enable=wrong-arg-types |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42522
diff
changeset
|
262 |
18638
047110c0e2a8
worker: allow a function to be run in multiple worker processes
Bryan O'Sullivan <bryano@fb.com>
parents:
18637
diff
changeset
|
263 def cleanup(): |
047110c0e2a8
worker: allow a function to be run in multiple worker processes
Bryan O'Sullivan <bryano@fb.com>
parents:
18637
diff
changeset
|
264 signal.signal(signal.SIGINT, oldhandler) |
30416
c27614f2dec1
worker: stop using a separate thread waiting for children
Jun Wu <quark@fb.com>
parents:
30415
diff
changeset
|
265 waitforworkers() |
30415
e8fb03cfbbde
worker: add a SIGCHLD handler to collect worker immediately
Jun Wu <quark@fb.com>
parents:
30414
diff
changeset
|
266 signal.signal(signal.SIGCHLD, oldchldhandler) |
38740
c08ea1e219c0
worker: call selector.close() to release polling resources
Yuya Nishihara <yuya@tcha.org>
parents:
38731
diff
changeset
|
267 selector.close() |
40988
03f7d0822ec1
worker: do not swallow exception occurred in main process
Yuya Nishihara <yuya@tcha.org>
parents:
38740
diff
changeset
|
268 return problem[0] |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42522
diff
changeset
|
269 |
18638
047110c0e2a8
worker: allow a function to be run in multiple worker processes
Bryan O'Sullivan <bryano@fb.com>
parents:
18637
diff
changeset
|
270 try: |
38729
9e6afe7fca31
worker: use one pipe per posix worker and select() in parent process
Danny Hooper <hooper@google.com>
parents:
38535
diff
changeset
|
271 openpipes = len(pipes) |
9e6afe7fca31
worker: use one pipe per posix worker and select() in parent process
Danny Hooper <hooper@google.com>
parents:
38535
diff
changeset
|
272 while openpipes > 0: |
9e6afe7fca31
worker: use one pipe per posix worker and select() in parent process
Danny Hooper <hooper@google.com>
parents:
38535
diff
changeset
|
273 for key, events in selector.select(): |
9e6afe7fca31
worker: use one pipe per posix worker and select() in parent process
Danny Hooper <hooper@google.com>
parents:
38535
diff
changeset
|
274 try: |
49899
3eef8baf6b92
worker: avoid reading 1 byte at a time from the OS pipe
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
49545
diff
changeset
|
275 for res in load_all(key.fileobj): |
3eef8baf6b92
worker: avoid reading 1 byte at a time from the OS pipe
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
49545
diff
changeset
|
276 if hasretval and res[0]: |
3eef8baf6b92
worker: avoid reading 1 byte at a time from the OS pipe
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
49545
diff
changeset
|
277 retval.update(res[1]) |
3eef8baf6b92
worker: avoid reading 1 byte at a time from the OS pipe
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
49545
diff
changeset
|
278 else: |
3eef8baf6b92
worker: avoid reading 1 byte at a time from the OS pipe
Arseniy Alekseyev <aalekseyev@janestreet.com>
parents:
49545
diff
changeset
|
279 yield res |
38729
9e6afe7fca31
worker: use one pipe per posix worker and select() in parent process
Danny Hooper <hooper@google.com>
parents:
38535
diff
changeset
|
280 except EOFError: |
9e6afe7fca31
worker: use one pipe per posix worker and select() in parent process
Danny Hooper <hooper@google.com>
parents:
38535
diff
changeset
|
281 selector.unregister(key.fileobj) |
49280
311fcc5a65f6
thirdparty: remove Python 2-specific selectors2 copy
Manuel Jacob <me@manueljacob.de>
parents:
49269
diff
changeset
|
282 # pytype: disable=attribute-error |
38729
9e6afe7fca31
worker: use one pipe per posix worker and select() in parent process
Danny Hooper <hooper@google.com>
parents:
38535
diff
changeset
|
283 key.fileobj.close() |
49280
311fcc5a65f6
thirdparty: remove Python 2-specific selectors2 copy
Manuel Jacob <me@manueljacob.de>
parents:
49269
diff
changeset
|
284 # pytype: enable=attribute-error |
38729
9e6afe7fca31
worker: use one pipe per posix worker and select() in parent process
Danny Hooper <hooper@google.com>
parents:
38535
diff
changeset
|
285 openpipes -= 1 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42522
diff
changeset
|
286 except: # re-raises |
18709
9955fc5ee24b
worker: handle worker failures more aggressively
Bryan O'Sullivan <bryano@fb.com>
parents:
18708
diff
changeset
|
287 killworkers() |
18638
047110c0e2a8
worker: allow a function to be run in multiple worker processes
Bryan O'Sullivan <bryano@fb.com>
parents:
18637
diff
changeset
|
288 cleanup() |
047110c0e2a8
worker: allow a function to be run in multiple worker processes
Bryan O'Sullivan <bryano@fb.com>
parents:
18637
diff
changeset
|
289 raise |
40988
03f7d0822ec1
worker: do not swallow exception occurred in main process
Yuya Nishihara <yuya@tcha.org>
parents:
38740
diff
changeset
|
290 status = cleanup() |
03f7d0822ec1
worker: do not swallow exception occurred in main process
Yuya Nishihara <yuya@tcha.org>
parents:
38740
diff
changeset
|
291 if status: |
03f7d0822ec1
worker: do not swallow exception occurred in main process
Yuya Nishihara <yuya@tcha.org>
parents:
38740
diff
changeset
|
292 if status < 0: |
03f7d0822ec1
worker: do not swallow exception occurred in main process
Yuya Nishihara <yuya@tcha.org>
parents:
38740
diff
changeset
|
293 os.kill(os.getpid(), -status) |
45825
8f07f5a9c3de
worker: raise exception instead of calling sys.exit() with child's code
Martin von Zweigbergk <martinvonz@google.com>
parents:
45390
diff
changeset
|
294 raise error.WorkerError(status) |
42455
5ca136bbd3f6
worker: support parallelization of functions with return values
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
40989
diff
changeset
|
295 if hasretval: |
42522
d29db0a0c4eb
update: fix spurious unclean status bug shown by previous commit
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
42455
diff
changeset
|
296 yield True, retval |
18638
047110c0e2a8
worker: allow a function to be run in multiple worker processes
Bryan O'Sullivan <bryano@fb.com>
parents:
18637
diff
changeset
|
297 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42522
diff
changeset
|
298 |
18707
d1a2b086d058
worker: on error, exit similarly to the first failing worker
Bryan O'Sullivan <bryano@fb.com>
parents:
18638
diff
changeset
|
299 def _posixexitstatus(code): |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
45825
diff
changeset
|
300 """convert a posix exit status into the same form returned by |
18707
d1a2b086d058
worker: on error, exit similarly to the first failing worker
Bryan O'Sullivan <bryano@fb.com>
parents:
18638
diff
changeset
|
301 os.spawnv |
d1a2b086d058
worker: on error, exit similarly to the first failing worker
Bryan O'Sullivan <bryano@fb.com>
parents:
18638
diff
changeset
|
302 |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
45825
diff
changeset
|
303 returns None if the process was stopped instead of exiting""" |
18707
d1a2b086d058
worker: on error, exit similarly to the first failing worker
Bryan O'Sullivan <bryano@fb.com>
parents:
18638
diff
changeset
|
304 if os.WIFEXITED(code): |
d1a2b086d058
worker: on error, exit similarly to the first failing worker
Bryan O'Sullivan <bryano@fb.com>
parents:
18638
diff
changeset
|
305 return os.WEXITSTATUS(code) |
d1a2b086d058
worker: on error, exit similarly to the first failing worker
Bryan O'Sullivan <bryano@fb.com>
parents:
18638
diff
changeset
|
306 elif os.WIFSIGNALED(code): |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42522
diff
changeset
|
307 return -(os.WTERMSIG(code)) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42522
diff
changeset
|
308 |
18707
d1a2b086d058
worker: on error, exit similarly to the first failing worker
Bryan O'Sullivan <bryano@fb.com>
parents:
18638
diff
changeset
|
309 |
49545
3556f0392808
lfs: avoid closing connections when the worker doesn't fork
Matt Harbison <matt_harbison@yahoo.com>
parents:
49309
diff
changeset
|
310 def _windowsworker(ui, func, staticargs, args, hasretval, prefork=None): |
35427
02b36e860e0b
workers: implemented worker on windows
Wojciech Lis <wlis@fb.com>
parents:
34646
diff
changeset
|
311 class Worker(threading.Thread): |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42522
diff
changeset
|
312 def __init__( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42522
diff
changeset
|
313 self, taskqueue, resultqueue, func, staticargs, *args, **kwargs |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42522
diff
changeset
|
314 ): |
40443
909c31805f54
py3: roll up threading.Thread constructor args into **kwargs
Matt Harbison <matt_harbison@yahoo.com>
parents:
38740
diff
changeset
|
315 threading.Thread.__init__(self, *args, **kwargs) |
35427
02b36e860e0b
workers: implemented worker on windows
Wojciech Lis <wlis@fb.com>
parents:
34646
diff
changeset
|
316 self._taskqueue = taskqueue |
02b36e860e0b
workers: implemented worker on windows
Wojciech Lis <wlis@fb.com>
parents:
34646
diff
changeset
|
317 self._resultqueue = resultqueue |
02b36e860e0b
workers: implemented worker on windows
Wojciech Lis <wlis@fb.com>
parents:
34646
diff
changeset
|
318 self._func = func |
02b36e860e0b
workers: implemented worker on windows
Wojciech Lis <wlis@fb.com>
parents:
34646
diff
changeset
|
319 self._staticargs = staticargs |
35428
71427ff1dff8
workers: handling exceptions in windows workers
Wojciech Lis <wlis@fb.com>
parents:
35427
diff
changeset
|
320 self._interrupted = False |
35432
86b8cc1f244e
worker: make windows workers daemons
Wojciech Lis <wlis@fb.com>
parents:
35431
diff
changeset
|
321 self.daemon = True |
35428
71427ff1dff8
workers: handling exceptions in windows workers
Wojciech Lis <wlis@fb.com>
parents:
35427
diff
changeset
|
322 self.exception = None |
71427ff1dff8
workers: handling exceptions in windows workers
Wojciech Lis <wlis@fb.com>
parents:
35427
diff
changeset
|
323 |
71427ff1dff8
workers: handling exceptions in windows workers
Wojciech Lis <wlis@fb.com>
parents:
35427
diff
changeset
|
324 def interrupt(self): |
71427ff1dff8
workers: handling exceptions in windows workers
Wojciech Lis <wlis@fb.com>
parents:
35427
diff
changeset
|
325 self._interrupted = True |
35427
02b36e860e0b
workers: implemented worker on windows
Wojciech Lis <wlis@fb.com>
parents:
34646
diff
changeset
|
326 |
02b36e860e0b
workers: implemented worker on windows
Wojciech Lis <wlis@fb.com>
parents:
34646
diff
changeset
|
327 def run(self): |
35428
71427ff1dff8
workers: handling exceptions in windows workers
Wojciech Lis <wlis@fb.com>
parents:
35427
diff
changeset
|
328 try: |
71427ff1dff8
workers: handling exceptions in windows workers
Wojciech Lis <wlis@fb.com>
parents:
35427
diff
changeset
|
329 while not self._taskqueue.empty(): |
71427ff1dff8
workers: handling exceptions in windows workers
Wojciech Lis <wlis@fb.com>
parents:
35427
diff
changeset
|
330 try: |
42522
d29db0a0c4eb
update: fix spurious unclean status bug shown by previous commit
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
42455
diff
changeset
|
331 args = self._taskqueue.get_nowait() |
35428
71427ff1dff8
workers: handling exceptions in windows workers
Wojciech Lis <wlis@fb.com>
parents:
35427
diff
changeset
|
332 for res in self._func(*self._staticargs + (args,)): |
42522
d29db0a0c4eb
update: fix spurious unclean status bug shown by previous commit
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
42455
diff
changeset
|
333 self._resultqueue.put(res) |
35428
71427ff1dff8
workers: handling exceptions in windows workers
Wojciech Lis <wlis@fb.com>
parents:
35427
diff
changeset
|
334 # threading doesn't provide a native way to |
71427ff1dff8
workers: handling exceptions in windows workers
Wojciech Lis <wlis@fb.com>
parents:
35427
diff
changeset
|
335 # interrupt execution. handle it manually at every |
71427ff1dff8
workers: handling exceptions in windows workers
Wojciech Lis <wlis@fb.com>
parents:
35427
diff
changeset
|
336 # iteration. |
71427ff1dff8
workers: handling exceptions in windows workers
Wojciech Lis <wlis@fb.com>
parents:
35427
diff
changeset
|
337 if self._interrupted: |
71427ff1dff8
workers: handling exceptions in windows workers
Wojciech Lis <wlis@fb.com>
parents:
35427
diff
changeset
|
338 return |
37844
8fb9985382be
pycompat: export queue module instead of symbols in module (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36835
diff
changeset
|
339 except pycompat.queue.Empty: |
35428
71427ff1dff8
workers: handling exceptions in windows workers
Wojciech Lis <wlis@fb.com>
parents:
35427
diff
changeset
|
340 break |
71427ff1dff8
workers: handling exceptions in windows workers
Wojciech Lis <wlis@fb.com>
parents:
35427
diff
changeset
|
341 except Exception as e: |
71427ff1dff8
workers: handling exceptions in windows workers
Wojciech Lis <wlis@fb.com>
parents:
35427
diff
changeset
|
342 # store the exception such that the main thread can resurface |
71427ff1dff8
workers: handling exceptions in windows workers
Wojciech Lis <wlis@fb.com>
parents:
35427
diff
changeset
|
343 # it as if the func was running without workers. |
71427ff1dff8
workers: handling exceptions in windows workers
Wojciech Lis <wlis@fb.com>
parents:
35427
diff
changeset
|
344 self.exception = e |
71427ff1dff8
workers: handling exceptions in windows workers
Wojciech Lis <wlis@fb.com>
parents:
35427
diff
changeset
|
345 raise |
71427ff1dff8
workers: handling exceptions in windows workers
Wojciech Lis <wlis@fb.com>
parents:
35427
diff
changeset
|
346 |
71427ff1dff8
workers: handling exceptions in windows workers
Wojciech Lis <wlis@fb.com>
parents:
35427
diff
changeset
|
347 threads = [] |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42522
diff
changeset
|
348 |
35432
86b8cc1f244e
worker: make windows workers daemons
Wojciech Lis <wlis@fb.com>
parents:
35431
diff
changeset
|
349 def trykillworkers(): |
86b8cc1f244e
worker: make windows workers daemons
Wojciech Lis <wlis@fb.com>
parents:
35431
diff
changeset
|
350 # Allow up to 1 second to clean worker threads nicely |
86b8cc1f244e
worker: make windows workers daemons
Wojciech Lis <wlis@fb.com>
parents:
35431
diff
changeset
|
351 cleanupend = time.time() + 1 |
35428
71427ff1dff8
workers: handling exceptions in windows workers
Wojciech Lis <wlis@fb.com>
parents:
35427
diff
changeset
|
352 for t in threads: |
71427ff1dff8
workers: handling exceptions in windows workers
Wojciech Lis <wlis@fb.com>
parents:
35427
diff
changeset
|
353 t.interrupt() |
71427ff1dff8
workers: handling exceptions in windows workers
Wojciech Lis <wlis@fb.com>
parents:
35427
diff
changeset
|
354 for t in threads: |
35432
86b8cc1f244e
worker: make windows workers daemons
Wojciech Lis <wlis@fb.com>
parents:
35431
diff
changeset
|
355 remainingtime = cleanupend - time.time() |
86b8cc1f244e
worker: make windows workers daemons
Wojciech Lis <wlis@fb.com>
parents:
35431
diff
changeset
|
356 t.join(remainingtime) |
35428
71427ff1dff8
workers: handling exceptions in windows workers
Wojciech Lis <wlis@fb.com>
parents:
35427
diff
changeset
|
357 if t.is_alive(): |
35432
86b8cc1f244e
worker: make windows workers daemons
Wojciech Lis <wlis@fb.com>
parents:
35431
diff
changeset
|
358 # pass over the workers joining failure. it is more |
86b8cc1f244e
worker: make windows workers daemons
Wojciech Lis <wlis@fb.com>
parents:
35431
diff
changeset
|
359 # important to surface the inital exception than the |
86b8cc1f244e
worker: make windows workers daemons
Wojciech Lis <wlis@fb.com>
parents:
35431
diff
changeset
|
360 # fact that one of workers may be processing a large |
86b8cc1f244e
worker: make windows workers daemons
Wojciech Lis <wlis@fb.com>
parents:
35431
diff
changeset
|
361 # task and does not get to handle the interruption. |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42522
diff
changeset
|
362 ui.warn( |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42522
diff
changeset
|
363 _( |
43077
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
364 b"failed to kill worker threads while " |
687b865b95ad
formatting: byteify all mercurial/ and hgext/ string literals
Augie Fackler <augie@google.com>
parents:
43076
diff
changeset
|
365 b"handling an exception\n" |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42522
diff
changeset
|
366 ) |
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42522
diff
changeset
|
367 ) |
35432
86b8cc1f244e
worker: make windows workers daemons
Wojciech Lis <wlis@fb.com>
parents:
35431
diff
changeset
|
368 return |
35427
02b36e860e0b
workers: implemented worker on windows
Wojciech Lis <wlis@fb.com>
parents:
34646
diff
changeset
|
369 |
02b36e860e0b
workers: implemented worker on windows
Wojciech Lis <wlis@fb.com>
parents:
34646
diff
changeset
|
370 workers = _numworkers(ui) |
37844
8fb9985382be
pycompat: export queue module instead of symbols in module (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36835
diff
changeset
|
371 resultqueue = pycompat.queue.Queue() |
8fb9985382be
pycompat: export queue module instead of symbols in module (API)
Gregory Szorc <gregory.szorc@gmail.com>
parents:
36835
diff
changeset
|
372 taskqueue = pycompat.queue.Queue() |
42522
d29db0a0c4eb
update: fix spurious unclean status bug shown by previous commit
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
42455
diff
changeset
|
373 retval = {} |
35427
02b36e860e0b
workers: implemented worker on windows
Wojciech Lis <wlis@fb.com>
parents:
34646
diff
changeset
|
374 # partition work to more pieces than workers to minimize the chance |
02b36e860e0b
workers: implemented worker on windows
Wojciech Lis <wlis@fb.com>
parents:
34646
diff
changeset
|
375 # of uneven distribution of large tasks between the workers |
42522
d29db0a0c4eb
update: fix spurious unclean status bug shown by previous commit
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
42455
diff
changeset
|
376 for pargs in partition(args, workers * 20): |
35427
02b36e860e0b
workers: implemented worker on windows
Wojciech Lis <wlis@fb.com>
parents:
34646
diff
changeset
|
377 taskqueue.put(pargs) |
02b36e860e0b
workers: implemented worker on windows
Wojciech Lis <wlis@fb.com>
parents:
34646
diff
changeset
|
378 for _i in range(workers): |
02b36e860e0b
workers: implemented worker on windows
Wojciech Lis <wlis@fb.com>
parents:
34646
diff
changeset
|
379 t = Worker(taskqueue, resultqueue, func, staticargs) |
02b36e860e0b
workers: implemented worker on windows
Wojciech Lis <wlis@fb.com>
parents:
34646
diff
changeset
|
380 threads.append(t) |
02b36e860e0b
workers: implemented worker on windows
Wojciech Lis <wlis@fb.com>
parents:
34646
diff
changeset
|
381 t.start() |
35432
86b8cc1f244e
worker: make windows workers daemons
Wojciech Lis <wlis@fb.com>
parents:
35431
diff
changeset
|
382 try: |
86b8cc1f244e
worker: make windows workers daemons
Wojciech Lis <wlis@fb.com>
parents:
35431
diff
changeset
|
383 while len(threads) > 0: |
86b8cc1f244e
worker: make windows workers daemons
Wojciech Lis <wlis@fb.com>
parents:
35431
diff
changeset
|
384 while not resultqueue.empty(): |
42522
d29db0a0c4eb
update: fix spurious unclean status bug shown by previous commit
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
42455
diff
changeset
|
385 res = resultqueue.get() |
42455
5ca136bbd3f6
worker: support parallelization of functions with return values
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
40989
diff
changeset
|
386 if hasretval and res[0]: |
42522
d29db0a0c4eb
update: fix spurious unclean status bug shown by previous commit
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
42455
diff
changeset
|
387 retval.update(res[1]) |
42455
5ca136bbd3f6
worker: support parallelization of functions with return values
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
40989
diff
changeset
|
388 else: |
5ca136bbd3f6
worker: support parallelization of functions with return values
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
40989
diff
changeset
|
389 yield res |
35432
86b8cc1f244e
worker: make windows workers daemons
Wojciech Lis <wlis@fb.com>
parents:
35431
diff
changeset
|
390 threads[0].join(0.05) |
86b8cc1f244e
worker: make windows workers daemons
Wojciech Lis <wlis@fb.com>
parents:
35431
diff
changeset
|
391 finishedthreads = [_t for _t in threads if not _t.is_alive()] |
86b8cc1f244e
worker: make windows workers daemons
Wojciech Lis <wlis@fb.com>
parents:
35431
diff
changeset
|
392 for t in finishedthreads: |
86b8cc1f244e
worker: make windows workers daemons
Wojciech Lis <wlis@fb.com>
parents:
35431
diff
changeset
|
393 if t.exception is not None: |
86b8cc1f244e
worker: make windows workers daemons
Wojciech Lis <wlis@fb.com>
parents:
35431
diff
changeset
|
394 raise t.exception |
86b8cc1f244e
worker: make windows workers daemons
Wojciech Lis <wlis@fb.com>
parents:
35431
diff
changeset
|
395 threads.remove(t) |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42522
diff
changeset
|
396 except (Exception, KeyboardInterrupt): # re-raises |
35432
86b8cc1f244e
worker: make windows workers daemons
Wojciech Lis <wlis@fb.com>
parents:
35431
diff
changeset
|
397 trykillworkers() |
86b8cc1f244e
worker: make windows workers daemons
Wojciech Lis <wlis@fb.com>
parents:
35431
diff
changeset
|
398 raise |
35427
02b36e860e0b
workers: implemented worker on windows
Wojciech Lis <wlis@fb.com>
parents:
34646
diff
changeset
|
399 while not resultqueue.empty(): |
42522
d29db0a0c4eb
update: fix spurious unclean status bug shown by previous commit
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
42455
diff
changeset
|
400 res = resultqueue.get() |
42455
5ca136bbd3f6
worker: support parallelization of functions with return values
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
40989
diff
changeset
|
401 if hasretval and res[0]: |
42522
d29db0a0c4eb
update: fix spurious unclean status bug shown by previous commit
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
42455
diff
changeset
|
402 retval.update(res[1]) |
42455
5ca136bbd3f6
worker: support parallelization of functions with return values
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
40989
diff
changeset
|
403 else: |
5ca136bbd3f6
worker: support parallelization of functions with return values
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
40989
diff
changeset
|
404 yield res |
5ca136bbd3f6
worker: support parallelization of functions with return values
Valentin Gatien-Baron <vgatien-baron@janestreet.com>
parents:
40989
diff
changeset
|
405 if hasretval: |
42522
d29db0a0c4eb
update: fix spurious unclean status bug shown by previous commit
Valentin Gatien-Baron <valentin.gatienbaron@gmail.com>
parents:
42455
diff
changeset
|
406 yield True, retval |
35427
02b36e860e0b
workers: implemented worker on windows
Wojciech Lis <wlis@fb.com>
parents:
34646
diff
changeset
|
407 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42522
diff
changeset
|
408 |
35427
02b36e860e0b
workers: implemented worker on windows
Wojciech Lis <wlis@fb.com>
parents:
34646
diff
changeset
|
409 if pycompat.iswindows: |
02b36e860e0b
workers: implemented worker on windows
Wojciech Lis <wlis@fb.com>
parents:
34646
diff
changeset
|
410 _platformworker = _windowsworker |
02b36e860e0b
workers: implemented worker on windows
Wojciech Lis <wlis@fb.com>
parents:
34646
diff
changeset
|
411 else: |
18638
047110c0e2a8
worker: allow a function to be run in multiple worker processes
Bryan O'Sullivan <bryano@fb.com>
parents:
18637
diff
changeset
|
412 _platformworker = _posixworker |
18707
d1a2b086d058
worker: on error, exit similarly to the first failing worker
Bryan O'Sullivan <bryano@fb.com>
parents:
18638
diff
changeset
|
413 _exitstatus = _posixexitstatus |
18638
047110c0e2a8
worker: allow a function to be run in multiple worker processes
Bryan O'Sullivan <bryano@fb.com>
parents:
18637
diff
changeset
|
414 |
43076
2372284d9457
formatting: blacken the codebase
Augie Fackler <augie@google.com>
parents:
42522
diff
changeset
|
415 |
18637
ac4dbceeb14a
worker: partition a list (of tasks) into equal-sized chunks
Bryan O'Sullivan <bryano@fb.com>
parents:
18636
diff
changeset
|
416 def partition(lst, nslices): |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
45825
diff
changeset
|
417 """partition a list into N slices of roughly equal size |
28181
f8efc8a3a991
worker: change partition strategy to every Nth element
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
418 |
f8efc8a3a991
worker: change partition strategy to every Nth element
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
419 The current strategy takes every Nth element from the input. If |
f8efc8a3a991
worker: change partition strategy to every Nth element
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
420 we ever write workers that need to preserve grouping in input |
f8efc8a3a991
worker: change partition strategy to every Nth element
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
421 we should consider allowing callers to specify a partition strategy. |
28292
3eb7faf6d958
worker: document poor partitioning scheme impact
Gregory Szorc <gregory.szorc@gmail.com>
parents:
28181
diff
changeset
|
422 |
46819
d4ba4d51f85f
contributor: change mentions of mpm to olivia
Raphaël Gomès <rgomes@octobus.net>
parents:
46698
diff
changeset
|
423 olivia is not a fan of this partitioning strategy when files are involved. |
28292
3eb7faf6d958
worker: document poor partitioning scheme impact
Gregory Szorc <gregory.szorc@gmail.com>
parents:
28181
diff
changeset
|
424 In his words: |
3eb7faf6d958
worker: document poor partitioning scheme impact
Gregory Szorc <gregory.szorc@gmail.com>
parents:
28181
diff
changeset
|
425 |
3eb7faf6d958
worker: document poor partitioning scheme impact
Gregory Szorc <gregory.szorc@gmail.com>
parents:
28181
diff
changeset
|
426 Single-threaded Mercurial makes a point of creating and visiting |
3eb7faf6d958
worker: document poor partitioning scheme impact
Gregory Szorc <gregory.szorc@gmail.com>
parents:
28181
diff
changeset
|
427 files in a fixed order (alphabetical). When creating files in order, |
3eb7faf6d958
worker: document poor partitioning scheme impact
Gregory Szorc <gregory.szorc@gmail.com>
parents:
28181
diff
changeset
|
428 a typical filesystem is likely to allocate them on nearby regions on |
3eb7faf6d958
worker: document poor partitioning scheme impact
Gregory Szorc <gregory.szorc@gmail.com>
parents:
28181
diff
changeset
|
429 disk. Thus, when revisiting in the same order, locality is maximized |
3eb7faf6d958
worker: document poor partitioning scheme impact
Gregory Szorc <gregory.szorc@gmail.com>
parents:
28181
diff
changeset
|
430 and various forms of OS and disk-level caching and read-ahead get a |
3eb7faf6d958
worker: document poor partitioning scheme impact
Gregory Szorc <gregory.szorc@gmail.com>
parents:
28181
diff
changeset
|
431 chance to work. |
3eb7faf6d958
worker: document poor partitioning scheme impact
Gregory Szorc <gregory.szorc@gmail.com>
parents:
28181
diff
changeset
|
432 |
3eb7faf6d958
worker: document poor partitioning scheme impact
Gregory Szorc <gregory.szorc@gmail.com>
parents:
28181
diff
changeset
|
433 This effect can be quite significant on spinning disks. I discovered it |
3eb7faf6d958
worker: document poor partitioning scheme impact
Gregory Szorc <gregory.szorc@gmail.com>
parents:
28181
diff
changeset
|
434 circa Mercurial v0.4 when revlogs were named by hashes of filenames. |
3eb7faf6d958
worker: document poor partitioning scheme impact
Gregory Szorc <gregory.szorc@gmail.com>
parents:
28181
diff
changeset
|
435 Tarring a repo and copying it to another disk effectively randomized |
3eb7faf6d958
worker: document poor partitioning scheme impact
Gregory Szorc <gregory.szorc@gmail.com>
parents:
28181
diff
changeset
|
436 the revlog ordering on disk by sorting the revlogs by hash and suddenly |
3eb7faf6d958
worker: document poor partitioning scheme impact
Gregory Szorc <gregory.szorc@gmail.com>
parents:
28181
diff
changeset
|
437 performance of my kernel checkout benchmark dropped by ~10x because the |
3eb7faf6d958
worker: document poor partitioning scheme impact
Gregory Szorc <gregory.szorc@gmail.com>
parents:
28181
diff
changeset
|
438 "working set" of sectors visited no longer fit in the drive's cache and |
3eb7faf6d958
worker: document poor partitioning scheme impact
Gregory Szorc <gregory.szorc@gmail.com>
parents:
28181
diff
changeset
|
439 the workload switched from streaming to random I/O. |
3eb7faf6d958
worker: document poor partitioning scheme impact
Gregory Szorc <gregory.szorc@gmail.com>
parents:
28181
diff
changeset
|
440 |
3eb7faf6d958
worker: document poor partitioning scheme impact
Gregory Szorc <gregory.szorc@gmail.com>
parents:
28181
diff
changeset
|
441 What we should really be doing is have workers read filenames from a |
3eb7faf6d958
worker: document poor partitioning scheme impact
Gregory Szorc <gregory.szorc@gmail.com>
parents:
28181
diff
changeset
|
442 ordered queue. This preserves locality and also keeps any worker from |
3eb7faf6d958
worker: document poor partitioning scheme impact
Gregory Szorc <gregory.szorc@gmail.com>
parents:
28181
diff
changeset
|
443 getting more than one file out of balance. |
45942
89a2afe31e82
formating: upgrade to black 20.8b1
Augie Fackler <raf@durin42.com>
parents:
45825
diff
changeset
|
444 """ |
28181
f8efc8a3a991
worker: change partition strategy to every Nth element
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
445 for i in range(nslices): |
f8efc8a3a991
worker: change partition strategy to every Nth element
Gregory Szorc <gregory.szorc@gmail.com>
parents:
26587
diff
changeset
|
446 yield lst[i::nslices] |