Mercurial > hg
annotate contrib/packaging/hg-docker @ 43044:f9d35f01b8b3
setup: build extensions in parallel by default
The build_ext distutils command in Python 3.5+ has a "parallel"
option that controls whether to build extensions in parallel. It
is disabled by default (None) and can be set to an integer value
for number of cores or True to indicate use all available CPU
cores.
This commit changes our build_ext command override to set
"parallel" to True unless a value has been provided by the caller.
On my machine, this makes `python setup.py build_ext` 1-4s faster.
It is worth noting that at this time, each individual source file
constituting the extension is still built serially. For Mercurial,
this means that we can't build faster than the slowest-to-build
extension, which is the zstd extension by a long shot. This means
that setup.py is still not very efficient at utilizing multiple
cores. But we're better than before.
Differential Revision: https://phab.mercurial-scm.org/D6923
# no-check-commit because of foo_bar naming
author | Gregory Szorc <gregory.szorc@gmail.com> |
---|---|
date | Mon, 30 Sep 2019 17:26:41 -0700 |
parents | aaad36b88298 |
children | 99e231afc29c |
rev | line source |
---|---|
38458
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1 #!/usr/bin/env python3 |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
2 # |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
3 # Copyright 2018 Gregory Szorc <gregory.szorc@gmail.com> |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
4 # |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
5 # This software may be used and distributed according to the terms of the |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
6 # GNU General Public License version 2 or any later version. |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
7 |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
8 import argparse |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
9 import pathlib |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
10 import shutil |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
11 import subprocess |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
12 import sys |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
13 |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
14 def get_docker() -> str: |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
15 docker = shutil.which('docker.io') or shutil.which('docker') |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
16 if not docker: |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
17 print('could not find docker executable') |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
18 return 1 |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
19 |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
20 try: |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
21 out = subprocess.check_output([docker, '-h'], stderr=subprocess.STDOUT) |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
22 |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
23 if b'Jansens' in out: |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
24 print('%s is the Docking System Tray; try installing docker.io' % |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
25 docker) |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
26 sys.exit(1) |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
27 except subprocess.CalledProcessError as e: |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
28 print('error calling `%s -h`: %s' % (docker, e.output)) |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
29 sys.exit(1) |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
30 |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
31 out = subprocess.check_output([docker, 'version'], |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
32 stderr=subprocess.STDOUT) |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
33 |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
34 lines = out.splitlines() |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
35 if not any(l.startswith((b'Client:', b'Client version:')) for l in lines): |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
36 print('`%s version` does not look like Docker' % docker) |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
37 sys.exit(1) |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
38 |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
39 if not any(l.startswith((b'Server:', b'Server version:')) for l in lines): |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
40 print('`%s version` does not look like Docker' % docker) |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
41 sys.exit(1) |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
42 |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
43 return docker |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
44 |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
45 def get_dockerfile(path: pathlib.Path, args: list) -> bytes: |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
46 with path.open('rb') as fh: |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
47 df = fh.read() |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
48 |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
49 for k, v in args: |
41234
4c0d4bbdc395
packaging: allow running packaging with custom uid+gid for CentOS
Mathias De Mare <mathias.de_mare@nokia.com>
parents:
41233
diff
changeset
|
50 df = df.replace(bytes('%%%s%%' % k.decode(), 'utf-8'), v) |
38458
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
51 |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
52 return df |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
53 |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
54 def build_docker_image(dockerfile: pathlib.Path, params: list, tag: str): |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
55 """Build a Docker image from a templatized Dockerfile.""" |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
56 docker = get_docker() |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
57 |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
58 dockerfile_path = pathlib.Path(dockerfile) |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
59 |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
60 dockerfile = get_dockerfile(dockerfile_path, params) |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
61 |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
62 print('building Dockerfile:') |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
63 print(dockerfile.decode('utf-8', 'replace')) |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
64 |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
65 args = [ |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
66 docker, |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
67 'build', |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
68 '--build-arg', 'http_proxy', |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
69 '--build-arg', 'https_proxy', |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
70 '--tag', tag, |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
71 '-', |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
72 ] |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
73 |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
74 print('executing: %r' % args) |
41233
92b3811fd15f
hg-docker: fix Python 3.4 compatibility (for CentOS 7)
Mathias De Mare <mathias.de_mare@nokia.com>
parents:
38458
diff
changeset
|
75 p = subprocess.Popen(args, stdin=subprocess.PIPE) |
92b3811fd15f
hg-docker: fix Python 3.4 compatibility (for CentOS 7)
Mathias De Mare <mathias.de_mare@nokia.com>
parents:
38458
diff
changeset
|
76 p.communicate(input=dockerfile) |
92b3811fd15f
hg-docker: fix Python 3.4 compatibility (for CentOS 7)
Mathias De Mare <mathias.de_mare@nokia.com>
parents:
38458
diff
changeset
|
77 if p.returncode: |
92b3811fd15f
hg-docker: fix Python 3.4 compatibility (for CentOS 7)
Mathias De Mare <mathias.de_mare@nokia.com>
parents:
38458
diff
changeset
|
78 raise subprocess.CalledProcessException( |
41759
aaad36b88298
cleanup: use () to wrap long lines instead of \
Augie Fackler <augie@google.com>
parents:
41234
diff
changeset
|
79 p.returncode, 'failed to build docker image: %s %s' |
41233
92b3811fd15f
hg-docker: fix Python 3.4 compatibility (for CentOS 7)
Mathias De Mare <mathias.de_mare@nokia.com>
parents:
38458
diff
changeset
|
80 % (p.stdout, p.stderr)) |
38458
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
81 |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
82 def command_build(args): |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
83 build_args = [] |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
84 for arg in args.build_arg: |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
85 k, v = arg.split('=', 1) |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
86 build_args.append((k.encode('utf-8'), v.encode('utf-8'))) |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
87 |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
88 build_docker_image(pathlib.Path(args.dockerfile), |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
89 build_args, |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
90 args.tag) |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
91 |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
92 def command_docker(args): |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
93 print(get_docker()) |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
94 |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
95 def main() -> int: |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
96 parser = argparse.ArgumentParser() |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
97 |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
98 subparsers = parser.add_subparsers(title='subcommands') |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
99 |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
100 build = subparsers.add_parser('build', help='Build a Docker image') |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
101 build.set_defaults(func=command_build) |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
102 build.add_argument('--build-arg', action='append', default=[], |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
103 help='Substitution to perform in Dockerfile; ' |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
104 'format: key=value') |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
105 build.add_argument('dockerfile', help='path to Dockerfile to use') |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
106 build.add_argument('tag', help='Tag to apply to created image') |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
107 |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
108 docker = subparsers.add_parser('docker-path', help='Resolve path to Docker') |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
109 docker.set_defaults(func=command_docker) |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
110 |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
111 args = parser.parse_args() |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
112 |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
113 return args.func(args) |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
114 |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
115 if __name__ == '__main__': |
e5916f1236f3
packaging: replace dockerlib.sh with a Python script
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
116 sys.exit(main()) |