Mercurial > hg
comparison contrib/automation/hgautomation/windows.py @ 43076:2372284d9457
formatting: blacken the codebase
This is using my patch to black
(https://github.com/psf/black/pull/826) so we don't un-wrap collection
literals.
Done with:
hg files 'set:**.py - mercurial/thirdparty/** - "contrib/python-zstandard/**"' | xargs black -S
# skip-blame mass-reformatting only
# no-check-commit reformats foo_bar functions
Differential Revision: https://phab.mercurial-scm.org/D6971
author | Augie Fackler <augie@google.com> |
---|---|
date | Sun, 06 Oct 2019 09:45:02 -0400 |
parents | 92593d72e10b |
children | 081a77df7bc6 |
comparison
equal
deleted
inserted
replaced
43075:57875cf423c9 | 43076:2372284d9457 |
---|---|
13 import pathlib | 13 import pathlib |
14 import re | 14 import re |
15 import subprocess | 15 import subprocess |
16 import tempfile | 16 import tempfile |
17 | 17 |
18 from .pypi import ( | 18 from .pypi import upload as pypi_upload |
19 upload as pypi_upload, | 19 from .winrm import run_powershell |
20 ) | |
21 from .winrm import ( | |
22 run_powershell, | |
23 ) | |
24 | 20 |
25 | 21 |
26 # PowerShell commands to activate a Visual Studio 2008 environment. | 22 # PowerShell commands to activate a Visual Studio 2008 environment. |
27 # This is essentially a port of vcvarsall.bat to PowerShell. | 23 # This is essentially a port of vcvarsall.bat to PowerShell. |
28 ACTIVATE_VC9_AMD64 = r''' | 24 ACTIVATE_VC9_AMD64 = r''' |
115 MERCURIAL_SCM_BASE_URL = 'https://mercurial-scm.org/release/windows' | 111 MERCURIAL_SCM_BASE_URL = 'https://mercurial-scm.org/release/windows' |
116 | 112 |
117 X86_USER_AGENT_PATTERN = '.*Windows.*' | 113 X86_USER_AGENT_PATTERN = '.*Windows.*' |
118 X64_USER_AGENT_PATTERN = '.*Windows.*(WOW|x)64.*' | 114 X64_USER_AGENT_PATTERN = '.*Windows.*(WOW|x)64.*' |
119 | 115 |
120 X86_EXE_DESCRIPTION = ('Mercurial {version} Inno Setup installer - x86 Windows ' | 116 X86_EXE_DESCRIPTION = ( |
121 '- does not require admin rights') | 117 'Mercurial {version} Inno Setup installer - x86 Windows ' |
122 X64_EXE_DESCRIPTION = ('Mercurial {version} Inno Setup installer - x64 Windows ' | 118 '- does not require admin rights' |
123 '- does not require admin rights') | 119 ) |
124 X86_MSI_DESCRIPTION = ('Mercurial {version} MSI installer - x86 Windows ' | 120 X64_EXE_DESCRIPTION = ( |
125 '- requires admin rights') | 121 'Mercurial {version} Inno Setup installer - x64 Windows ' |
126 X64_MSI_DESCRIPTION = ('Mercurial {version} MSI installer - x64 Windows ' | 122 '- does not require admin rights' |
127 '- requires admin rights') | 123 ) |
124 X86_MSI_DESCRIPTION = ( | |
125 'Mercurial {version} MSI installer - x86 Windows ' '- requires admin rights' | |
126 ) | |
127 X64_MSI_DESCRIPTION = ( | |
128 'Mercurial {version} MSI installer - x64 Windows ' '- requires admin rights' | |
129 ) | |
130 | |
128 | 131 |
129 def get_vc_prefix(arch): | 132 def get_vc_prefix(arch): |
130 if arch == 'x86': | 133 if arch == 'x86': |
131 return ACTIVATE_VC9_X86 | 134 return ACTIVATE_VC9_X86 |
132 elif arch == 'x64': | 135 elif arch == 'x64': |
156 ssh_dir = temp_dir / '.ssh' | 159 ssh_dir = temp_dir / '.ssh' |
157 ssh_dir.mkdir() | 160 ssh_dir.mkdir() |
158 ssh_dir.chmod(0o0700) | 161 ssh_dir.chmod(0o0700) |
159 | 162 |
160 # Generate SSH key to use for communication. | 163 # Generate SSH key to use for communication. |
161 subprocess.run([ | 164 subprocess.run( |
162 'ssh-keygen', '-t', 'rsa', '-b', '4096', '-N', '', | 165 [ |
163 '-f', str(ssh_dir / 'id_rsa')], | 166 'ssh-keygen', |
164 check=True, capture_output=True) | 167 '-t', |
168 'rsa', | |
169 '-b', | |
170 '4096', | |
171 '-N', | |
172 '', | |
173 '-f', | |
174 str(ssh_dir / 'id_rsa'), | |
175 ], | |
176 check=True, | |
177 capture_output=True, | |
178 ) | |
165 | 179 |
166 # Add it to ~/.ssh/authorized_keys on remote. | 180 # Add it to ~/.ssh/authorized_keys on remote. |
167 # This assumes the file doesn't already exist. | 181 # This assumes the file doesn't already exist. |
168 authorized_keys = r'c:\Users\Administrator\.ssh\authorized_keys' | 182 authorized_keys = r'c:\Users\Administrator\.ssh\authorized_keys' |
169 winrm_client.execute_cmd(r'mkdir c:\Users\Administrator\.ssh') | 183 winrm_client.execute_cmd(r'mkdir c:\Users\Administrator\.ssh') |
180 fh.write(' StrictHostKeyChecking no\n') | 194 fh.write(' StrictHostKeyChecking no\n') |
181 fh.write(' UserKnownHostsFile %s\n' % (ssh_dir / 'known_hosts')) | 195 fh.write(' UserKnownHostsFile %s\n' % (ssh_dir / 'known_hosts')) |
182 fh.write(' IdentityFile %s\n' % (ssh_dir / 'id_rsa')) | 196 fh.write(' IdentityFile %s\n' % (ssh_dir / 'id_rsa')) |
183 | 197 |
184 if not (hg_repo / '.hg').is_dir(): | 198 if not (hg_repo / '.hg').is_dir(): |
185 raise Exception('%s is not a Mercurial repository; ' | 199 raise Exception( |
186 'synchronization not yet supported' % hg_repo) | 200 '%s is not a Mercurial repository; ' |
201 'synchronization not yet supported' % hg_repo | |
202 ) | |
187 | 203 |
188 env = dict(os.environ) | 204 env = dict(os.environ) |
189 env['HGPLAIN'] = '1' | 205 env['HGPLAIN'] = '1' |
190 env['HGENCODING'] = 'utf-8' | 206 env['HGENCODING'] = 'utf-8' |
191 | 207 |
192 hg_bin = hg_repo / 'hg' | 208 hg_bin = hg_repo / 'hg' |
193 | 209 |
194 res = subprocess.run( | 210 res = subprocess.run( |
195 ['python2.7', str(hg_bin), 'log', '-r', revision, '-T', '{node}'], | 211 ['python2.7', str(hg_bin), 'log', '-r', revision, '-T', '{node}'], |
196 cwd=str(hg_repo), env=env, check=True, capture_output=True) | 212 cwd=str(hg_repo), |
213 env=env, | |
214 check=True, | |
215 capture_output=True, | |
216 ) | |
197 | 217 |
198 full_revision = res.stdout.decode('ascii') | 218 full_revision = res.stdout.decode('ascii') |
199 | 219 |
200 args = [ | 220 args = [ |
201 'python2.7', hg_bin, | 221 'python2.7', |
202 '--config', 'ui.ssh=ssh -F %s' % ssh_config, | 222 hg_bin, |
203 '--config', 'ui.remotecmd=c:/hgdev/venv-bootstrap/Scripts/hg.exe', | 223 '--config', |
224 'ui.ssh=ssh -F %s' % ssh_config, | |
225 '--config', | |
226 'ui.remotecmd=c:/hgdev/venv-bootstrap/Scripts/hg.exe', | |
204 # Also ensure .hgtags changes are present so auto version | 227 # Also ensure .hgtags changes are present so auto version |
205 # calculation works. | 228 # calculation works. |
206 'push', '-f', '-r', full_revision, '-r', 'file(.hgtags)', | 229 'push', |
230 '-f', | |
231 '-r', | |
232 full_revision, | |
233 '-r', | |
234 'file(.hgtags)', | |
207 'ssh://%s/c:/hgdev/src' % public_ip, | 235 'ssh://%s/c:/hgdev/src' % public_ip, |
208 ] | 236 ] |
209 | 237 |
210 res = subprocess.run(args, cwd=str(hg_repo), env=env) | 238 res = subprocess.run(args, cwd=str(hg_repo), env=env) |
211 | 239 |
212 # Allow 1 (no-op) to not trigger error. | 240 # Allow 1 (no-op) to not trigger error. |
213 if res.returncode not in (0, 1): | 241 if res.returncode not in (0, 1): |
214 res.check_returncode() | 242 res.check_returncode() |
215 | 243 |
216 run_powershell(winrm_client, | 244 run_powershell( |
217 HG_UPDATE_CLEAN.format(revision=full_revision)) | 245 winrm_client, HG_UPDATE_CLEAN.format(revision=full_revision) |
246 ) | |
218 | 247 |
219 # TODO detect dirty local working directory and synchronize accordingly. | 248 # TODO detect dirty local working directory and synchronize accordingly. |
220 | 249 |
221 | 250 |
222 def purge_hg(winrm_client): | 251 def purge_hg(winrm_client): |
248 dest = dest_path / latest | 277 dest = dest_path / latest |
249 print('copying %s to %s' % (source, dest)) | 278 print('copying %s to %s' % (source, dest)) |
250 winrm_client.fetch(source, str(dest)) | 279 winrm_client.fetch(source, str(dest)) |
251 | 280 |
252 | 281 |
253 def build_inno_installer(winrm_client, arch: str, dest_path: pathlib.Path, | 282 def build_inno_installer( |
254 version=None): | 283 winrm_client, arch: str, dest_path: pathlib.Path, version=None |
284 ): | |
255 """Build the Inno Setup installer on a remote machine. | 285 """Build the Inno Setup installer on a remote machine. |
256 | 286 |
257 Using a WinRM client, remote commands are executed to build | 287 Using a WinRM client, remote commands are executed to build |
258 a Mercurial Inno Setup installer. | 288 a Mercurial Inno Setup installer. |
259 """ | 289 """ |
261 | 291 |
262 extra_args = [] | 292 extra_args = [] |
263 if version: | 293 if version: |
264 extra_args.extend(['--version', version]) | 294 extra_args.extend(['--version', version]) |
265 | 295 |
266 ps = get_vc_prefix(arch) + BUILD_INNO.format(arch=arch, | 296 ps = get_vc_prefix(arch) + BUILD_INNO.format( |
267 extra_args=' '.join(extra_args)) | 297 arch=arch, extra_args=' '.join(extra_args) |
298 ) | |
268 run_powershell(winrm_client, ps) | 299 run_powershell(winrm_client, ps) |
269 copy_latest_dist(winrm_client, '*.exe', dest_path) | 300 copy_latest_dist(winrm_client, '*.exe', dest_path) |
270 | 301 |
271 | 302 |
272 def build_wheel(winrm_client, arch: str, dest_path: pathlib.Path): | 303 def build_wheel(winrm_client, arch: str, dest_path: pathlib.Path): |
279 ps = get_vc_prefix(arch) + BUILD_WHEEL.format(arch=arch) | 310 ps = get_vc_prefix(arch) + BUILD_WHEEL.format(arch=arch) |
280 run_powershell(winrm_client, ps) | 311 run_powershell(winrm_client, ps) |
281 copy_latest_dist(winrm_client, '*.whl', dest_path) | 312 copy_latest_dist(winrm_client, '*.whl', dest_path) |
282 | 313 |
283 | 314 |
284 def build_wix_installer(winrm_client, arch: str, dest_path: pathlib.Path, | 315 def build_wix_installer( |
285 version=None): | 316 winrm_client, arch: str, dest_path: pathlib.Path, version=None |
317 ): | |
286 """Build the WiX installer on a remote machine. | 318 """Build the WiX installer on a remote machine. |
287 | 319 |
288 Using a WinRM client, remote commands are executed to build a WiX installer. | 320 Using a WinRM client, remote commands are executed to build a WiX installer. |
289 """ | 321 """ |
290 print('Building WiX installer for %s' % arch) | 322 print('Building WiX installer for %s' % arch) |
291 extra_args = [] | 323 extra_args = [] |
292 if version: | 324 if version: |
293 extra_args.extend(['--version', version]) | 325 extra_args.extend(['--version', version]) |
294 | 326 |
295 ps = get_vc_prefix(arch) + BUILD_WIX.format(arch=arch, | 327 ps = get_vc_prefix(arch) + BUILD_WIX.format( |
296 extra_args=' '.join(extra_args)) | 328 arch=arch, extra_args=' '.join(extra_args) |
329 ) | |
297 run_powershell(winrm_client, ps) | 330 run_powershell(winrm_client, ps) |
298 copy_latest_dist(winrm_client, '*.msi', dest_path) | 331 copy_latest_dist(winrm_client, '*.msi', dest_path) |
299 | 332 |
300 | 333 |
301 def run_tests(winrm_client, python_version, arch, test_flags=''): | 334 def run_tests(winrm_client, python_version, arch, test_flags=''): |
305 ``arch`` is ``x86`` or ``x64``. | 338 ``arch`` is ``x86`` or ``x64``. |
306 ``test_flags`` is a str representing extra arguments to pass to | 339 ``test_flags`` is a str representing extra arguments to pass to |
307 ``run-tests.py``. | 340 ``run-tests.py``. |
308 """ | 341 """ |
309 if not re.match(r'\d\.\d', python_version): | 342 if not re.match(r'\d\.\d', python_version): |
310 raise ValueError(r'python_version must be \d.\d; got %s' % | 343 raise ValueError( |
311 python_version) | 344 r'python_version must be \d.\d; got %s' % python_version |
345 ) | |
312 | 346 |
313 if arch not in ('x86', 'x64'): | 347 if arch not in ('x86', 'x64'): |
314 raise ValueError('arch must be x86 or x64; got %s' % arch) | 348 raise ValueError('arch must be x86 or x64; got %s' % arch) |
315 | 349 |
316 python_path = 'python%s-%s' % (python_version.replace('.', ''), arch) | 350 python_path = 'python%s-%s' % (python_version.replace('.', ''), arch) |
317 | 351 |
318 ps = RUN_TESTS.format( | 352 ps = RUN_TESTS.format(python_path=python_path, test_flags=test_flags or '',) |
319 python_path=python_path, | |
320 test_flags=test_flags or '', | |
321 ) | |
322 | 353 |
323 run_powershell(winrm_client, ps) | 354 run_powershell(winrm_client, ps) |
324 | 355 |
325 | 356 |
326 def resolve_wheel_artifacts(dist_path: pathlib.Path, version: str): | 357 def resolve_wheel_artifacts(dist_path: pathlib.Path, version: str): |
372 ( | 403 ( |
373 '10', | 404 '10', |
374 version, | 405 version, |
375 X64_USER_AGENT_PATTERN, | 406 X64_USER_AGENT_PATTERN, |
376 '%s/%s' % (MERCURIAL_SCM_BASE_URL, x64_msi_filename), | 407 '%s/%s' % (MERCURIAL_SCM_BASE_URL, x64_msi_filename), |
377 X64_MSI_DESCRIPTION.format(version=version) | 408 X64_MSI_DESCRIPTION.format(version=version), |
378 ) | 409 ), |
379 ) | 410 ) |
380 | 411 |
381 lines = ['\t'.join(e) for e in entries] | 412 lines = ['\t'.join(e) for e in entries] |
382 | 413 |
383 return '\n'.join(lines) + '\n' | 414 return '\n'.join(lines) + '\n' |
394 | 425 |
395 print('uploading wheels to PyPI (you may be prompted for credentials)') | 426 print('uploading wheels to PyPI (you may be prompted for credentials)') |
396 pypi_upload(wheel_paths) | 427 pypi_upload(wheel_paths) |
397 | 428 |
398 | 429 |
399 def publish_artifacts_mercurial_scm_org(dist_path: pathlib.Path, version: str, | 430 def publish_artifacts_mercurial_scm_org( |
400 ssh_username=None): | 431 dist_path: pathlib.Path, version: str, ssh_username=None |
432 ): | |
401 """Publish Windows release artifacts to mercurial-scm.org.""" | 433 """Publish Windows release artifacts to mercurial-scm.org.""" |
402 all_paths = resolve_all_artifacts(dist_path, version) | 434 all_paths = resolve_all_artifacts(dist_path, version) |
403 | 435 |
404 for p in all_paths: | 436 for p in all_paths: |
405 if not p.exists(): | 437 if not p.exists(): |
434 | 466 |
435 latest_dat_path = '/var/www/release/windows/latest.dat' | 467 latest_dat_path = '/var/www/release/windows/latest.dat' |
436 | 468 |
437 now = datetime.datetime.utcnow() | 469 now = datetime.datetime.utcnow() |
438 backup_path = dist_path / ( | 470 backup_path = dist_path / ( |
439 'latest-windows-%s.dat' % now.strftime('%Y%m%dT%H%M%S')) | 471 'latest-windows-%s.dat' % now.strftime('%Y%m%dT%H%M%S') |
472 ) | |
440 print('backing up %s to %s' % (latest_dat_path, backup_path)) | 473 print('backing up %s to %s' % (latest_dat_path, backup_path)) |
441 | 474 |
442 with sftp.open(latest_dat_path, 'rb') as fh: | 475 with sftp.open(latest_dat_path, 'rb') as fh: |
443 latest_dat_old = fh.read() | 476 latest_dat_old = fh.read() |
444 | 477 |
451 | 484 |
452 with sftp.open(latest_dat_path, 'wb') as fh: | 485 with sftp.open(latest_dat_path, 'wb') as fh: |
453 fh.write(latest_dat_content.encode('ascii')) | 486 fh.write(latest_dat_content.encode('ascii')) |
454 | 487 |
455 | 488 |
456 def publish_artifacts(dist_path: pathlib.Path, version: str, | 489 def publish_artifacts( |
457 pypi=True, mercurial_scm_org=True, | 490 dist_path: pathlib.Path, |
458 ssh_username=None): | 491 version: str, |
492 pypi=True, | |
493 mercurial_scm_org=True, | |
494 ssh_username=None, | |
495 ): | |
459 """Publish Windows release artifacts. | 496 """Publish Windows release artifacts. |
460 | 497 |
461 Files are found in `dist_path`. We will look for files with version string | 498 Files are found in `dist_path`. We will look for files with version string |
462 `version`. | 499 `version`. |
463 | 500 |
466 """ | 503 """ |
467 if pypi: | 504 if pypi: |
468 publish_artifacts_pypi(dist_path, version) | 505 publish_artifacts_pypi(dist_path, version) |
469 | 506 |
470 if mercurial_scm_org: | 507 if mercurial_scm_org: |
471 publish_artifacts_mercurial_scm_org(dist_path, version, | 508 publish_artifacts_mercurial_scm_org( |
472 ssh_username=ssh_username) | 509 dist_path, version, ssh_username=ssh_username |
510 ) |