Mercurial > hg
annotate contrib/automation/hgautomation/aws.py @ 42284:195dcc10b3d7
automation: move image operations to own functions
An upcoming commit will need this functionality with slightly different
values and it is enough code to not want to duplicate. Let's refactor
into standalone functions so it can be reused.
Differential Revision: https://phab.mercurial-scm.org/D6318
author | Gregory Szorc <gregory.szorc@gmail.com> |
---|---|
date | Tue, 23 Apr 2019 21:57:32 -0700 |
parents | e570106beda1 |
children | 65b3ef162b39 |
rev | line source |
---|---|
42024
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
1 # aws.py - Automation code for Amazon Web Services |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
2 # |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
3 # Copyright 2019 Gregory Szorc <gregory.szorc@gmail.com> |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
4 # |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
5 # This software may be used and distributed according to the terms of the |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
6 # GNU General Public License version 2 or any later version. |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
7 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
8 # no-check-code because Python 3 native. |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
9 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
10 import contextlib |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
11 import copy |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
12 import hashlib |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
13 import json |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
14 import os |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
15 import pathlib |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
16 import subprocess |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
17 import time |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
18 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
19 import boto3 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
20 import botocore.exceptions |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
21 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
22 from .winrm import ( |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
23 run_powershell, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
24 wait_for_winrm, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
25 ) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
26 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
27 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
28 SOURCE_ROOT = pathlib.Path(os.path.abspath(__file__)).parent.parent.parent.parent |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
29 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
30 INSTALL_WINDOWS_DEPENDENCIES = (SOURCE_ROOT / 'contrib' / |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
31 'install-windows-dependencies.ps1') |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
32 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
33 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
34 KEY_PAIRS = { |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
35 'automation', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
36 } |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
37 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
38 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
39 SECURITY_GROUPS = { |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
40 'windows-dev-1': { |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
41 'description': 'Mercurial Windows instances that perform build automation', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
42 'ingress': [ |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
43 { |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
44 'FromPort': 22, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
45 'ToPort': 22, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
46 'IpProtocol': 'tcp', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
47 'IpRanges': [ |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
48 { |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
49 'CidrIp': '0.0.0.0/0', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
50 'Description': 'SSH from entire Internet', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
51 }, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
52 ], |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
53 }, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
54 { |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
55 'FromPort': 3389, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
56 'ToPort': 3389, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
57 'IpProtocol': 'tcp', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
58 'IpRanges': [ |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
59 { |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
60 'CidrIp': '0.0.0.0/0', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
61 'Description': 'RDP from entire Internet', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
62 }, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
63 ], |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
64 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
65 }, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
66 { |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
67 'FromPort': 5985, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
68 'ToPort': 5986, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
69 'IpProtocol': 'tcp', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
70 'IpRanges': [ |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
71 { |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
72 'CidrIp': '0.0.0.0/0', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
73 'Description': 'PowerShell Remoting (Windows Remote Management)', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
74 }, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
75 ], |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
76 } |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
77 ], |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
78 }, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
79 } |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
80 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
81 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
82 IAM_ROLES = { |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
83 'ephemeral-ec2-role-1': { |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
84 'description': 'Mercurial temporary EC2 instances', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
85 'policy_arns': [ |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
86 'arn:aws:iam::aws:policy/service-role/AmazonEC2RoleforSSM', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
87 ], |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
88 }, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
89 } |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
90 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
91 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
92 ASSUME_ROLE_POLICY_DOCUMENT = ''' |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
93 { |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
94 "Version": "2012-10-17", |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
95 "Statement": [ |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
96 { |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
97 "Effect": "Allow", |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
98 "Principal": { |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
99 "Service": "ec2.amazonaws.com" |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
100 }, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
101 "Action": "sts:AssumeRole" |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
102 } |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
103 ] |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
104 } |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
105 '''.strip() |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
106 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
107 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
108 IAM_INSTANCE_PROFILES = { |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
109 'ephemeral-ec2-1': { |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
110 'roles': [ |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
111 'ephemeral-ec2-role-1', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
112 ], |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
113 } |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
114 } |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
115 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
116 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
117 # User Data for Windows EC2 instance. Mainly used to set the password |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
118 # and configure WinRM. |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
119 # Inspired by the User Data script used by Packer |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
120 # (from https://www.packer.io/intro/getting-started/build-image.html). |
42064
0e9066db5e44
automation: use raw strings when there are backslashes
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42024
diff
changeset
|
121 WINDOWS_USER_DATA = r''' |
42024
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
122 <powershell> |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
123 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
124 # TODO enable this once we figure out what is failing. |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
125 #$ErrorActionPreference = "stop" |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
126 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
127 # Set administrator password |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
128 net user Administrator "%s" |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
129 wmic useraccount where "name='Administrator'" set PasswordExpires=FALSE |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
130 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
131 # First, make sure WinRM can't be connected to |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
132 netsh advfirewall firewall set rule name="Windows Remote Management (HTTP-In)" new enable=yes action=block |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
133 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
134 # Delete any existing WinRM listeners |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
135 winrm delete winrm/config/listener?Address=*+Transport=HTTP 2>$Null |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
136 winrm delete winrm/config/listener?Address=*+Transport=HTTPS 2>$Null |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
137 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
138 # Create a new WinRM listener and configure |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
139 winrm create winrm/config/listener?Address=*+Transport=HTTP |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
140 winrm set winrm/config/winrs '@{MaxMemoryPerShellMB="0"}' |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
141 winrm set winrm/config '@{MaxTimeoutms="7200000"}' |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
142 winrm set winrm/config/service '@{AllowUnencrypted="true"}' |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
143 winrm set winrm/config/service '@{MaxConcurrentOperationsPerUser="12000"}' |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
144 winrm set winrm/config/service/auth '@{Basic="true"}' |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
145 winrm set winrm/config/client/auth '@{Basic="true"}' |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
146 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
147 # Configure UAC to allow privilege elevation in remote shells |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
148 $Key = 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Policies\System' |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
149 $Setting = 'LocalAccountTokenFilterPolicy' |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
150 Set-ItemProperty -Path $Key -Name $Setting -Value 1 -Force |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
151 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
152 # Configure and restart the WinRM Service; Enable the required firewall exception |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
153 Stop-Service -Name WinRM |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
154 Set-Service -Name WinRM -StartupType Automatic |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
155 netsh advfirewall firewall set rule name="Windows Remote Management (HTTP-In)" new action=allow localip=any remoteip=any |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
156 Start-Service -Name WinRM |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
157 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
158 # Disable firewall on private network interfaces so prompts don't appear. |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
159 Set-NetFirewallProfile -Name private -Enabled false |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
160 </powershell> |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
161 '''.lstrip() |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
162 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
163 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
164 WINDOWS_BOOTSTRAP_POWERSHELL = ''' |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
165 Write-Output "installing PowerShell dependencies" |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
166 Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.201 -Force |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
167 Set-PSRepository -Name PSGallery -InstallationPolicy Trusted |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
168 Install-Module -Name OpenSSHUtils -RequiredVersion 0.0.2.0 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
169 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
170 Write-Output "installing OpenSSL server" |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
171 Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
172 # Various tools will attempt to use older versions of .NET. So we enable |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
173 # the feature that provides them so it doesn't have to be auto-enabled |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
174 # later. |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
175 Write-Output "enabling .NET Framework feature" |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
176 Install-WindowsFeature -Name Net-Framework-Core |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
177 ''' |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
178 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
179 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
180 class AWSConnection: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
181 """Manages the state of a connection with AWS.""" |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
182 |
42277
dd6a9723ae2b
automation: don't create resources when deleting things
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42276
diff
changeset
|
183 def __init__(self, automation, region: str, ensure_ec2_state: bool=True): |
42024
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
184 self.automation = automation |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
185 self.local_state_path = automation.state_path |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
186 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
187 self.prefix = 'hg-' |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
188 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
189 self.session = boto3.session.Session(region_name=region) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
190 self.ec2client = self.session.client('ec2') |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
191 self.ec2resource = self.session.resource('ec2') |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
192 self.iamclient = self.session.client('iam') |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
193 self.iamresource = self.session.resource('iam') |
42277
dd6a9723ae2b
automation: don't create resources when deleting things
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42276
diff
changeset
|
194 self.security_groups = {} |
42024
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
195 |
42277
dd6a9723ae2b
automation: don't create resources when deleting things
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42276
diff
changeset
|
196 if ensure_ec2_state: |
dd6a9723ae2b
automation: don't create resources when deleting things
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42276
diff
changeset
|
197 ensure_key_pairs(automation.state_path, self.ec2resource) |
dd6a9723ae2b
automation: don't create resources when deleting things
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42276
diff
changeset
|
198 self.security_groups = ensure_security_groups(self.ec2resource) |
42278
8dc22a209420
automation: wait for instance profiles and roles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42277
diff
changeset
|
199 ensure_iam_state(self.iamclient, self.iamresource) |
42024
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
200 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
201 def key_pair_path_private(self, name): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
202 """Path to a key pair private key file.""" |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
203 return self.local_state_path / 'keys' / ('keypair-%s' % name) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
204 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
205 def key_pair_path_public(self, name): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
206 return self.local_state_path / 'keys' / ('keypair-%s.pub' % name) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
207 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
208 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
209 def rsa_key_fingerprint(p: pathlib.Path): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
210 """Compute the fingerprint of an RSA private key.""" |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
211 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
212 # TODO use rsa package. |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
213 res = subprocess.run( |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
214 ['openssl', 'pkcs8', '-in', str(p), '-nocrypt', '-topk8', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
215 '-outform', 'DER'], |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
216 capture_output=True, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
217 check=True) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
218 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
219 sha1 = hashlib.sha1(res.stdout).hexdigest() |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
220 return ':'.join(a + b for a, b in zip(sha1[::2], sha1[1::2])) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
221 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
222 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
223 def ensure_key_pairs(state_path: pathlib.Path, ec2resource, prefix='hg-'): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
224 remote_existing = {} |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
225 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
226 for kpi in ec2resource.key_pairs.all(): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
227 if kpi.name.startswith(prefix): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
228 remote_existing[kpi.name[len(prefix):]] = kpi.key_fingerprint |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
229 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
230 # Validate that we have these keys locally. |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
231 key_path = state_path / 'keys' |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
232 key_path.mkdir(exist_ok=True, mode=0o700) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
233 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
234 def remove_remote(name): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
235 print('deleting key pair %s' % name) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
236 key = ec2resource.KeyPair(name) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
237 key.delete() |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
238 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
239 def remove_local(name): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
240 pub_full = key_path / ('keypair-%s.pub' % name) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
241 priv_full = key_path / ('keypair-%s' % name) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
242 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
243 print('removing %s' % pub_full) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
244 pub_full.unlink() |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
245 print('removing %s' % priv_full) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
246 priv_full.unlink() |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
247 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
248 local_existing = {} |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
249 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
250 for f in sorted(os.listdir(key_path)): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
251 if not f.startswith('keypair-') or not f.endswith('.pub'): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
252 continue |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
253 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
254 name = f[len('keypair-'):-len('.pub')] |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
255 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
256 pub_full = key_path / f |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
257 priv_full = key_path / ('keypair-%s' % name) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
258 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
259 with open(pub_full, 'r', encoding='ascii') as fh: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
260 data = fh.read() |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
261 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
262 if not data.startswith('ssh-rsa '): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
263 print('unexpected format for key pair file: %s; removing' % |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
264 pub_full) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
265 pub_full.unlink() |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
266 priv_full.unlink() |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
267 continue |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
268 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
269 local_existing[name] = rsa_key_fingerprint(priv_full) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
270 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
271 for name in sorted(set(remote_existing) | set(local_existing)): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
272 if name not in local_existing: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
273 actual = '%s%s' % (prefix, name) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
274 print('remote key %s does not exist locally' % name) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
275 remove_remote(actual) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
276 del remote_existing[name] |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
277 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
278 elif name not in remote_existing: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
279 print('local key %s does not exist remotely' % name) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
280 remove_local(name) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
281 del local_existing[name] |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
282 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
283 elif remote_existing[name] != local_existing[name]: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
284 print('key fingerprint mismatch for %s; ' |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
285 'removing from local and remote' % name) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
286 remove_local(name) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
287 remove_remote('%s%s' % (prefix, name)) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
288 del local_existing[name] |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
289 del remote_existing[name] |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
290 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
291 missing = KEY_PAIRS - set(remote_existing) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
292 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
293 for name in sorted(missing): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
294 actual = '%s%s' % (prefix, name) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
295 print('creating key pair %s' % actual) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
296 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
297 priv_full = key_path / ('keypair-%s' % name) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
298 pub_full = key_path / ('keypair-%s.pub' % name) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
299 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
300 kp = ec2resource.create_key_pair(KeyName=actual) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
301 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
302 with priv_full.open('w', encoding='ascii') as fh: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
303 fh.write(kp.key_material) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
304 fh.write('\n') |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
305 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
306 priv_full.chmod(0o0600) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
307 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
308 # SSH public key can be extracted via `ssh-keygen`. |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
309 with pub_full.open('w', encoding='ascii') as fh: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
310 subprocess.run( |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
311 ['ssh-keygen', '-y', '-f', str(priv_full)], |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
312 stdout=fh, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
313 check=True) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
314 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
315 pub_full.chmod(0o0600) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
316 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
317 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
318 def delete_instance_profile(profile): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
319 for role in profile.roles: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
320 print('removing role %s from instance profile %s' % (role.name, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
321 profile.name)) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
322 profile.remove_role(RoleName=role.name) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
323 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
324 print('deleting instance profile %s' % profile.name) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
325 profile.delete() |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
326 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
327 |
42278
8dc22a209420
automation: wait for instance profiles and roles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42277
diff
changeset
|
328 def ensure_iam_state(iamclient, iamresource, prefix='hg-'): |
42024
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
329 """Ensure IAM state is in sync with our canonical definition.""" |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
330 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
331 remote_profiles = {} |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
332 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
333 for profile in iamresource.instance_profiles.all(): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
334 if profile.name.startswith(prefix): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
335 remote_profiles[profile.name[len(prefix):]] = profile |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
336 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
337 for name in sorted(set(remote_profiles) - set(IAM_INSTANCE_PROFILES)): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
338 delete_instance_profile(remote_profiles[name]) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
339 del remote_profiles[name] |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
340 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
341 remote_roles = {} |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
342 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
343 for role in iamresource.roles.all(): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
344 if role.name.startswith(prefix): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
345 remote_roles[role.name[len(prefix):]] = role |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
346 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
347 for name in sorted(set(remote_roles) - set(IAM_ROLES)): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
348 role = remote_roles[name] |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
349 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
350 print('removing role %s' % role.name) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
351 role.delete() |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
352 del remote_roles[name] |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
353 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
354 # We've purged remote state that doesn't belong. Create missing |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
355 # instance profiles and roles. |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
356 for name in sorted(set(IAM_INSTANCE_PROFILES) - set(remote_profiles)): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
357 actual = '%s%s' % (prefix, name) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
358 print('creating IAM instance profile %s' % actual) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
359 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
360 profile = iamresource.create_instance_profile( |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
361 InstanceProfileName=actual) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
362 remote_profiles[name] = profile |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
363 |
42278
8dc22a209420
automation: wait for instance profiles and roles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42277
diff
changeset
|
364 waiter = iamclient.get_waiter('instance_profile_exists') |
8dc22a209420
automation: wait for instance profiles and roles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42277
diff
changeset
|
365 waiter.wait(InstanceProfileName=actual) |
8dc22a209420
automation: wait for instance profiles and roles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42277
diff
changeset
|
366 print('IAM instance profile %s is available' % actual) |
8dc22a209420
automation: wait for instance profiles and roles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42277
diff
changeset
|
367 |
42024
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
368 for name in sorted(set(IAM_ROLES) - set(remote_roles)): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
369 entry = IAM_ROLES[name] |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
370 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
371 actual = '%s%s' % (prefix, name) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
372 print('creating IAM role %s' % actual) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
373 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
374 role = iamresource.create_role( |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
375 RoleName=actual, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
376 Description=entry['description'], |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
377 AssumeRolePolicyDocument=ASSUME_ROLE_POLICY_DOCUMENT, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
378 ) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
379 |
42278
8dc22a209420
automation: wait for instance profiles and roles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42277
diff
changeset
|
380 waiter = iamclient.get_waiter('role_exists') |
8dc22a209420
automation: wait for instance profiles and roles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42277
diff
changeset
|
381 waiter.wait(RoleName=actual) |
8dc22a209420
automation: wait for instance profiles and roles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42277
diff
changeset
|
382 print('IAM role %s is available' % actual) |
8dc22a209420
automation: wait for instance profiles and roles
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42277
diff
changeset
|
383 |
42024
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
384 remote_roles[name] = role |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
385 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
386 for arn in entry['policy_arns']: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
387 print('attaching policy %s to %s' % (arn, role.name)) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
388 role.attach_policy(PolicyArn=arn) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
389 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
390 # Now reconcile state of profiles. |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
391 for name, meta in sorted(IAM_INSTANCE_PROFILES.items()): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
392 profile = remote_profiles[name] |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
393 wanted = {'%s%s' % (prefix, role) for role in meta['roles']} |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
394 have = {role.name for role in profile.roles} |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
395 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
396 for role in sorted(have - wanted): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
397 print('removing role %s from %s' % (role, profile.name)) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
398 profile.remove_role(RoleName=role) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
399 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
400 for role in sorted(wanted - have): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
401 print('adding role %s to %s' % (role, profile.name)) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
402 profile.add_role(RoleName=role) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
403 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
404 |
42284
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
405 def find_image(ec2resource, owner_id, name): |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
406 """Find an AMI by its owner ID and name.""" |
42024
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
407 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
408 images = ec2resource.images.filter( |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
409 Filters=[ |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
410 { |
42284
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
411 'Name': 'owner-id', |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
412 'Values': [owner_id], |
42024
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
413 }, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
414 { |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
415 'Name': 'state', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
416 'Values': ['available'], |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
417 }, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
418 { |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
419 'Name': 'image-type', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
420 'Values': ['machine'], |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
421 }, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
422 { |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
423 'Name': 'name', |
42284
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
424 'Values': [name], |
42024
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
425 }, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
426 ]) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
427 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
428 for image in images: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
429 return image |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
430 |
42284
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
431 raise Exception('unable to find image for %s' % name) |
42024
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
432 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
433 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
434 def ensure_security_groups(ec2resource, prefix='hg-'): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
435 """Ensure all necessary Mercurial security groups are present. |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
436 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
437 All security groups are prefixed with ``hg-`` by default. Any security |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
438 groups having this prefix but aren't in our list are deleted. |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
439 """ |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
440 existing = {} |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
441 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
442 for group in ec2resource.security_groups.all(): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
443 if group.group_name.startswith(prefix): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
444 existing[group.group_name[len(prefix):]] = group |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
445 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
446 purge = set(existing) - set(SECURITY_GROUPS) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
447 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
448 for name in sorted(purge): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
449 group = existing[name] |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
450 print('removing legacy security group: %s' % group.group_name) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
451 group.delete() |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
452 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
453 security_groups = {} |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
454 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
455 for name, group in sorted(SECURITY_GROUPS.items()): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
456 if name in existing: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
457 security_groups[name] = existing[name] |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
458 continue |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
459 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
460 actual = '%s%s' % (prefix, name) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
461 print('adding security group %s' % actual) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
462 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
463 group_res = ec2resource.create_security_group( |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
464 Description=group['description'], |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
465 GroupName=actual, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
466 ) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
467 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
468 group_res.authorize_ingress( |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
469 IpPermissions=group['ingress'], |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
470 ) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
471 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
472 security_groups[name] = group_res |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
473 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
474 return security_groups |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
475 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
476 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
477 def terminate_ec2_instances(ec2resource, prefix='hg-'): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
478 """Terminate all EC2 instances managed by us.""" |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
479 waiting = [] |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
480 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
481 for instance in ec2resource.instances.all(): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
482 if instance.state['Name'] == 'terminated': |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
483 continue |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
484 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
485 for tag in instance.tags or []: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
486 if tag['Key'] == 'Name' and tag['Value'].startswith(prefix): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
487 print('terminating %s' % instance.id) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
488 instance.terminate() |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
489 waiting.append(instance) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
490 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
491 for instance in waiting: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
492 instance.wait_until_terminated() |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
493 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
494 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
495 def remove_resources(c, prefix='hg-'): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
496 """Purge all of our resources in this EC2 region.""" |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
497 ec2resource = c.ec2resource |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
498 iamresource = c.iamresource |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
499 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
500 terminate_ec2_instances(ec2resource, prefix=prefix) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
501 |
42275
730edbd836d8
automation: only iterate over our AMIs
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42064
diff
changeset
|
502 for image in ec2resource.images.filter(Owners=['self']): |
42024
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
503 if image.name.startswith(prefix): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
504 remove_ami(ec2resource, image) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
505 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
506 for group in ec2resource.security_groups.all(): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
507 if group.group_name.startswith(prefix): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
508 print('removing security group %s' % group.group_name) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
509 group.delete() |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
510 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
511 for profile in iamresource.instance_profiles.all(): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
512 if profile.name.startswith(prefix): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
513 delete_instance_profile(profile) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
514 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
515 for role in iamresource.roles.all(): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
516 if role.name.startswith(prefix): |
42276
fcb97cb91ff8
automation: detach policies before deleting role
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42275
diff
changeset
|
517 for p in role.attached_policies.all(): |
fcb97cb91ff8
automation: detach policies before deleting role
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42275
diff
changeset
|
518 print('detaching policy %s from %s' % (p.arn, role.name)) |
fcb97cb91ff8
automation: detach policies before deleting role
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42275
diff
changeset
|
519 role.detach_policy(PolicyArn=p.arn) |
fcb97cb91ff8
automation: detach policies before deleting role
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42275
diff
changeset
|
520 |
42024
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
521 print('removing role %s' % role.name) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
522 role.delete() |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
523 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
524 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
525 def wait_for_ip_addresses(instances): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
526 """Wait for the public IP addresses of an iterable of instances.""" |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
527 for instance in instances: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
528 while True: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
529 if not instance.public_ip_address: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
530 time.sleep(2) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
531 instance.reload() |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
532 continue |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
533 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
534 print('public IP address for %s: %s' % ( |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
535 instance.id, instance.public_ip_address)) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
536 break |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
537 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
538 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
539 def remove_ami(ec2resource, image): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
540 """Remove an AMI and its underlying snapshots.""" |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
541 snapshots = [] |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
542 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
543 for device in image.block_device_mappings: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
544 if 'Ebs' in device: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
545 snapshots.append(ec2resource.Snapshot(device['Ebs']['SnapshotId'])) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
546 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
547 print('deregistering %s' % image.id) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
548 image.deregister() |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
549 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
550 for snapshot in snapshots: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
551 print('deleting snapshot %s' % snapshot.id) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
552 snapshot.delete() |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
553 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
554 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
555 def wait_for_ssm(ssmclient, instances): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
556 """Wait for SSM to come online for an iterable of instance IDs.""" |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
557 while True: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
558 res = ssmclient.describe_instance_information( |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
559 Filters=[ |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
560 { |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
561 'Key': 'InstanceIds', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
562 'Values': [i.id for i in instances], |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
563 }, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
564 ], |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
565 ) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
566 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
567 available = len(res['InstanceInformationList']) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
568 wanted = len(instances) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
569 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
570 print('%d/%d instances available in SSM' % (available, wanted)) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
571 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
572 if available == wanted: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
573 return |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
574 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
575 time.sleep(2) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
576 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
577 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
578 def run_ssm_command(ssmclient, instances, document_name, parameters): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
579 """Run a PowerShell script on an EC2 instance.""" |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
580 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
581 res = ssmclient.send_command( |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
582 InstanceIds=[i.id for i in instances], |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
583 DocumentName=document_name, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
584 Parameters=parameters, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
585 CloudWatchOutputConfig={ |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
586 'CloudWatchOutputEnabled': True, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
587 }, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
588 ) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
589 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
590 command_id = res['Command']['CommandId'] |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
591 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
592 for instance in instances: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
593 while True: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
594 try: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
595 res = ssmclient.get_command_invocation( |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
596 CommandId=command_id, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
597 InstanceId=instance.id, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
598 ) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
599 except botocore.exceptions.ClientError as e: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
600 if e.response['Error']['Code'] == 'InvocationDoesNotExist': |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
601 print('could not find SSM command invocation; waiting') |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
602 time.sleep(1) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
603 continue |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
604 else: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
605 raise |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
606 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
607 if res['Status'] == 'Success': |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
608 break |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
609 elif res['Status'] in ('Pending', 'InProgress', 'Delayed'): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
610 time.sleep(2) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
611 else: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
612 raise Exception('command failed on %s: %s' % ( |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
613 instance.id, res['Status'])) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
614 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
615 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
616 @contextlib.contextmanager |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
617 def temporary_ec2_instances(ec2resource, config): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
618 """Create temporary EC2 instances. |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
619 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
620 This is a proxy to ``ec2client.run_instances(**config)`` that takes care of |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
621 managing the lifecycle of the instances. |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
622 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
623 When the context manager exits, the instances are terminated. |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
624 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
625 The context manager evaluates to the list of data structures |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
626 describing each created instance. The instances may not be available |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
627 for work immediately: it is up to the caller to wait for the instance |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
628 to start responding. |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
629 """ |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
630 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
631 ids = None |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
632 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
633 try: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
634 res = ec2resource.create_instances(**config) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
635 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
636 ids = [i.id for i in res] |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
637 print('started instances: %s' % ' '.join(ids)) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
638 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
639 yield res |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
640 finally: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
641 if ids: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
642 print('terminating instances: %s' % ' '.join(ids)) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
643 for instance in res: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
644 instance.terminate() |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
645 print('terminated %d instances' % len(ids)) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
646 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
647 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
648 @contextlib.contextmanager |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
649 def create_temp_windows_ec2_instances(c: AWSConnection, config): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
650 """Create temporary Windows EC2 instances. |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
651 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
652 This is a higher-level wrapper around ``create_temp_ec2_instances()`` that |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
653 configures the Windows instance for Windows Remote Management. The emitted |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
654 instances will have a ``winrm_client`` attribute containing a |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
655 ``pypsrp.client.Client`` instance bound to the instance. |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
656 """ |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
657 if 'IamInstanceProfile' in config: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
658 raise ValueError('IamInstanceProfile cannot be provided in config') |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
659 if 'UserData' in config: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
660 raise ValueError('UserData cannot be provided in config') |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
661 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
662 password = c.automation.default_password() |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
663 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
664 config = copy.deepcopy(config) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
665 config['IamInstanceProfile'] = { |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
666 'Name': 'hg-ephemeral-ec2-1', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
667 } |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
668 config.setdefault('TagSpecifications', []).append({ |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
669 'ResourceType': 'instance', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
670 'Tags': [{'Key': 'Name', 'Value': 'hg-temp-windows'}], |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
671 }) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
672 config['UserData'] = WINDOWS_USER_DATA % password |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
673 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
674 with temporary_ec2_instances(c.ec2resource, config) as instances: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
675 wait_for_ip_addresses(instances) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
676 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
677 print('waiting for Windows Remote Management service...') |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
678 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
679 for instance in instances: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
680 client = wait_for_winrm(instance.public_ip_address, 'Administrator', password) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
681 print('established WinRM connection to %s' % instance.id) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
682 instance.winrm_client = client |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
683 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
684 yield instances |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
685 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
686 |
42284
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
687 def resolve_fingerprint(fingerprint): |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
688 fingerprint = json.dumps(fingerprint, sort_keys=True) |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
689 return hashlib.sha256(fingerprint.encode('utf-8')).hexdigest() |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
690 |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
691 |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
692 def find_and_reconcile_image(ec2resource, name, fingerprint): |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
693 """Attempt to find an existing EC2 AMI with a name and fingerprint. |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
694 |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
695 If an image with the specified fingerprint is found, it is returned. |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
696 Otherwise None is returned. |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
697 |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
698 Existing images for the specified name that don't have the specified |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
699 fingerprint or are missing required metadata or deleted. |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
700 """ |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
701 # Find existing AMIs with this name and delete the ones that are invalid. |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
702 # Store a reference to a good image so it can be returned one the |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
703 # image state is reconciled. |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
704 images = ec2resource.images.filter( |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
705 Filters=[{'Name': 'name', 'Values': [name]}]) |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
706 |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
707 existing_image = None |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
708 |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
709 for image in images: |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
710 if image.tags is None: |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
711 print('image %s for %s lacks required tags; removing' % ( |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
712 image.id, image.name)) |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
713 remove_ami(ec2resource, image) |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
714 else: |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
715 tags = {t['Key']: t['Value'] for t in image.tags} |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
716 |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
717 if tags.get('HGIMAGEFINGERPRINT') == fingerprint: |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
718 existing_image = image |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
719 else: |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
720 print('image %s for %s has wrong fingerprint; removing' % ( |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
721 image.id, image.name)) |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
722 remove_ami(ec2resource, image) |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
723 |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
724 return existing_image |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
725 |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
726 |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
727 def create_ami_from_instance(ec2client, instance, name, description, |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
728 fingerprint): |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
729 """Create an AMI from a running instance. |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
730 |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
731 Returns the ``ec2resource.Image`` representing the created AMI. |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
732 """ |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
733 instance.stop() |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
734 |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
735 ec2client.get_waiter('instance_stopped').wait( |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
736 InstanceIds=[instance.id], |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
737 WaiterConfig={ |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
738 'Delay': 5, |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
739 }) |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
740 print('%s is stopped' % instance.id) |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
741 |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
742 image = instance.create_image( |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
743 Name=name, |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
744 Description=description, |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
745 ) |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
746 |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
747 image.create_tags(Tags=[ |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
748 { |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
749 'Key': 'HGIMAGEFINGERPRINT', |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
750 'Value': fingerprint, |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
751 }, |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
752 ]) |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
753 |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
754 print('waiting for image %s' % image.id) |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
755 |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
756 ec2client.get_waiter('image_available').wait( |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
757 ImageIds=[image.id], |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
758 ) |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
759 |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
760 print('image %s available as %s' % (image.id, image.name)) |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
761 |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
762 return image |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
763 |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
764 |
42024
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
765 def ensure_windows_dev_ami(c: AWSConnection, prefix='hg-'): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
766 """Ensure Windows Development AMI is available and up-to-date. |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
767 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
768 If necessary, a modern AMI will be built by starting a temporary EC2 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
769 instance and bootstrapping it. |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
770 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
771 Obsolete AMIs will be deleted so there is only a single AMI having the |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
772 desired name. |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
773 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
774 Returns an ``ec2.Image`` of either an existing AMI or a newly-built |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
775 one. |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
776 """ |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
777 ec2client = c.ec2client |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
778 ec2resource = c.ec2resource |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
779 ssmclient = c.session.client('ssm') |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
780 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
781 name = '%s%s' % (prefix, 'windows-dev') |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
782 |
42284
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
783 image = find_image(ec2resource, |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
784 '801119661308', |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
785 'Windows_Server-2019-English-Full-Base-2019.02.13') |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
786 |
42024
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
787 config = { |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
788 'BlockDeviceMappings': [ |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
789 { |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
790 'DeviceName': '/dev/sda1', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
791 'Ebs': { |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
792 'DeleteOnTermination': True, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
793 'VolumeSize': 32, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
794 'VolumeType': 'gp2', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
795 }, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
796 } |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
797 ], |
42284
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
798 'ImageId': image.id, |
42024
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
799 'InstanceInitiatedShutdownBehavior': 'stop', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
800 'InstanceType': 't3.medium', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
801 'KeyName': '%sautomation' % prefix, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
802 'MaxCount': 1, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
803 'MinCount': 1, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
804 'SecurityGroupIds': [c.security_groups['windows-dev-1'].id], |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
805 } |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
806 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
807 commands = [ |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
808 # Need to start the service so sshd_config is generated. |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
809 'Start-Service sshd', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
810 'Write-Output "modifying sshd_config"', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
811 r'$content = Get-Content C:\ProgramData\ssh\sshd_config', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
812 '$content = $content -replace "Match Group administrators","" -replace "AuthorizedKeysFile __PROGRAMDATA__/ssh/administrators_authorized_keys",""', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
813 r'$content | Set-Content C:\ProgramData\ssh\sshd_config', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
814 'Import-Module OpenSSHUtils', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
815 r'Repair-SshdConfigPermission C:\ProgramData\ssh\sshd_config -Confirm:$false', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
816 'Restart-Service sshd', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
817 'Write-Output "installing OpenSSL client"', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
818 'Add-WindowsCapability -Online -Name OpenSSH.Client~~~~0.0.1.0', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
819 'Set-Service -Name sshd -StartupType "Automatic"', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
820 'Write-Output "OpenSSH server running"', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
821 ] |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
822 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
823 with INSTALL_WINDOWS_DEPENDENCIES.open('r', encoding='utf-8') as fh: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
824 commands.extend(l.rstrip() for l in fh) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
825 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
826 # Disable Windows Defender when bootstrapping because it just slows |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
827 # things down. |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
828 commands.insert(0, 'Set-MpPreference -DisableRealtimeMonitoring $true') |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
829 commands.append('Set-MpPreference -DisableRealtimeMonitoring $false') |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
830 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
831 # Compute a deterministic fingerprint to determine whether image needs |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
832 # to be regenerated. |
42284
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
833 fingerprint = resolve_fingerprint({ |
42024
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
834 'instance_config': config, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
835 'user_data': WINDOWS_USER_DATA, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
836 'initial_bootstrap': WINDOWS_BOOTSTRAP_POWERSHELL, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
837 'bootstrap_commands': commands, |
42284
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
838 }) |
42024
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
839 |
42284
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
840 existing_image = find_and_reconcile_image(ec2resource, name, fingerprint) |
42024
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
841 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
842 if existing_image: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
843 return existing_image |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
844 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
845 print('no suitable Windows development image found; creating one...') |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
846 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
847 with create_temp_windows_ec2_instances(c, config) as instances: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
848 assert len(instances) == 1 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
849 instance = instances[0] |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
850 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
851 wait_for_ssm(ssmclient, [instance]) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
852 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
853 # On first boot, install various Windows updates. |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
854 # We would ideally use PowerShell Remoting for this. However, there are |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
855 # trust issues that make it difficult to invoke Windows Update |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
856 # remotely. So we use SSM, which has a mechanism for running Windows |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
857 # Update. |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
858 print('installing Windows features...') |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
859 run_ssm_command( |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
860 ssmclient, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
861 [instance], |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
862 'AWS-RunPowerShellScript', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
863 { |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
864 'commands': WINDOWS_BOOTSTRAP_POWERSHELL.split('\n'), |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
865 }, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
866 ) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
867 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
868 # Reboot so all updates are fully applied. |
42280
e570106beda1
automation: shore up rebooting behavior
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42278
diff
changeset
|
869 # |
e570106beda1
automation: shore up rebooting behavior
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42278
diff
changeset
|
870 # We don't use instance.reboot() here because it is asynchronous and |
e570106beda1
automation: shore up rebooting behavior
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42278
diff
changeset
|
871 # we don't know when exactly the instance has rebooted. It could take |
e570106beda1
automation: shore up rebooting behavior
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42278
diff
changeset
|
872 # a while to stop and we may start trying to interact with the instance |
e570106beda1
automation: shore up rebooting behavior
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42278
diff
changeset
|
873 # before it has rebooted. |
42024
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
874 print('rebooting instance %s' % instance.id) |
42280
e570106beda1
automation: shore up rebooting behavior
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42278
diff
changeset
|
875 instance.stop() |
e570106beda1
automation: shore up rebooting behavior
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42278
diff
changeset
|
876 ec2client.get_waiter('instance_stopped').wait( |
e570106beda1
automation: shore up rebooting behavior
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42278
diff
changeset
|
877 InstanceIds=[instance.id], |
e570106beda1
automation: shore up rebooting behavior
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42278
diff
changeset
|
878 WaiterConfig={ |
e570106beda1
automation: shore up rebooting behavior
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42278
diff
changeset
|
879 'Delay': 5, |
e570106beda1
automation: shore up rebooting behavior
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42278
diff
changeset
|
880 }) |
42024
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
881 |
42280
e570106beda1
automation: shore up rebooting behavior
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42278
diff
changeset
|
882 instance.start() |
e570106beda1
automation: shore up rebooting behavior
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42278
diff
changeset
|
883 wait_for_ip_addresses([instance]) |
e570106beda1
automation: shore up rebooting behavior
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42278
diff
changeset
|
884 |
e570106beda1
automation: shore up rebooting behavior
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42278
diff
changeset
|
885 # There is a race condition here between the User Data PS script running |
e570106beda1
automation: shore up rebooting behavior
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42278
diff
changeset
|
886 # and us connecting to WinRM. This can manifest as |
e570106beda1
automation: shore up rebooting behavior
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42278
diff
changeset
|
887 # "AuthorizationManager check failed" failures during run_powershell(). |
e570106beda1
automation: shore up rebooting behavior
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42278
diff
changeset
|
888 # TODO figure out a workaround. |
42024
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
889 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
890 print('waiting for Windows Remote Management to come back...') |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
891 client = wait_for_winrm(instance.public_ip_address, 'Administrator', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
892 c.automation.default_password()) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
893 print('established WinRM connection to %s' % instance.id) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
894 instance.winrm_client = client |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
895 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
896 print('bootstrapping instance...') |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
897 run_powershell(instance.winrm_client, '\n'.join(commands)) |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
898 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
899 print('bootstrap completed; stopping %s to create image' % instance.id) |
42284
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
900 return create_ami_from_instance(ec2client, instance, name, |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
901 'Mercurial Windows development environment', |
195dcc10b3d7
automation: move image operations to own functions
Gregory Szorc <gregory.szorc@gmail.com>
parents:
42280
diff
changeset
|
902 fingerprint) |
42024
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
903 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
904 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
905 @contextlib.contextmanager |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
906 def temporary_windows_dev_instances(c: AWSConnection, image, instance_type, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
907 prefix='hg-', disable_antivirus=False): |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
908 """Create a temporary Windows development EC2 instance. |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
909 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
910 Context manager resolves to the list of ``EC2.Instance`` that were created. |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
911 """ |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
912 config = { |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
913 'BlockDeviceMappings': [ |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
914 { |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
915 'DeviceName': '/dev/sda1', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
916 'Ebs': { |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
917 'DeleteOnTermination': True, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
918 'VolumeSize': 32, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
919 'VolumeType': 'gp2', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
920 }, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
921 } |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
922 ], |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
923 'ImageId': image.id, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
924 'InstanceInitiatedShutdownBehavior': 'stop', |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
925 'InstanceType': instance_type, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
926 'KeyName': '%sautomation' % prefix, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
927 'MaxCount': 1, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
928 'MinCount': 1, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
929 'SecurityGroupIds': [c.security_groups['windows-dev-1'].id], |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
930 } |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
931 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
932 with create_temp_windows_ec2_instances(c, config) as instances: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
933 if disable_antivirus: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
934 for instance in instances: |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
935 run_powershell( |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
936 instance.winrm_client, |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
937 'Set-MpPreference -DisableRealtimeMonitoring $true') |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
938 |
b05a3e28cf24
automation: perform tasks on remote machines
Gregory Szorc <gregory.szorc@gmail.com>
parents:
diff
changeset
|
939 yield instances |