Mercurial > hg
comparison contrib/automation/hgautomation/aws.py @ 42284:195dcc10b3d7
automation: move image operations to own functions
An upcoming commit will need this functionality with slightly different
values and it is enough code to not want to duplicate. Let's refactor
into standalone functions so it can be reused.
Differential Revision: https://phab.mercurial-scm.org/D6318
author | Gregory Szorc <gregory.szorc@gmail.com> |
---|---|
date | Tue, 23 Apr 2019 21:57:32 -0700 |
parents | e570106beda1 |
children | 65b3ef162b39 |
comparison
equal
deleted
inserted
replaced
42283:d137a3d5ad41 | 42284:195dcc10b3d7 |
---|---|
400 for role in sorted(wanted - have): | 400 for role in sorted(wanted - have): |
401 print('adding role %s to %s' % (role, profile.name)) | 401 print('adding role %s to %s' % (role, profile.name)) |
402 profile.add_role(RoleName=role) | 402 profile.add_role(RoleName=role) |
403 | 403 |
404 | 404 |
405 def find_windows_server_2019_image(ec2resource): | 405 def find_image(ec2resource, owner_id, name): |
406 """Find the Amazon published Windows Server 2019 base image.""" | 406 """Find an AMI by its owner ID and name.""" |
407 | 407 |
408 images = ec2resource.images.filter( | 408 images = ec2resource.images.filter( |
409 Filters=[ | 409 Filters=[ |
410 { | 410 { |
411 'Name': 'owner-alias', | 411 'Name': 'owner-id', |
412 'Values': ['amazon'], | 412 'Values': [owner_id], |
413 }, | 413 }, |
414 { | 414 { |
415 'Name': 'state', | 415 'Name': 'state', |
416 'Values': ['available'], | 416 'Values': ['available'], |
417 }, | 417 }, |
419 'Name': 'image-type', | 419 'Name': 'image-type', |
420 'Values': ['machine'], | 420 'Values': ['machine'], |
421 }, | 421 }, |
422 { | 422 { |
423 'Name': 'name', | 423 'Name': 'name', |
424 'Values': ['Windows_Server-2019-English-Full-Base-2019.02.13'], | 424 'Values': [name], |
425 }, | 425 }, |
426 ]) | 426 ]) |
427 | 427 |
428 for image in images: | 428 for image in images: |
429 return image | 429 return image |
430 | 430 |
431 raise Exception('unable to find Windows Server 2019 image') | 431 raise Exception('unable to find image for %s' % name) |
432 | 432 |
433 | 433 |
434 def ensure_security_groups(ec2resource, prefix='hg-'): | 434 def ensure_security_groups(ec2resource, prefix='hg-'): |
435 """Ensure all necessary Mercurial security groups are present. | 435 """Ensure all necessary Mercurial security groups are present. |
436 | 436 |
682 instance.winrm_client = client | 682 instance.winrm_client = client |
683 | 683 |
684 yield instances | 684 yield instances |
685 | 685 |
686 | 686 |
687 def resolve_fingerprint(fingerprint): | |
688 fingerprint = json.dumps(fingerprint, sort_keys=True) | |
689 return hashlib.sha256(fingerprint.encode('utf-8')).hexdigest() | |
690 | |
691 | |
692 def find_and_reconcile_image(ec2resource, name, fingerprint): | |
693 """Attempt to find an existing EC2 AMI with a name and fingerprint. | |
694 | |
695 If an image with the specified fingerprint is found, it is returned. | |
696 Otherwise None is returned. | |
697 | |
698 Existing images for the specified name that don't have the specified | |
699 fingerprint or are missing required metadata or deleted. | |
700 """ | |
701 # Find existing AMIs with this name and delete the ones that are invalid. | |
702 # Store a reference to a good image so it can be returned one the | |
703 # image state is reconciled. | |
704 images = ec2resource.images.filter( | |
705 Filters=[{'Name': 'name', 'Values': [name]}]) | |
706 | |
707 existing_image = None | |
708 | |
709 for image in images: | |
710 if image.tags is None: | |
711 print('image %s for %s lacks required tags; removing' % ( | |
712 image.id, image.name)) | |
713 remove_ami(ec2resource, image) | |
714 else: | |
715 tags = {t['Key']: t['Value'] for t in image.tags} | |
716 | |
717 if tags.get('HGIMAGEFINGERPRINT') == fingerprint: | |
718 existing_image = image | |
719 else: | |
720 print('image %s for %s has wrong fingerprint; removing' % ( | |
721 image.id, image.name)) | |
722 remove_ami(ec2resource, image) | |
723 | |
724 return existing_image | |
725 | |
726 | |
727 def create_ami_from_instance(ec2client, instance, name, description, | |
728 fingerprint): | |
729 """Create an AMI from a running instance. | |
730 | |
731 Returns the ``ec2resource.Image`` representing the created AMI. | |
732 """ | |
733 instance.stop() | |
734 | |
735 ec2client.get_waiter('instance_stopped').wait( | |
736 InstanceIds=[instance.id], | |
737 WaiterConfig={ | |
738 'Delay': 5, | |
739 }) | |
740 print('%s is stopped' % instance.id) | |
741 | |
742 image = instance.create_image( | |
743 Name=name, | |
744 Description=description, | |
745 ) | |
746 | |
747 image.create_tags(Tags=[ | |
748 { | |
749 'Key': 'HGIMAGEFINGERPRINT', | |
750 'Value': fingerprint, | |
751 }, | |
752 ]) | |
753 | |
754 print('waiting for image %s' % image.id) | |
755 | |
756 ec2client.get_waiter('image_available').wait( | |
757 ImageIds=[image.id], | |
758 ) | |
759 | |
760 print('image %s available as %s' % (image.id, image.name)) | |
761 | |
762 return image | |
763 | |
764 | |
687 def ensure_windows_dev_ami(c: AWSConnection, prefix='hg-'): | 765 def ensure_windows_dev_ami(c: AWSConnection, prefix='hg-'): |
688 """Ensure Windows Development AMI is available and up-to-date. | 766 """Ensure Windows Development AMI is available and up-to-date. |
689 | 767 |
690 If necessary, a modern AMI will be built by starting a temporary EC2 | 768 If necessary, a modern AMI will be built by starting a temporary EC2 |
691 instance and bootstrapping it. | 769 instance and bootstrapping it. |
699 ec2client = c.ec2client | 777 ec2client = c.ec2client |
700 ec2resource = c.ec2resource | 778 ec2resource = c.ec2resource |
701 ssmclient = c.session.client('ssm') | 779 ssmclient = c.session.client('ssm') |
702 | 780 |
703 name = '%s%s' % (prefix, 'windows-dev') | 781 name = '%s%s' % (prefix, 'windows-dev') |
782 | |
783 image = find_image(ec2resource, | |
784 '801119661308', | |
785 'Windows_Server-2019-English-Full-Base-2019.02.13') | |
704 | 786 |
705 config = { | 787 config = { |
706 'BlockDeviceMappings': [ | 788 'BlockDeviceMappings': [ |
707 { | 789 { |
708 'DeviceName': '/dev/sda1', | 790 'DeviceName': '/dev/sda1', |
711 'VolumeSize': 32, | 793 'VolumeSize': 32, |
712 'VolumeType': 'gp2', | 794 'VolumeType': 'gp2', |
713 }, | 795 }, |
714 } | 796 } |
715 ], | 797 ], |
716 'ImageId': find_windows_server_2019_image(ec2resource).id, | 798 'ImageId': image.id, |
717 'InstanceInitiatedShutdownBehavior': 'stop', | 799 'InstanceInitiatedShutdownBehavior': 'stop', |
718 'InstanceType': 't3.medium', | 800 'InstanceType': 't3.medium', |
719 'KeyName': '%sautomation' % prefix, | 801 'KeyName': '%sautomation' % prefix, |
720 'MaxCount': 1, | 802 'MaxCount': 1, |
721 'MinCount': 1, | 803 'MinCount': 1, |
746 commands.insert(0, 'Set-MpPreference -DisableRealtimeMonitoring $true') | 828 commands.insert(0, 'Set-MpPreference -DisableRealtimeMonitoring $true') |
747 commands.append('Set-MpPreference -DisableRealtimeMonitoring $false') | 829 commands.append('Set-MpPreference -DisableRealtimeMonitoring $false') |
748 | 830 |
749 # Compute a deterministic fingerprint to determine whether image needs | 831 # Compute a deterministic fingerprint to determine whether image needs |
750 # to be regenerated. | 832 # to be regenerated. |
751 fingerprint = { | 833 fingerprint = resolve_fingerprint({ |
752 'instance_config': config, | 834 'instance_config': config, |
753 'user_data': WINDOWS_USER_DATA, | 835 'user_data': WINDOWS_USER_DATA, |
754 'initial_bootstrap': WINDOWS_BOOTSTRAP_POWERSHELL, | 836 'initial_bootstrap': WINDOWS_BOOTSTRAP_POWERSHELL, |
755 'bootstrap_commands': commands, | 837 'bootstrap_commands': commands, |
756 } | 838 }) |
757 | 839 |
758 fingerprint = json.dumps(fingerprint, sort_keys=True) | 840 existing_image = find_and_reconcile_image(ec2resource, name, fingerprint) |
759 fingerprint = hashlib.sha256(fingerprint.encode('utf-8')).hexdigest() | |
760 | |
761 # Find existing AMIs with this name and delete the ones that are invalid. | |
762 # Store a reference to a good image so it can be returned one the | |
763 # image state is reconciled. | |
764 images = ec2resource.images.filter( | |
765 Filters=[{'Name': 'name', 'Values': [name]}]) | |
766 | |
767 existing_image = None | |
768 | |
769 for image in images: | |
770 if image.tags is None: | |
771 print('image %s for %s lacks required tags; removing' % ( | |
772 image.id, image.name)) | |
773 remove_ami(ec2resource, image) | |
774 else: | |
775 tags = {t['Key']: t['Value'] for t in image.tags} | |
776 | |
777 if tags.get('HGIMAGEFINGERPRINT') == fingerprint: | |
778 existing_image = image | |
779 else: | |
780 print('image %s for %s has wrong fingerprint; removing' % ( | |
781 image.id, image.name)) | |
782 remove_ami(ec2resource, image) | |
783 | 841 |
784 if existing_image: | 842 if existing_image: |
785 return existing_image | 843 return existing_image |
786 | 844 |
787 print('no suitable Windows development image found; creating one...') | 845 print('no suitable Windows development image found; creating one...') |
837 | 895 |
838 print('bootstrapping instance...') | 896 print('bootstrapping instance...') |
839 run_powershell(instance.winrm_client, '\n'.join(commands)) | 897 run_powershell(instance.winrm_client, '\n'.join(commands)) |
840 | 898 |
841 print('bootstrap completed; stopping %s to create image' % instance.id) | 899 print('bootstrap completed; stopping %s to create image' % instance.id) |
842 instance.stop() | 900 return create_ami_from_instance(ec2client, instance, name, |
843 | 901 'Mercurial Windows development environment', |
844 ec2client.get_waiter('instance_stopped').wait( | 902 fingerprint) |
845 InstanceIds=[instance.id], | |
846 WaiterConfig={ | |
847 'Delay': 5, | |
848 }) | |
849 print('%s is stopped' % instance.id) | |
850 | |
851 image = instance.create_image( | |
852 Name=name, | |
853 Description='Mercurial Windows development environment', | |
854 ) | |
855 | |
856 image.create_tags(Tags=[ | |
857 { | |
858 'Key': 'HGIMAGEFINGERPRINT', | |
859 'Value': fingerprint, | |
860 }, | |
861 ]) | |
862 | |
863 print('waiting for image %s' % image.id) | |
864 | |
865 ec2client.get_waiter('image_available').wait( | |
866 ImageIds=[image.id], | |
867 ) | |
868 | |
869 print('image %s available as %s' % (image.id, image.name)) | |
870 | |
871 return image | |
872 | 903 |
873 | 904 |
874 @contextlib.contextmanager | 905 @contextlib.contextmanager |
875 def temporary_windows_dev_instances(c: AWSConnection, image, instance_type, | 906 def temporary_windows_dev_instances(c: AWSConnection, image, instance_type, |
876 prefix='hg-', disable_antivirus=False): | 907 prefix='hg-', disable_antivirus=False): |