amazon-web-services - 在 AWS 上运行具有不同标志的相同 Docker 映像并保存结果

Question

我有一个实验，我想运行 100 次不同的时间，每次都有一个命令行标志设置为不同的整数值。每个实验都会将结果输出到一个文本文件中。每个实验大约需要 2 个小时，并且彼此独立。

我目前有一个 Docker 映像，可以在提供命令行标志时运行实验。

我很好奇是否有办法编写一个脚本来启动 100 个 AWS 实例（每个可能的标志值一个），运行 Docker 映像，然后将结果输出到某个地方的共享文本文件。这可能吗？我对 AWS 非常缺乏经验，所以我不确定这是否是正确的工具或需要哪些步骤（除了构建 Docker 映像）。

谢谢。

标签： amazon-web-servicesdockeramazon-ec2

Answer 1

你可以使用vagrant和vagrant-aws插件来启动实例和Docker Provisioner到pull你的图像/run容器或Ansible Provisioner。例如：

.
├── playbook.yml
└── Vagrantfile

Vagrantfile: _

# -*- mode: ruby -*-
# vi: set ft=ruby :

VAGRANTFILE_API_VERSION = "2"
Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
  N = 100
  (1..N).each do |server_id|
    config.vm.box = "dummy"
    config.ssh.forward_agent = true
    config.vm.define "server#{server_id}" do |server|
      server.vm.provider :aws do |aws, override|
        aws.access_key_id = ENV["AWS_ACCESS_KEY_ID"]
        aws.secret_access_key = ENV["AWS_SECRET_ACCESS_KEY"]
        aws.instance_type = "t2.micro"
        aws.block_device_mapping = [
          {
            "DeviceName" => "/dev/sda1",
            "Ebs.VolumeSize" => 30
          }
        ]
        aws.tags = {
          "Name" => "node#{server_id}.example.com",
          "Environment" => "stage"
        }
        aws.subnet_id = "subnet-d65893b0"
        aws.security_groups = [
          "sg-deadbeef"
        ]
        aws.region = "eu-west-1"
        aws.region_config "eu-west-1" do |region|
          region.ami = "ami-0635ad49b5839867c"
          region.keypair_name = "ubuntu"
        end
        aws.monitoring = true
        aws.associate_public_ip = false
        aws.ssh_host_attribute = :private_ip_address
        override.ssh.username = "ubuntu"
        override.ssh.private_key_path = ENV["HOME"] + "/.ssh/id_rsa"
        override.ssh.forward_agent = true
      end
      if server_id == N
        server.vm.provision :ansible do |ansible|
          ansible.limit = "all"
          ansible.playbook = "playbook.yml"
          ansible.compatibility_mode = "2.0"
          ansible.raw_ssh_args = "-o ForwardAgent=yes"
          ansible.extra_vars = {
            "ansible_python_interpreter": "/usr/bin/python3"
          }
        end
      end
    end
  end
end

注意：此示例从提示和技巧执行 ansible 并行执行。

: _ansible playbook.yml

- hosts: all
  pre_tasks:
  - name: get instance facts
    local_action:
      module: ec2_instance_facts
      filters:
        private-dns-name: '{{ ansible_fqdn }}'
        "tag:Environment": stage
    register: _ec2_instance_facts

  - name: add route53 entry
    local_action:
      module: route53
      state: present
      private_zone: yes
      zone: 'example.com'
      record: '{{ _ec2_instance_facts.instances[0].tags["Name"] }}'
      type: A
      ttl: 7200
      value: '{{ _ec2_instance_facts.instances[0].private_ip_address }}'
      wait: yes
      overwrite: yes

  tasks:
  - name: install build requirements
    apt:
      name: ['python3-pip', 'python3-socks', 'git']
      state: present
      update_cache: yes
    become: true

  - name: apt install docker requirements
    apt:
      name: ['apt-transport-https', 'ca-certificates', 'curl', 'gnupg-agent', 'software-properties-common']
      state: present
    become: true

  - name: add docker apt key
    apt_key:
      url: https://download.docker.com/linux/ubuntu/gpg
      state: present
    become: true

  - name: add docker apt repository
    apt_repository:
      repo: 'deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable'
      state: present
    become: true

  - name: apt install docker-ce
    apt:
      name: ['docker-ce', 'docker-ce-cli', 'containerd.io']
      state: present
      update_cache: yes
    become: true

  - name: get docker-compose
    get_url:
      url: 'https://github.com/docker/compose/releases/download/1.24.1/docker-compose-{{ ansible_system }}-{{ ansible_userspace_architecture }}'
      dest: /usr/local/bin/docker-compose
      mode: '0755'
    become: true

  - name: pip install docker and boto3
    pip:
      name: ['boto3', 'docker', 'docker-compose']
      executable: pip3

  - name: create docker config directory
    file:
      path: /etc/docker
      state: directory
    become: true

  - name: copy docker daemon.json
    copy:
      content: |
        {
            "group": "docker",
            "log-driver": "journald",
            "live-restore": true,
            "experimental": true,
            "insecure-registries" : [],
            "features": { "buildkit": true }
        }
      dest: /etc/docker/daemon.json
    become: true

  - name: enable docker service
    service:
      name: docker
      enabled: yes
    become: true

  - name: add ubuntu user to docker group
    user:
      name: ubuntu
      groups: docker
      append: yes
    become: true

  - name: restart docker daemon
    systemd:
      state: restarted
      daemon_reload: yes
      name: docker
      no_block: yes
    become: true

  # pull your images then run your containers