aws-recipes.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Init"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ncluster version 0.1.73\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import sys\n",
    "import time\n",
    "import boto3\n",
    "import ncluster\n",
    "import boto3\n",
    "import base64\n",
    "import pickle\n",
    "import shlex\n",
    "from ncluster import aws_util as u\n",
    "from ncluster import util as util\n",
    "import subprocess\n",
    "\n",
    "from pprint import pprint as pprint\n",
    "import importlib\n",
    "\n",
    "sys.path.insert(0, os.environ['HOME'])\n",
    "from credentials import *   # go_yaro, go_mind, go_andrew\n",
    "\n",
    "\n",
    "def ossystem(cmd, shell=True):\n",
    "    \"\"\"Like os.system, but returns output of command as string.\"\"\"\n",
    "    p = subprocess.Popen(cmd, shell=shell, stdout=subprocess.PIPE,\n",
    "                         stderr=subprocess.STDOUT)\n",
    "    (stdout, stderr) = p.communicate()\n",
    "    return stdout.decode('ascii')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Filters=[{'Name': 'instance-state-name', 'Values': ['stopped']}])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def easy_filter()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Instances"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Launching EFA enabled instance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "import importlib\n",
    "client = u.get_ec2_client()\n",
    "ec2 = u.get_ec2_resource()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "  image = u.lookup_image('dlami23-efa')\n",
    "  keypair = u.get_keypair()\n",
    "  security_group = u.get_security_group()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ['NCLUSTER_ZONE']='us-east-1d'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "subnet=u.get_subnet()\n",
    "placement_group = ''\n",
    "name='efa-programmatic'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "instance_type='p3dn.24xlarge'\n",
    "disk_size=500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ami-03147512dd1ffee88'"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image.id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ncluster-yaroslavvb'"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keypair.name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'sg-01494dff7a2a16363'"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "security_group.id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'subnet-047707529f095f614'"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subnet.id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ami-03147512dd1ffee88'"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image.id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ncluster-yaroslavvb'"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keypair.name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "boto3.resources.factory.ec2.SecurityGroup"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(security_group)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "keypair_name='ncluster-yaroslavvb'\n",
    "instance_type='p3dn.24xlarge'\n",
    "instance_type='c5n.18xlarge'\n",
    "security_group_id='sg-0139018179276c4ba'\n",
    "subnet_id='subnet-047707529f095f614'\n",
    "image_id='ami-03147512dd1ffee88'\n",
    "zone='us-east-1d'\n",
    "\n",
    "args = {'ImageId': image_id,\n",
    "        'InstanceType': instance_type,\n",
    "        'MinCount': 1,\n",
    "        'MaxCount': 1,\n",
    "        'KeyName': keypair_name}\n",
    "\n",
    "args['TagSpecifications'] = [{\n",
    "  'ResourceType': 'instance',\n",
    "  'Tags': [{\n",
    "    'Key': 'Name',\n",
    "    'Value': name\n",
    "  }]\n",
    "}]\n",
    "\n",
    "subnet = u.get_subnet()\n",
    "args['NetworkInterfaces'] = [{'SubnetId': subnet_id,\n",
    "                              'DeviceIndex': 0,\n",
    "                              'DeleteOnTermination': True,\n",
    "                              'InterfaceType':'efa',\n",
    "                              'Groups': [security_group_id]}]\n",
    "placement_specs = {'AvailabilityZone': zone}\n",
    "\n",
    "args['Placement'] = placement_specs\n",
    "args['Monitoring'] = {'Enabled': True}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "instances = ec2.create_instances(**args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.9.122'"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "boto3.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting boto3\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/8e/5e/d109e3ec9a6b43887ed9de1bd16141ad8e4442aaecff6f90fca05429c43a/boto3-1.9.159-py2.py3-none-any.whl (128kB)\n",
      "\u001b[K    100% |████████████████████████████████| 133kB 240kB/s ta 0:00:01\n",
      "\u001b[?25hCollecting botocore<1.13.0,>=1.12.159 (from boto3)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/52/bb/113f5083f145c41750d5b6f171a76c20f2e2b55a7bd9d02f55e849559619/botocore-1.12.159-py2.py3-none-any.whl (5.5MB)\n",
      "\u001b[K    100% |████████████████████████████████| 5.5MB 1.3MB/s ta 0:00:011\n",
      "\u001b[?25hRequirement already satisfied, skipping upgrade: jmespath<1.0.0,>=0.7.1 in /Users/yaroslavvb/anaconda3/envs/main/lib/python3.6/site-packages (from boto3) (0.9.4)\n",
      "Requirement already satisfied, skipping upgrade: s3transfer<0.3.0,>=0.2.0 in /Users/yaroslavvb/anaconda3/envs/main/lib/python3.6/site-packages (from boto3) (0.2.0)\n",
      "Requirement already satisfied, skipping upgrade: docutils>=0.10 in /Users/yaroslavvb/anaconda3/envs/main/lib/python3.6/site-packages (from botocore<1.13.0,>=1.12.159->boto3) (0.14)\n",
      "Requirement already satisfied, skipping upgrade: urllib3<1.26,>=1.20; python_version >= \"3.4\" in /Users/yaroslavvb/anaconda3/envs/main/lib/python3.6/site-packages (from botocore<1.13.0,>=1.12.159->boto3) (1.24.1)\n",
      "Requirement already satisfied, skipping upgrade: python-dateutil<3.0.0,>=2.1; python_version >= \"2.7\" in /Users/yaroslavvb/anaconda3/envs/main/lib/python3.6/site-packages (from botocore<1.13.0,>=1.12.159->boto3) (2.8.0)\n",
      "Requirement already satisfied, skipping upgrade: six>=1.5 in /Users/yaroslavvb/anaconda3/envs/main/lib/python3.6/site-packages (from python-dateutil<3.0.0,>=2.1; python_version >= \"2.7\"->botocore<1.13.0,>=1.12.159->boto3) (1.12.0)\n",
      "Installing collected packages: botocore, boto3\n",
      "  Found existing installation: botocore 1.12.122\n",
      "    Uninstalling botocore-1.12.122:\n",
      "      Successfully uninstalled botocore-1.12.122\n",
      "  Found existing installation: boto3 1.9.122\n",
      "    Uninstalling boto3-1.9.122:\n",
      "      Successfully uninstalled boto3-1.9.122\n",
      "Successfully installed boto3-1.9.159 botocore-1.12.159\n"
     ]
    }
   ],
   "source": [
    "!pip install -U boto3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Launching EFA2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "ename": "ClientError",
     "evalue": "An error occurred (InvalidParameterCombination) when calling the RunInstances operation: Network interfaces and an instance-level security groups may not be specified on the same request",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mClientError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-45-3ffda00fd701>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m \u001b[0mec2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mboto3\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSession\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresource\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'ec2'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 24\u001b[0;31m \u001b[0minstances\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mec2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcreate_instances\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/envs/main/lib/python3.6/site-packages/boto3/resources/factory.py\u001b[0m in \u001b[0;36mdo_action\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    518\u001b[0m             \u001b[0;31m# instance via ``self``.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    519\u001b[0m             \u001b[0;32mdef\u001b[0m \u001b[0mdo_action\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 520\u001b[0;31m                 \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0maction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    521\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    522\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'load'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/main/lib/python3.6/site-packages/boto3/resources/action.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, parent, *args, **kwargs)\u001b[0m\n\u001b[1;32m     81\u001b[0m                     operation_name, params)\n\u001b[1;32m     82\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 83\u001b[0;31m         \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmeta\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moperation_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     84\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     85\u001b[0m         \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdebug\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Response: %r'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/main/lib/python3.6/site-packages/botocore/client.py\u001b[0m in \u001b[0;36m_api_call\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    355\u001b[0m                     \"%s() only accepts keyword arguments.\" % py_operation_name)\n\u001b[1;32m    356\u001b[0m             \u001b[0;31m# The \"self\" in this scope is referring to the BaseClient.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 357\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_api_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moperation_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    358\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    359\u001b[0m         \u001b[0m_api_call\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpy_operation_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/main/lib/python3.6/site-packages/botocore/client.py\u001b[0m in \u001b[0;36m_make_api_call\u001b[0;34m(self, operation_name, api_params)\u001b[0m\n\u001b[1;32m    659\u001b[0m             \u001b[0merror_code\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparsed_response\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Error\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Code\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    660\u001b[0m             \u001b[0merror_class\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexceptions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_code\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merror_code\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 661\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0merror_class\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparsed_response\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moperation_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    662\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    663\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mparsed_response\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mClientError\u001b[0m: An error occurred (InvalidParameterCombination) when calling the RunInstances operation: Network interfaces and an instance-level security groups may not be specified on the same request"
     ]
    }
   ],
   "source": [
    "import boto3\n",
    "instance_type='p3dn.24xlarge'\n",
    "\n",
    "subnet_id='subnet-07e59d5b61d434fd4'\n",
    "name='efa-programmatic'\n",
    "image_id='ami-03147512dd1ffee88'\n",
    "security_group_id='sg-01494dff7a2a16363'\n",
    "\n",
    "args = {'ImageId': image_id,\n",
    "        'InstanceType': instance_type,\n",
    "        'MinCount': 1,\n",
    "        'MaxCount': 1,\n",
    "        'SecurityGroupIds': [security_group_id],\n",
    "        'KeyName': 'ncluster-yaroslavvb'}\n",
    "\n",
    "args['NetworkInterfaces'] = [{'SubnetId': subnet_id,\n",
    "                               'DeviceIndex': 0,\n",
    "                               'DeleteOnTermination': True,\n",
    "                               'InterfaceType':'efa',\n",
    "                               'Groups': [security_group_id]}]\n",
    "placement_specs = {'AvailabilityZone': 'us-east-1b'}\n",
    "\n",
    "ec2 = boto3.Session().resource('ec2')\n",
    "instances = ec2.create_instances(**args)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Checking disks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using region  us-east-1\n"
     ]
    }
   ],
   "source": [
    "instance=u.lookup_instance('demo')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "volumes = list(instance.volumes.all())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "volume = volumes[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'AttachTime': datetime.datetime(2019, 6, 23, 22, 34, 54, tzinfo=tzutc()),\n",
       "  'Device': '/dev/sda1',\n",
       "  'InstanceId': 'i-048f463c7019b821e',\n",
       "  'State': 'attached',\n",
       "  'VolumeId': 'vol-00af8a22453ab79f0',\n",
       "  'DeleteOnTermination': True}]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "volume.attachments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "500"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "volume.size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'vol-00af8a22453ab79f0'"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "volume.id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'gp2'"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "volume.volume_type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ec2.Volume.snapshotsCollectionManager(ec2.Volume(id='vol-00af8a22453ab79f0'), ec2.Snapshot)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "volume.snapshots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/dev/sda1'"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "volume.attachments[0]['Device']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " /dev/sda1        500        gp2          vol-00af8a22453ab79f0 \n",
      " /dev/xvda        500        gp2          vol-063f8a82cc1ebc300 \n"
     ]
    }
   ],
   "source": [
    "instance = u.lookup_instances('efa-image11')\n",
    "for volume in instance.volumes.all():\n",
    "    device = volume.attachments[0]['Device']\n",
    "    print(f\"{device:>10s} {volume.size:>10d} {volume.volume_type:>10s} {volume.id:>30s} \")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Checking security group"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using region  us-east-1\n"
     ]
    },
    {
     "ename": "AssertionError",
     "evalue": "Didn't find any instances matching 'pair'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAssertionError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-7-f6efe1139d19>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0minstance\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mu\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlookup_instance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'pair'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/Dropbox/git0/ncluster/ncluster/aws_util.py\u001b[0m in \u001b[0;36mlookup_instance\u001b[0;34m(fragment)\u001b[0m\n\u001b[1;32m    460\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    461\u001b[0m   \u001b[0minstances\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlookup_instances\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfragment\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 462\u001b[0;31m   \u001b[0;32massert\u001b[0m \u001b[0minstances\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34mf\"Didn't find any instances matching '{fragment}'\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    463\u001b[0m   \u001b[0mnames\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mget_name\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0minstances\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    464\u001b[0m   \u001b[0;32massert\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minstances\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34mf\"Found multiple instances matching fragment {fragment}: {','.join(names)}\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAssertionError\u001b[0m: Didn't find any instances matching 'pair'"
     ]
    }
   ],
   "source": [
    "instance=u.lookup_instance('pair')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "''"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "instance.placement_group.name"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Filtering instances by state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['MindscribeWeb-env',\n",
       " 'launch_interact-gpt1-ryanmckinney',\n",
       " 'voice-server-auto',\n",
       " 'voice-server-dev-server',\n",
       " 'voice-server-auto']"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[u.get_name(i) for i in ec2.instances.filter(Filters=[{'Name': 'instance-state-name', 'Values': ['running']}])]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Network"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Getting default VPC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ncluster import aws_util as u"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ec2.Vpc(id='vpc-0a007bc3954ed0c9d')"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "u.get_vpc()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "client = u.get_ec2_client()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Vpcs': [{'CidrBlock': '172.31.0.0/16',\n",
       "   'DhcpOptionsId': 'dopt-086557b289697d4b4',\n",
       "   'State': 'available',\n",
       "   'VpcId': 'vpc-d5c91aaf',\n",
       "   'OwnerId': '331439827203',\n",
       "   'InstanceTenancy': 'default',\n",
       "   'CidrBlockAssociationSet': [{'AssociationId': 'vpc-cidr-assoc-f6a70b9a',\n",
       "     'CidrBlock': '172.31.0.0/16',\n",
       "     'CidrBlockState': {'State': 'associated'}}],\n",
       "   'IsDefault': True},\n",
       "  {'CidrBlock': '192.168.0.0/16',\n",
       "   'DhcpOptionsId': 'dopt-086557b289697d4b4',\n",
       "   'State': 'available',\n",
       "   'VpcId': 'vpc-0a007bc3954ed0c9d',\n",
       "   'OwnerId': '331439827203',\n",
       "   'InstanceTenancy': 'default',\n",
       "   'CidrBlockAssociationSet': [{'AssociationId': 'vpc-cidr-assoc-0a30b722e61658fc8',\n",
       "     'CidrBlock': '192.168.0.0/16',\n",
       "     'CidrBlockState': {'State': 'associated'}}],\n",
       "   'IsDefault': False,\n",
       "   'Tags': [{'Key': 'Name', 'Value': 'ncluster'}]}],\n",
       " 'ResponseMetadata': {'RequestId': 'c0cb05a1-8ee4-4e30-a137-4f074ccbd48a',\n",
       "  'HTTPStatusCode': 200,\n",
       "  'HTTPHeaders': {'content-type': 'text/xml;charset=UTF-8',\n",
       "   'content-length': '1902',\n",
       "   'date': 'Mon, 03 Jun 2019 20:46:11 GMT',\n",
       "   'server': 'AmazonEC2'},\n",
       "  'RetryAttempts': 0}}"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "client.describe_vpcs()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ec2.Vpc(id='vpc-d5c91aaf')"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "u.get_default_vpc()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create EFA-friendly security group"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. check existing security group that works\n",
    "# 2. create new security group, double check that it looks the same"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ['AWS_DEFAULT_REGION']='us-east-1'\n",
    "importlib.reload(boto3)\n",
    "d = u.get_security_group_dict()\n",
    "good_group=d['ncluster']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'FromPort': 6006,\n",
       "  'IpProtocol': 'tcp',\n",
       "  'IpRanges': [{'CidrIp': '0.0.0.0/0'}],\n",
       "  'Ipv6Ranges': [],\n",
       "  'PrefixListIds': [],\n",
       "  'ToPort': 6016,\n",
       "  'UserIdGroupPairs': []},\n",
       " {'FromPort': 0,\n",
       "  'IpProtocol': 'tcp',\n",
       "  'IpRanges': [],\n",
       "  'Ipv6Ranges': [],\n",
       "  'PrefixListIds': [],\n",
       "  'ToPort': 65535,\n",
       "  'UserIdGroupPairs': [{'GroupId': 'sg-01494dff7a2a16363',\n",
       "    'UserId': '331439827203'},\n",
       "   {'GroupId': 'sg-09a65a86d87d54560',\n",
       "    'PeeringStatus': 'active',\n",
       "    'UserId': '331439827203',\n",
       "    'VpcId': 'vpc-0a007bc3954ed0c9d',\n",
       "    'VpcPeeringConnectionId': 'pcx-087644c72415b62e1'}]},\n",
       " {'FromPort': 8888,\n",
       "  'IpProtocol': 'tcp',\n",
       "  'IpRanges': [{'CidrIp': '0.0.0.0/0'}],\n",
       "  'Ipv6Ranges': [],\n",
       "  'PrefixListIds': [],\n",
       "  'ToPort': 8899,\n",
       "  'UserIdGroupPairs': []},\n",
       " {'IpProtocol': '-1',\n",
       "  'IpRanges': [],\n",
       "  'Ipv6Ranges': [],\n",
       "  'PrefixListIds': [],\n",
       "  'UserIdGroupPairs': [{'Description': 'efa',\n",
       "    'GroupId': 'sg-01494dff7a2a16363',\n",
       "    'UserId': '331439827203'}]},\n",
       " {'FromPort': 60000,\n",
       "  'IpProtocol': 'udp',\n",
       "  'IpRanges': [{'CidrIp': '0.0.0.0/0'}],\n",
       "  'Ipv6Ranges': [],\n",
       "  'PrefixListIds': [],\n",
       "  'ToPort': 61000,\n",
       "  'UserIdGroupPairs': []},\n",
       " {'FromPort': 22,\n",
       "  'IpProtocol': 'tcp',\n",
       "  'IpRanges': [{'CidrIp': '0.0.0.0/0'}],\n",
       "  'Ipv6Ranges': [],\n",
       "  'PrefixListIds': [],\n",
       "  'ToPort': 22,\n",
       "  'UserIdGroupPairs': []},\n",
       " {'FromPort': 6379,\n",
       "  'IpProtocol': 'tcp',\n",
       "  'IpRanges': [{'CidrIp': '0.0.0.0/0'}],\n",
       "  'Ipv6Ranges': [],\n",
       "  'PrefixListIds': [],\n",
       "  'ToPort': 6379,\n",
       "  'UserIdGroupPairs': []},\n",
       " {'FromPort': 2049,\n",
       "  'IpProtocol': 'udp',\n",
       "  'IpRanges': [{'CidrIp': '0.0.0.0/0', 'Description': 'NFS-udp'}],\n",
       "  'Ipv6Ranges': [{'CidrIpv6': '::/0', 'Description': 'NFS-udp'}],\n",
       "  'PrefixListIds': [],\n",
       "  'ToPort': 2049,\n",
       "  'UserIdGroupPairs': []},\n",
       " {'FromPort': 0,\n",
       "  'IpProtocol': 'udp',\n",
       "  'IpRanges': [],\n",
       "  'Ipv6Ranges': [],\n",
       "  'PrefixListIds': [],\n",
       "  'ToPort': 65535,\n",
       "  'UserIdGroupPairs': [{'GroupId': 'sg-01494dff7a2a16363',\n",
       "    'UserId': '331439827203'},\n",
       "   {'GroupId': 'sg-09a65a86d87d54560',\n",
       "    'PeeringStatus': 'active',\n",
       "    'UserId': '331439827203',\n",
       "    'VpcId': 'vpc-0a007bc3954ed0c9d',\n",
       "    'VpcPeeringConnectionId': 'pcx-087644c72415b62e1'}]},\n",
       " {'FromPort': -1,\n",
       "  'IpProtocol': 'icmp',\n",
       "  'IpRanges': [{'CidrIp': '0.0.0.0/0'}],\n",
       "  'Ipv6Ranges': [],\n",
       "  'PrefixListIds': [],\n",
       "  'ToPort': -1,\n",
       "  'UserIdGroupPairs': [{'GroupId': 'sg-01494dff7a2a16363',\n",
       "    'UserId': '331439827203'},\n",
       "   {'GroupId': 'sg-09a65a86d87d54560',\n",
       "    'PeeringStatus': 'active',\n",
       "    'UserId': '331439827203',\n",
       "    'VpcId': 'vpc-0a007bc3954ed0c9d',\n",
       "    'VpcPeeringConnectionId': 'pcx-087644c72415b62e1'}]},\n",
       " {'FromPort': 2049,\n",
       "  'IpProtocol': 'tcp',\n",
       "  'IpRanges': [{'CidrIp': '0.0.0.0/0', 'Description': 'NFS'}],\n",
       "  'Ipv6Ranges': [{'CidrIpv6': '::/0', 'Description': 'NFS'}],\n",
       "  'PrefixListIds': [],\n",
       "  'ToPort': 2049,\n",
       "  'UserIdGroupPairs': []}]"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "group.ip_permissions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ['AWS_DEFAULT_REGION']='us-west-2'\n",
    "importlib.reload(boto3)\n",
    "d = u.get_security_group_dict()\n",
    "bad_group=d['ncluster']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'sg-01494dff7a2a16363'"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "good_group.id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'sg-0481f4230bc91ab83'"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bad_group.id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'sg-01494dff7a2a16363'"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "good_group.id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'FromPort': 6006,\n",
       "  'IpProtocol': 'tcp',\n",
       "  'IpRanges': [{'CidrIp': '0.0.0.0/0'}],\n",
       "  'Ipv6Ranges': [],\n",
       "  'PrefixListIds': [],\n",
       "  'ToPort': 6016,\n",
       "  'UserIdGroupPairs': []},\n",
       " {'FromPort': 0,\n",
       "  'IpProtocol': 'tcp',\n",
       "  'IpRanges': [],\n",
       "  'Ipv6Ranges': [],\n",
       "  'PrefixListIds': [],\n",
       "  'ToPort': 65535,\n",
       "  'UserIdGroupPairs': [{'GroupId': 'sg-01494dff7a2a16363',\n",
       "    'UserId': '331439827203'},\n",
       "   {'GroupId': 'sg-09a65a86d87d54560',\n",
       "    'PeeringStatus': 'active',\n",
       "    'UserId': '331439827203',\n",
       "    'VpcId': 'vpc-0a007bc3954ed0c9d',\n",
       "    'VpcPeeringConnectionId': 'pcx-087644c72415b62e1'}]},\n",
       " {'FromPort': 8888,\n",
       "  'IpProtocol': 'tcp',\n",
       "  'IpRanges': [{'CidrIp': '0.0.0.0/0'}],\n",
       "  'Ipv6Ranges': [],\n",
       "  'PrefixListIds': [],\n",
       "  'ToPort': 8899,\n",
       "  'UserIdGroupPairs': []},\n",
       " {'IpProtocol': '-1',\n",
       "  'IpRanges': [],\n",
       "  'Ipv6Ranges': [],\n",
       "  'PrefixListIds': [],\n",
       "  'UserIdGroupPairs': [{'Description': 'efa',\n",
       "    'GroupId': 'sg-01494dff7a2a16363',\n",
       "    'UserId': '331439827203'}]},\n",
       " {'FromPort': 60000,\n",
       "  'IpProtocol': 'udp',\n",
       "  'IpRanges': [{'CidrIp': '0.0.0.0/0'}],\n",
       "  'Ipv6Ranges': [],\n",
       "  'PrefixListIds': [],\n",
       "  'ToPort': 61000,\n",
       "  'UserIdGroupPairs': []},\n",
       " {'FromPort': 22,\n",
       "  'IpProtocol': 'tcp',\n",
       "  'IpRanges': [{'CidrIp': '0.0.0.0/0'}],\n",
       "  'Ipv6Ranges': [],\n",
       "  'PrefixListIds': [],\n",
       "  'ToPort': 22,\n",
       "  'UserIdGroupPairs': []},\n",
       " {'FromPort': 6379,\n",
       "  'IpProtocol': 'tcp',\n",
       "  'IpRanges': [{'CidrIp': '0.0.0.0/0'}],\n",
       "  'Ipv6Ranges': [],\n",
       "  'PrefixListIds': [],\n",
       "  'ToPort': 6379,\n",
       "  'UserIdGroupPairs': []},\n",
       " {'FromPort': 2049,\n",
       "  'IpProtocol': 'udp',\n",
       "  'IpRanges': [{'CidrIp': '0.0.0.0/0', 'Description': 'NFS-udp'}],\n",
       "  'Ipv6Ranges': [{'CidrIpv6': '::/0', 'Description': 'NFS-udp'}],\n",
       "  'PrefixListIds': [],\n",
       "  'ToPort': 2049,\n",
       "  'UserIdGroupPairs': []},\n",
       " {'FromPort': 0,\n",
       "  'IpProtocol': 'udp',\n",
       "  'IpRanges': [],\n",
       "  'Ipv6Ranges': [],\n",
       "  'PrefixListIds': [],\n",
       "  'ToPort': 65535,\n",
       "  'UserIdGroupPairs': [{'GroupId': 'sg-01494dff7a2a16363',\n",
       "    'UserId': '331439827203'},\n",
       "   {'GroupId': 'sg-09a65a86d87d54560',\n",
       "    'PeeringStatus': 'active',\n",
       "    'UserId': '331439827203',\n",
       "    'VpcId': 'vpc-0a007bc3954ed0c9d',\n",
       "    'VpcPeeringConnectionId': 'pcx-087644c72415b62e1'}]},\n",
       " {'FromPort': -1,\n",
       "  'IpProtocol': 'icmp',\n",
       "  'IpRanges': [{'CidrIp': '0.0.0.0/0'}],\n",
       "  'Ipv6Ranges': [],\n",
       "  'PrefixListIds': [],\n",
       "  'ToPort': -1,\n",
       "  'UserIdGroupPairs': [{'GroupId': 'sg-01494dff7a2a16363',\n",
       "    'UserId': '331439827203'},\n",
       "   {'GroupId': 'sg-09a65a86d87d54560',\n",
       "    'PeeringStatus': 'active',\n",
       "    'UserId': '331439827203',\n",
       "    'VpcId': 'vpc-0a007bc3954ed0c9d',\n",
       "    'VpcPeeringConnectionId': 'pcx-087644c72415b62e1'}]},\n",
       " {'FromPort': 2049,\n",
       "  'IpProtocol': 'tcp',\n",
       "  'IpRanges': [{'CidrIp': '0.0.0.0/0', 'Description': 'NFS'}],\n",
       "  'Ipv6Ranges': [{'CidrIpv6': '::/0', 'Description': 'NFS'}],\n",
       "  'PrefixListIds': [],\n",
       "  'ToPort': 2049,\n",
       "  'UserIdGroupPairs': []}]"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "good_group.ip_permissions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### What correct rule looks like"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "{'IpProtocol': '-1',\n",
    "  'IpRanges': [],\n",
    "  'Ipv6Ranges': [],\n",
    "  'PrefixListIds': [],\n",
    "  'UserIdGroupPairs': [{'Description': 'efa',\n",
    "    'GroupId': 'sg-01494dff7a2a16363',\n",
    "    'UserId': '331439827203'}]},"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'sg-069bb8d7ea3950cfc'"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "group.id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\r\n",
      "    \"GroupId\": \"sg-0d9123fbedb51266f\"\r\n",
      "}\r\n"
     ]
    }
   ],
   "source": [
    "!aws ec2 create-security-group --group-name MySecurityGroup --description \"My security group\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'IpProtocol': '-1',\n",
       "  'IpRanges': [{'CidrIp': '0.0.0.0/0'}],\n",
       "  'Ipv6Ranges': [],\n",
       "  'PrefixListIds': [],\n",
       "  'UserIdGroupPairs': [{'Description': 'efa',\n",
       "    'GroupId': 'sg-01494dff7a2a16363',\n",
       "    'UserId': '331439827203'}]}]"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "good_group.ip_permissions_egress"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "{'IpProtocol': '-1',\n",
    "  'IpRanges': [],\n",
    "  'Ipv6Ranges': [],\n",
    "  'PrefixListIds': [],\n",
    "  'UserIdGroupPairs': [{'Description': 'efa',\n",
    "    'GroupId': 'sg-01494dff7a2a16363',\n",
    "    'UserId': '331439827203'}]},"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'IpProtocol': '-1',\n",
       "  'IpRanges': [{'CidrIp': '0.0.0.0/0'}],\n",
       "  'Ipv6Ranges': [],\n",
       "  'PrefixListIds': [],\n",
       "  'UserIdGroupPairs': []}]"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bad_group.ip_permissions_egress"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'IpProtocol': '-1',\n",
       "  'IpRanges': [{'CidrIp': '0.0.0.0/0'}],\n",
       "  'Ipv6Ranges': [],\n",
       "  'PrefixListIds': [],\n",
       "  'UserIdGroupPairs': [{'Description': 'efa',\n",
       "    'GroupId': 'sg-0481f4230bc91ab83',\n",
       "    'UserId': '331439827203'}]}]"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bad_group.reload()\n",
    "bad_group.ip_permissions_egress"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "other_security_group=bad_group"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "rule = {\n",
    "\"IpProtocol\": \"-1\",\n",
    "\"PrefixListIds\": [],\n",
    "'UserIdGroupPairs': [{'Description': 'efa',\n",
    "                      'GroupId': other_security_group.id,\n",
    "                      'UserId': u.get_account_number()}]\n",
    "}\n",
    "bad_group.authorize_egress(IpPermissions=[rule])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ResponseMetadata': {'RequestId': 'c537d59e-f7ca-432b-b6f7-434e0eef6715',\n",
       "  'HTTPStatusCode': 200,\n",
       "  'HTTPHeaders': {'content-type': 'text/xml;charset=UTF-8',\n",
       "   'content-length': '257',\n",
       "   'date': 'Fri, 07 Jun 2019 22:34:25 GMT',\n",
       "   'server': 'AmazonEC2'},\n",
       "  'RetryAttempts': 0}}"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bad_group.authorize_egress(IpPermissions=[rule])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "!aws ec2 authorize-security-group-ingress --group-id sg-0d9123fbedb51266f --protocol all --port all --cidr 0.0.0.0/0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\r\n",
      "    \"SecurityGroups\": [\r\n",
      "        {\r\n",
      "            \"Description\": \"My security group\",\r\n",
      "            \"GroupName\": \"MySecurityGroup\",\r\n",
      "            \"IpPermissions\": [\r\n",
      "                {\r\n",
      "                    \"IpProtocol\": \"-1\",\r\n",
      "                    \"IpRanges\": [\r\n",
      "                        {\r\n",
      "                            \"CidrIp\": \"0.0.0.0/0\"\r\n",
      "                        }\r\n",
      "                    ],\r\n",
      "                    \"Ipv6Ranges\": [],\r\n",
      "                    \"PrefixListIds\": [],\r\n",
      "                    \"UserIdGroupPairs\": []\r\n",
      "                }\r\n",
      "            ],\r\n",
      "            \"OwnerId\": \"331439827203\",\r\n",
      "            \"GroupId\": \"sg-0d9123fbedb51266f\",\r\n",
      "            \"IpPermissionsEgress\": [\r\n",
      "                {\r\n",
      "                    \"IpProtocol\": \"-1\",\r\n",
      "                    \"IpRanges\": [\r\n",
      "                        {\r\n",
      "                            \"CidrIp\": \"0.0.0.0/0\"\r\n",
      "                        }\r\n",
      "                    ],\r\n",
      "                    \"Ipv6Ranges\": [],\r\n",
      "                    \"PrefixListIds\": [],\r\n",
      "                    \"UserIdGroupPairs\": []\r\n",
      "                }\r\n",
      "            ],\r\n",
      "            \"VpcId\": \"vpc-d5c91aaf\"\r\n",
      "        }\r\n",
      "    ]\r\n",
      "}\r\n"
     ]
    }
   ],
   "source": [
    "!aws ec2 describe-security-groups --group-names MySecurityGroup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create one through boto3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "vpc = u.get_default_vpc()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "security_group_name='MySecurityGroup3'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "ec2 = u.get_ec2_resource()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "group2=ec2.create_security_group(GroupName=security_group_name,\n",
    "                          Description=security_group_name,\n",
    "                          VpcId=vpc.id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'sg-0f8536aa45d3ab3ee'"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "group2.id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "rule={'IpProtocol': '-1',\n",
    "      'IpRanges': [],\n",
    "      'Ipv6Ranges': [],\n",
    "      'PrefixListIds': [],\n",
    "      'UserIdGroupPairs': [{'Description': 'efa',\n",
    "        'GroupId': group2.id,\n",
    "        'UserId': u.get_account_number()}]\n",
    "     },"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "rule2= {\n",
    "                    \"IpProtocol\": \"-1\",\n",
    "                    \"IpRanges\": [\n",
    "                        {\n",
    "                            \"CidrIp\": \"0.0.0.0/0\"\n",
    "                        }\n",
    "                    ],\n",
    "                    \"Ipv6Ranges\": [],\n",
    "                    \"PrefixListIds\": [],\n",
    "                    'UserIdGroupPairs': [{'Description': 'efa',\n",
    "        'GroupId': group2.id,\n",
    "        'UserId': u.get_account_number()}]\n",
    "                }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ncluster'"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "group.description"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vpc.is_default"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.9.159'"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "boto3.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pip install"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "response = group2.authorize_ingress(IpPermissions=[rule2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ResponseMetadata': {'RequestId': 'b99465f3-fc28-41c7-99b0-b0e73be00641',\n",
       "  'HTTPStatusCode': 200,\n",
       "  'HTTPHeaders': {'content-type': 'text/xml;charset=UTF-8',\n",
       "   'content-length': '259',\n",
       "   'date': 'Fri, 07 Jun 2019 00:52:03 GMT',\n",
       "   'server': 'AmazonEC2'},\n",
       "  'RetryAttempts': 0}}"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "response"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Account-level"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Limits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "client = u.get_ec2_client()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'AccountAttributes': [{'AttributeName': 'supported-platforms',\n",
       "   'AttributeValues': [{'AttributeValue': 'VPC'}]},\n",
       "  {'AttributeName': 'vpc-max-security-groups-per-interface',\n",
       "   'AttributeValues': [{'AttributeValue': '5'}]},\n",
       "  {'AttributeName': 'max-elastic-ips',\n",
       "   'AttributeValues': [{'AttributeValue': '5'}]},\n",
       "  {'AttributeName': 'max-instances',\n",
       "   'AttributeValues': [{'AttributeValue': '300'}]},\n",
       "  {'AttributeName': 'vpc-max-elastic-ips',\n",
       "   'AttributeValues': [{'AttributeValue': '5'}]},\n",
       "  {'AttributeName': 'default-vpc',\n",
       "   'AttributeValues': [{'AttributeValue': 'vpc-d5c91aaf'}]}],\n",
       " 'ResponseMetadata': {'RequestId': 'c576523f-f2e6-436f-ba84-f96edadac907',\n",
       "  'HTTPStatusCode': 200,\n",
       "  'HTTPHeaders': {'content-type': 'text/xml;charset=UTF-8',\n",
       "   'content-length': '1858',\n",
       "   'date': 'Fri, 21 Jun 2019 15:46:24 GMT',\n",
       "   'server': 'AmazonEC2'},\n",
       "  'RetryAttempts': 0}}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "client.describe_account_attributes()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Account name"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- https://docs.aws.amazon.com/IAM/latest/APIReference/API_ListAccountAliases.html \n",
    "- https://docs.aws.amazon.com/cli/latest/reference/iam/list-account-aliases.html "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['keyapp']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "go_mind()\n",
    "importlib.reload(boto3)\n",
    "iam_client = boto3.client('iam')\n",
    "iam_client.list_account_aliases()\n",
    "iam_client.list_account_aliases()['AccountAliases']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'yaroslavvb'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import getpass\n",
    "getpass.getuser()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'yaroslavvb'"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "iam_client.get_user()['User']['UserName']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'AccountAliases': ['yacht'],\n",
       " 'IsTruncated': False,\n",
       " 'ResponseMetadata': {'RequestId': '8901298c-8a1d-11e9-a205-2d1e7619d282',\n",
       "  'HTTPStatusCode': 200,\n",
       "  'HTTPHeaders': {'x-amzn-requestid': '8901298c-8a1d-11e9-a205-2d1e7619d282',\n",
       "   'content-type': 'text/xml',\n",
       "   'content-length': '384',\n",
       "   'date': 'Sat, 08 Jun 2019 18:45:06 GMT'},\n",
       "  'RetryAttempts': 0}}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# yaro\n",
    "importlib.reload(boto3)\n",
    "go_yaro()\n",
    "iam_client = boto3.client('iam')\n",
    "iam_client.list_account_aliases()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'AccountAliases': ['antics'],\n",
       " 'IsTruncated': False,\n",
       " 'ResponseMetadata': {'RequestId': '8e8fe37b-8a1d-11e9-831f-2d10654543da',\n",
       "  'HTTPStatusCode': 200,\n",
       "  'HTTPHeaders': {'x-amzn-requestid': '8e8fe37b-8a1d-11e9-831f-2d10654543da',\n",
       "   'content-type': 'text/xml',\n",
       "   'content-length': '385',\n",
       "   'date': 'Sat, 08 Jun 2019 18:45:15 GMT'},\n",
       "  'RetryAttempts': 0}}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# andrew\n",
    "go_andrew()\n",
    "importlib.reload(boto3)\n",
    "iam_client = boto3.client('iam')\n",
    "iam_client.list_account_aliases()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Account': '979440106638',\n",
      " 'Arn': 'arn:aws:iam::979440106638:user/yaroslavvb',\n",
      " 'ResponseMetadata': {'HTTPHeaders': {'content-length': '407',\n",
      "                                      'content-type': 'text/xml',\n",
      "                                      'date': 'Sat, 08 Jun 2019 18:45:44 GMT',\n",
      "                                      'x-amzn-requestid': '9fc0fc18-8a1d-11e9-8f18-af3e36405676'},\n",
      "                      'HTTPStatusCode': 200,\n",
      "                      'RequestId': '9fc0fc18-8a1d-11e9-8f18-af3e36405676',\n",
      "                      'RetryAttempts': 0},\n",
      " 'UserId': 'AIDA6ICZN7SHO54UXNLS3'}\n"
     ]
    }
   ],
   "source": [
    "# mindscribe\n",
    "go_mind()\n",
    "importlib.reload(boto3)\n",
    "sts_client = boto3.client('sts')\n",
    "pprint(sts_client.get_caller_identity())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Valid zones for p3dn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Valid zones for  yacht p3dn.24xlarge\n",
      "eu-west-1a: 9.739100\n",
      "eu-west-1b: 9.739100\n",
      "ap-northeast-1b: 42.783000\n",
      "us-east-1b: 9.363600\n",
      "us-east-1d: 9.363600\n",
      "us-west-2b: 9.363600\n",
      "us-west-2c: 9.363600\n",
      "{'eu-west-1a': '9.739100', 'eu-west-1b': '9.739100', 'ap-northeast-1b': '42.783000', 'us-east-1b': '9.363600', 'us-east-1d': '9.363600', 'us-west-2b': '9.363600', 'us-west-2c': '9.363600'}\n"
     ]
    }
   ],
   "source": [
    "import boto3\n",
    "importlib.reload(boto3)\n",
    "\n",
    "def valid_zones(prefix, init_func, instanceType):\n",
    "    product = 'Linux/UNIX (Amazon VPC)'\n",
    "    result = {}\n",
    "    print(\"Valid zones for \", prefix, instanceType)\n",
    "    for region in boto3.client('ec2').describe_regions()['Regions']:\n",
    "        importlib.reload(boto3)\n",
    "        client = boto3.client('ec2', region_name=region['RegionName'])\n",
    "        for zone in [z['ZoneName'] for z in client.describe_availability_zones()['AvailabilityZones'] if z['State'] == 'available']:\n",
    "            try:\n",
    "                price = client.describe_spot_price_history(InstanceTypes=[instanceType],\n",
    "                                                           MaxResults=1,\n",
    "                                                           ProductDescriptions=[product],\n",
    "                                                           AvailabilityZone=zone)['SpotPriceHistory'][0]['SpotPrice']\n",
    "                print(\"%s: %s\" % (zone, price))\n",
    "                result[zone] = price\n",
    "            except IndexError: pass\n",
    "    return result\n",
    "\n",
    "print(valid_zones('yacht', go_yacht, 'p3dn.24xlarge'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Valid zones for  yacht p3.16xlarge\n",
      "eu-west-2a: 8.613600\n",
      "eu-west-2b: 8.613600\n",
      "eu-west-1a: 7.932000\n",
      "eu-west-1b: 7.932000\n",
      "eu-west-1c: 26.440000\n",
      "ap-northeast-2a: 11.954400\n",
      "ap-northeast-2c: 11.954400\n",
      "ap-northeast-1a: 10.065600\n",
      "ap-northeast-1c: 10.065600\n",
      "ca-central-1b: 8.078400\n",
      "ap-southeast-1a: 10.161600\n",
      "ap-southeast-1b: 10.161600\n",
      "ap-southeast-2b: 10.161600\n",
      "eu-central-1a: 9.175200\n",
      "eu-central-1b: 9.175200\n",
      "us-east-1a: 24.480000\n",
      "us-east-1b: 7.344000\n",
      "us-east-1c: 7.344000\n",
      "us-east-1d: 7.344000\n",
      "us-east-1f: 7.344000\n",
      "us-east-2a: 7.344000\n",
      "us-east-2b: 7.344000\n",
      "us-west-2a: 7.344000\n",
      "us-west-2b: 7.344000\n",
      "us-west-2c: 7.344000\n",
      "Valid zones for  yacht p3dn.24xlarge\n",
      "eu-west-1a: 9.739100\n",
      "eu-west-1c: 9.739100\n",
      "ap-northeast-1a: 12.834900\n",
      "us-east-1b: 9.363600\n",
      "us-east-1c: 9.363600\n",
      "us-west-2b: 9.363600\n",
      "us-west-2c: 9.363600\n",
      "Valid zones for  antics p3.16xlarge\n",
      "eu-west-2a: 8.613600\n",
      "eu-west-2b: 8.613600\n",
      "eu-west-1a: 7.932000\n",
      "eu-west-1b: 7.932000\n",
      "eu-west-1c: 26.440000\n",
      "ap-northeast-2a: 11.954400\n",
      "ap-northeast-2c: 11.954400\n",
      "ap-northeast-1a: 10.065600\n",
      "ap-northeast-1c: 10.065600\n",
      "ca-central-1b: 8.078400\n",
      "ap-southeast-1a: 10.161600\n",
      "ap-southeast-1b: 10.161600\n",
      "ap-southeast-2b: 10.161600\n",
      "eu-central-1a: 9.175200\n",
      "eu-central-1b: 9.175200\n",
      "us-east-1a: 24.480000\n",
      "us-east-1b: 7.344000\n",
      "us-east-1c: 7.344000\n",
      "us-east-1d: 7.344000\n",
      "us-east-1f: 7.344000\n",
      "us-east-2a: 7.344000\n",
      "us-east-2b: 7.344000\n",
      "us-west-2a: 7.344000\n",
      "us-west-2b: 7.344000\n",
      "us-west-2c: 7.344000\n",
      "Valid zones for  antics p3dn.24xlarge\n",
      "eu-west-1a: 9.739100\n",
      "eu-west-1c: 9.739100\n",
      "ap-northeast-1a: 12.834900\n",
      "us-east-1b: 9.363600\n",
      "us-east-1c: 9.363600\n",
      "us-west-2b: 9.363600\n",
      "us-west-2c: 9.363600\n",
      "Valid zones for  keyapp p3.16xlarge\n",
      "eu-west-2a: 8.613600\n",
      "eu-west-2b: 8.613600\n",
      "eu-west-1a: 7.932000\n",
      "eu-west-1b: 7.932000\n",
      "eu-west-1c: 26.440000\n",
      "ap-northeast-2a: 11.954400\n",
      "ap-northeast-2c: 11.954400\n",
      "ap-northeast-1a: 10.065600\n",
      "ap-northeast-1c: 10.065600\n",
      "ca-central-1b: 8.078400\n",
      "ap-southeast-1a: 10.161600\n",
      "ap-southeast-1b: 10.161600\n",
      "ap-southeast-2b: 10.161600\n",
      "eu-central-1a: 9.175200\n",
      "eu-central-1b: 9.175200\n",
      "us-east-1a: 24.480000\n",
      "us-east-1b: 7.344000\n",
      "us-east-1c: 7.344000\n",
      "us-east-1d: 7.344000\n",
      "us-east-1f: 7.344000\n",
      "us-east-2a: 7.344000\n",
      "us-east-2b: 7.344000\n",
      "us-west-2a: 7.344000\n",
      "us-west-2b: 7.344000\n",
      "us-west-2c: 7.344000\n",
      "Valid zones for  keyapp p3dn.24xlarge\n",
      "eu-west-1a: 9.739100\n",
      "eu-west-1c: 9.739100\n",
      "ap-northeast-1a: 12.834900\n",
      "us-east-1b: 9.363600\n",
      "us-east-1c: 9.363600\n",
      "us-west-2b: 9.363600\n",
      "us-west-2c: 9.363600\n",
      "{('yacht', 'p3.16xlarge'): {'eu-west-2a': '8.613600', 'eu-west-2b': '8.613600', 'eu-west-1a': '7.932000', 'eu-west-1b': '7.932000', 'eu-west-1c': '26.440000', 'ap-northeast-2a': '11.954400', 'ap-northeast-2c': '11.954400', 'ap-northeast-1a': '10.065600', 'ap-northeast-1c': '10.065600', 'ca-central-1b': '8.078400', 'ap-southeast-1a': '10.161600', 'ap-southeast-1b': '10.161600', 'ap-southeast-2b': '10.161600', 'eu-central-1a': '9.175200', 'eu-central-1b': '9.175200', 'us-east-1a': '24.480000', 'us-east-1b': '7.344000', 'us-east-1c': '7.344000', 'us-east-1d': '7.344000', 'us-east-1f': '7.344000', 'us-east-2a': '7.344000', 'us-east-2b': '7.344000', 'us-west-2a': '7.344000', 'us-west-2b': '7.344000', 'us-west-2c': '7.344000'}, ('yacht', 'p3dn.24xlarge'): {'eu-west-1a': '9.739100', 'eu-west-1c': '9.739100', 'ap-northeast-1a': '12.834900', 'us-east-1b': '9.363600', 'us-east-1c': '9.363600', 'us-west-2b': '9.363600', 'us-west-2c': '9.363600'}, ('antics', 'p3.16xlarge'): {'eu-west-2a': '8.613600', 'eu-west-2b': '8.613600', 'eu-west-1a': '7.932000', 'eu-west-1b': '7.932000', 'eu-west-1c': '26.440000', 'ap-northeast-2a': '11.954400', 'ap-northeast-2c': '11.954400', 'ap-northeast-1a': '10.065600', 'ap-northeast-1c': '10.065600', 'ca-central-1b': '8.078400', 'ap-southeast-1a': '10.161600', 'ap-southeast-1b': '10.161600', 'ap-southeast-2b': '10.161600', 'eu-central-1a': '9.175200', 'eu-central-1b': '9.175200', 'us-east-1a': '24.480000', 'us-east-1b': '7.344000', 'us-east-1c': '7.344000', 'us-east-1d': '7.344000', 'us-east-1f': '7.344000', 'us-east-2a': '7.344000', 'us-east-2b': '7.344000', 'us-west-2a': '7.344000', 'us-west-2b': '7.344000', 'us-west-2c': '7.344000'}, ('antics', 'p3dn.24xlarge'): {'eu-west-1a': '9.739100', 'eu-west-1c': '9.739100', 'ap-northeast-1a': '12.834900', 'us-east-1b': '9.363600', 'us-east-1c': '9.363600', 'us-west-2b': '9.363600', 'us-west-2c': '9.363600'}, ('keyapp', 'p3.16xlarge'): {'eu-west-2a': '8.613600', 'eu-west-2b': '8.613600', 'eu-west-1a': '7.932000', 'eu-west-1b': '7.932000', 'eu-west-1c': '26.440000', 'ap-northeast-2a': '11.954400', 'ap-northeast-2c': '11.954400', 'ap-northeast-1a': '10.065600', 'ap-northeast-1c': '10.065600', 'ca-central-1b': '8.078400', 'ap-southeast-1a': '10.161600', 'ap-southeast-1b': '10.161600', 'ap-southeast-2b': '10.161600', 'eu-central-1a': '9.175200', 'eu-central-1b': '9.175200', 'us-east-1a': '24.480000', 'us-east-1b': '7.344000', 'us-east-1c': '7.344000', 'us-east-1d': '7.344000', 'us-east-1f': '7.344000', 'us-east-2a': '7.344000', 'us-east-2b': '7.344000', 'us-west-2a': '7.344000', 'us-west-2b': '7.344000', 'us-west-2c': '7.344000'}, ('keyapp', 'p3dn.24xlarge'): {'eu-west-1a': '9.739100', 'eu-west-1c': '9.739100', 'ap-northeast-1a': '12.834900', 'us-east-1b': '9.363600', 'us-east-1c': '9.363600', 'us-west-2b': '9.363600', 'us-west-2c': '9.363600'}}\n"
     ]
    }
   ],
   "source": [
    "# get valid zones for accounts\n",
    "valid_zone_dict = {}\n",
    "for (prefix, init) in [('yacht', go_yacht), ('antics', go_antics), ('keyapp', go_keyapp)]:\n",
    "    importlib.reload(boto3)\n",
    "\n",
    "    for instance_type in ['p3.16xlarge', 'p3dn.24xlarge']:\n",
    "        valid_zone_dict[(prefix, instance_type)] = valid_zones(prefix, init, instance_type)\n",
    "print(valid_zone_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = {('yacht', 'p3.16xlarge'): {'eu-west-2a': '8.613600', 'eu-west-2b': '8.613600', 'eu-west-1a': '26.440000', 'eu-west-1b': '7.932000', 'eu-west-1c': '7.932000', 'ap-northeast-2a': '11.954400', 'ap-northeast-2c': '11.954400', 'ap-northeast-1b': '10.065600', 'ap-northeast-1c': '10.065600', 'ca-central-1b': '8.078400', 'ap-southeast-1a': '10.161600', 'ap-southeast-1b': '10.161600', 'ap-southeast-2a': '10.161600', 'eu-central-1a': '9.175200', 'eu-central-1b': '9.175200', 'us-east-1a': '24.480000', 'us-east-1b': '7.344000', 'us-east-1d': '7.344000', 'us-east-1e': '7.344000', 'us-east-1f': '7.344000', 'us-east-2a': '7.471400', 'us-east-2b': '7.344000', 'us-west-2a': '7.344000', 'us-west-2b': '7.344000', 'us-west-2c': '7.344000'}, ('yacht', 'p3dn.24xlarge'): {'eu-west-1a': '9.739100', 'eu-west-1b': '9.739100', 'ap-northeast-1b': '12.834900', 'us-east-1b': '9.363600', 'us-east-1d': '9.363600', 'us-west-2b': '9.363600', 'us-west-2c': '9.363600'}, ('antics', 'p3.16xlarge'): {'eu-west-2a': '8.613600', 'eu-west-2b': '8.613600', 'eu-west-1a': '26.440000', 'eu-west-1b': '7.932000', 'eu-west-1c': '7.932000', 'ap-northeast-2a': '11.954400', 'ap-northeast-2c': '11.954400', 'ap-northeast-1b': '10.065600', 'ap-northeast-1c': '10.065600', 'ca-central-1b': '8.078400', 'ap-southeast-1a': '10.161600', 'ap-southeast-1b': '10.161600', 'ap-southeast-2a': '10.161600', 'eu-central-1a': '9.175200', 'eu-central-1b': '9.175200', 'us-east-1a': '24.480000', 'us-east-1b': '7.344000', 'us-east-1d': '7.344000', 'us-east-1e': '7.344000', 'us-east-1f': '7.344000', 'us-east-2a': '7.471400', 'us-east-2b': '7.344000', 'us-west-2a': '7.344000', 'us-west-2b': '7.344000', 'us-west-2c': '7.344000'}, ('antics', 'p3dn.24xlarge'): {'eu-west-1a': '9.739100', 'eu-west-1b': '9.739100', 'ap-northeast-1b': '12.834900', 'us-east-1b': '9.363600', 'us-east-1d': '9.363600', 'us-west-2b': '9.363600', 'us-west-2c': '9.363600'}, ('keyapp', 'p3.16xlarge'): {'eu-west-2a': '8.613600', 'eu-west-2b': '8.613600', 'eu-west-1a': '26.440000', 'eu-west-1b': '7.932000', 'eu-west-1c': '7.932000', 'ap-northeast-2a': '11.954400', 'ap-northeast-2c': '11.954400', 'ap-northeast-1b': '10.065600', 'ap-northeast-1c': '10.065600', 'ca-central-1b': '8.078400', 'ap-southeast-1a': '10.161600', 'ap-southeast-1b': '10.161600', 'ap-southeast-2a': '10.161600', 'eu-central-1a': '9.175200', 'eu-central-1b': '9.175200', 'us-east-1a': '24.480000', 'us-east-1b': '7.344000', 'us-east-1d': '7.344000', 'us-east-1e': '7.344000', 'us-east-1f': '7.344000', 'us-east-2a': '7.471400', 'us-east-2b': '7.344000', 'us-west-2a': '7.344000', 'us-west-2b': '7.344000', 'us-west-2c': '7.344000'}, ('keyapp', 'p3dn.24xlarge'): {'eu-west-1a': '9.739100', 'eu-west-1b': '9.739100', 'ap-northeast-1b': '12.834900', 'us-east-1b': '9.363600', 'us-east-1d': '9.363600', 'us-west-2b': '9.363600', 'us-west-2c': '9.363600'}}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'eu-west-2a': '8.613600',\n",
       " 'eu-west-2b': '8.613600',\n",
       " 'eu-west-1a': '26.440000',\n",
       " 'eu-west-1b': '7.932000',\n",
       " 'eu-west-1c': '7.932000',\n",
       " 'ap-northeast-2a': '11.954400',\n",
       " 'ap-northeast-2c': '11.954400',\n",
       " 'ap-northeast-1b': '10.065600',\n",
       " 'ap-northeast-1c': '10.065600',\n",
       " 'ca-central-1b': '8.078400',\n",
       " 'ap-southeast-1a': '10.161600',\n",
       " 'ap-southeast-1b': '10.161600',\n",
       " 'ap-southeast-2a': '10.161600',\n",
       " 'eu-central-1a': '9.175200',\n",
       " 'eu-central-1b': '9.175200',\n",
       " 'us-east-1a': '24.480000',\n",
       " 'us-east-1b': '7.344000',\n",
       " 'us-east-1d': '7.344000',\n",
       " 'us-east-1e': '7.344000',\n",
       " 'us-east-1f': '7.344000',\n",
       " 'us-east-2a': '7.471400',\n",
       " 'us-east-2b': '7.344000',\n",
       " 'us-west-2a': '7.344000',\n",
       " 'us-west-2b': '7.344000',\n",
       " 'us-west-2c': '7.344000'}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d[('yacht', 'p3.16xlarge')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'eu-west-1a': '9.739100',\n",
       " 'eu-west-1b': '9.739100',\n",
       " 'ap-northeast-1b': '12.834900',\n",
       " 'us-east-1b': '9.363600',\n",
       " 'us-east-1d': '9.363600',\n",
       " 'us-west-2b': '9.363600',\n",
       " 'us-west-2c': '9.363600'}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d[('keyapp', 'p3dn.24xlarge')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'eu-west-1a': '9.739100',\n",
       " 'eu-west-1b': '9.739100',\n",
       " 'ap-northeast-1b': '12.834900',\n",
       " 'us-east-1b': '9.363600',\n",
       " 'us-east-1d': '9.363600',\n",
       " 'us-west-2b': '9.363600',\n",
       " 'us-west-2c': '9.363600'}"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d[('antics', 'p3dn.24xlarge')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'eu-west-1a': '9.739100',\n",
       " 'eu-west-1b': '9.739100',\n",
       " 'ap-northeast-1b': '12.834900',\n",
       " 'us-east-1b': '9.363600',\n",
       " 'us-east-1d': '9.363600',\n",
       " 'us-west-2b': '9.363600',\n",
       " 'us-west-2c': '9.363600'}"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d[('yacht', 'p3dn.24xlarge')]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "ename": "IndexError",
     "evalue": "list index out of range",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-80-9364c187a168>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mimages\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mec2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimages\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0;34m[\u001b[0m\u001b[0mim\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mim\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mimages\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mim\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mid\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'ami-a9d09ed1'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m: list index out of range"
     ]
    }
   ],
   "source": [
    "images = list(ec2.images.all())\n",
    "[im for im in images if im.id == 'ami-a9d09ed1'][0].name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "amazon_ami='ami-0cc96feef8c6bbff3'\n",
    "ec2 = u.get_ec2_resource()\n",
    "images = list(ec2.images.all())\n",
    "[im for im in images if im.id == amazon_ami][0].name\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get root device name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "images = list(ec2.images.filter(ImageIds=['ami-0cc96feef8c6bbff3']));\n",
    "device_name = images[0].root_device_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "image = images[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Amazon Linux 2 AMI 2.0.20190612 x86_64 HVM gp2'"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image.description"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/dev/xvda'"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image.root_device_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "client = u.get_ec2_client()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Images': [{'Architecture': 'x86_64',\n",
       "   'CreationDate': '2019-06-11T07:15:13.000Z',\n",
       "   'ImageId': 'ami-0cc96feef8c6bbff3',\n",
       "   'ImageLocation': 'amazon/amzn2-ami-hvm-2.0.20190612-x86_64-gp2',\n",
       "   'ImageType': 'machine',\n",
       "   'Public': True,\n",
       "   'OwnerId': '137112412989',\n",
       "   'State': 'available',\n",
       "   'BlockDeviceMappings': [{'DeviceName': '/dev/xvda',\n",
       "     'Ebs': {'DeleteOnTermination': True,\n",
       "      'SnapshotId': 'snap-08f742c6ac4f80dda',\n",
       "      'VolumeSize': 8,\n",
       "      'VolumeType': 'gp2',\n",
       "      'Encrypted': False}}],\n",
       "   'Description': 'Amazon Linux 2 AMI 2.0.20190612 x86_64 HVM gp2',\n",
       "   'EnaSupport': True,\n",
       "   'Hypervisor': 'xen',\n",
       "   'ImageOwnerAlias': 'amazon',\n",
       "   'Name': 'amzn2-ami-hvm-2.0.20190612-x86_64-gp2',\n",
       "   'RootDeviceName': '/dev/xvda',\n",
       "   'RootDeviceType': 'ebs',\n",
       "   'SriovNetSupport': 'simple',\n",
       "   'VirtualizationType': 'hvm'}],\n",
       " 'ResponseMetadata': {'RequestId': '5b4677b0-8357-4c84-bfca-4a6914874746',\n",
       "  'HTTPStatusCode': 200,\n",
       "  'HTTPHeaders': {'content-type': 'text/xml;charset=UTF-8',\n",
       "   'content-length': '1727',\n",
       "   'date': 'Tue, 09 Jul 2019 17:25:01 GMT',\n",
       "   'server': 'AmazonEC2'},\n",
       "  'RetryAttempts': 0}}"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "client.describe_images(ImageIds=['ami-0cc96feef8c6bbff3'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ebs'"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image.root_device_type"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Volumes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "ec2 = u.get_ec2_resource()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'noname'"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "u.get_name(v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'gp2'"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v.volume_type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "volumes = list(ec2.volumes.all())\n",
    "for v in volumes:\n",
    "  print(v.id, u.get_name(v), v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "    volume_names = set(u.get_name(v) for v in ec2.volumes.all())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'2prepare-image', 'imagenet18', 'noname'}"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "volume_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'05'"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f\"{5:02d}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vol-0d0f0b89221e5e821 noname 0.one_small_machine_wiki_state\n",
      "vol-0417b338eca4208dd 2prepare-image 2prepare-image\n",
      "vol-0caa9c257d63d99bc noname 4prepare\n",
      "vol-053292dec5fff7965 noname tensorboard-east\n",
      "vol-0b3370c415f9dc7cd noname 0.one-imagenet\n",
      "vol-0e7e0928ccc431214 noname shell\n",
      "vol-0f2293283ba7a254e noname pytorch_aws\n",
      "vol-021050b054b05c6a4 noname sshtest\n",
      "vol-0fef8436b3b5c0e99 noname mnist\n",
      "vol-0ff25f669b4da1a84 noname ncluster_launch\n",
      "vol-07f0daec1852ab6f1 noname efs_test\n",
      "vol-0bddd83e9f307b0c3 noname shell\n",
      "vol-0cd4da6eabe52cebb noname shell\n",
      "vol-09fe7efe1b43e8e6b noname mnist\n",
      "vol-04191ce2971c80d14 noname 1.mnist_distributed\n",
      "vol-096daf0d410040175 noname 0.mnist_distributed\n",
      "vol-0b7341b3dbe9da443 noname 0.mnist_distributed2\n",
      "vol-09e4b8c3b7b590f57 noname 1.mnist_distributed2\n",
      "vol-083dd0297aab919ae noname 2test\n",
      "vol-01dd504f31802f2ab noname test\n",
      "vol-05acff587a1a279b8 noname test2\n",
      "vol-045366b5c3a90e53c noname imagenet_prep\n",
      "vol-02c4b1c5ebe4db908 noname imagenet-prep\n",
      "vol-0fd20d716517c942d imagenet18 <unattached>\n",
      "vol-0ddc6ef3d0f57fe6d noname grover-checkpoints\n",
      "vol-06e70db102aa4d4cc noname 0.2imagenet\n",
      "vol-001ac894268e0a891 noname tensorboard\n",
      "vol-01c48e1cc0676ba08 noname 1.multi-imagenet00\n",
      "vol-0925c55a1dd96fbc4 noname 3.multi-imagenet00\n",
      "vol-08ffffce4f3738a77 noname 2.multi-imagenet00\n",
      "vol-0fb2e947428404acc noname 0.multi-imagenet00\n",
      "vol-02c5e2b7bfd164f49 noname imagenet-p3dn\n",
      "vol-09459ca540d4a43d1 noname 1.two-imagenet\n",
      "vol-0a74c9ada4c644db3 noname 0.two-imagenet\n",
      "vol-0695b61a472256744 noname 1.four-imagenet\n",
      "vol-069d252f9079ced1e noname 2.four-imagenet\n",
      "vol-09b24a5bfe5f0beb1 noname 3.four-imagenet\n",
      "vol-0fd9c9d205170c755 noname 0.four-imagenet\n"
     ]
    }
   ],
   "source": [
    "volumes = list(ec2.volumes.all())\n",
    "for v in volumes:\n",
    "  if v.attachments:\n",
    "    machine = u.get_name(ec2.Instance(v.attachments[0]['InstanceId']))\n",
    "  else:\n",
    "    machine = '<unattached>'\n",
    "  print(v.id, u.get_name(v), machine)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ec2.volumes.filter()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using region  us-east-1\n"
     ]
    }
   ],
   "source": [
    "instance = u.lookup_instance('imagenet-prep')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "instance.attach_volume(Device='/dev/xvdh')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "ec2.create_volume?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "v =volumes[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1000"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v.size"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Keypairs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Delete security group"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/Users/yaroslavvb/.ncluster/ncluster-yaroslavvb-979440106638-us-east-1.pem'"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "u.get_keypair_fn()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/Users/yaroslavvb/.ncluster/ncluster-yaroslavvb-979440106638-us-east-1.pem'"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "u.get_keypair_fn()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "pairs = u.get_keypair_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'dnew': ec2.KeyPairInfo(name='dnew'),\n",
       " 'ncluster-ben': ec2.KeyPairInfo(name='ncluster-ben'),\n",
       " 'ncluster-darius': ec2.KeyPairInfo(name='ncluster-darius'),\n",
       " 'ncluster-dlam': ec2.KeyPairInfo(name='ncluster-dlam'),\n",
       " 'ncluster-fakeben': ec2.KeyPairInfo(name='ncluster-fakeben'),\n",
       " 'ncluster-ubuntu': ec2.KeyPairInfo(name='ncluster-ubuntu'),\n",
       " 'ncluster-ubuntu2': ec2.KeyPairInfo(name='ncluster-ubuntu2'),\n",
       " 'ncluster-yaroslav': ec2.KeyPairInfo(name='ncluster-yaroslav'),\n",
       " 'ncluster-yaroslavvb': ec2.KeyPairInfo(name='ncluster-yaroslavvb')}"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ncluster-yaroslavvb'"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "u.get_keypair_name()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'224u-ec2': ec2.KeyPairInfo(name='224u-ec2'),\n",
       " 'aws-eb': ec2.KeyPairInfo(name='aws-eb'),\n",
       " 'dlam': ec2.KeyPairInfo(name='dlam'),\n",
       " 'ec2-test': ec2.KeyPairInfo(name='ec2-test'),\n",
       " 'ncluster-darius': ec2.KeyPairInfo(name='ncluster-darius'),\n",
       " 'ncluster-ryanmckinney': ec2.KeyPairInfo(name='ncluster-ryanmckinney'),\n",
       " 'ncluster-ubuntu': ec2.KeyPairInfo(name='ncluster-ubuntu')}"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pairs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fingerprinting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/yaroslavvb/.ncluster/ncluster-yaroslavvb-979440106638-us-east-1.pem\n",
      "50:b9:04:03:b9:79:53:1d:cd:d2:1f:82:7b:ef:20:63:cb:19:26:26\n"
     ]
    }
   ],
   "source": [
    "pair = pairs['ncluster-yaroslavvb']\n",
    "print(u.get_keypair_fn())\n",
    "print(pair.key_fingerprint)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "openssl pkcs8 -in /Users/yaroslavvb/.ncluster/ncluster-yaroslavvb-979440106638-us-east-1.pem -inform PEM -outform DER -topk8 -nocrypt | openssl sha1 -c\n"
     ]
    }
   ],
   "source": [
    "print(f'openssl pkcs8 -in {u.get_keypair_fn()} -inform PEM -outform DER -topk8 -nocrypt | openssl sha1 -c')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/bin/sh: openssl pkcs8 -in /Users/yaroslavvb/.ncluster/ncluster-yaroslavvb-979440106638-us-east-1.pem -inform PEM -outform DER -topk8 -nocrypt | openssl sha1 -c: No such file or directory\r\n"
     ]
    }
   ],
   "source": [
    "!'openssl pkcs8 -in /Users/yaroslavvb/.ncluster/ncluster-yaroslavvb-979440106638-us-east-1.pem -inform PEM -outform DER -topk8 -nocrypt | openssl sha1 -c'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'(stdin)= f2:8c:2a:ea:a3:da:ca:b0:7e:a0:d5:54:d3:5a:86:19:82:1e:b1:40\\n'"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ossystem(f'openssl pkcs8 -in {u.get_keypair_fn()} -inform PEM -outform DER -topk8 -nocrypt | openssl sha1 -c')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Not the same :("
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Deleting keypairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Deleting keypair ncluster-yaroslavvb\n",
      "Warning, the following instances will be no longer accessible:\n",
      "gpubox i-00a15a75a001b9fab\n",
      "train_squad-large-yaroslavvb i-038ab3bf2c5feeafc\n",
      "Proceed? (y/N)y\n"
     ]
    }
   ],
   "source": [
    "key_name = 'ncluster-yaroslavvb'\n",
    "keypair = pairs[key_name]\n",
    "print(f\"Deleting keypair {key_name}\")\n",
    "ec2 = u.get_ec2_resource()\n",
    "instance_list = []\n",
    "for instance in ec2.instances.all():\n",
    "    if instance.key_name == key_name:\n",
    "        instance_list.append(instance)\n",
    "if instance_list:\n",
    "    print(\"Warning, after deleting keypair, the following instances will be no longer accessible:\")\n",
    "    for i in instance_list:\n",
    "        print(u.get_name(i), i.id)\n",
    "    answer = input(\"Proceed? (y/N)\")\n",
    "else:\n",
    "    answer = \"y\"\n",
    "if answer.lower() == 'y':\n",
    "    keypair.delete()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'i-038ab3bf2c5feeafc'"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "instance.id"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scratch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}