View Javadoc
1   /**
2    * Copyright 2004-2014 The Kuali Foundation
3    *
4    * Licensed under the Educational Community License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    * http://www.opensource.org/licenses/ecl2.php
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   * See the License for the specific language governing permissions and
14   * limitations under the License.
15   */
16  package org.kuali.common.aws.ec2.impl;
17  
18  import static com.amazonaws.services.ec2.model.VolumeAttachmentState.Attached;
19  import static com.amazonaws.services.ec2.model.VolumeState.Available;
20  import static com.amazonaws.services.ec2.model.VolumeState.InUse;
21  import static com.google.common.base.Optional.absent;
22  import static com.google.common.base.Optional.fromNullable;
23  import static com.google.common.base.Preconditions.checkArgument;
24  import static com.google.common.base.Preconditions.checkNotNull;
25  import static com.google.common.base.Preconditions.checkState;
26  import static com.google.common.base.Stopwatch.createStarted;
27  import static com.google.common.collect.Lists.newArrayList;
28  import static java.lang.String.format;
29  import static java.util.Arrays.asList;
30  import static java.util.Collections.singletonList;
31  import static java.util.concurrent.TimeUnit.MILLISECONDS;
32  import static org.kuali.common.aws.ec2.model.InstanceStateName.RUNNING;
33  import static org.kuali.common.aws.ec2.model.InstanceStateName.STOPPED;
34  import static org.kuali.common.aws.ec2.model.InstanceStateName.TERMINATED;
35  import static org.kuali.common.core.collect.Iterables.getSingleElement;
36  import static org.kuali.common.util.CollectionUtils.hasBlanks;
37  import static org.kuali.common.util.CollectionUtils.isEmpty;
38  import static org.kuali.common.util.FormatUtils.getMillisAsInt;
39  import static org.kuali.common.util.FormatUtils.getTime;
40  import static org.kuali.common.util.base.Exceptions.illegalState;
41  import static org.kuali.common.util.base.Precondition.checkMin;
42  import static org.kuali.common.util.base.Precondition.checkNotBlank;
43  import static org.kuali.common.util.base.Precondition.checkNotNull;
44  import static org.kuali.common.util.base.Threads.checkedWait;
45  import static org.kuali.common.util.base.Threads.sleep;
46  import static org.kuali.common.util.log.Loggers.newLogger;
47  
48  import java.util.ArrayList;
49  import java.util.Collection;
50  import java.util.Collections;
51  import java.util.HashSet;
52  import java.util.List;
53  import java.util.Set;
54  
55  import org.apache.commons.lang3.StringUtils;
56  import org.kuali.common.aws.ec2.api.EC2Service;
57  import org.kuali.common.aws.ec2.model.CreateAMIRequest;
58  import org.kuali.common.aws.ec2.model.CreateVolumeRequest;
59  import org.kuali.common.aws.ec2.model.EC2ServiceContext;
60  import org.kuali.common.aws.ec2.model.InstanceStateName;
61  import org.kuali.common.aws.ec2.model.LaunchInstanceContext;
62  import org.kuali.common.aws.ec2.model.Regions;
63  import org.kuali.common.aws.ec2.model.RootVolume;
64  import org.kuali.common.aws.ec2.model.VolumeRequest;
65  import org.kuali.common.aws.ec2.model.security.KualiSecurityGroup;
66  import org.kuali.common.aws.ec2.model.security.Permission;
67  import org.kuali.common.aws.ec2.model.security.Protocol;
68  import org.kuali.common.aws.ec2.model.security.SetPermissionsResult;
69  import org.kuali.common.aws.ec2.model.status.InstanceStatusType;
70  import org.kuali.common.aws.ec2.model.status.InstanceStatusValue;
71  import org.kuali.common.aws.ec2.util.LaunchUtils;
72  import org.kuali.common.core.base.TimedInterval;
73  import org.kuali.common.core.ssh.PublicKey;
74  import org.kuali.common.util.Assert;
75  import org.kuali.common.util.CollectionUtils;
76  import org.kuali.common.util.FormatUtils;
77  import org.kuali.common.util.SetUtils;
78  import org.kuali.common.util.condition.Condition;
79  import org.kuali.common.util.wait.DefaultWaitService;
80  import org.kuali.common.util.wait.WaitContext;
81  import org.kuali.common.util.wait.WaitResult;
82  import org.kuali.common.util.wait.WaitService;
83  import org.slf4j.Logger;
84  
85  import com.amazonaws.auth.AWSCredentials;
86  import com.amazonaws.regions.RegionUtils;
87  import com.amazonaws.services.ec2.AmazonEC2Client;
88  import com.amazonaws.services.ec2.model.AuthorizeSecurityGroupIngressRequest;
89  import com.amazonaws.services.ec2.model.BlockDeviceMapping;
90  import com.amazonaws.services.ec2.model.CopyImageRequest;
91  import com.amazonaws.services.ec2.model.CopyImageResult;
92  import com.amazonaws.services.ec2.model.CreateSecurityGroupRequest;
93  import com.amazonaws.services.ec2.model.CreateSnapshotRequest;
94  import com.amazonaws.services.ec2.model.CreateSnapshotResult;
95  import com.amazonaws.services.ec2.model.CreateTagsRequest;
96  import com.amazonaws.services.ec2.model.DeleteSnapshotRequest;
97  import com.amazonaws.services.ec2.model.DeleteVolumeRequest;
98  import com.amazonaws.services.ec2.model.DeregisterImageRequest;
99  import com.amazonaws.services.ec2.model.DescribeImagesRequest;
100 import com.amazonaws.services.ec2.model.DescribeImagesResult;
101 import com.amazonaws.services.ec2.model.DescribeInstanceStatusRequest;
102 import com.amazonaws.services.ec2.model.DescribeInstanceStatusResult;
103 import com.amazonaws.services.ec2.model.DescribeInstancesRequest;
104 import com.amazonaws.services.ec2.model.DescribeInstancesResult;
105 import com.amazonaws.services.ec2.model.DescribeKeyPairsResult;
106 import com.amazonaws.services.ec2.model.DescribeSecurityGroupsRequest;
107 import com.amazonaws.services.ec2.model.DescribeSecurityGroupsResult;
108 import com.amazonaws.services.ec2.model.DescribeSnapshotsRequest;
109 import com.amazonaws.services.ec2.model.DescribeSnapshotsResult;
110 import com.amazonaws.services.ec2.model.DescribeVolumesRequest;
111 import com.amazonaws.services.ec2.model.DescribeVolumesResult;
112 import com.amazonaws.services.ec2.model.EbsBlockDevice;
113 import com.amazonaws.services.ec2.model.EbsInstanceBlockDevice;
114 import com.amazonaws.services.ec2.model.Image;
115 import com.amazonaws.services.ec2.model.ImportKeyPairRequest;
116 import com.amazonaws.services.ec2.model.ImportKeyPairResult;
117 import com.amazonaws.services.ec2.model.Instance;
118 import com.amazonaws.services.ec2.model.InstanceBlockDeviceMapping;
119 import com.amazonaws.services.ec2.model.InstanceStatus;
120 import com.amazonaws.services.ec2.model.InstanceStatusDetails;
121 import com.amazonaws.services.ec2.model.InstanceStatusSummary;
122 import com.amazonaws.services.ec2.model.IpPermission;
123 import com.amazonaws.services.ec2.model.KeyPairInfo;
124 import com.amazonaws.services.ec2.model.ModifyInstanceAttributeRequest;
125 import com.amazonaws.services.ec2.model.Placement;
126 import com.amazonaws.services.ec2.model.Region;
127 import com.amazonaws.services.ec2.model.RegisterImageRequest;
128 import com.amazonaws.services.ec2.model.RegisterImageResult;
129 import com.amazonaws.services.ec2.model.Reservation;
130 import com.amazonaws.services.ec2.model.RevokeSecurityGroupIngressRequest;
131 import com.amazonaws.services.ec2.model.RunInstancesRequest;
132 import com.amazonaws.services.ec2.model.RunInstancesResult;
133 import com.amazonaws.services.ec2.model.SecurityGroup;
134 import com.amazonaws.services.ec2.model.Snapshot;
135 import com.amazonaws.services.ec2.model.StartInstancesRequest;
136 import com.amazonaws.services.ec2.model.StopInstancesRequest;
137 import com.amazonaws.services.ec2.model.Tag;
138 import com.amazonaws.services.ec2.model.TerminateInstancesRequest;
139 import com.amazonaws.services.ec2.model.Volume;
140 import com.amazonaws.services.ec2.model.VolumeAttachment;
141 import com.amazonaws.services.ec2.model.VolumeAttachmentState;
142 import com.amazonaws.services.ec2.model.VolumeState;
143 import com.amazonaws.services.ec2.model.VolumeType;
144 import com.google.common.base.Optional;
145 import com.google.common.base.Stopwatch;
146 import com.google.common.collect.ImmutableList;
147 
148 /**
149  * This service implementation performs operations using a single set of AWS credentials on a single EC2 region.
150  */
151 public final class DefaultEC2Service implements EC2Service {
152 
153 	private static final Logger logger = newLogger();
154 	private static final String SNAPSHOT_COMPLETED_STATE = "completed";
155 	private static final String AMI_AVAILABLE_STATE = "available";
156 	private static int SLEEP_MILLIS = 100;
157 
158 	// Don't expose the AmazonEC2Client object via a getter, it's not thread safe
159 	private final AmazonEC2Client client;
160 
161 	private final EC2ServiceContext context;
162 	private final WaitService service;
163 
164 	public DefaultEC2Service(AWSCredentials credentials) {
165 		this(credentials, new Region().withRegionName(Regions.DEFAULT_REGION.getName()));
166 	}
167 
168 	public DefaultEC2Service(AWSCredentials credentials, String region) {
169 		this(credentials, new Region().withRegionName(region));
170 	}
171 
172 	public DefaultEC2Service(AWSCredentials credentials, Region region) {
173 		this(EC2ServiceContext.create(credentials, region), new DefaultWaitService());
174 	}
175 
176 	public DefaultEC2Service(EC2ServiceContext context, WaitService service) {
177 		this.context = checkNotNull(context, "context");
178 		this.service = checkNotNull(service, "service");
179 		this.client = LaunchUtils.getClient(context);
180 	}
181 
182 	@Override
183 	public List<Volume> listVolumes() {
184 		return ImmutableList.copyOf(client.describeVolumes().getVolumes());
185 	}
186 
187 	@Override
188 	public Volume getVolume(String volumeId) {
189 		DescribeVolumesRequest dvr = new DescribeVolumesRequest();
190 		dvr.setVolumeIds(asList(volumeId));
191 		DescribeVolumesResult result = client.describeVolumes(dvr);
192 		return getSingleElement(result.getVolumes());
193 	}
194 
195 	/**
196 	 * Return true if this volume is attached to anything. ie if volume.getAttachments() is not empty
197 	 */
198 	private boolean isAttached(String volumeId) {
199 		Volume volume = getVolume(volumeId);
200 		List<VolumeAttachment> attachments = volume.getAttachments();
201 		return !isEmpty(attachments);
202 	}
203 
204 	private String waitForVolumeState(String volumeId, VolumeState desiredState, long timeout) {
205 		VolumeState currentState = null;
206 		Stopwatch sw = createStarted();
207 		while (!desiredState.equals(currentState)) {
208 			checkedWait(sw, timeout, SLEEP_MILLIS);
209 			DescribeVolumesRequest dvr = new DescribeVolumesRequest();
210 			dvr.setVolumeIds(asList(volumeId));
211 			DescribeVolumesResult result = client.describeVolumes(dvr);
212 			Volume volume = getSingleElement(result.getVolumes());
213 			currentState = VolumeState.fromValue(volume.getState());
214 		}
215 		return volumeId;
216 	}
217 
218 	private String waitForDetached(String volumeId, long timeout) {
219 		checkNotBlank(volumeId, "volumeId");
220 		checkMin(timeout, 1, "timeout");
221 		boolean attached = isAttached(volumeId);
222 		Stopwatch sw = createStarted();
223 		while (attached) {
224 			checkedWait(sw, timeout, SLEEP_MILLIS);
225 			attached = isAttached(volumeId);
226 		}
227 		return waitForVolumeState(volumeId, Available, timeout - sw.elapsed(MILLISECONDS));
228 	}
229 
230 	@Override
231 	public String detachVolume(VolumeRequest request) {
232 		com.amazonaws.services.ec2.model.DetachVolumeRequest dvr = new com.amazonaws.services.ec2.model.DetachVolumeRequest();
233 		dvr.setInstanceId(request.getInstanceId());
234 		dvr.setVolumeId(request.getVolumeId());
235 		dvr.setDevice(request.getDevice());
236 		client.detachVolume(dvr);
237 		return waitForDetached(request.getVolumeId(), request.getTimeout());
238 	}
239 
240 	@Override
241 	public String getAccessKey() {
242 		return context.getCredentials().getAWSAccessKeyId();
243 	}
244 
245 	@Override
246 	public String getRegion() {
247 		return context.getRegion();
248 	}
249 
250 	@Override
251 	public String copyAmi(String region, String ami) {
252 		return copyAmi(region, ami, Optional.<String> absent());
253 	}
254 
255 	@Override
256 	public String copyAmi(String region, String ami, String name) {
257 		return copyAmi(region, ami, Optional.of(name));
258 	}
259 
260 	public String copyAmi(String region, String ami, Optional<String> name) {
261 		checkNotBlank(region, "region");
262 		checkNotBlank(ami, "ami");
263 		checkNotBlank(name, "name");
264 		checkNotNull(RegionUtils.getRegion(region), "region %s is unknown", region);
265 		CopyImageRequest request = new CopyImageRequest();
266 		request.setSourceImageId(ami);
267 		request.setSourceRegion(region);
268 		if (name.isPresent()) {
269 			request.setName(name.get());
270 		}
271 		CopyImageResult result = client.copyImage(request);
272 		sleep(1000);
273 
274 		int millis = getMillisAsInt(System.getProperty("ec2.copyAmiMaxWait", "6h"));
275 		waitForAmiState(result.getImageId(), AMI_AVAILABLE_STATE, millis);
276 		return result.getImageId();
277 	}
278 
279 	@Override
280 	public List<Image> getImages() {
281 		return client.describeImages().getImages();
282 	}
283 
284 	@Override
285 	public void deleteSnapshot(String snapshotId) {
286 		DeleteSnapshotRequest request = new DeleteSnapshotRequest();
287 		request.setSnapshotId(snapshotId);
288 		client.deleteSnapshot(request);
289 	}
290 
291 	@Override
292 	public void purgeAmi(String imageId) {
293 		Image image = getImage(imageId);
294 		List<String> snapshotIds = newArrayList();
295 		List<BlockDeviceMapping> mappings = image.getBlockDeviceMappings();
296 		for (BlockDeviceMapping mapping : mappings) {
297 			Optional<EbsBlockDevice> ebsBlockDevice = fromNullable(mapping.getEbs());
298 			if (ebsBlockDevice.isPresent()) {
299 				snapshotIds.add(ebsBlockDevice.get().getSnapshotId());
300 			}
301 		}
302 		DeregisterImageRequest request = new DeregisterImageRequest();
303 		request.setImageId(imageId);
304 		client.deregisterImage(request);
305 		for (String snapshotId : snapshotIds) {
306 			deleteSnapshot(snapshotId);
307 		}
308 	}
309 
310 	@Override
311 	public Image createAmi(CreateAMIRequest request) {
312 		Instance instance = getInstance(request.getInstanceId());
313 		String rootVolumeId = getRootVolumeId(instance);
314 		Snapshot snapshot = createSnapshot(rootVolumeId, request.getDescription(), request.getTimeoutMillis());
315 		tag(snapshot.getSnapshotId(), request.getName());
316 		return createAmi(request, instance, snapshot);
317 	}
318 
319 	protected Image createAmi(CreateAMIRequest create, Instance instance, Snapshot snapshot) {
320 		BlockDeviceMapping rootVolumeMapping = getRootVolumeMapping(instance, snapshot.getSnapshotId(), create.getRootVolume());
321 		List<BlockDeviceMapping> mappings = newArrayList();
322 		mappings.add(rootVolumeMapping);
323 		for (BlockDeviceMapping mapping : create.getAdditionalMappings()) {
324 			mappings.add(mapping);
325 		}
326 
327 		RegisterImageRequest request = new RegisterImageRequest();
328 		request.setName(create.getName().getValue());
329 		request.setDescription(create.getDescription());
330 		request.setArchitecture(instance.getArchitecture());
331 		request.setRootDeviceName(instance.getRootDeviceName());
332 		request.setKernelId(instance.getKernelId());
333 		request.setBlockDeviceMappings(mappings);
334 		RegisterImageResult result = client.registerImage(request);
335 		String imageId = result.getImageId();
336 		waitForAmiState(imageId, AMI_AVAILABLE_STATE, create.getTimeoutMillis());
337 		tag(imageId, create.getName());
338 		return getImage(imageId);
339 	}
340 
341 	protected BlockDeviceMapping getRootVolumeMapping(Instance instance, String snapshotId, RootVolume rootVolume) {
342 		InstanceBlockDeviceMapping ibdm = getRootVolumeMapping(instance);
343 
344 		EbsBlockDevice ebs = new EbsBlockDevice();
345 		ebs.setDeleteOnTermination(rootVolume.getDeleteOnTermination().orNull());
346 		ebs.setSnapshotId(snapshotId);
347 		ebs.setVolumeSize(rootVolume.getSizeInGigabytes().orNull());
348 
349 		BlockDeviceMapping bdm = new BlockDeviceMapping();
350 		bdm.setDeviceName(ibdm.getDeviceName());
351 		bdm.setEbs(ebs);
352 		return bdm;
353 	}
354 
355 	protected BlockDeviceMapping convert(InstanceBlockDeviceMapping mapping, String snapshotId, int sizeInGigabytes) {
356 		BlockDeviceMapping converted = new BlockDeviceMapping();
357 		converted.setDeviceName(mapping.getDeviceName());
358 		converted.setEbs(convert(mapping.getEbs(), snapshotId, sizeInGigabytes));
359 		return converted;
360 	}
361 
362 	protected EbsBlockDevice convert(EbsInstanceBlockDevice device, String snapshotId, int sizeInGigabytes) {
363 		EbsBlockDevice converted = new EbsBlockDevice();
364 		converted.setDeleteOnTermination(device.getDeleteOnTermination());
365 		converted.setSnapshotId(snapshotId);
366 		converted.setVolumeSize(sizeInGigabytes);
367 		return converted;
368 	}
369 
370 	protected Optional<Tag> getTag(List<Tag> tags, String key) {
371 		for (Tag tag : tags) {
372 			if (tag.getKey().equals(key)) {
373 				return Optional.of(tag);
374 			}
375 		}
376 		return absent();
377 	}
378 
379 	protected String getRootVolumeId(Instance instance) {
380 		checkNotNull(instance, "instance");
381 		InstanceBlockDeviceMapping rootMapping = getRootVolumeMapping(instance);
382 		return rootMapping.getEbs().getVolumeId();
383 	}
384 
385 	protected InstanceBlockDeviceMapping getRootVolumeMapping(Instance instance) {
386 		checkNotNull(instance, "instance");
387 		String rootDeviceName = instance.getRootDeviceName();
388 		List<InstanceBlockDeviceMapping> mappings = instance.getBlockDeviceMappings();
389 		for (InstanceBlockDeviceMapping mapping : mappings) {
390 			if (rootDeviceName.equals(mapping.getDeviceName())) {
391 				return mapping;
392 			}
393 		}
394 		throw illegalState("Unable to locate the root volume mapping for [%s]", instance.getInstanceId());
395 	}
396 
397 	@Override
398 	public Snapshot createSnapshot(String volumeId, String description, int timeoutMillis) {
399 		CreateSnapshotRequest request = new CreateSnapshotRequest(volumeId, description);
400 		CreateSnapshotResult result = client.createSnapshot(request);
401 		waitForSnapshotState(result.getSnapshot().getSnapshotId(), SNAPSHOT_COMPLETED_STATE, timeoutMillis);
402 		return result.getSnapshot();
403 	}
404 
405 	@Override
406 	public Snapshot getSnapshot(String snapshotId) {
407 		DescribeSnapshotsRequest request = new DescribeSnapshotsRequest();
408 		request.setSnapshotIds(singletonList(snapshotId));
409 		DescribeSnapshotsResult result = client.describeSnapshots(request);
410 		List<Snapshot> snapshots = result.getSnapshots();
411 		checkState(snapshots.size() == 1, "expected 1 snapshot, but there were %s instead", snapshots.size());
412 		return snapshots.get(0);
413 	}
414 
415 	protected void waitForSnapshotState(String snapshotId, String state, int timeoutMillis) {
416 		Condition condition = new SnapshotStateCondition(this, snapshotId, state);
417 		WaitContext waitContext = getWaitContext(timeoutMillis);
418 		Object[] args = { FormatUtils.getTime(waitContext.getTimeoutMillis()), snapshotId, state };
419 		logger.info(format("waiting up to %s for snapshot [%s] to reach state '%s'", args));
420 		WaitResult result = service.wait(waitContext, condition);
421 		Object[] resultArgs = { snapshotId, state, FormatUtils.getTime(result.getElapsed()) };
422 		logger.info(format("snapshot [%s] is now '%s' - %s", resultArgs));
423 	}
424 
425 	protected void waitForAmiState(String imageId, String state, int timeoutMillis) {
426 		Condition condition = new AmiStateCondition(this, imageId, state);
427 		WaitContext waitContext = getWaitContext(timeoutMillis);
428 		Object[] args = { FormatUtils.getTime(waitContext.getTimeoutMillis()), imageId, state };
429 		logger.info(format("waiting up to %s for image [%s] to reach state '%s'", args));
430 		WaitResult result = service.wait(waitContext, condition);
431 		Object[] resultArgs = { imageId, state, FormatUtils.getTime(result.getElapsed()) };
432 		logger.info(format("ami [%s] is now '%s' - %s", resultArgs));
433 	}
434 
435 	@Override
436 	public List<Image> getMyImages() {
437 		DescribeImagesRequest request = new DescribeImagesRequest();
438 		request.withOwners(AmiOwner.SELF.getValue());
439 		DescribeImagesResult result = client.describeImages(request);
440 		return result.getImages();
441 	}
442 
443 	protected void checkSizeEquals(Collection<?> c, int size) {
444 		checkState(c.size() == size, "expected size of %s, was %s instead", size, c.size());
445 	}
446 
447 	@Override
448 	public Image getImage(String imageId) {
449 		DescribeImagesRequest request = new DescribeImagesRequest();
450 		request.setImageIds(singletonList(imageId));
451 		DescribeImagesResult result = client.describeImages(request);
452 		List<Image> images = result.getImages();
453 		checkSizeEquals(images, 1);
454 		return images.get(0);
455 	}
456 
457 	@Override
458 	public String importKey(String keyName, String publicKey) {
459 		checkNotBlank(keyName, "keyName");
460 		checkNotBlank(publicKey, "publicKey");
461 		ImportKeyPairRequest request = new ImportKeyPairRequest(keyName, publicKey);
462 		ImportKeyPairResult result = client.importKeyPair(request);
463 		return result.getKeyFingerprint();
464 	}
465 
466 	@Override
467 	public boolean isExistingKey(String keyName) {
468 		checkNotBlank(keyName, "keyName");
469 		DescribeKeyPairsResult result = client.describeKeyPairs();
470 		List<KeyPairInfo> keys = result.getKeyPairs();
471 		Optional<KeyPairInfo> optional = getKeyPairInfo(keyName, keys);
472 		return optional.isPresent();
473 	}
474 
475 	protected Optional<KeyPairInfo> getKeyPairInfo(String name, List<KeyPairInfo> list) {
476 		for (KeyPairInfo element : list) {
477 			logger.debug("fingerprint - {}, name - {}", element.getKeyFingerprint(), element.getKeyName());
478 			if (name.equals(element.getKeyName())) {
479 				return Optional.of(element);
480 			}
481 		}
482 		return Optional.<KeyPairInfo> absent();
483 	}
484 
485 	@Override
486 	public List<String> getSecurityGroupNames() {
487 		DescribeSecurityGroupsResult result = client.describeSecurityGroups();
488 		List<String> names = newArrayList();
489 		for (SecurityGroup group : result.getSecurityGroups()) {
490 			names.add(group.getGroupName());
491 		}
492 		Collections.sort(names);
493 		return ImmutableList.copyOf(names);
494 	}
495 
496 	@Override
497 	public void createSecurityGroup(KualiSecurityGroup group) {
498 		checkNotNull(group, "group");
499 		CreateSecurityGroupRequest request = new CreateSecurityGroupRequest();
500 		request.setGroupName(group.getName());
501 		if (group.getDescription().isPresent()) {
502 			request.setDescription(group.getDescription().get());
503 		}
504 		client.createSecurityGroup(request);
505 		setPermissions(group.getName(), group.getPermissions());
506 	}
507 
508 	@Override
509 	public boolean isExistingSecurityGroup(String name) {
510 		checkNotBlank(name, "name");
511 		return getSecurityGroup(name).isPresent();
512 	}
513 
514 	@Override
515 	public SetPermissionsResult setPermissions(String securityGroupName, List<Permission> permissions) {
516 		checkNotBlank(securityGroupName, "securityGroupName");
517 		checkNotNull(permissions, "permissions");
518 		Optional<SecurityGroup> optional = getSecurityGroup(securityGroupName);
519 		checkState(optional.isPresent(), "Security group [%s] does not exist", securityGroupName);
520 		SecurityGroup group = optional.get();
521 		List<IpPermission> oldPerms = group.getIpPermissions();
522 		List<Permission> oldPermissions = getPermissions(oldPerms);
523 
524 		Set<Permission> newSet = new HashSet<Permission>(permissions);
525 		Set<Permission> oldSet = new HashSet<Permission>(oldPermissions);
526 
527 		Set<Permission> adds = SetUtils.difference(newSet, oldSet);
528 		Set<Permission> deletes = SetUtils.difference(oldSet, newSet);
529 		Set<Permission> existing = SetUtils.intersection(newSet, oldSet);
530 
531 		// Delete any permissions that are not in the list, but exist in the security group
532 		if (deletes.size() > 0) {
533 			RevokeSecurityGroupIngressRequest revoker = new RevokeSecurityGroupIngressRequest(securityGroupName, getIpPermissions(deletes));
534 			client.revokeSecurityGroupIngress(revoker);
535 		}
536 
537 		// Add any permissions that are in the list but don't exist in the security group
538 		if (adds.size() > 0) {
539 			AuthorizeSecurityGroupIngressRequest authorizer = new AuthorizeSecurityGroupIngressRequest();
540 			authorizer.withGroupName(securityGroupName).withIpPermissions(getIpPermissions(adds));
541 			client.authorizeSecurityGroupIngress(authorizer);
542 		}
543 
544 		return new SetPermissionsResult(adds, deletes, existing);
545 	}
546 
547 	@Override
548 	public Optional<SecurityGroup> getSecurityGroup(String name) {
549 		checkNotBlank(name, "name");
550 		List<String> names = getSecurityGroupNames();
551 		if (names.contains(name)) {
552 			DescribeSecurityGroupsRequest request = new DescribeSecurityGroupsRequest();
553 			request.setGroupNames(Collections.singletonList(name));
554 			DescribeSecurityGroupsResult result = client.describeSecurityGroups(request);
555 			List<SecurityGroup> groups = result.getSecurityGroups();
556 			checkState(groups.size() == 1, "Expected exactly 1 security group but there were %s instead", groups.size());
557 			SecurityGroup group = groups.get(0);
558 			return Optional.of(group);
559 		} else {
560 			return Optional.<SecurityGroup> absent();
561 		}
562 	}
563 
564 	protected List<IpPermission> getIpPermissions(Collection<Permission> permissions) {
565 		List<IpPermission> newPerms = new ArrayList<IpPermission>();
566 		for (Permission perm : permissions) {
567 			IpPermission newPerm = getIpPermission(perm);
568 			newPerms.add(newPerm);
569 		}
570 		return ImmutableList.copyOf(newPerms);
571 	}
572 
573 	protected List<Permission> getPermissions(Collection<IpPermission> permissions) {
574 		List<Permission> newPerms = new ArrayList<Permission>();
575 		for (IpPermission perm : permissions) {
576 			Permission newPerm = getPermission(perm);
577 			newPerms.add(newPerm);
578 		}
579 		return ImmutableList.copyOf(newPerms);
580 	}
581 
582 	protected Permission getPermission(IpPermission perm) {
583 		checkArgument(CollectionUtils.isEmpty(perm.getUserIdGroupPairs()), "User id / group pairs are not supported");
584 		String protocolName = perm.getIpProtocol();
585 		Integer fromPort = perm.getFromPort();
586 		Integer toPort = perm.getToPort();
587 		List<String> ipRanges = perm.getIpRanges();
588 		Assert.noNulls(fromPort, toPort, ipRanges);
589 		Assert.noBlanks(protocolName);
590 		Assert.isTrue(fromPort.equals(toPort), "port ranges are not supported");
591 		Protocol protocol = Protocol.valueOf(protocolName.toUpperCase());
592 		return Permission.builder(fromPort).withCidrNotations(ipRanges).withProtocol(protocol).build();
593 	}
594 
595 	protected IpPermission getIpPermission(Permission perm) {
596 		IpPermission newPerm = new IpPermission();
597 		newPerm.withIpRanges(perm.getCidrNotations());
598 		newPerm.withIpProtocol(perm.getProtocol().getValue());
599 		newPerm.withFromPort(perm.getPort());
600 		newPerm.withToPort(perm.getPort());
601 		return newPerm;
602 	}
603 
604 	@Override
605 	public Instance getInstance(String instanceId) {
606 		checkNotBlank(instanceId, "instanceId");
607 		return getSingleElement(getInstances(ImmutableList.of(instanceId)));
608 	}
609 
610 	@Override
611 	public List<Instance> getInstances(List<String> instanceIds) {
612 		checkNotNull(instanceIds);
613 		checkArgument(!isEmpty(instanceIds), "must supply at least one instance id");
614 		DescribeInstancesRequest request = new DescribeInstancesRequest().withInstanceIds(instanceIds);
615 		DescribeInstancesResult result = client.describeInstances(request);
616 		return getInstances(result);
617 	}
618 
619 	protected List<Instance> getInstances(DescribeInstancesResult result) {
620 		List<Instance> list = newArrayList();
621 		for (Reservation reservation : result.getReservations()) {
622 			list.addAll(reservation.getInstances());
623 		}
624 		return ImmutableList.copyOf(list);
625 	}
626 
627 	@Override
628 	public List<Instance> getInstances() {
629 		return getInstances(client.describeInstances());
630 	}
631 
632 	@Override
633 	public Instance launchInstance(LaunchInstanceContext context) {
634 		checkNotNull(context, "context");
635 
636 		// Connect to AWS and ask them to create an instance
637 		Instance instance = issueRunInstanceRequest(context);
638 
639 		// Was getting some flaky behavior from AWS without a small delay after issuing the RunInstancesRequest
640 		// Since it generally takes a few minutes for the instance to spin up, pausing here for 1 second should be ok
641 		sleep(this.context.getInitialPauseMillis());
642 
643 		// Tag the instance
644 		tag(instance.getInstanceId(), context.getTags());
645 
646 		// Wait for confirmation that the instance is online and functioning
647 		waitForOnlineConfirmation(instance, context);
648 
649 		// Return the fully populated instance object
650 		return getInstance(instance.getInstanceId());
651 	}
652 
653 	@Override
654 	public void allowTermination(String instanceId) {
655 		Assert.noBlanks(instanceId);
656 		preventTermination(instanceId, false);
657 	}
658 
659 	@Override
660 	public void preventTermination(String instanceId) {
661 		Assert.noBlanks(instanceId);
662 		preventTermination(instanceId, true);
663 	}
664 
665 	@Override
666 	public Instance startInstance(String instanceId) {
667 		checkNotBlank(instanceId, "instanceId");
668 		StartInstancesRequest request = new StartInstancesRequest();
669 		request.setInstanceIds(singletonList(instanceId));
670 		client.startInstances(request);
671 		WaitContext waitContext = getWaitContext(context.getTerminationTimeoutMillis());
672 		Object[] args = { FormatUtils.getTime(waitContext.getTimeoutMillis()), instanceId, RUNNING.getValue() };
673 		logger.info("Waiting up to {} for [{}] to start", args);
674 		Condition online = new IsOnlineCondition(this, instanceId);
675 		WaitResult result = service.wait(waitContext, online);
676 		Object[] resultArgs = { instanceId, getTime(result.getElapsed()) };
677 		logger.info("[{}] has been started - {}", resultArgs);
678 		return getInstance(instanceId);
679 	}
680 
681 	@Override
682 	public void stopInstance(String instanceId) {
683 		checkNotBlank(instanceId, "instanceId");
684 		StopInstancesRequest request = new StopInstancesRequest();
685 		request.setInstanceIds(singletonList(instanceId));
686 		client.stopInstances(request);
687 		WaitContext waitContext = getWaitContext(context.getTerminationTimeoutMillis());
688 		Object[] args = { FormatUtils.getTime(waitContext.getTimeoutMillis()), instanceId, STOPPED.getValue() };
689 		logger.info("Waiting up to {} for [{}] to stop", args);
690 		Condition condition = new InstanceStateCondition(this, instanceId, STOPPED);
691 		WaitResult result = service.wait(waitContext, condition);
692 		Object[] resultArgs = { instanceId, getTime(result.getElapsed()) };
693 		logger.info("[{}] has been stopped - {}", resultArgs);
694 	}
695 
696 	@Override
697 	public void terminateInstance(String instanceId) {
698 		checkNotBlank(instanceId, "instanceId");
699 		TerminateInstancesRequest request = new TerminateInstancesRequest();
700 		request.setInstanceIds(singletonList(instanceId));
701 		client.terminateInstances(request);
702 		WaitContext waitContext = getWaitContext(context.getTerminationTimeoutMillis());
703 		Object[] args = { FormatUtils.getTime(waitContext.getTimeoutMillis()), instanceId, TERMINATED.getValue() };
704 		logger.info("Waiting up to {} for [{}] to terminate", args);
705 		Condition condition = new InstanceStateCondition(this, instanceId, TERMINATED);
706 		WaitResult result = service.wait(waitContext, condition);
707 		Object[] resultArgs = { instanceId, getTime(result.getElapsed()) };
708 		logger.info("[{}] has been terminated - {}", resultArgs);
709 	}
710 
711 	@Override
712 	public void terminateInstancesNoWait(List<String> instanceIds) {
713 		checkArgument(!isEmpty(instanceIds), "must provide at least 1 instance id");
714 		checkArgument(!hasBlanks(instanceIds), "blank instance id's not allowed");
715 		TerminateInstancesRequest request = new TerminateInstancesRequest().withInstanceIds(instanceIds);
716 		client.terminateInstances(request);
717 	}
718 
719 	@Override
720 	public void terminateInstanceNoWait(String instanceId) {
721 		checkNotBlank(instanceId, "instanceId");
722 		terminateInstancesNoWait(singletonList(instanceId));
723 	}
724 
725 	@Override
726 	public void tag(String resourceId, List<Tag> tags) {
727 		checkNotBlank(resourceId, "resourceId");
728 		checkNotNull(tags, "tags");
729 		if (CollectionUtils.isEmpty(tags)) {
730 			return;
731 		}
732 		List<String> resources = Collections.singletonList(resourceId);
733 		CreateTagsRequest request = new CreateTagsRequest(resources, tags);
734 		client.createTags(request);
735 	}
736 
737 	@Override
738 	public void tag(String resourceId, Tag tag) {
739 		tag(resourceId, ImmutableList.of(tag));
740 	}
741 
742 	@Override
743 	public boolean isOnline(String instanceId) {
744 		return new IsOnlineCondition(this, instanceId).isTrue();
745 	}
746 
747 	@Override
748 	public String getStatus(String instanceId, InstanceStatusType type, String statusName) {
749 		List<InstanceStatus> statuses = getStatusList(instanceId);
750 		return getStatus(statuses, type, statusName);
751 	}
752 
753 	protected void preventTermination(String instanceId, boolean preventTermination) {
754 		Assert.noBlanks(instanceId);
755 		ModifyInstanceAttributeRequest request = new ModifyInstanceAttributeRequest();
756 		request.withInstanceId(instanceId);
757 
758 		// EC2 instances can normally be terminated by a single API call
759 		// Disabling API termination forces 2 API calls. (1 to re-enable termination, and a 2nd one to actually terminate the instance)
760 		request.withDisableApiTermination(preventTermination);
761 
762 		client.modifyInstanceAttribute(request);
763 	}
764 
765 	protected List<InstanceStatus> getStatusList(String instanceId) {
766 		DescribeInstanceStatusRequest request = new DescribeInstanceStatusRequest();
767 		request.setInstanceIds(Collections.singletonList(instanceId));
768 		DescribeInstanceStatusResult result = client.describeInstanceStatus(request);
769 		return result.getInstanceStatuses();
770 	}
771 
772 	protected String getStatus(List<InstanceStatus> statuses, InstanceStatusType type, String name) {
773 		for (InstanceStatus status : statuses) {
774 			InstanceStatusSummary summary = getSummary(status, type);
775 			Optional<String> detail = getStatusDetail(summary, name);
776 			if (detail.isPresent()) {
777 				return detail.get();
778 			}
779 		}
780 		return InstanceStatusValue.UNKNOWN.getValue();
781 	}
782 
783 	protected InstanceStatusSummary getSummary(InstanceStatus status, InstanceStatusType type) {
784 		switch (type) {
785 		case INSTANCE:
786 			return status.getInstanceStatus();
787 		case SYSTEM:
788 			return status.getSystemStatus();
789 		default:
790 			throw new IllegalArgumentException("[" + type + "] is unknown");
791 		}
792 	}
793 
794 	protected Optional<String> getStatusDetail(InstanceStatusSummary summary, String name) {
795 		List<InstanceStatusDetails> details = summary.getDetails();
796 		for (InstanceStatusDetails detail : details) {
797 			if (name.equals(detail.getName())) {
798 				return Optional.of(detail.getStatus());
799 			}
800 		}
801 		return Optional.absent();
802 	}
803 
804 	protected Instance issueRunInstanceRequest(LaunchInstanceContext context) {
805 		PublicKey publicKey = context.getPublicKey();
806 		if (!isExistingKey(publicKey.getName())) {
807 			logger.info("Importing key [{}]", publicKey.getName());
808 			importKey(publicKey.getName(), publicKey.getValue());
809 		}
810 
811 		List<String> securityGroupNames = getSecurityGroupNames();
812 		for (KualiSecurityGroup securityGroup : context.getSecurityGroups()) {
813 			if (!securityGroupNames.contains(securityGroup.getName())) {
814 				logger.info("Creating security group {}", securityGroup.getName());
815 				createSecurityGroup(securityGroup);
816 			}
817 			if (context.isOverrideExistingSecurityGroupPermissions()) {
818 				SetPermissionsResult result = setPermissions(securityGroup.getName(), securityGroup.getPermissions());
819 				logPermissionChanges(securityGroup, result.getDeletes(), "deleted");
820 				logPermissionChanges(securityGroup, result.getAdds(), "added");
821 			}
822 		}
823 
824 		RunInstancesRequest request = getRunInstanceRequest(context);
825 		RunInstancesResult result = client.runInstances(request);
826 		Reservation r = result.getReservation();
827 		List<Instance> instances = r.getInstances();
828 		checkState(instances.size() == 1, "Expected exactly 1 instance but there were %s instead", instances.size());
829 		return instances.get(0);
830 	}
831 
832 	protected void logPermissionChanges(KualiSecurityGroup group, List<Permission> perms, String changeDescription) {
833 		for (Permission perm : perms) {
834 			String port = StringUtils.leftPad(perm.getPort() + "", 5);
835 			String permDescription = "port:" + port + ", protocol:" + perm.getProtocol() + ", CIDR:" + CollectionUtils.asCSV(perm.getCidrNotations());
836 			Object[] args = { group.getName(), StringUtils.rightPad(changeDescription, 7, " "), permDescription };
837 			logger.info("Security Group:[{}] - permission {} [{}]", args);
838 		}
839 	}
840 
841 	protected WaitContext getWaitContext(int timeout) {
842 		int sleep = context.getSleepIntervalMillis();
843 		int pause = context.getInitialPauseMillis();
844 		return WaitContext.builder(timeout).sleepMillis(sleep).initialPauseMillis(pause).build();
845 	}
846 
847 	protected void waitForOnlineConfirmation(Instance instance, LaunchInstanceContext context) {
848 		InstanceStateName running = InstanceStateName.RUNNING;
849 		WaitContext waitContext = getWaitContext(context.getTimeoutMillis());
850 		Object[] args = { FormatUtils.getTime(waitContext.getTimeoutMillis()), instance.getInstanceId(), running.getValue() };
851 		logger.info("Waiting up to {} for [{}] to come online", args);
852 		Condition online = new IsOnlineCondition(this, instance.getInstanceId());
853 		WaitResult result = service.wait(waitContext, online);
854 		Object[] resultArgs = { instance.getInstanceId(), FormatUtils.getTime(result.getElapsed()) };
855 		logger.info("[{}] is now online - {}", resultArgs);
856 	}
857 
858 	protected List<String> getNames(List<KualiSecurityGroup> groups) {
859 		// Extract the names of any security groups into a list of strings
860 		List<String> names = new ArrayList<String>();
861 		for (KualiSecurityGroup group : groups) {
862 			names.add(group.getName());
863 		}
864 		Collections.sort(names);
865 		return ImmutableList.copyOf(names);
866 	}
867 
868 	/**
869 	 * Return a request that spins up exactly one instance.
870 	 */
871 	protected RunInstancesRequest getRunInstanceRequest(LaunchInstanceContext context) {
872 		RunInstancesRequest rir = new RunInstancesRequest();
873 		rir.setMaxCount(1);
874 		rir.setMinCount(1);
875 		rir.setImageId(context.getAmi());
876 		rir.setKeyName(context.getPublicKey().getName());
877 		rir.setSecurityGroups(getNames(context.getSecurityGroups()));
878 		rir.setInstanceType(context.getType());
879 		rir.setDisableApiTermination(context.isPreventTermination());
880 		rir.setEbsOptimized(context.isEbsOptimized());
881 		rir.setMonitoring(context.isEnableMonitoring());
882 
883 		// Update the request with an availability zone (if one has been supplied)
884 		if (context.getAvailabilityZone().isPresent()) {
885 			String zone = context.getAvailabilityZone().get();
886 			Placement placement = new Placement(zone);
887 			rir.setPlacement(placement);
888 		}
889 
890 		// Update the request with custom root volume settings (if any have been supplied)
891 
892 		// Get the list of block device mappings associated with this AMI after updating the BlockDeviceMapping for the root volume
893 		List<BlockDeviceMapping> mappings = getUpdatedBlockDeviceMappings(context);
894 		for (BlockDeviceMapping mapping : mappings) {
895 			EbsBlockDevice device = mapping.getEbs();
896 			if (device != null) {
897 				// This is just straight goofy.
898 				// If a snapshot id is present, the encrypted flag must be "null"
899 				// It's not good enough to set it to "false", it must be set to "null"
900 				Optional<String> snapshotId = Optional.of(device.getSnapshotId());
901 				if (snapshotId.isPresent()) {
902 					device.setEncrypted(null);
903 				}
904 			}
905 		}
906 
907 		// Store the block device mappings on the request
908 		rir.setBlockDeviceMappings(mappings);
909 		return rir;
910 	}
911 
912 	protected List<BlockDeviceMapping> getUpdatedBlockDeviceMappings(LaunchInstanceContext context) {
913 		// Get an Image object from Amazon for the AMI we are working with
914 		Image ami = getAmi(context.getAmi());
915 
916 		// Update the root volume as needed
917 		if (context.getRootVolume().isPresent()) {
918 			updateRootBlockDeviceMapping(ami, context.getRootVolume().get());
919 		}
920 
921 		// Store all of the existing block device mappings
922 		List<BlockDeviceMapping> mappings = newArrayList(ami.getBlockDeviceMappings());
923 
924 		// Cycle through the additional mappings, updating existing mappings with new mappings as we go
925 		for (BlockDeviceMapping additionalMapping : context.getAdditionalMappings()) {
926 
927 			// Look for a match in the existing mappings
928 			Optional<BlockDeviceMapping> optional = findMatch(mappings, additionalMapping);
929 
930 			// Check to see if we found a match
931 			if (optional.isPresent()) {
932 				// If so, override any existing block device settings with the new settings
933 				BlockDeviceMapping existing = optional.get();
934 				existing.setDeviceName(additionalMapping.getDeviceName());
935 				existing.setEbs(additionalMapping.getEbs());
936 				existing.setNoDevice(additionalMapping.getNoDevice());
937 				existing.setVirtualName(additionalMapping.getVirtualName());
938 			} else {
939 				// Otherwise just add the new mapping
940 				mappings.add(additionalMapping);
941 			}
942 		}
943 
944 		// Return the updated list
945 		return mappings;
946 	}
947 
948 	protected Optional<BlockDeviceMapping> findMatch(List<BlockDeviceMapping> mappings, BlockDeviceMapping mapping) {
949 		for (BlockDeviceMapping element : mappings) {
950 			if (element.getDeviceName().equals(mapping.getDeviceName())) {
951 				return Optional.of(element);
952 			}
953 		}
954 		return absent();
955 	}
956 
957 	protected void updateRootBlockDeviceMapping(Image ami, RootVolume rootVolume) {
958 
959 		// Extract the default root block device mapping specific to this AMI
960 		BlockDeviceMapping mapping = getRootBlockDeviceMapping(ami);
961 
962 		// Extract the block device
963 		EbsBlockDevice device = mapping.getEbs();
964 
965 		// If a size in gigabytes has been provided, update the device with the new size
966 		if (rootVolume.getSizeInGigabytes().isPresent()) {
967 			int sizeInGigabytes = rootVolume.getSizeInGigabytes().get();
968 			device.setVolumeSize(sizeInGigabytes);
969 		}
970 
971 		// If the delete on termination setting has been provided, update the device with the new setting
972 		if (rootVolume.getDeleteOnTermination().isPresent()) {
973 			boolean deleteOnTermination = rootVolume.getDeleteOnTermination().get();
974 			device.setDeleteOnTermination(deleteOnTermination);
975 		}
976 
977 		// If the root volume type has been provided, update the device with the new setting
978 		if (rootVolume.getType().isPresent()) {
979 			VolumeType type = rootVolume.getType().get();
980 			device.setVolumeType(type);
981 		}
982 
983 	}
984 
985 	protected BlockDeviceMapping getRootBlockDeviceMapping(Image image) {
986 		String rootDeviceName = image.getRootDeviceName();
987 		List<BlockDeviceMapping> mappings = image.getBlockDeviceMappings();
988 		for (BlockDeviceMapping mapping : mappings) {
989 			String deviceName = mapping.getDeviceName();
990 			if (rootDeviceName.equals(deviceName)) {
991 				return mapping;
992 			}
993 		}
994 		throw illegalState("Could not locate the root block device mapping for AMI [%s]", image.getImageId());
995 	}
996 
997 	public Image getAmi(String ami) {
998 		checkNotBlank(ami, "ami");
999 		DescribeImagesRequest request = new DescribeImagesRequest();
1000 		request.setImageIds(singletonList(ami));
1001 		DescribeImagesResult result = client.describeImages(request);
1002 		List<Image> images = result.getImages();
1003 		checkState(images.size() == 1, "Expected exactly 1 image but there were %s instead", images.size());
1004 		return images.get(0);
1005 	}
1006 
1007 	public EC2ServiceContext getContext() {
1008 		return context;
1009 	}
1010 
1011 	@Override
1012 	public String deleteVolume(String volumeId) {
1013 		checkNotBlank(volumeId, "volumeId");
1014 		DeleteVolumeRequest dvr = new DeleteVolumeRequest();
1015 		dvr.setVolumeId(volumeId);
1016 		// Don't bother with checking that the volume got deleted
1017 		// Unless an exception is thrown, Amazon assures us that the volume will get nuked
1018 		client.deleteVolume(dvr);
1019 		return volumeId;
1020 	}
1021 
1022 	@Override
1023 	public String attachVolume(VolumeRequest request) {
1024 		checkNotNull(request, "request");
1025 		com.amazonaws.services.ec2.model.AttachVolumeRequest avr = new com.amazonaws.services.ec2.model.AttachVolumeRequest();
1026 		avr.setInstanceId(request.getInstanceId());
1027 		avr.setVolumeId(request.getVolumeId());
1028 		avr.setDevice(request.getDevice());
1029 		client.attachVolume(avr);
1030 		return waitForAttached(request.getVolumeId(), request.getTimeout());
1031 	}
1032 
1033 	@Override
1034 	public String createVolume(String zone, int size) {
1035 		return createVolume(CreateVolumeRequest.build(zone, size));
1036 	}
1037 
1038 	@Override
1039 	public String createVolume(CreateVolumeRequest request) {
1040 		checkNotNull(request, "request");
1041 		com.amazonaws.services.ec2.model.CreateVolumeRequest cvr = new com.amazonaws.services.ec2.model.CreateVolumeRequest();
1042 		cvr.setAvailabilityZone(request.getZone());
1043 		cvr.setSize(request.getSize());
1044 		cvr.setVolumeType(request.getType());
1045 		String volumeId = client.createVolume(cvr).getVolume().getVolumeId();
1046 		return waitForVolumeState(volumeId, request.getRequiredState(), request.getTimeout());
1047 	}
1048 
1049 	/**
1050 	 * Wait until the first attachment has the desired state
1051 	 */
1052 	private TimedInterval waitForAttachmentState(String volumeId, VolumeAttachmentState desiredState, long timeout) {
1053 		Stopwatch sw = createStarted();
1054 		Volume volume = getVolume(volumeId);
1055 		List<VolumeAttachment> attachments = volume.getAttachments();
1056 		VolumeAttachment first = getSingleElement(attachments);
1057 		VolumeAttachmentState currentState = VolumeAttachmentState.fromValue(first.getState());
1058 		while (!currentState.equals(desiredState)) {
1059 			checkedWait(sw, timeout, SLEEP_MILLIS);
1060 			volume = getVolume(volumeId);
1061 			attachments = volume.getAttachments();
1062 			first = getSingleElement(attachments);
1063 			currentState = VolumeAttachmentState.fromValue(first.getState());
1064 		}
1065 		return TimedInterval.build(sw);
1066 	}
1067 
1068 	/**
1069 	 * Wait for 3 things:<br>
1070 	 * 
1071 	 * 1 - volume.getAttachments() to return a non-empty list<br>
1072 	 * 2 - The first VolumeAttachment in volume.getAttachments() to have a state of "attached"<br>
1073 	 * 3 - The volume to have a state of "in-use"<br>
1074 	 */
1075 	private String waitForAttached(String volumeId, long timeout) {
1076 		checkNotBlank(volumeId, "volumeId");
1077 		checkMin(timeout, 1, "timeout");
1078 		TimedInterval timing1 = waitForAttachments(volumeId, timeout);
1079 		TimedInterval timing2 = waitForAttachmentState(volumeId, Attached, timeout - timing1.getElapsed());
1080 		return waitForVolumeState(volumeId, InUse, timeout - (timing1.getElapsed() + timing2.getElapsed()));
1081 	}
1082 
1083 	/**
1084 	 * Wait until volume.getAttachments() has at least one element
1085 	 */
1086 	private TimedInterval waitForAttachments(String volumeId, long timeout) {
1087 		Stopwatch sw = createStarted();
1088 		boolean attached = isAttached(volumeId);
1089 		while (!attached) {
1090 			checkedWait(sw, timeout, SLEEP_MILLIS);
1091 			attached = isAttached(volumeId);
1092 		}
1093 		return TimedInterval.build(sw);
1094 	}
1095 
1096 }