1 package org.kuali.maven.ec2;
2
3 import java.util.ArrayList;
4 import java.util.Collection;
5 import java.util.Collections;
6 import java.util.List;
7 import java.util.Properties;
8
9 import org.apache.commons.lang.StringUtils;
10 import org.kuali.maven.ec2.pojo.ImageComparator;
11 import org.kuali.maven.ec2.state.ImageStateRetriever;
12 import org.kuali.maven.ec2.state.InstanceStateRetriever;
13 import org.kuali.maven.ec2.state.SnapshotStateRetriever;
14 import org.kuali.maven.ec2.state.StateRetriever;
15 import org.slf4j.Logger;
16 import org.slf4j.LoggerFactory;
17
18 import com.amazonaws.auth.AWSCredentials;
19 import com.amazonaws.auth.BasicAWSCredentials;
20 import com.amazonaws.services.ec2.AmazonEC2Client;
21 import com.amazonaws.services.ec2.model.BlockDeviceMapping;
22 import com.amazonaws.services.ec2.model.CreateSnapshotRequest;
23 import com.amazonaws.services.ec2.model.CreateSnapshotResult;
24 import com.amazonaws.services.ec2.model.CreateTagsRequest;
25 import com.amazonaws.services.ec2.model.DeleteSnapshotRequest;
26 import com.amazonaws.services.ec2.model.DeregisterImageRequest;
27 import com.amazonaws.services.ec2.model.DescribeImagesRequest;
28 import com.amazonaws.services.ec2.model.DescribeImagesResult;
29 import com.amazonaws.services.ec2.model.DescribeInstancesRequest;
30 import com.amazonaws.services.ec2.model.DescribeInstancesResult;
31 import com.amazonaws.services.ec2.model.DescribeSnapshotsRequest;
32 import com.amazonaws.services.ec2.model.DescribeSnapshotsResult;
33 import com.amazonaws.services.ec2.model.EbsBlockDevice;
34 import com.amazonaws.services.ec2.model.Filter;
35 import com.amazonaws.services.ec2.model.Image;
36 import com.amazonaws.services.ec2.model.Instance;
37 import com.amazonaws.services.ec2.model.RegisterImageRequest;
38 import com.amazonaws.services.ec2.model.RegisterImageResult;
39 import com.amazonaws.services.ec2.model.Reservation;
40 import com.amazonaws.services.ec2.model.RunInstancesRequest;
41 import com.amazonaws.services.ec2.model.RunInstancesResult;
42 import com.amazonaws.services.ec2.model.Snapshot;
43 import com.amazonaws.services.ec2.model.Tag;
44 import com.amazonaws.services.ec2.model.TerminateInstancesRequest;
45
46 public class EC2Utils {
47
48 private static final Logger logger = LoggerFactory.getLogger(EC2Utils.class);
49
50 AmazonEC2Client client;
51
52 private EC2Utils(AWSCredentials credentials) {
53 this.client = new AmazonEC2Client(credentials);
54 }
55
56 public static EC2Utils getInstance(String accessKey, String secretKey) {
57 AWSCredentials credentials = getCredentials(accessKey, secretKey);
58 return getInstance(credentials);
59 }
60
61 public static EC2Utils getInstance(AWSCredentials credentials) {
62 return new EC2Utils(credentials);
63 }
64
65 public static EC2Utils getImage(String accessKey, String secretKey) {
66 AWSCredentials credentials = getCredentials(accessKey, secretKey);
67 return getImage(credentials);
68 }
69
70 public static EC2Utils getImage(AWSCredentials credentials) {
71 return new EC2Utils(credentials);
72 }
73
74 public void cleanupSlaveImages(String key, String prefix, String device, int min) {
75
76 if (min < 1) {
77 min = 1;
78 }
79 List<Image> images = getEC2ImagesOwnedByMe();
80 Collections.sort(images, new ImageComparator());
81 List<SlaveTag> slaveTags = new ArrayList<SlaveTag>();
82 for (Image image : images) {
83 if (containsTag(image.getTags(), key, prefix)) {
84 Tag tag = getTag(image.getTags(), key, prefix);
85 SlaveTag slaveTag = getSlaveTag(image, tag, device);
86 slaveTags.add(slaveTag);
87 }
88 }
89 int size = slaveTags.size();
90 if (size <= min) {
91 logger.info("Retaining all slave images since there are only " + size + " and " + min + " must be retained");
92 return;
93 }
94 Collections.sort(slaveTags);
95 Collections.reverse(slaveTags);
96 List<SlaveTag> delete = new ArrayList<SlaveTag>();
97 for (int i = min; i < size; i++) {
98 delete.add(slaveTags.get(i));
99 }
100 logger.info("Retaining " + min + " slave images");
101 logger.info("Deleting " + delete.size() + " slave images");
102 for (SlaveTag st : delete) {
103 logger.info("Deleting " + st.getSequence() + " - " + st.getImageId() + " - " + st.getSnapshotId());
104 deRegisterImage(st.getImageId());
105 deleteSnapshot(st.getSnapshotId());
106 }
107 }
108
109 public boolean containsTag(List<Tag> tags, String key, String prefix) {
110 for (Tag tag : tags) {
111 if (matches(tag, key, prefix)) {
112 return true;
113 }
114 }
115 return false;
116 }
117
118 public String getSnapshotId(Image image, String deviceName) {
119 List<BlockDeviceMapping> mappings = image.getBlockDeviceMappings();
120 for (BlockDeviceMapping mapping : mappings) {
121 if (deviceName.equals(mapping.getDeviceName())) {
122 EbsBlockDevice ebd = mapping.getEbs();
123 return ebd.getSnapshotId();
124 }
125 }
126 return null;
127 }
128
129 public SlaveTag getSlaveTag(Image image, Tag tag, String device) {
130 String[] tokens = StringUtils.splitByWholeSeparator(tag.getValue(), " - ");
131 String snapshotId = getSnapshotId(image, device);
132
133 SlaveTag slaveTag = new SlaveTag();
134 slaveTag.setImageId(image.getImageId());
135 slaveTag.setKey(tag.getKey());
136 slaveTag.setLabel(tokens[0]);
137 slaveTag.setDate(tokens[1]);
138 slaveTag.setSequence(new Integer(tokens[2]));
139 slaveTag.setSnapshotId(snapshotId);
140 return slaveTag;
141 }
142
143 public boolean matches(Tag tag, String key, String prefix) {
144 return key.equals(tag.getKey()) && tag.getValue().startsWith(prefix);
145 }
146
147 public Tag getTag(List<Tag> tags, String key, String prefix) {
148 for (Tag tag : tags) {
149 if (matches(tag, key, prefix)) {
150 return tag;
151 }
152 }
153 return null;
154 }
155
156 public Image getImage(String imageId) {
157 DescribeImagesRequest request = new DescribeImagesRequest();
158 request.setImageIds(Collections.singletonList(imageId));
159 DescribeImagesResult result = client.describeImages(request);
160 List<Image> images = result.getImages();
161 if (isEmpty(images)) {
162 throw new IllegalArgumentException("Unable to locate '" + imageId + "'");
163 }
164 if (images.size() > 1) {
165 throw new IllegalArgumentException("Found " + images.size() + " matching '" + imageId + "'");
166 }
167
168 return images.get(0);
169 }
170
171 public RegisterImageResult registerImage(RegisterImageRequest request, WaitControl wc) {
172 RegisterImageResult result = client.registerImage(request);
173 if (wc.isWait()) {
174 String id = result.getImageId();
175 int timeout = wc.getTimeout();
176 StateRetriever sr = new ImageStateRetriever(this, id);
177 logger.info("Waiting up to " + timeout + " seconds for '" + id + "' to reach state '" + wc.getState() + "'");
178 waitForState(sr, wc);
179 } else {
180 logger.info("Created image " + result.getImageId());
181 }
182 return result;
183 }
184
185 public void terminate(String instanceId, WaitControl wc) {
186 TerminateInstancesRequest request = new TerminateInstancesRequest();
187 request.setInstanceIds(Collections.singletonList(instanceId));
188 client.terminateInstances(request);
189 if (wc.isWait()) {
190 StateRetriever sr = new InstanceStateRetriever(this, instanceId);
191 logger.info("Waiting up to " + wc.getTimeout() + " seconds for " + instanceId + " to terminate");
192 waitForState(sr, wc);
193 } else {
194 logger.info("Terminated " + instanceId);
195 }
196 }
197
198 public Instance wait(Instance i, WaitControl wc, Properties props) {
199 if (wc.isWait()) {
200 StateRetriever sr = new InstanceStateRetriever(this, i.getInstanceId());
201 logger.info("Waiting up to " + wc.getTimeout() + " seconds for " + i.getInstanceId() + " to start");
202 waitForState(sr, wc);
203 Instance running = getEC2Instance(i.getInstanceId());
204 String id = i.getInstanceId();
205 String dns = running.getPublicDnsName();
206 String name = getTagValue(running, "Name");
207 logger.info("EC2 Instance: " + name + " (" + id + ") " + dns);
208 props.setProperty("ec2.instance.dns", running.getPublicDnsName());
209 return running;
210 } else {
211 logger.info("Launched " + i.getInstanceId());
212 return i;
213 }
214 }
215
216 public void createTags(Instance instance, List<Tag> tags) {
217 if (isEmpty(tags)) {
218 return;
219 }
220 CreateTagsRequest request = new CreateTagsRequest();
221 request.setResources(Collections.singletonList(instance.getInstanceId()));
222 request.setTags(tags);
223 client.createTags(request);
224 }
225
226 public Instance getSingleEC2Instance(RunInstancesRequest request) {
227 RunInstancesResult result = client.runInstances(request);
228 Reservation r = result.getReservation();
229 List<Instance> instances = r.getInstances();
230 return instances.get(0);
231 }
232
233 protected Filter getFilterFromTag(String tag, String value) {
234 Filter filter = new Filter();
235 filter.setName("tag:" + tag);
236 filter.setValues(Collections.singletonList(value));
237 return filter;
238 }
239
240 protected DescribeInstancesRequest getDescribeInstancesRequest(Tag tag) {
241 DescribeInstancesRequest request = new DescribeInstancesRequest();
242 Filter filter = getFilterFromTag(tag.getKey(), tag.getValue());
243 request.setFilters(Collections.singletonList(filter));
244 return request;
245 }
246
247 protected int validate(List<Instance> instances, Tag tag, boolean failIfNotFound) {
248 int size = instances.size();
249 String msg = tag.getKey() + "=" + tag.getValue() + " matched " + size + " instances";
250 if (size == 1) {
251 return size;
252 }
253 if (size > 1) {
254 throw new IllegalStateException(msg);
255 }
256
257 if (failIfNotFound) {
258 throw new IllegalStateException(msg);
259 } else {
260 logger.info(msg);
261 }
262 return size;
263 }
264
265 public Instance findInstanceFromTag(Tag tag, boolean failIfNotFound) {
266 DescribeInstancesRequest request = getDescribeInstancesRequest(tag);
267 DescribeInstancesResult result = client.describeInstances(request);
268 List<Instance> instances = getAllInstances(result.getReservations());
269 int size = validate(instances, tag, failIfNotFound);
270 if (size == 1) {
271 return instances.get(0);
272 } else {
273 return null;
274 }
275 }
276
277 public List<Instance> getEC2Instances() {
278 return getEC2Instances(Collections.<String> emptyList());
279 }
280
281 public List<Instance> getEC2Instances(List<String> instanceIds) {
282 DescribeInstancesRequest request = new DescribeInstancesRequest();
283 request.setInstanceIds(instanceIds);
284 DescribeInstancesResult result = client.describeInstances(request);
285 return getAllInstances(result.getReservations());
286 }
287
288 public List<Image> getEC2Images(List<String> imageIds) {
289 DescribeImagesRequest request = new DescribeImagesRequest();
290 request.setImageIds(imageIds);
291 DescribeImagesResult result = client.describeImages(request);
292 return result.getImages();
293 }
294
295 public List<Image> getEC2ImagesOwnedByMe() {
296 DescribeImagesRequest request = new DescribeImagesRequest();
297 request.withOwners("self");
298 DescribeImagesResult result = client.describeImages(request);
299
300 return result.getImages();
301 }
302
303 public void deleteSnapshot(String snapshotId) {
304 DeleteSnapshotRequest request = new DeleteSnapshotRequest();
305 request.setSnapshotId(snapshotId);
306 client.deleteSnapshot(request);
307 }
308
309 public void deRegisterImage(String ImageId) {
310 DeregisterImageRequest request = new DeregisterImageRequest();
311 request.setImageId(ImageId);
312 client.deregisterImage(request);
313 }
314
315 public List<Snapshot> getEC2SnapshotsbyTag(Tag tag) {
316 DescribeSnapshotsRequest request = new DescribeSnapshotsRequest();
317 Filter filter = getFilterFromTag(tag.getKey(), tag.getValue());
318 request.setFilters(Collections.singletonList(filter));
319 DescribeSnapshotsResult result = client.describeSnapshots(request);
320 return result.getSnapshots();
321 }
322
323 public Snapshot createSnapshot(String volumeId, String description, WaitControl wc) {
324 CreateSnapshotRequest request = new CreateSnapshotRequest(volumeId, description);
325 CreateSnapshotResult result = client.createSnapshot(request);
326 String id = result.getSnapshot().getSnapshotId();
327 if (wc.isWait()) {
328 StateRetriever sr = new SnapshotStateRetriever(this, id);
329 logger.info("Waiting up to " + wc.getTimeout() + " seconds for snapshot '" + id + "' to complete");
330 waitForState(sr, wc);
331 } else {
332 logger.info("Completed " + id);
333 }
334 return result.getSnapshot();
335 }
336
337 public void tag(String id, String name, String value) {
338 tag(id, new Tag(name, value));
339 }
340
341 public void tag(String id, Tag tag) {
342 tag(id, Collections.singletonList(tag));
343 }
344
345
346
347
348
349 public void tag(String id, List<Tag> tags) {
350 if (isEmpty(tags)) {
351 return;
352 }
353 CreateTagsRequest request = new CreateTagsRequest();
354 request.setResources(Collections.singletonList(id));
355 request.setTags(tags);
356 client.createTags(request);
357 logger.info("Tagged '" + id + "' with " + tags.size() + " tags");
358 }
359
360 public static AWSCredentials getCredentials(String accessKey, String secretKey) {
361 return new BasicAWSCredentials(accessKey, secretKey);
362 }
363
364 public static AmazonEC2Client getEC2Client(String accessKey, String secretKey) {
365 AWSCredentials credentials = getCredentials(accessKey, secretKey);
366 return new AmazonEC2Client(credentials);
367 }
368
369 public String getTagValue(Instance i, String tag) {
370 return getTagValue(i.getTags(), tag);
371 }
372
373 public String getTagValue(Image i, String tag) {
374 return getTagValue(i.getTags(), tag);
375 }
376
377 public String getTagValue(List<Tag> tags, String tag) {
378 for (Tag t : tags) {
379 if (t.getKey().equals(tag)) {
380 return t.getValue();
381 }
382 }
383 return "";
384 }
385
386 public void waitForState(StateRetriever retriever, WaitControl wc) {
387 long now = System.currentTimeMillis();
388 long timeout = now + wc.getTimeout() * 1000;
389
390
391 sleep(wc.getInitialPause());
392 while (true) {
393 now = System.currentTimeMillis();
394 if (now > timeout) {
395 throw new IllegalStateException("Timed out waiting for state '" + wc.getState() + "'");
396 }
397 long remaining = (timeout - now) / 1000;
398 String newState = retriever.getState();
399 if (newState.equals(wc.getState())) {
400 logger.info("Success!!! state=" + newState);
401 break;
402 } else {
403 logger.info(newState + " - " + remaining + "s");
404 sleep(wc.getSleep());
405 }
406 }
407 }
408
409 public static final void sleep(int millis) {
410 try {
411 Thread.sleep(millis);
412 } catch (InterruptedException e) {
413 throw new IllegalStateException(e);
414 }
415 }
416
417 public List<Instance> getAllInstances(List<Reservation> reservations) {
418 List<Instance> instances = new ArrayList<Instance>();
419 for (Reservation r : reservations) {
420 instances.addAll(r.getInstances());
421 }
422 return instances;
423 }
424
425 public Snapshot getSnapshot(String snapshotId) {
426 DescribeSnapshotsRequest request = new DescribeSnapshotsRequest();
427 request.setSnapshotIds(Collections.singletonList(snapshotId));
428 DescribeSnapshotsResult result = client.describeSnapshots(request);
429 List<Snapshot> snapshots = result.getSnapshots();
430 return snapshots.get(0);
431 }
432
433 public Instance getEC2Instance(String instanceId) {
434 DescribeInstancesRequest request = new DescribeInstancesRequest();
435 request.setInstanceIds(Collections.singletonList(instanceId));
436 DescribeInstancesResult result = client.describeInstances(request);
437 List<Reservation> reservations = result.getReservations();
438 Reservation r = reservations.get(0);
439 List<Instance> instances = r.getInstances();
440 return instances.get(0);
441 }
442
443 public static final boolean isEmpty(Collection<?> c) {
444 return c == null || c.size() == 0;
445 }
446
447 }